1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright 1993 by Theodore Ts'o.
4 */
5 #include <linux/module.h>
6 #include <linux/moduleparam.h>
7 #include <linux/sched.h>
8 #include <linux/fs.h>
9 #include <linux/pagemap.h>
10 #include <linux/file.h>
11 #include <linux/stat.h>
12 #include <linux/errno.h>
13 #include <linux/major.h>
14 #include <linux/wait.h>
15 #include <linux/blkpg.h>
16 #include <linux/init.h>
17 #include <linux/swap.h>
18 #include <linux/slab.h>
19 #include <linux/compat.h>
20 #include <linux/suspend.h>
21 #include <linux/freezer.h>
22 #include <linux/mutex.h>
23 #include <linux/writeback.h>
24 #include <linux/completion.h>
25 #include <linux/highmem.h>
26 #include <linux/splice.h>
27 #include <linux/sysfs.h>
28 #include <linux/miscdevice.h>
29 #include <linux/falloc.h>
30 #include <linux/uio.h>
31 #include <linux/ioprio.h>
32 #include <linux/blk-cgroup.h>
33 #include <linux/sched/mm.h>
34 #include <linux/statfs.h>
35 #include <linux/uaccess.h>
36 #include <linux/blk-mq.h>
37 #include <linux/spinlock.h>
38 #include <uapi/linux/loop.h>
39
40 /* Possible states of device */
41 enum {
42 Lo_unbound,
43 Lo_bound,
44 Lo_rundown,
45 Lo_deleting,
46 };
47
48 struct loop_device {
49 int lo_number;
50 loff_t lo_offset;
51 loff_t lo_sizelimit;
52 int lo_flags;
53 char lo_file_name[LO_NAME_SIZE];
54
55 struct file *lo_backing_file;
56 unsigned int lo_min_dio_size;
57 unsigned int lo_dio_mem_align;
58 struct block_device *lo_device;
59
60 gfp_t old_gfp_mask;
61
62 spinlock_t lo_lock;
63 int lo_state;
64 spinlock_t lo_work_lock;
65 struct workqueue_struct *workqueue;
66 struct work_struct rootcg_work;
67 struct list_head rootcg_cmd_list;
68 struct list_head idle_worker_list;
69 struct rb_root worker_tree;
70 struct timer_list timer;
71 bool sysfs_inited;
72
73 struct request_queue *lo_queue;
74 struct blk_mq_tag_set tag_set;
75 struct gendisk *lo_disk;
76 struct mutex lo_mutex;
77 bool idr_visible;
78 };
79
80 struct loop_cmd {
81 struct list_head list_entry;
82 bool use_aio; /* use AIO interface to handle I/O */
83 atomic_t ref; /* only for aio */
84 long ret;
85 struct kiocb iocb;
86 struct bio_vec *bvec;
87 struct cgroup_subsys_state *blkcg_css;
88 struct cgroup_subsys_state *memcg_css;
89 };
90
91 #define LOOP_IDLE_WORKER_TIMEOUT (60 * HZ)
92 #define LOOP_DEFAULT_HW_Q_DEPTH 128
93
94 static DEFINE_IDR(loop_index_idr);
95 static DEFINE_MUTEX(loop_ctl_mutex);
96 static DEFINE_MUTEX(loop_validate_mutex);
97
98 /**
99 * loop_global_lock_killable() - take locks for safe loop_validate_file() test
100 *
101 * @lo: struct loop_device
102 * @global: true if @lo is about to bind another "struct loop_device", false otherwise
103 *
104 * Returns 0 on success, -EINTR otherwise.
105 *
106 * Since loop_validate_file() traverses on other "struct loop_device" if
107 * is_loop_device() is true, we need a global lock for serializing concurrent
108 * loop_configure()/loop_change_fd()/__loop_clr_fd() calls.
109 */
loop_global_lock_killable(struct loop_device * lo,bool global)110 static int loop_global_lock_killable(struct loop_device *lo, bool global)
111 {
112 int err;
113
114 if (global) {
115 err = mutex_lock_killable(&loop_validate_mutex);
116 if (err)
117 return err;
118 }
119 err = mutex_lock_killable(&lo->lo_mutex);
120 if (err && global)
121 mutex_unlock(&loop_validate_mutex);
122 return err;
123 }
124
125 /**
126 * loop_global_unlock() - release locks taken by loop_global_lock_killable()
127 *
128 * @lo: struct loop_device
129 * @global: true if @lo was about to bind another "struct loop_device", false otherwise
130 */
loop_global_unlock(struct loop_device * lo,bool global)131 static void loop_global_unlock(struct loop_device *lo, bool global)
132 {
133 mutex_unlock(&lo->lo_mutex);
134 if (global)
135 mutex_unlock(&loop_validate_mutex);
136 }
137
138 static int max_part;
139 static int part_shift;
140
lo_calculate_size(struct loop_device * lo,struct file * file)141 static loff_t lo_calculate_size(struct loop_device *lo, struct file *file)
142 {
143 loff_t loopsize;
144 int ret;
145
146 if (S_ISBLK(file_inode(file)->i_mode)) {
147 loopsize = i_size_read(file->f_mapping->host);
148 } else {
149 struct kstat stat;
150
151 /*
152 * Get the accurate file size. This provides better results than
153 * cached inode data, particularly for network filesystems where
154 * metadata may be stale.
155 */
156 ret = vfs_getattr_nosec(&file->f_path, &stat, STATX_SIZE, 0);
157 if (ret)
158 return 0;
159
160 loopsize = stat.size;
161 }
162
163 if (lo->lo_offset > 0)
164 loopsize -= lo->lo_offset;
165 /* offset is beyond i_size, weird but possible */
166 if (loopsize < 0)
167 return 0;
168 if (lo->lo_sizelimit > 0 && lo->lo_sizelimit < loopsize)
169 loopsize = lo->lo_sizelimit;
170 /*
171 * Unfortunately, if we want to do I/O on the device,
172 * the number of 512-byte sectors has to fit into a sector_t.
173 */
174 return loopsize >> 9;
175 }
176
177 /*
178 * We support direct I/O only if lo_offset is aligned with the logical I/O size
179 * of backing device, and the logical block size of loop is bigger than that of
180 * the backing device.
181 */
lo_can_use_dio(struct loop_device * lo)182 static bool lo_can_use_dio(struct loop_device *lo)
183 {
184 if (!(lo->lo_backing_file->f_mode & FMODE_CAN_ODIRECT))
185 return false;
186 if (queue_logical_block_size(lo->lo_queue) < lo->lo_min_dio_size)
187 return false;
188 if (lo->lo_offset & (lo->lo_min_dio_size - 1))
189 return false;
190 return true;
191 }
192
193 /*
194 * Direct I/O can be enabled either by using an O_DIRECT file descriptor, or by
195 * passing in the LO_FLAGS_DIRECT_IO flag from userspace. It will be silently
196 * disabled when the device block size is too small or the offset is unaligned.
197 *
198 * loop_get_status will always report the effective LO_FLAGS_DIRECT_IO flag and
199 * not the originally passed in one.
200 */
loop_update_dio(struct loop_device * lo)201 static inline void loop_update_dio(struct loop_device *lo)
202 {
203 lockdep_assert_held(&lo->lo_mutex);
204 WARN_ON_ONCE(lo->lo_state == Lo_bound &&
205 lo->lo_queue->mq_freeze_depth == 0);
206
207 if ((lo->lo_flags & LO_FLAGS_DIRECT_IO) && !lo_can_use_dio(lo))
208 lo->lo_flags &= ~LO_FLAGS_DIRECT_IO;
209 }
210
211 /**
212 * loop_set_size() - sets device size and notifies userspace
213 * @lo: struct loop_device to set the size for
214 * @size: new size of the loop device
215 *
216 * Callers must validate that the size passed into this function fits into
217 * a sector_t, eg using loop_validate_size()
218 */
loop_set_size(struct loop_device * lo,loff_t size)219 static void loop_set_size(struct loop_device *lo, loff_t size)
220 {
221 if (!set_capacity_and_notify(lo->lo_disk, size))
222 kobject_uevent(&disk_to_dev(lo->lo_disk)->kobj, KOBJ_CHANGE);
223 }
224
loop_clear_limits(struct loop_device * lo,int mode)225 static void loop_clear_limits(struct loop_device *lo, int mode)
226 {
227 struct queue_limits lim = queue_limits_start_update(lo->lo_queue);
228
229 if (mode & FALLOC_FL_ZERO_RANGE)
230 lim.max_write_zeroes_sectors = 0;
231
232 if (mode & FALLOC_FL_PUNCH_HOLE) {
233 lim.max_hw_discard_sectors = 0;
234 lim.discard_granularity = 0;
235 }
236
237 /*
238 * XXX: this updates the queue limits without freezing the queue, which
239 * is against the locking protocol and dangerous. But we can't just
240 * freeze the queue as we're inside the ->queue_rq method here. So this
241 * should move out into a workqueue unless we get the file operations to
242 * advertise if they support specific fallocate operations.
243 */
244 queue_limits_commit_update(lo->lo_queue, &lim);
245 }
246
lo_fallocate(struct loop_device * lo,struct request * rq,loff_t pos,int mode)247 static int lo_fallocate(struct loop_device *lo, struct request *rq, loff_t pos,
248 int mode)
249 {
250 /*
251 * We use fallocate to manipulate the space mappings used by the image
252 * a.k.a. discard/zerorange.
253 */
254 struct file *file = lo->lo_backing_file;
255 int ret;
256
257 mode |= FALLOC_FL_KEEP_SIZE;
258
259 if (!bdev_max_discard_sectors(lo->lo_device))
260 return -EOPNOTSUPP;
261
262 ret = file->f_op->fallocate(file, mode, pos, blk_rq_bytes(rq));
263 if (unlikely(ret && ret != -EINVAL && ret != -EOPNOTSUPP))
264 return -EIO;
265
266 /*
267 * We initially configure the limits in a hope that fallocate is
268 * supported and clear them here if that turns out not to be true.
269 */
270 if (unlikely(ret == -EOPNOTSUPP))
271 loop_clear_limits(lo, mode);
272
273 return ret;
274 }
275
lo_req_flush(struct loop_device * lo,struct request * rq)276 static int lo_req_flush(struct loop_device *lo, struct request *rq)
277 {
278 int ret = vfs_fsync(lo->lo_backing_file, 0);
279 if (unlikely(ret && ret != -EINVAL))
280 ret = -EIO;
281
282 return ret;
283 }
284
lo_complete_rq(struct request * rq)285 static void lo_complete_rq(struct request *rq)
286 {
287 struct loop_cmd *cmd = blk_mq_rq_to_pdu(rq);
288 blk_status_t ret = BLK_STS_OK;
289
290 if (cmd->ret < 0 || cmd->ret == blk_rq_bytes(rq) ||
291 req_op(rq) != REQ_OP_READ) {
292 if (cmd->ret < 0)
293 ret = errno_to_blk_status(cmd->ret);
294 goto end_io;
295 }
296
297 /*
298 * Short READ - if we got some data, advance our request and
299 * retry it. If we got no data, end the rest with EIO.
300 */
301 if (cmd->ret) {
302 blk_update_request(rq, BLK_STS_OK, cmd->ret);
303 cmd->ret = 0;
304 blk_mq_requeue_request(rq, true);
305 } else {
306 struct bio *bio = rq->bio;
307
308 while (bio) {
309 zero_fill_bio(bio);
310 bio = bio->bi_next;
311 }
312
313 ret = BLK_STS_IOERR;
314 end_io:
315 blk_mq_end_request(rq, ret);
316 }
317 }
318
lo_rw_aio_do_completion(struct loop_cmd * cmd)319 static void lo_rw_aio_do_completion(struct loop_cmd *cmd)
320 {
321 struct request *rq = blk_mq_rq_from_pdu(cmd);
322
323 if (!atomic_dec_and_test(&cmd->ref))
324 return;
325 kfree(cmd->bvec);
326 cmd->bvec = NULL;
327 if (req_op(rq) == REQ_OP_WRITE)
328 kiocb_end_write(&cmd->iocb);
329 if (likely(!blk_should_fake_timeout(rq->q)))
330 blk_mq_complete_request(rq);
331 }
332
lo_rw_aio_complete(struct kiocb * iocb,long ret)333 static void lo_rw_aio_complete(struct kiocb *iocb, long ret)
334 {
335 struct loop_cmd *cmd = container_of(iocb, struct loop_cmd, iocb);
336
337 cmd->ret = ret;
338 lo_rw_aio_do_completion(cmd);
339 }
340
lo_rw_aio(struct loop_device * lo,struct loop_cmd * cmd,loff_t pos,int rw)341 static int lo_rw_aio(struct loop_device *lo, struct loop_cmd *cmd,
342 loff_t pos, int rw)
343 {
344 struct iov_iter iter;
345 struct req_iterator rq_iter;
346 struct request *rq = blk_mq_rq_from_pdu(cmd);
347 struct file *file = lo->lo_backing_file;
348 unsigned int nr_bvec;
349 int ret;
350
351 nr_bvec = blk_rq_nr_bvec(rq);
352
353 if (rq->bio != rq->biotail) {
354 struct bio_vec tmp, *bvec;
355
356 cmd->bvec = kmalloc_objs(*cmd->bvec, nr_bvec, GFP_NOIO);
357 if (!cmd->bvec)
358 return -EIO;
359
360 /*
361 * The bios of the request may be started from the middle of
362 * the 'bvec' because of bio splitting, so we can't directly
363 * copy bio->bi_iov_vec to new bvec. The rq_for_each_bvec
364 * API will take care of all details for us.
365 */
366 bvec = cmd->bvec;
367 rq_for_each_bvec(tmp, rq, rq_iter) {
368 *bvec = tmp;
369 bvec++;
370 }
371 iov_iter_bvec(&iter, rw, cmd->bvec, nr_bvec, blk_rq_bytes(rq));
372 iter.iov_offset = 0;
373 } else {
374 /*
375 * Same here, this bio may be started from the middle of the
376 * 'bvec' because of bio splitting, so offset from the bvec
377 * must be passed to iov iterator
378 */
379 iov_iter_bvec(&iter, rw,
380 __bvec_iter_bvec(rq->bio->bi_io_vec, rq->bio->bi_iter),
381 nr_bvec, blk_rq_bytes(rq));
382 iter.iov_offset = rq->bio->bi_iter.bi_offset;
383 }
384 atomic_set(&cmd->ref, 2);
385
386 cmd->iocb.ki_pos = pos;
387 cmd->iocb.ki_filp = file;
388 cmd->iocb.ki_ioprio = req_get_ioprio(rq);
389 if (cmd->use_aio) {
390 cmd->iocb.ki_complete = lo_rw_aio_complete;
391 cmd->iocb.ki_flags = IOCB_DIRECT;
392 } else {
393 cmd->iocb.ki_complete = NULL;
394 cmd->iocb.ki_flags = 0;
395 }
396
397 if (rw == ITER_SOURCE) {
398 kiocb_start_write(&cmd->iocb);
399 ret = file->f_op->write_iter(&cmd->iocb, &iter);
400 } else
401 ret = file->f_op->read_iter(&cmd->iocb, &iter);
402
403 lo_rw_aio_do_completion(cmd);
404
405 if (ret != -EIOCBQUEUED)
406 lo_rw_aio_complete(&cmd->iocb, ret);
407 return -EIOCBQUEUED;
408 }
409
do_req_filebacked(struct loop_device * lo,struct request * rq)410 static int do_req_filebacked(struct loop_device *lo, struct request *rq)
411 {
412 struct loop_cmd *cmd = blk_mq_rq_to_pdu(rq);
413 loff_t pos = ((loff_t) blk_rq_pos(rq) << 9) + lo->lo_offset;
414
415 switch (req_op(rq)) {
416 case REQ_OP_FLUSH:
417 return lo_req_flush(lo, rq);
418 case REQ_OP_WRITE_ZEROES:
419 /*
420 * If the caller doesn't want deallocation, call zeroout to
421 * write zeroes the range. Otherwise, punch them out.
422 */
423 return lo_fallocate(lo, rq, pos,
424 (rq->cmd_flags & REQ_NOUNMAP) ?
425 FALLOC_FL_ZERO_RANGE :
426 FALLOC_FL_PUNCH_HOLE);
427 case REQ_OP_DISCARD:
428 return lo_fallocate(lo, rq, pos, FALLOC_FL_PUNCH_HOLE);
429 case REQ_OP_WRITE:
430 return lo_rw_aio(lo, cmd, pos, ITER_SOURCE);
431 case REQ_OP_READ:
432 return lo_rw_aio(lo, cmd, pos, ITER_DEST);
433 default:
434 WARN_ON_ONCE(1);
435 return -EIO;
436 }
437 }
438
loop_reread_partitions(struct loop_device * lo)439 static void loop_reread_partitions(struct loop_device *lo)
440 {
441 int rc;
442
443 mutex_lock(&lo->lo_disk->open_mutex);
444 rc = bdev_disk_changed(lo->lo_disk, false);
445 mutex_unlock(&lo->lo_disk->open_mutex);
446 if (rc)
447 pr_warn("%s: partition scan of loop%d (%s) failed (rc=%d)\n",
448 __func__, lo->lo_number, lo->lo_file_name, rc);
449 }
450
loop_update_dio_alignment(struct loop_device * lo)451 static void loop_update_dio_alignment(struct loop_device *lo)
452 {
453 struct file *file = lo->lo_backing_file;
454 struct block_device *sb_bdev = file->f_mapping->host->i_sb->s_bdev;
455 struct kstat st;
456
457 /*
458 * Use the dio alignment of the file system if provided. The incomoing
459 * request's bio_vec is forwarded to the backing file unchanged, so its
460 * required memory alignment becomes the device's dma_alignment when
461 * used for direct-io. The file system reports zeroed alignments if the
462 * file can't be used for direct-io at all, so fall back to the block
463 * device limits in that case.
464 */
465 if (!vfs_getattr(&file->f_path, &st, STATX_DIOALIGN, 0) &&
466 (st.result_mask & STATX_DIOALIGN) && st.dio_mem_align) {
467 lo->lo_min_dio_size = st.dio_offset_align;
468 lo->lo_dio_mem_align = min(st.dio_mem_align - 1, PAGE_SIZE - 1);
469 return;
470 }
471
472 /*
473 * In a perfect world this wouldn't be needed, but as of Linux 6.13 only
474 * a handful of file systems support the STATX_DIOALIGN flag.
475 */
476 if (sb_bdev) {
477 lo->lo_min_dio_size = bdev_logical_block_size(sb_bdev);
478 lo->lo_dio_mem_align = bdev_dma_alignment(sb_bdev);
479 return;
480 }
481
482 lo->lo_min_dio_size = SECTOR_SIZE;
483 lo->lo_dio_mem_align = SECTOR_SIZE - 1;
484 }
485
is_loop_device(struct file * file)486 static inline int is_loop_device(struct file *file)
487 {
488 struct inode *i = file->f_mapping->host;
489
490 return i && S_ISBLK(i->i_mode) && imajor(i) == LOOP_MAJOR;
491 }
492
loop_validate_file(struct file * file,struct block_device * bdev)493 static int loop_validate_file(struct file *file, struct block_device *bdev)
494 {
495 struct inode *inode = file->f_mapping->host;
496 struct file *f = file;
497
498 /* Avoid recursion */
499 while (is_loop_device(f)) {
500 struct loop_device *l;
501
502 lockdep_assert_held(&loop_validate_mutex);
503 if (f->f_mapping->host->i_rdev == bdev->bd_dev)
504 return -EBADF;
505
506 l = I_BDEV(f->f_mapping->host)->bd_disk->private_data;
507 if (l->lo_state != Lo_bound)
508 return -EINVAL;
509 /* Order wrt setting lo->lo_backing_file in loop_configure(). */
510 rmb();
511 f = l->lo_backing_file;
512 }
513 if (!S_ISREG(inode->i_mode) && !S_ISBLK(inode->i_mode))
514 return -EINVAL;
515 return 0;
516 }
517
loop_assign_backing_file(struct loop_device * lo,struct file * file)518 static void loop_assign_backing_file(struct loop_device *lo, struct file *file)
519 {
520 lo->lo_backing_file = file;
521 lo->old_gfp_mask = mapping_gfp_mask(file->f_mapping);
522 mapping_set_gfp_mask(file->f_mapping,
523 lo->old_gfp_mask & ~(__GFP_IO | __GFP_FS));
524 if (lo->lo_backing_file->f_flags & O_DIRECT)
525 lo->lo_flags |= LO_FLAGS_DIRECT_IO;
526 loop_update_dio_alignment(lo);
527 }
528
loop_check_backing_file(struct file * file)529 static int loop_check_backing_file(struct file *file)
530 {
531 if (!file->f_op->read_iter)
532 return -EINVAL;
533
534 if ((file->f_mode & FMODE_WRITE) && !file->f_op->write_iter)
535 return -EINVAL;
536
537 return 0;
538 }
539
540 /*
541 * loop_change_fd switched the backing store of a loopback device to
542 * a new file. This is useful for operating system installers to free up
543 * the original file and in High Availability environments to switch to
544 * an alternative location for the content in case of server meltdown.
545 * This can only work if the loop device is used read-only, and if the
546 * new backing store is the same size and type as the old backing store.
547 */
loop_change_fd(struct loop_device * lo,struct block_device * bdev,unsigned int arg)548 static int loop_change_fd(struct loop_device *lo, struct block_device *bdev,
549 unsigned int arg)
550 {
551 struct file *file = fget(arg);
552 struct file *old_file;
553 unsigned int memflags;
554 int error;
555 bool partscan;
556 bool is_loop;
557
558 if (!file)
559 return -EBADF;
560
561 error = loop_check_backing_file(file);
562 if (error) {
563 fput(file);
564 return error;
565 }
566
567 /* suppress uevents while reconfiguring the device */
568 dev_set_uevent_suppress(disk_to_dev(lo->lo_disk), 1);
569
570 is_loop = is_loop_device(file);
571 error = loop_global_lock_killable(lo, is_loop);
572 if (error)
573 goto out_putf;
574 error = -ENXIO;
575 if (lo->lo_state != Lo_bound)
576 goto out_err;
577
578 /* the loop device has to be read-only */
579 error = -EINVAL;
580 if (!(lo->lo_flags & LO_FLAGS_READ_ONLY))
581 goto out_err;
582
583 error = loop_validate_file(file, bdev);
584 if (error)
585 goto out_err;
586
587 old_file = lo->lo_backing_file;
588
589 error = -EINVAL;
590
591 /* size of the new backing store needs to be the same */
592 if (lo_calculate_size(lo, file) != lo_calculate_size(lo, old_file))
593 goto out_err;
594
595 /*
596 * We might switch to direct I/O mode for the loop device, write back
597 * all dirty data the page cache now that so that the individual I/O
598 * operations don't have to do that.
599 */
600 vfs_fsync(file, 0);
601
602 /* and ... switch */
603 disk_force_media_change(lo->lo_disk);
604 memflags = blk_mq_freeze_queue(lo->lo_queue);
605 mapping_set_gfp_mask(old_file->f_mapping, lo->old_gfp_mask);
606 loop_assign_backing_file(lo, file);
607 loop_update_dio(lo);
608 blk_mq_unfreeze_queue(lo->lo_queue, memflags);
609 partscan = lo->lo_flags & LO_FLAGS_PARTSCAN;
610 loop_global_unlock(lo, is_loop);
611
612 /*
613 * Flush loop_validate_file() before fput(), for l->lo_backing_file
614 * might be pointing at old_file which might be the last reference.
615 */
616 if (!is_loop) {
617 mutex_lock(&loop_validate_mutex);
618 mutex_unlock(&loop_validate_mutex);
619 }
620 /*
621 * We must drop file reference outside of lo_mutex as dropping
622 * the file ref can take open_mutex which creates circular locking
623 * dependency.
624 */
625 fput(old_file);
626 dev_set_uevent_suppress(disk_to_dev(lo->lo_disk), 0);
627 if (partscan)
628 loop_reread_partitions(lo);
629
630 error = 0;
631 done:
632 kobject_uevent(&disk_to_dev(lo->lo_disk)->kobj, KOBJ_CHANGE);
633 return error;
634
635 out_err:
636 loop_global_unlock(lo, is_loop);
637 out_putf:
638 fput(file);
639 dev_set_uevent_suppress(disk_to_dev(lo->lo_disk), 0);
640 goto done;
641 }
642
643 /* loop sysfs attributes */
644
loop_attr_show(struct device * dev,char * page,ssize_t (* callback)(struct loop_device *,char *))645 static ssize_t loop_attr_show(struct device *dev, char *page,
646 ssize_t (*callback)(struct loop_device *, char *))
647 {
648 struct gendisk *disk = dev_to_disk(dev);
649 struct loop_device *lo = disk->private_data;
650
651 return callback(lo, page);
652 }
653
654 #define LOOP_ATTR_RO(_name) \
655 static ssize_t loop_attr_##_name##_show(struct loop_device *, char *); \
656 static ssize_t loop_attr_do_show_##_name(struct device *d, \
657 struct device_attribute *attr, char *b) \
658 { \
659 return loop_attr_show(d, b, loop_attr_##_name##_show); \
660 } \
661 static struct device_attribute loop_attr_##_name = \
662 __ATTR(_name, 0444, loop_attr_do_show_##_name, NULL);
663
loop_attr_backing_file_show(struct loop_device * lo,char * buf)664 static ssize_t loop_attr_backing_file_show(struct loop_device *lo, char *buf)
665 {
666 ssize_t ret;
667 char *p = NULL;
668
669 spin_lock_irq(&lo->lo_lock);
670 if (lo->lo_backing_file)
671 p = file_path(lo->lo_backing_file, buf, PAGE_SIZE - 1);
672 spin_unlock_irq(&lo->lo_lock);
673
674 if (IS_ERR_OR_NULL(p))
675 ret = PTR_ERR(p);
676 else {
677 ret = strlen(p);
678 memmove(buf, p, ret);
679 buf[ret++] = '\n';
680 buf[ret] = 0;
681 }
682
683 return ret;
684 }
685
loop_attr_offset_show(struct loop_device * lo,char * buf)686 static ssize_t loop_attr_offset_show(struct loop_device *lo, char *buf)
687 {
688 return sysfs_emit(buf, "%llu\n", (unsigned long long)lo->lo_offset);
689 }
690
loop_attr_sizelimit_show(struct loop_device * lo,char * buf)691 static ssize_t loop_attr_sizelimit_show(struct loop_device *lo, char *buf)
692 {
693 return sysfs_emit(buf, "%llu\n", (unsigned long long)lo->lo_sizelimit);
694 }
695
loop_attr_autoclear_show(struct loop_device * lo,char * buf)696 static ssize_t loop_attr_autoclear_show(struct loop_device *lo, char *buf)
697 {
698 int autoclear = (lo->lo_flags & LO_FLAGS_AUTOCLEAR);
699
700 return sysfs_emit(buf, "%s\n", autoclear ? "1" : "0");
701 }
702
loop_attr_partscan_show(struct loop_device * lo,char * buf)703 static ssize_t loop_attr_partscan_show(struct loop_device *lo, char *buf)
704 {
705 int partscan = (lo->lo_flags & LO_FLAGS_PARTSCAN);
706
707 return sysfs_emit(buf, "%s\n", partscan ? "1" : "0");
708 }
709
loop_attr_dio_show(struct loop_device * lo,char * buf)710 static ssize_t loop_attr_dio_show(struct loop_device *lo, char *buf)
711 {
712 int dio = (lo->lo_flags & LO_FLAGS_DIRECT_IO);
713
714 return sysfs_emit(buf, "%s\n", dio ? "1" : "0");
715 }
716
717 LOOP_ATTR_RO(backing_file);
718 LOOP_ATTR_RO(offset);
719 LOOP_ATTR_RO(sizelimit);
720 LOOP_ATTR_RO(autoclear);
721 LOOP_ATTR_RO(partscan);
722 LOOP_ATTR_RO(dio);
723
724 static struct attribute *loop_attrs[] = {
725 &loop_attr_backing_file.attr,
726 &loop_attr_offset.attr,
727 &loop_attr_sizelimit.attr,
728 &loop_attr_autoclear.attr,
729 &loop_attr_partscan.attr,
730 &loop_attr_dio.attr,
731 NULL,
732 };
733
734 static struct attribute_group loop_attribute_group = {
735 .name = "loop",
736 .attrs= loop_attrs,
737 };
738
loop_sysfs_init(struct loop_device * lo)739 static void loop_sysfs_init(struct loop_device *lo)
740 {
741 lo->sysfs_inited = !sysfs_create_group(&disk_to_dev(lo->lo_disk)->kobj,
742 &loop_attribute_group);
743 }
744
loop_sysfs_exit(struct loop_device * lo)745 static void loop_sysfs_exit(struct loop_device *lo)
746 {
747 if (lo->sysfs_inited)
748 sysfs_remove_group(&disk_to_dev(lo->lo_disk)->kobj,
749 &loop_attribute_group);
750 }
751
loop_get_discard_config(struct loop_device * lo,u32 * granularity,u32 * max_discard_sectors)752 static void loop_get_discard_config(struct loop_device *lo,
753 u32 *granularity, u32 *max_discard_sectors)
754 {
755 struct file *file = lo->lo_backing_file;
756 struct inode *inode = file->f_mapping->host;
757 struct kstatfs sbuf;
758
759 /*
760 * If the backing device is a block device, mirror its zeroing
761 * capability. Set the discard sectors to the block device's zeroing
762 * capabilities because loop discards result in blkdev_issue_zeroout(),
763 * not blkdev_issue_discard(). This maintains consistent behavior with
764 * file-backed loop devices: discarded regions read back as zero.
765 */
766 if (S_ISBLK(inode->i_mode)) {
767 struct block_device *bdev = I_BDEV(inode);
768
769 *max_discard_sectors = bdev_write_zeroes_sectors(bdev);
770 *granularity = bdev_discard_granularity(bdev);
771
772 /*
773 * We use punch hole to reclaim the free space used by the
774 * image a.k.a. discard.
775 */
776 } else if (file->f_op->fallocate && !vfs_statfs(&file->f_path, &sbuf)) {
777 *max_discard_sectors = UINT_MAX >> 9;
778 *granularity = sbuf.f_bsize;
779 }
780 }
781
782 struct loop_worker {
783 struct rb_node rb_node;
784 struct work_struct work;
785 struct list_head cmd_list;
786 struct list_head idle_list;
787 struct loop_device *lo;
788 struct cgroup_subsys_state *blkcg_css;
789 unsigned long last_ran_at;
790 };
791
792 static void loop_workfn(struct work_struct *work);
793
794 #ifdef CONFIG_BLK_CGROUP
queue_on_root_worker(struct cgroup_subsys_state * css)795 static inline int queue_on_root_worker(struct cgroup_subsys_state *css)
796 {
797 return !css || css == blkcg_root_css;
798 }
799 #else
queue_on_root_worker(struct cgroup_subsys_state * css)800 static inline int queue_on_root_worker(struct cgroup_subsys_state *css)
801 {
802 return !css;
803 }
804 #endif
805
loop_queue_work(struct loop_device * lo,struct loop_cmd * cmd)806 static void loop_queue_work(struct loop_device *lo, struct loop_cmd *cmd)
807 {
808 struct rb_node **node, *parent = NULL;
809 struct loop_worker *cur_worker, *worker = NULL;
810 struct work_struct *work;
811 struct list_head *cmd_list;
812
813 spin_lock_irq(&lo->lo_work_lock);
814
815 if (queue_on_root_worker(cmd->blkcg_css))
816 goto queue_work;
817
818 node = &lo->worker_tree.rb_node;
819
820 while (*node) {
821 parent = *node;
822 cur_worker = container_of(*node, struct loop_worker, rb_node);
823 if (cur_worker->blkcg_css == cmd->blkcg_css) {
824 worker = cur_worker;
825 break;
826 } else if ((long)cur_worker->blkcg_css < (long)cmd->blkcg_css) {
827 node = &(*node)->rb_left;
828 } else {
829 node = &(*node)->rb_right;
830 }
831 }
832 if (worker)
833 goto queue_work;
834
835 worker = kzalloc_obj(struct loop_worker, GFP_NOWAIT);
836 /*
837 * In the event we cannot allocate a worker, just queue on the
838 * rootcg worker and issue the I/O as the rootcg
839 */
840 if (!worker) {
841 cmd->blkcg_css = NULL;
842 if (cmd->memcg_css)
843 css_put(cmd->memcg_css);
844 cmd->memcg_css = NULL;
845 goto queue_work;
846 }
847
848 worker->blkcg_css = cmd->blkcg_css;
849 css_get(worker->blkcg_css);
850 INIT_WORK(&worker->work, loop_workfn);
851 INIT_LIST_HEAD(&worker->cmd_list);
852 INIT_LIST_HEAD(&worker->idle_list);
853 worker->lo = lo;
854 rb_link_node(&worker->rb_node, parent, node);
855 rb_insert_color(&worker->rb_node, &lo->worker_tree);
856 queue_work:
857 if (worker) {
858 /*
859 * We need to remove from the idle list here while
860 * holding the lock so that the idle timer doesn't
861 * free the worker
862 */
863 if (!list_empty(&worker->idle_list))
864 list_del_init(&worker->idle_list);
865 work = &worker->work;
866 cmd_list = &worker->cmd_list;
867 } else {
868 work = &lo->rootcg_work;
869 cmd_list = &lo->rootcg_cmd_list;
870 }
871 list_add_tail(&cmd->list_entry, cmd_list);
872 queue_work(lo->workqueue, work);
873 spin_unlock_irq(&lo->lo_work_lock);
874 }
875
loop_set_timer(struct loop_device * lo)876 static void loop_set_timer(struct loop_device *lo)
877 {
878 timer_reduce(&lo->timer, jiffies + LOOP_IDLE_WORKER_TIMEOUT);
879 }
880
loop_free_idle_workers(struct loop_device * lo,bool delete_all)881 static void loop_free_idle_workers(struct loop_device *lo, bool delete_all)
882 {
883 struct loop_worker *pos, *worker;
884
885 spin_lock_irq(&lo->lo_work_lock);
886 list_for_each_entry_safe(worker, pos, &lo->idle_worker_list,
887 idle_list) {
888 if (!delete_all &&
889 time_is_after_jiffies(worker->last_ran_at +
890 LOOP_IDLE_WORKER_TIMEOUT))
891 break;
892 list_del(&worker->idle_list);
893 rb_erase(&worker->rb_node, &lo->worker_tree);
894 css_put(worker->blkcg_css);
895 kfree(worker);
896 }
897 if (!list_empty(&lo->idle_worker_list))
898 loop_set_timer(lo);
899 spin_unlock_irq(&lo->lo_work_lock);
900 }
901
loop_free_idle_workers_timer(struct timer_list * timer)902 static void loop_free_idle_workers_timer(struct timer_list *timer)
903 {
904 struct loop_device *lo = container_of(timer, struct loop_device, timer);
905
906 return loop_free_idle_workers(lo, false);
907 }
908
909 /**
910 * loop_set_status_from_info - configure device from loop_info
911 * @lo: struct loop_device to configure
912 * @info: struct loop_info64 to configure the device with
913 *
914 * Configures the loop device parameters according to the passed
915 * in loop_info64 configuration.
916 */
917 static int
loop_set_status_from_info(struct loop_device * lo,const struct loop_info64 * info)918 loop_set_status_from_info(struct loop_device *lo,
919 const struct loop_info64 *info)
920 {
921 if ((unsigned int) info->lo_encrypt_key_size > LO_KEY_SIZE)
922 return -EINVAL;
923
924 switch (info->lo_encrypt_type) {
925 case LO_CRYPT_NONE:
926 break;
927 case LO_CRYPT_XOR:
928 pr_warn("support for the xor transformation has been removed.\n");
929 return -EINVAL;
930 case LO_CRYPT_CRYPTOAPI:
931 pr_warn("support for cryptoloop has been removed. Use dm-crypt instead.\n");
932 return -EINVAL;
933 default:
934 return -EINVAL;
935 }
936
937 /* Avoid assigning overflow values */
938 if (info->lo_offset > LLONG_MAX || info->lo_sizelimit > LLONG_MAX)
939 return -EOVERFLOW;
940
941 lo->lo_offset = info->lo_offset;
942 lo->lo_sizelimit = info->lo_sizelimit;
943
944 memcpy(lo->lo_file_name, info->lo_file_name, LO_NAME_SIZE);
945 lo->lo_file_name[LO_NAME_SIZE-1] = 0;
946 return 0;
947 }
948
loop_default_blocksize(struct loop_device * lo)949 static unsigned int loop_default_blocksize(struct loop_device *lo)
950 {
951 /* In case of direct I/O, match underlying minimum I/O size */
952 if (lo->lo_flags & LO_FLAGS_DIRECT_IO)
953 return lo->lo_min_dio_size;
954 return SECTOR_SIZE;
955 }
956
loop_set_dma_limit(struct loop_device * lo,struct queue_limits * lim)957 static void loop_set_dma_limit(struct loop_device *lo, struct queue_limits *lim)
958 {
959 /*
960 * Direct I/O forwards the user pages to the backing file unchanged, so
961 * track the backing's DMA alignment requirement as the mode is toggled.
962 */
963 if (lo->lo_flags & LO_FLAGS_DIRECT_IO)
964 lim->dma_alignment = max_t(unsigned int, lo->lo_dio_mem_align,
965 SECTOR_SIZE - 1);
966 else
967 lim->dma_alignment = SECTOR_SIZE - 1;
968 }
969
loop_update_limits(struct loop_device * lo,struct queue_limits * lim,unsigned int bsize)970 static void loop_update_limits(struct loop_device *lo, struct queue_limits *lim,
971 unsigned int bsize)
972 {
973 struct file *file = lo->lo_backing_file;
974 struct inode *inode = file->f_mapping->host;
975 struct block_device *backing_bdev = NULL;
976 u32 granularity = 0, max_discard_sectors = 0;
977
978 if (S_ISBLK(inode->i_mode))
979 backing_bdev = I_BDEV(inode);
980 else if (inode->i_sb->s_bdev)
981 backing_bdev = inode->i_sb->s_bdev;
982
983 if (!bsize)
984 bsize = loop_default_blocksize(lo);
985
986 loop_get_discard_config(lo, &granularity, &max_discard_sectors);
987
988 lim->logical_block_size = bsize;
989 lim->physical_block_size = bsize;
990 lim->io_min = bsize;
991 loop_set_dma_limit(lo, lim);
992 lim->features &= ~(BLK_FEAT_WRITE_CACHE | BLK_FEAT_ROTATIONAL);
993 if (file->f_op->fsync && !(lo->lo_flags & LO_FLAGS_READ_ONLY))
994 lim->features |= BLK_FEAT_WRITE_CACHE;
995 if (backing_bdev && bdev_rot(backing_bdev))
996 lim->features |= BLK_FEAT_ROTATIONAL;
997 lim->max_hw_discard_sectors = max_discard_sectors;
998 lim->max_write_zeroes_sectors = max_discard_sectors;
999 if (max_discard_sectors)
1000 lim->discard_granularity = granularity;
1001 else
1002 lim->discard_granularity = 0;
1003 }
1004
loop_configure(struct loop_device * lo,blk_mode_t mode,struct block_device * bdev,const struct loop_config * config)1005 static int loop_configure(struct loop_device *lo, blk_mode_t mode,
1006 struct block_device *bdev,
1007 const struct loop_config *config)
1008 {
1009 struct file *file = fget(config->fd);
1010 struct queue_limits lim;
1011 int error;
1012 loff_t size;
1013 bool partscan;
1014 bool is_loop;
1015
1016 if (!file)
1017 return -EBADF;
1018
1019 error = loop_check_backing_file(file);
1020 if (error) {
1021 fput(file);
1022 return error;
1023 }
1024
1025 is_loop = is_loop_device(file);
1026
1027 /* This is safe, since we have a reference from open(). */
1028 __module_get(THIS_MODULE);
1029
1030 /*
1031 * If we don't hold exclusive handle for the device, upgrade to it
1032 * here to avoid changing device under exclusive owner.
1033 */
1034 if (!(mode & BLK_OPEN_EXCL)) {
1035 error = bd_prepare_to_claim(bdev, loop_configure, NULL);
1036 if (error)
1037 goto out_putf;
1038 }
1039
1040 error = loop_global_lock_killable(lo, is_loop);
1041 if (error)
1042 goto out_bdev;
1043
1044 error = -EBUSY;
1045 if (lo->lo_state != Lo_unbound)
1046 goto out_unlock;
1047
1048 error = loop_validate_file(file, bdev);
1049 if (error)
1050 goto out_unlock;
1051
1052 if ((config->info.lo_flags & ~LOOP_CONFIGURE_SETTABLE_FLAGS) != 0) {
1053 error = -EINVAL;
1054 goto out_unlock;
1055 }
1056
1057 error = loop_set_status_from_info(lo, &config->info);
1058 if (error)
1059 goto out_unlock;
1060 lo->lo_flags = config->info.lo_flags;
1061
1062 if (!(file->f_mode & FMODE_WRITE) || !(mode & BLK_OPEN_WRITE) ||
1063 !file->f_op->write_iter)
1064 lo->lo_flags |= LO_FLAGS_READ_ONLY;
1065
1066 if (!lo->workqueue) {
1067 lo->workqueue = alloc_workqueue("loop%d",
1068 WQ_UNBOUND | WQ_FREEZABLE,
1069 0, lo->lo_number);
1070 if (!lo->workqueue) {
1071 error = -ENOMEM;
1072 goto out_unlock;
1073 }
1074 }
1075
1076 /* suppress uevents while reconfiguring the device */
1077 dev_set_uevent_suppress(disk_to_dev(lo->lo_disk), 1);
1078
1079 disk_force_media_change(lo->lo_disk);
1080 set_disk_ro(lo->lo_disk, (lo->lo_flags & LO_FLAGS_READ_ONLY) != 0);
1081
1082 lo->lo_device = bdev;
1083 loop_assign_backing_file(lo, file);
1084
1085 lim = queue_limits_start_update(lo->lo_queue);
1086 loop_update_limits(lo, &lim, config->block_size);
1087 /* No need to freeze the queue as the device isn't bound yet. */
1088 error = queue_limits_commit_update(lo->lo_queue, &lim);
1089 if (error)
1090 goto out_unlock;
1091
1092 /*
1093 * We might switch to direct I/O mode for the loop device, write back
1094 * all dirty data the page cache now that so that the individual I/O
1095 * operations don't have to do that.
1096 */
1097 vfs_fsync(file, 0);
1098
1099 loop_update_dio(lo);
1100 loop_sysfs_init(lo);
1101
1102 size = lo_calculate_size(lo, file);
1103 loop_set_size(lo, size);
1104
1105 /* Order wrt reading lo_state in loop_validate_file(). */
1106 wmb();
1107
1108 WRITE_ONCE(lo->lo_state, Lo_bound);
1109 if (part_shift)
1110 lo->lo_flags |= LO_FLAGS_PARTSCAN;
1111 partscan = lo->lo_flags & LO_FLAGS_PARTSCAN;
1112 if (partscan)
1113 clear_bit(GD_SUPPRESS_PART_SCAN, &lo->lo_disk->state);
1114
1115 dev_set_uevent_suppress(disk_to_dev(lo->lo_disk), 0);
1116 kobject_uevent(&disk_to_dev(lo->lo_disk)->kobj, KOBJ_CHANGE);
1117
1118 loop_global_unlock(lo, is_loop);
1119 if (partscan)
1120 loop_reread_partitions(lo);
1121
1122 if (!(mode & BLK_OPEN_EXCL))
1123 bd_abort_claiming(bdev, loop_configure);
1124
1125 return 0;
1126
1127 out_unlock:
1128 loop_global_unlock(lo, is_loop);
1129 out_bdev:
1130 if (!(mode & BLK_OPEN_EXCL))
1131 bd_abort_claiming(bdev, loop_configure);
1132 out_putf:
1133 fput(file);
1134 /* This is safe: open() is still holding a reference. */
1135 module_put(THIS_MODULE);
1136 return error;
1137 }
1138
__loop_clr_fd(struct loop_device * lo)1139 static void __loop_clr_fd(struct loop_device *lo)
1140 {
1141 struct queue_limits lim;
1142 struct file *filp;
1143 gfp_t gfp = lo->old_gfp_mask;
1144 int err;
1145
1146 spin_lock_irq(&lo->lo_lock);
1147 filp = lo->lo_backing_file;
1148 lo->lo_backing_file = NULL;
1149 spin_unlock_irq(&lo->lo_lock);
1150
1151 lo->lo_device = NULL;
1152 lo->lo_offset = 0;
1153 lo->lo_sizelimit = 0;
1154 memset(lo->lo_file_name, 0, LO_NAME_SIZE);
1155
1156 /*
1157 * Reset the block size to the default.
1158 *
1159 * No queue freezing needed because this is called from the final
1160 * ->release call only, so there can't be any outstanding I/O.
1161 */
1162 lim = queue_limits_start_update(lo->lo_queue);
1163 lim.logical_block_size = SECTOR_SIZE;
1164 lim.physical_block_size = SECTOR_SIZE;
1165 lim.io_min = SECTOR_SIZE;
1166 queue_limits_commit_update(lo->lo_queue, &lim);
1167
1168 invalidate_disk(lo->lo_disk);
1169 loop_sysfs_exit(lo);
1170 /* let user-space know about this change */
1171 kobject_uevent(&disk_to_dev(lo->lo_disk)->kobj, KOBJ_CHANGE);
1172 mapping_set_gfp_mask(filp->f_mapping, gfp);
1173 /* This is safe: open() is still holding a reference. */
1174 module_put(THIS_MODULE);
1175
1176 disk_force_media_change(lo->lo_disk);
1177
1178 /*
1179 * Remove all partitions, including partitions added manually with
1180 * BLKPG, which may exist even if LO_FLAGS_PARTSCAN is not set.
1181 *
1182 * open_mutex has been held already in release path, so don't acquire
1183 * it here.
1184 */
1185 err = bdev_disk_changed(lo->lo_disk, false);
1186 if (err)
1187 pr_warn("%s: partition scan of loop%d failed (rc=%d)\n",
1188 __func__, lo->lo_number, err);
1189 /* Device is gone, no point in returning error */
1190
1191 /*
1192 * lo->lo_state is set to Lo_unbound here after removing partitions has
1193 * finished. There cannot be anybody else entering __loop_clr_fd() as
1194 * Lo_rundown state protects us from all the other places trying to
1195 * change the 'lo' device.
1196 */
1197 lo->lo_flags = 0;
1198 if (!part_shift)
1199 set_bit(GD_SUPPRESS_PART_SCAN, &lo->lo_disk->state);
1200 mutex_lock(&lo->lo_mutex);
1201 WRITE_ONCE(lo->lo_state, Lo_unbound);
1202 mutex_unlock(&lo->lo_mutex);
1203
1204 /*
1205 * Need not hold lo_mutex to fput backing file. Calling fput holding
1206 * lo_mutex triggers a circular lock dependency possibility warning as
1207 * fput can take open_mutex which is usually taken before lo_mutex.
1208 */
1209 fput(filp);
1210 }
1211
loop_clr_fd(struct loop_device * lo)1212 static int loop_clr_fd(struct loop_device *lo)
1213 {
1214 int err;
1215
1216 /*
1217 * Since lo_ioctl() is called without locks held, it is possible that
1218 * loop_configure()/loop_change_fd() and loop_clr_fd() run in parallel.
1219 *
1220 * Therefore, use global lock when setting Lo_rundown state in order to
1221 * make sure that loop_validate_file() will fail if the "struct file"
1222 * which loop_configure()/loop_change_fd() found via fget() was this
1223 * loop device.
1224 */
1225 err = loop_global_lock_killable(lo, true);
1226 if (err)
1227 return err;
1228 if (lo->lo_state != Lo_bound) {
1229 loop_global_unlock(lo, true);
1230 return -ENXIO;
1231 }
1232 /*
1233 * Mark the device for removing the backing device on last close.
1234 * If we are the only opener, also switch the state to roundown here to
1235 * prevent new openers from coming in.
1236 */
1237
1238 lo->lo_flags |= LO_FLAGS_AUTOCLEAR;
1239 if (disk_openers(lo->lo_disk) == 1)
1240 WRITE_ONCE(lo->lo_state, Lo_rundown);
1241 loop_global_unlock(lo, true);
1242
1243 return 0;
1244 }
1245
1246 static int
loop_set_status(struct loop_device * lo,const struct loop_info64 * info)1247 loop_set_status(struct loop_device *lo, const struct loop_info64 *info)
1248 {
1249 int err;
1250 bool partscan = false;
1251 bool size_changed = false;
1252 unsigned int memflags;
1253
1254 err = mutex_lock_killable(&lo->lo_mutex);
1255 if (err)
1256 return err;
1257 if (lo->lo_state != Lo_bound) {
1258 err = -ENXIO;
1259 goto out_unlock;
1260 }
1261
1262 if (lo->lo_offset != info->lo_offset ||
1263 lo->lo_sizelimit != info->lo_sizelimit) {
1264 size_changed = true;
1265 sync_blockdev(lo->lo_device);
1266 invalidate_bdev(lo->lo_device);
1267 }
1268
1269 /* I/O needs to be drained before changing lo_offset or lo_sizelimit */
1270 memflags = blk_mq_freeze_queue(lo->lo_queue);
1271
1272 err = loop_set_status_from_info(lo, info);
1273 if (err)
1274 goto out_unfreeze;
1275
1276 partscan = !(lo->lo_flags & LO_FLAGS_PARTSCAN) &&
1277 (info->lo_flags & LO_FLAGS_PARTSCAN);
1278
1279 lo->lo_flags &= ~LOOP_SET_STATUS_CLEARABLE_FLAGS;
1280 lo->lo_flags |= (info->lo_flags & LOOP_SET_STATUS_SETTABLE_FLAGS);
1281
1282 /* update the direct I/O flag if lo_offset changed */
1283 loop_update_dio(lo);
1284
1285 out_unfreeze:
1286 blk_mq_unfreeze_queue(lo->lo_queue, memflags);
1287 if (partscan)
1288 clear_bit(GD_SUPPRESS_PART_SCAN, &lo->lo_disk->state);
1289 if (!err && size_changed) {
1290 loff_t new_size = lo_calculate_size(lo, lo->lo_backing_file);
1291 loop_set_size(lo, new_size);
1292 }
1293 out_unlock:
1294 mutex_unlock(&lo->lo_mutex);
1295 if (partscan)
1296 loop_reread_partitions(lo);
1297
1298 return err;
1299 }
1300
1301 static int
loop_get_status(struct loop_device * lo,struct loop_info64 * info)1302 loop_get_status(struct loop_device *lo, struct loop_info64 *info)
1303 {
1304 struct path path;
1305 struct kstat stat;
1306 int ret;
1307
1308 ret = mutex_lock_killable(&lo->lo_mutex);
1309 if (ret)
1310 return ret;
1311 if (lo->lo_state != Lo_bound) {
1312 mutex_unlock(&lo->lo_mutex);
1313 return -ENXIO;
1314 }
1315
1316 memset(info, 0, sizeof(*info));
1317 info->lo_number = lo->lo_number;
1318 info->lo_offset = lo->lo_offset;
1319 info->lo_sizelimit = lo->lo_sizelimit;
1320 info->lo_flags = lo->lo_flags;
1321 memcpy(info->lo_file_name, lo->lo_file_name, LO_NAME_SIZE);
1322
1323 /* Drop lo_mutex while we call into the filesystem. */
1324 path = lo->lo_backing_file->f_path;
1325 path_get(&path);
1326 mutex_unlock(&lo->lo_mutex);
1327 ret = vfs_getattr(&path, &stat, STATX_INO, AT_STATX_SYNC_AS_STAT);
1328 if (!ret) {
1329 info->lo_device = huge_encode_dev(stat.dev);
1330 info->lo_inode = stat.ino;
1331 info->lo_rdevice = huge_encode_dev(stat.rdev);
1332 }
1333 path_put(&path);
1334 return ret;
1335 }
1336
1337 static void
loop_info64_from_old(const struct loop_info * info,struct loop_info64 * info64)1338 loop_info64_from_old(const struct loop_info *info, struct loop_info64 *info64)
1339 {
1340 memset(info64, 0, sizeof(*info64));
1341 info64->lo_number = info->lo_number;
1342 info64->lo_device = info->lo_device;
1343 info64->lo_inode = info->lo_inode;
1344 info64->lo_rdevice = info->lo_rdevice;
1345 info64->lo_offset = info->lo_offset;
1346 info64->lo_sizelimit = 0;
1347 info64->lo_flags = info->lo_flags;
1348 memcpy(info64->lo_file_name, info->lo_name, LO_NAME_SIZE);
1349 }
1350
1351 static int
loop_info64_to_old(const struct loop_info64 * info64,struct loop_info * info)1352 loop_info64_to_old(const struct loop_info64 *info64, struct loop_info *info)
1353 {
1354 memset(info, 0, sizeof(*info));
1355 info->lo_number = info64->lo_number;
1356 info->lo_device = info64->lo_device;
1357 info->lo_inode = info64->lo_inode;
1358 info->lo_rdevice = info64->lo_rdevice;
1359 info->lo_offset = info64->lo_offset;
1360 info->lo_flags = info64->lo_flags;
1361 memcpy(info->lo_name, info64->lo_file_name, LO_NAME_SIZE);
1362
1363 /* error in case values were truncated */
1364 if (info->lo_device != info64->lo_device ||
1365 info->lo_rdevice != info64->lo_rdevice ||
1366 info->lo_inode != info64->lo_inode ||
1367 info->lo_offset != info64->lo_offset)
1368 return -EOVERFLOW;
1369
1370 return 0;
1371 }
1372
1373 static int
loop_set_status_old(struct loop_device * lo,const struct loop_info __user * arg)1374 loop_set_status_old(struct loop_device *lo, const struct loop_info __user *arg)
1375 {
1376 struct loop_info info;
1377 struct loop_info64 info64;
1378
1379 if (copy_from_user(&info, arg, sizeof (struct loop_info)))
1380 return -EFAULT;
1381 loop_info64_from_old(&info, &info64);
1382 return loop_set_status(lo, &info64);
1383 }
1384
1385 static int
loop_set_status64(struct loop_device * lo,const struct loop_info64 __user * arg)1386 loop_set_status64(struct loop_device *lo, const struct loop_info64 __user *arg)
1387 {
1388 struct loop_info64 info64;
1389
1390 if (copy_from_user(&info64, arg, sizeof (struct loop_info64)))
1391 return -EFAULT;
1392 return loop_set_status(lo, &info64);
1393 }
1394
1395 static int
loop_get_status_old(struct loop_device * lo,struct loop_info __user * arg)1396 loop_get_status_old(struct loop_device *lo, struct loop_info __user *arg) {
1397 struct loop_info info;
1398 struct loop_info64 info64;
1399 int err;
1400
1401 if (!arg)
1402 return -EINVAL;
1403 err = loop_get_status(lo, &info64);
1404 if (!err)
1405 err = loop_info64_to_old(&info64, &info);
1406 if (!err && copy_to_user(arg, &info, sizeof(info)))
1407 err = -EFAULT;
1408
1409 return err;
1410 }
1411
1412 static int
loop_get_status64(struct loop_device * lo,struct loop_info64 __user * arg)1413 loop_get_status64(struct loop_device *lo, struct loop_info64 __user *arg) {
1414 struct loop_info64 info64;
1415 int err;
1416
1417 if (!arg)
1418 return -EINVAL;
1419 err = loop_get_status(lo, &info64);
1420 if (!err && copy_to_user(arg, &info64, sizeof(info64)))
1421 err = -EFAULT;
1422
1423 return err;
1424 }
1425
loop_set_capacity(struct loop_device * lo)1426 static int loop_set_capacity(struct loop_device *lo)
1427 {
1428 loff_t size;
1429
1430 if (unlikely(lo->lo_state != Lo_bound))
1431 return -ENXIO;
1432
1433 size = lo_calculate_size(lo, lo->lo_backing_file);
1434 loop_set_size(lo, size);
1435
1436 return 0;
1437 }
1438
loop_set_dio(struct loop_device * lo,unsigned long arg)1439 static int loop_set_dio(struct loop_device *lo, unsigned long arg)
1440 {
1441 bool use_dio = !!arg;
1442 unsigned int memflags;
1443 struct queue_limits lim;
1444
1445 if (lo->lo_state != Lo_bound)
1446 return -ENXIO;
1447 if (use_dio == !!(lo->lo_flags & LO_FLAGS_DIRECT_IO))
1448 return 0;
1449
1450 if (use_dio) {
1451 if (!lo_can_use_dio(lo))
1452 return -EINVAL;
1453 /* flush dirty pages before starting to use direct I/O */
1454 vfs_fsync(lo->lo_backing_file, 0);
1455 }
1456
1457 lim = queue_limits_start_update(lo->lo_queue);
1458 memflags = blk_mq_freeze_queue(lo->lo_queue);
1459 if (use_dio)
1460 lo->lo_flags |= LO_FLAGS_DIRECT_IO;
1461 else
1462 lo->lo_flags &= ~LO_FLAGS_DIRECT_IO;
1463 loop_set_dma_limit(lo, &lim);
1464 queue_limits_commit_update(lo->lo_queue, &lim);
1465 blk_mq_unfreeze_queue(lo->lo_queue, memflags);
1466 return 0;
1467 }
1468
loop_set_block_size(struct loop_device * lo,blk_mode_t mode,struct block_device * bdev,unsigned long arg)1469 static int loop_set_block_size(struct loop_device *lo, blk_mode_t mode,
1470 struct block_device *bdev, unsigned long arg)
1471 {
1472 struct queue_limits lim;
1473 unsigned int memflags;
1474 int err = 0;
1475
1476 /*
1477 * If we don't hold exclusive handle for the device, upgrade to it
1478 * here to avoid changing device under exclusive owner.
1479 */
1480 if (!(mode & BLK_OPEN_EXCL)) {
1481 err = bd_prepare_to_claim(bdev, loop_set_block_size, NULL);
1482 if (err)
1483 return err;
1484 }
1485
1486 err = mutex_lock_killable(&lo->lo_mutex);
1487 if (err)
1488 goto abort_claim;
1489
1490 if (lo->lo_state != Lo_bound) {
1491 err = -ENXIO;
1492 goto unlock;
1493 }
1494
1495 if (lo->lo_queue->limits.logical_block_size == arg)
1496 goto unlock;
1497
1498 sync_blockdev(lo->lo_device);
1499 invalidate_bdev(lo->lo_device);
1500
1501 lim = queue_limits_start_update(lo->lo_queue);
1502 loop_update_limits(lo, &lim, arg);
1503
1504 memflags = blk_mq_freeze_queue(lo->lo_queue);
1505 err = queue_limits_commit_update(lo->lo_queue, &lim);
1506 loop_update_dio(lo);
1507 blk_mq_unfreeze_queue(lo->lo_queue, memflags);
1508
1509 unlock:
1510 mutex_unlock(&lo->lo_mutex);
1511 abort_claim:
1512 if (!(mode & BLK_OPEN_EXCL))
1513 bd_abort_claiming(bdev, loop_set_block_size);
1514 return err;
1515 }
1516
lo_simple_ioctl(struct loop_device * lo,unsigned int cmd,unsigned long arg)1517 static int lo_simple_ioctl(struct loop_device *lo, unsigned int cmd,
1518 unsigned long arg)
1519 {
1520 int err;
1521
1522 err = mutex_lock_killable(&lo->lo_mutex);
1523 if (err)
1524 return err;
1525 switch (cmd) {
1526 case LOOP_SET_CAPACITY:
1527 err = loop_set_capacity(lo);
1528 break;
1529 case LOOP_SET_DIRECT_IO:
1530 err = loop_set_dio(lo, arg);
1531 break;
1532 default:
1533 err = -EINVAL;
1534 }
1535 mutex_unlock(&lo->lo_mutex);
1536 return err;
1537 }
1538
lo_ioctl(struct block_device * bdev,blk_mode_t mode,unsigned int cmd,unsigned long arg)1539 static int lo_ioctl(struct block_device *bdev, blk_mode_t mode,
1540 unsigned int cmd, unsigned long arg)
1541 {
1542 struct loop_device *lo = bdev->bd_disk->private_data;
1543 void __user *argp = (void __user *) arg;
1544 int err;
1545
1546 switch (cmd) {
1547 case LOOP_SET_FD: {
1548 /*
1549 * Legacy case - pass in a zeroed out struct loop_config with
1550 * only the file descriptor set , which corresponds with the
1551 * default parameters we'd have used otherwise.
1552 */
1553 struct loop_config config;
1554
1555 memset(&config, 0, sizeof(config));
1556 config.fd = arg;
1557
1558 return loop_configure(lo, mode, bdev, &config);
1559 }
1560 case LOOP_CONFIGURE: {
1561 struct loop_config config;
1562
1563 if (copy_from_user(&config, argp, sizeof(config)))
1564 return -EFAULT;
1565
1566 return loop_configure(lo, mode, bdev, &config);
1567 }
1568 case LOOP_CHANGE_FD:
1569 return loop_change_fd(lo, bdev, arg);
1570 case LOOP_CLR_FD:
1571 return loop_clr_fd(lo);
1572 case LOOP_SET_STATUS:
1573 err = -EPERM;
1574 if ((mode & BLK_OPEN_WRITE) || capable(CAP_SYS_ADMIN))
1575 err = loop_set_status_old(lo, argp);
1576 break;
1577 case LOOP_GET_STATUS:
1578 return loop_get_status_old(lo, argp);
1579 case LOOP_SET_STATUS64:
1580 err = -EPERM;
1581 if ((mode & BLK_OPEN_WRITE) || capable(CAP_SYS_ADMIN))
1582 err = loop_set_status64(lo, argp);
1583 break;
1584 case LOOP_GET_STATUS64:
1585 return loop_get_status64(lo, argp);
1586 case LOOP_SET_BLOCK_SIZE:
1587 if (!(mode & BLK_OPEN_WRITE) && !capable(CAP_SYS_ADMIN))
1588 return -EPERM;
1589 return loop_set_block_size(lo, mode, bdev, arg);
1590 case LOOP_SET_CAPACITY:
1591 case LOOP_SET_DIRECT_IO:
1592 if (!(mode & BLK_OPEN_WRITE) && !capable(CAP_SYS_ADMIN))
1593 return -EPERM;
1594 fallthrough;
1595 default:
1596 err = lo_simple_ioctl(lo, cmd, arg);
1597 break;
1598 }
1599
1600 return err;
1601 }
1602
1603 #ifdef CONFIG_COMPAT
1604 struct compat_loop_info {
1605 compat_int_t lo_number; /* ioctl r/o */
1606 compat_dev_t lo_device; /* ioctl r/o */
1607 compat_ulong_t lo_inode; /* ioctl r/o */
1608 compat_dev_t lo_rdevice; /* ioctl r/o */
1609 compat_int_t lo_offset;
1610 compat_int_t lo_encrypt_type; /* obsolete, ignored */
1611 compat_int_t lo_encrypt_key_size; /* ioctl w/o */
1612 compat_int_t lo_flags; /* ioctl r/o */
1613 char lo_name[LO_NAME_SIZE];
1614 unsigned char lo_encrypt_key[LO_KEY_SIZE]; /* ioctl w/o */
1615 compat_ulong_t lo_init[2];
1616 char reserved[4];
1617 };
1618
1619 /*
1620 * Transfer 32-bit compatibility structure in userspace to 64-bit loop info
1621 * - noinlined to reduce stack space usage in main part of driver
1622 */
1623 static noinline int
loop_info64_from_compat(const struct compat_loop_info __user * arg,struct loop_info64 * info64)1624 loop_info64_from_compat(const struct compat_loop_info __user *arg,
1625 struct loop_info64 *info64)
1626 {
1627 struct compat_loop_info info;
1628
1629 if (copy_from_user(&info, arg, sizeof(info)))
1630 return -EFAULT;
1631
1632 memset(info64, 0, sizeof(*info64));
1633 info64->lo_number = info.lo_number;
1634 info64->lo_device = info.lo_device;
1635 info64->lo_inode = info.lo_inode;
1636 info64->lo_rdevice = info.lo_rdevice;
1637 info64->lo_offset = info.lo_offset;
1638 info64->lo_sizelimit = 0;
1639 info64->lo_flags = info.lo_flags;
1640 memcpy(info64->lo_file_name, info.lo_name, LO_NAME_SIZE);
1641 return 0;
1642 }
1643
1644 /*
1645 * Transfer 64-bit loop info to 32-bit compatibility structure in userspace
1646 * - noinlined to reduce stack space usage in main part of driver
1647 */
1648 static noinline int
loop_info64_to_compat(const struct loop_info64 * info64,struct compat_loop_info __user * arg)1649 loop_info64_to_compat(const struct loop_info64 *info64,
1650 struct compat_loop_info __user *arg)
1651 {
1652 struct compat_loop_info info;
1653
1654 memset(&info, 0, sizeof(info));
1655 info.lo_number = info64->lo_number;
1656 info.lo_device = info64->lo_device;
1657 info.lo_inode = info64->lo_inode;
1658 info.lo_rdevice = info64->lo_rdevice;
1659 info.lo_offset = info64->lo_offset;
1660 info.lo_flags = info64->lo_flags;
1661 memcpy(info.lo_name, info64->lo_file_name, LO_NAME_SIZE);
1662
1663 /* error in case values were truncated */
1664 if (info.lo_device != info64->lo_device ||
1665 info.lo_rdevice != info64->lo_rdevice ||
1666 info.lo_inode != info64->lo_inode ||
1667 info.lo_offset != info64->lo_offset)
1668 return -EOVERFLOW;
1669
1670 if (copy_to_user(arg, &info, sizeof(info)))
1671 return -EFAULT;
1672 return 0;
1673 }
1674
1675 static int
loop_set_status_compat(struct loop_device * lo,const struct compat_loop_info __user * arg)1676 loop_set_status_compat(struct loop_device *lo,
1677 const struct compat_loop_info __user *arg)
1678 {
1679 struct loop_info64 info64;
1680 int ret;
1681
1682 ret = loop_info64_from_compat(arg, &info64);
1683 if (ret < 0)
1684 return ret;
1685 return loop_set_status(lo, &info64);
1686 }
1687
1688 static int
loop_get_status_compat(struct loop_device * lo,struct compat_loop_info __user * arg)1689 loop_get_status_compat(struct loop_device *lo,
1690 struct compat_loop_info __user *arg)
1691 {
1692 struct loop_info64 info64;
1693 int err;
1694
1695 if (!arg)
1696 return -EINVAL;
1697 err = loop_get_status(lo, &info64);
1698 if (!err)
1699 err = loop_info64_to_compat(&info64, arg);
1700 return err;
1701 }
1702
lo_compat_ioctl(struct block_device * bdev,blk_mode_t mode,unsigned int cmd,unsigned long arg)1703 static int lo_compat_ioctl(struct block_device *bdev, blk_mode_t mode,
1704 unsigned int cmd, unsigned long arg)
1705 {
1706 struct loop_device *lo = bdev->bd_disk->private_data;
1707 int err;
1708
1709 switch(cmd) {
1710 case LOOP_SET_STATUS:
1711 err = loop_set_status_compat(lo,
1712 (const struct compat_loop_info __user *)arg);
1713 break;
1714 case LOOP_GET_STATUS:
1715 err = loop_get_status_compat(lo,
1716 (struct compat_loop_info __user *)arg);
1717 break;
1718 case LOOP_SET_CAPACITY:
1719 case LOOP_CLR_FD:
1720 case LOOP_GET_STATUS64:
1721 case LOOP_SET_STATUS64:
1722 case LOOP_CONFIGURE:
1723 arg = (unsigned long) compat_ptr(arg);
1724 fallthrough;
1725 case LOOP_SET_FD:
1726 case LOOP_CHANGE_FD:
1727 case LOOP_SET_BLOCK_SIZE:
1728 case LOOP_SET_DIRECT_IO:
1729 err = lo_ioctl(bdev, mode, cmd, arg);
1730 break;
1731 default:
1732 err = -ENOIOCTLCMD;
1733 break;
1734 }
1735 return err;
1736 }
1737 #endif
1738
lo_open(struct gendisk * disk,blk_mode_t mode)1739 static int lo_open(struct gendisk *disk, blk_mode_t mode)
1740 {
1741 struct loop_device *lo = disk->private_data;
1742 int err;
1743
1744 err = mutex_lock_killable(&lo->lo_mutex);
1745 if (err)
1746 return err;
1747
1748 if (lo->lo_state == Lo_deleting || lo->lo_state == Lo_rundown)
1749 err = -ENXIO;
1750 mutex_unlock(&lo->lo_mutex);
1751 return err;
1752 }
1753
lo_release(struct gendisk * disk)1754 static void lo_release(struct gendisk *disk)
1755 {
1756 struct loop_device *lo = disk->private_data;
1757 bool need_clear = false;
1758
1759 if (disk_openers(disk) > 0)
1760 return;
1761 /*
1762 * Clear the backing device information if this is the last close of
1763 * a device that's been marked for auto clear, or on which LOOP_CLR_FD
1764 * has been called.
1765 */
1766
1767 mutex_lock(&lo->lo_mutex);
1768 if (lo->lo_state == Lo_bound && (lo->lo_flags & LO_FLAGS_AUTOCLEAR))
1769 WRITE_ONCE(lo->lo_state, Lo_rundown);
1770
1771 need_clear = (lo->lo_state == Lo_rundown);
1772 mutex_unlock(&lo->lo_mutex);
1773
1774 if (need_clear)
1775 __loop_clr_fd(lo);
1776 }
1777
lo_free_disk(struct gendisk * disk)1778 static void lo_free_disk(struct gendisk *disk)
1779 {
1780 struct loop_device *lo = disk->private_data;
1781
1782 if (lo->workqueue)
1783 destroy_workqueue(lo->workqueue);
1784 loop_free_idle_workers(lo, true);
1785 timer_shutdown_sync(&lo->timer);
1786 mutex_destroy(&lo->lo_mutex);
1787 kfree(lo);
1788 }
1789
1790 static const struct block_device_operations lo_fops = {
1791 .owner = THIS_MODULE,
1792 .open = lo_open,
1793 .release = lo_release,
1794 .ioctl = lo_ioctl,
1795 #ifdef CONFIG_COMPAT
1796 .compat_ioctl = lo_compat_ioctl,
1797 #endif
1798 .free_disk = lo_free_disk,
1799 };
1800
1801 /*
1802 * And now the modules code and kernel interface.
1803 */
1804
1805 /*
1806 * If max_loop is specified, create that many devices upfront.
1807 * This also becomes a hard limit. If max_loop is not specified,
1808 * the default isn't a hard limit (as before commit 85c50197716c
1809 * changed the default value from 0 for max_loop=0 reasons), just
1810 * create CONFIG_BLK_DEV_LOOP_MIN_COUNT loop devices at module
1811 * init time. Loop devices can be requested on-demand with the
1812 * /dev/loop-control interface, or be instantiated by accessing
1813 * a 'dead' device node.
1814 */
1815 static int max_loop = CONFIG_BLK_DEV_LOOP_MIN_COUNT;
1816
1817 #ifdef CONFIG_BLOCK_LEGACY_AUTOLOAD
1818 static bool max_loop_specified;
1819
max_loop_param_set_int(const char * val,const struct kernel_param * kp)1820 static int max_loop_param_set_int(const char *val,
1821 const struct kernel_param *kp)
1822 {
1823 int ret;
1824
1825 ret = param_set_int(val, kp);
1826 if (ret < 0)
1827 return ret;
1828
1829 max_loop_specified = true;
1830 return 0;
1831 }
1832
1833 static const struct kernel_param_ops max_loop_param_ops = {
1834 .set = max_loop_param_set_int,
1835 .get = param_get_int,
1836 };
1837
1838 module_param_cb(max_loop, &max_loop_param_ops, &max_loop, 0444);
1839 MODULE_PARM_DESC(max_loop, "Maximum number of loop devices");
1840 #else
1841 module_param(max_loop, int, 0444);
1842 MODULE_PARM_DESC(max_loop, "Initial number of loop devices");
1843 #endif
1844
1845 module_param(max_part, int, 0444);
1846 MODULE_PARM_DESC(max_part, "Maximum number of partitions per loop device");
1847
1848 static int hw_queue_depth = LOOP_DEFAULT_HW_Q_DEPTH;
1849
loop_set_hw_queue_depth(const char * s,const struct kernel_param * p)1850 static int loop_set_hw_queue_depth(const char *s, const struct kernel_param *p)
1851 {
1852 int qd, ret;
1853
1854 ret = kstrtoint(s, 0, &qd);
1855 if (ret < 0)
1856 return ret;
1857 if (qd < 1)
1858 return -EINVAL;
1859 hw_queue_depth = qd;
1860 return 0;
1861 }
1862
1863 static const struct kernel_param_ops loop_hw_qdepth_param_ops = {
1864 .set = loop_set_hw_queue_depth,
1865 .get = param_get_int,
1866 };
1867
1868 device_param_cb(hw_queue_depth, &loop_hw_qdepth_param_ops, &hw_queue_depth, 0444);
1869 MODULE_PARM_DESC(hw_queue_depth, "Queue depth for each hardware queue. Default: " __stringify(LOOP_DEFAULT_HW_Q_DEPTH));
1870
1871 MODULE_DESCRIPTION("Loopback device support");
1872 MODULE_LICENSE("GPL");
1873 MODULE_ALIAS_BLOCKDEV_MAJOR(LOOP_MAJOR);
1874
loop_queue_rq(struct blk_mq_hw_ctx * hctx,const struct blk_mq_queue_data * bd)1875 static blk_status_t loop_queue_rq(struct blk_mq_hw_ctx *hctx,
1876 const struct blk_mq_queue_data *bd)
1877 {
1878 struct request *rq = bd->rq;
1879 struct loop_cmd *cmd = blk_mq_rq_to_pdu(rq);
1880 struct loop_device *lo = rq->q->queuedata;
1881
1882 blk_mq_start_request(rq);
1883
1884 if (data_race(READ_ONCE(lo->lo_state)) != Lo_bound)
1885 return BLK_STS_IOERR;
1886
1887 switch (req_op(rq)) {
1888 case REQ_OP_FLUSH:
1889 case REQ_OP_DISCARD:
1890 case REQ_OP_WRITE_ZEROES:
1891 cmd->use_aio = false;
1892 break;
1893 default:
1894 cmd->use_aio = lo->lo_flags & LO_FLAGS_DIRECT_IO;
1895 break;
1896 }
1897
1898 /* always use the first bio's css */
1899 cmd->blkcg_css = NULL;
1900 cmd->memcg_css = NULL;
1901 #ifdef CONFIG_BLK_CGROUP
1902 if (rq->bio) {
1903 cmd->blkcg_css = bio_blkcg_css(rq->bio);
1904 #ifdef CONFIG_MEMCG
1905 if (cmd->blkcg_css) {
1906 cmd->memcg_css =
1907 cgroup_get_e_css(cmd->blkcg_css->cgroup,
1908 &memory_cgrp_subsys);
1909 }
1910 #endif
1911 }
1912 #endif
1913 loop_queue_work(lo, cmd);
1914
1915 return BLK_STS_OK;
1916 }
1917
loop_handle_cmd(struct loop_cmd * cmd)1918 static void loop_handle_cmd(struct loop_cmd *cmd)
1919 {
1920 struct cgroup_subsys_state *cmd_blkcg_css = cmd->blkcg_css;
1921 struct cgroup_subsys_state *cmd_memcg_css = cmd->memcg_css;
1922 struct request *rq = blk_mq_rq_from_pdu(cmd);
1923 const bool write = op_is_write(req_op(rq));
1924 struct loop_device *lo = rq->q->queuedata;
1925 int ret = 0;
1926 struct mem_cgroup *old_memcg = NULL;
1927
1928 if (write && (lo->lo_flags & LO_FLAGS_READ_ONLY)) {
1929 ret = -EIO;
1930 goto failed;
1931 }
1932
1933 /* We can block in this context, so ignore REQ_NOWAIT. */
1934 if (rq->cmd_flags & REQ_NOWAIT)
1935 rq->cmd_flags &= ~REQ_NOWAIT;
1936
1937 if (cmd_blkcg_css)
1938 kthread_associate_blkcg(cmd_blkcg_css);
1939 if (cmd_memcg_css)
1940 old_memcg = set_active_memcg(
1941 mem_cgroup_from_css(cmd_memcg_css));
1942
1943 /*
1944 * do_req_filebacked() may call blk_mq_complete_request() synchronously
1945 * or asynchronously if using aio. Hence, do not touch 'cmd' after
1946 * do_req_filebacked() has returned unless we are sure that 'cmd' has
1947 * not yet been completed.
1948 */
1949 ret = do_req_filebacked(lo, rq);
1950
1951 if (cmd_blkcg_css)
1952 kthread_associate_blkcg(NULL);
1953
1954 if (cmd_memcg_css) {
1955 set_active_memcg(old_memcg);
1956 css_put(cmd_memcg_css);
1957 }
1958 failed:
1959 /* complete non-aio request */
1960 if (ret != -EIOCBQUEUED) {
1961 if (ret == -EOPNOTSUPP)
1962 cmd->ret = ret;
1963 else
1964 cmd->ret = ret ? -EIO : 0;
1965 if (likely(!blk_should_fake_timeout(rq->q)))
1966 blk_mq_complete_request(rq);
1967 }
1968 }
1969
loop_process_work(struct loop_worker * worker,struct list_head * cmd_list,struct loop_device * lo)1970 static void loop_process_work(struct loop_worker *worker,
1971 struct list_head *cmd_list, struct loop_device *lo)
1972 {
1973 int orig_flags = current->flags;
1974 struct loop_cmd *cmd;
1975
1976 current->flags |= PF_LOCAL_THROTTLE | PF_MEMALLOC_NOIO;
1977 spin_lock_irq(&lo->lo_work_lock);
1978 while (!list_empty(cmd_list)) {
1979 cmd = container_of(
1980 cmd_list->next, struct loop_cmd, list_entry);
1981 list_del(cmd_list->next);
1982 spin_unlock_irq(&lo->lo_work_lock);
1983
1984 loop_handle_cmd(cmd);
1985 cond_resched();
1986
1987 spin_lock_irq(&lo->lo_work_lock);
1988 }
1989
1990 /*
1991 * We only add to the idle list if there are no pending cmds
1992 * *and* the worker will not run again which ensures that it
1993 * is safe to free any worker on the idle list
1994 */
1995 if (worker && !work_pending(&worker->work)) {
1996 worker->last_ran_at = jiffies;
1997 list_add_tail(&worker->idle_list, &lo->idle_worker_list);
1998 loop_set_timer(lo);
1999 }
2000 spin_unlock_irq(&lo->lo_work_lock);
2001 current->flags = orig_flags;
2002 }
2003
loop_workfn(struct work_struct * work)2004 static void loop_workfn(struct work_struct *work)
2005 {
2006 struct loop_worker *worker =
2007 container_of(work, struct loop_worker, work);
2008 loop_process_work(worker, &worker->cmd_list, worker->lo);
2009 }
2010
loop_rootcg_workfn(struct work_struct * work)2011 static void loop_rootcg_workfn(struct work_struct *work)
2012 {
2013 struct loop_device *lo =
2014 container_of(work, struct loop_device, rootcg_work);
2015 loop_process_work(NULL, &lo->rootcg_cmd_list, lo);
2016 }
2017
2018 static const struct blk_mq_ops loop_mq_ops = {
2019 .queue_rq = loop_queue_rq,
2020 .complete = lo_complete_rq,
2021 };
2022
loop_add(int i)2023 static int loop_add(int i)
2024 {
2025 struct queue_limits lim = {
2026 /*
2027 * Random number picked from the historic block max_sectors cap.
2028 */
2029 .max_hw_sectors = 2560u,
2030 };
2031 struct loop_device *lo;
2032 struct gendisk *disk;
2033 int err;
2034
2035 err = -ENOMEM;
2036 lo = kzalloc_obj(*lo);
2037 if (!lo)
2038 goto out;
2039 lo->worker_tree = RB_ROOT;
2040 INIT_LIST_HEAD(&lo->idle_worker_list);
2041 timer_setup(&lo->timer, loop_free_idle_workers_timer, TIMER_DEFERRABLE);
2042 WRITE_ONCE(lo->lo_state, Lo_unbound);
2043
2044 err = mutex_lock_killable(&loop_ctl_mutex);
2045 if (err)
2046 goto out_free_dev;
2047
2048 /* allocate id, if @id >= 0, we're requesting that specific id */
2049 if (i >= 0) {
2050 err = idr_alloc(&loop_index_idr, lo, i, i + 1, GFP_KERNEL);
2051 if (err == -ENOSPC)
2052 err = -EEXIST;
2053 } else {
2054 err = idr_alloc(&loop_index_idr, lo, 0, 0, GFP_KERNEL);
2055 }
2056 mutex_unlock(&loop_ctl_mutex);
2057 if (err < 0)
2058 goto out_free_dev;
2059 i = err;
2060
2061 lo->tag_set.ops = &loop_mq_ops;
2062 lo->tag_set.nr_hw_queues = 1;
2063 lo->tag_set.queue_depth = hw_queue_depth;
2064 lo->tag_set.numa_node = NUMA_NO_NODE;
2065 lo->tag_set.cmd_size = sizeof(struct loop_cmd);
2066 lo->tag_set.flags = BLK_MQ_F_STACKING | BLK_MQ_F_NO_SCHED_BY_DEFAULT;
2067 lo->tag_set.driver_data = lo;
2068
2069 err = blk_mq_alloc_tag_set(&lo->tag_set);
2070 if (err)
2071 goto out_free_idr;
2072
2073 disk = lo->lo_disk = blk_mq_alloc_disk(&lo->tag_set, &lim, lo);
2074 if (IS_ERR(disk)) {
2075 err = PTR_ERR(disk);
2076 goto out_cleanup_tags;
2077 }
2078 lo->lo_queue = lo->lo_disk->queue;
2079
2080 /*
2081 * Disable partition scanning by default. The in-kernel partition
2082 * scanning can be requested individually per-device during its
2083 * setup. Userspace can always add and remove partitions from all
2084 * devices. The needed partition minors are allocated from the
2085 * extended minor space, the main loop device numbers will continue
2086 * to match the loop minors, regardless of the number of partitions
2087 * used.
2088 *
2089 * If max_part is given, partition scanning is globally enabled for
2090 * all loop devices. The minors for the main loop devices will be
2091 * multiples of max_part.
2092 *
2093 * Note: Global-for-all-devices, set-only-at-init, read-only module
2094 * parameteters like 'max_loop' and 'max_part' make things needlessly
2095 * complicated, are too static, inflexible and may surprise
2096 * userspace tools. Parameters like this in general should be avoided.
2097 */
2098 if (!part_shift)
2099 set_bit(GD_SUPPRESS_PART_SCAN, &disk->state);
2100 mutex_init(&lo->lo_mutex);
2101 lo->lo_number = i;
2102 spin_lock_init(&lo->lo_lock);
2103 spin_lock_init(&lo->lo_work_lock);
2104 INIT_WORK(&lo->rootcg_work, loop_rootcg_workfn);
2105 INIT_LIST_HEAD(&lo->rootcg_cmd_list);
2106 disk->major = LOOP_MAJOR;
2107 disk->first_minor = i << part_shift;
2108 disk->minors = 1 << part_shift;
2109 disk->fops = &lo_fops;
2110 disk->private_data = lo;
2111 disk->queue = lo->lo_queue;
2112 disk->events = DISK_EVENT_MEDIA_CHANGE;
2113 disk->event_flags = DISK_EVENT_FLAG_UEVENT;
2114 sprintf(disk->disk_name, "loop%d", i);
2115 /* Make this loop device reachable from pathname. */
2116 err = add_disk(disk);
2117 if (err)
2118 goto out_cleanup_disk;
2119
2120 /* Show this loop device. */
2121 mutex_lock(&loop_ctl_mutex);
2122 lo->idr_visible = true;
2123 mutex_unlock(&loop_ctl_mutex);
2124
2125 return i;
2126
2127 out_cleanup_disk:
2128 put_disk(disk);
2129 out_cleanup_tags:
2130 blk_mq_free_tag_set(&lo->tag_set);
2131 out_free_idr:
2132 mutex_lock(&loop_ctl_mutex);
2133 idr_remove(&loop_index_idr, i);
2134 mutex_unlock(&loop_ctl_mutex);
2135 out_free_dev:
2136 kfree(lo);
2137 out:
2138 return err;
2139 }
2140
loop_remove(struct loop_device * lo)2141 static void loop_remove(struct loop_device *lo)
2142 {
2143 /* Make this loop device unreachable from pathname. */
2144 del_gendisk(lo->lo_disk);
2145 blk_mq_free_tag_set(&lo->tag_set);
2146
2147 mutex_lock(&loop_ctl_mutex);
2148 idr_remove(&loop_index_idr, lo->lo_number);
2149 mutex_unlock(&loop_ctl_mutex);
2150
2151 put_disk(lo->lo_disk);
2152 }
2153
2154 #ifdef CONFIG_BLOCK_LEGACY_AUTOLOAD
loop_probe(dev_t dev)2155 static void loop_probe(dev_t dev)
2156 {
2157 int idx = MINOR(dev) >> part_shift;
2158
2159 if (max_loop_specified && max_loop && idx >= max_loop)
2160 return;
2161 loop_add(idx);
2162 }
2163 #else
2164 #define loop_probe NULL
2165 #endif /* !CONFIG_BLOCK_LEGACY_AUTOLOAD */
2166
loop_control_remove(int idx)2167 static int loop_control_remove(int idx)
2168 {
2169 struct loop_device *lo;
2170 int ret;
2171
2172 if (idx < 0) {
2173 pr_warn_once("deleting an unspecified loop device is not supported.\n");
2174 return -EINVAL;
2175 }
2176
2177 /* Hide this loop device for serialization. */
2178 ret = mutex_lock_killable(&loop_ctl_mutex);
2179 if (ret)
2180 return ret;
2181 lo = idr_find(&loop_index_idr, idx);
2182 if (!lo || !lo->idr_visible)
2183 ret = -ENODEV;
2184 else
2185 lo->idr_visible = false;
2186 mutex_unlock(&loop_ctl_mutex);
2187 if (ret)
2188 return ret;
2189
2190 /* Check whether this loop device can be removed. */
2191 ret = mutex_lock_killable(&lo->lo_mutex);
2192 if (ret)
2193 goto mark_visible;
2194 if (lo->lo_state != Lo_unbound || disk_openers(lo->lo_disk) > 0) {
2195 mutex_unlock(&lo->lo_mutex);
2196 ret = -EBUSY;
2197 goto mark_visible;
2198 }
2199 /* Mark this loop device as no more bound, but not quite unbound yet */
2200 WRITE_ONCE(lo->lo_state, Lo_deleting);
2201 mutex_unlock(&lo->lo_mutex);
2202
2203 loop_remove(lo);
2204 return 0;
2205
2206 mark_visible:
2207 /* Show this loop device again. */
2208 mutex_lock(&loop_ctl_mutex);
2209 lo->idr_visible = true;
2210 mutex_unlock(&loop_ctl_mutex);
2211 return ret;
2212 }
2213
loop_control_get_free(int idx)2214 static int loop_control_get_free(int idx)
2215 {
2216 struct loop_device *lo;
2217 int id, ret;
2218
2219 ret = mutex_lock_killable(&loop_ctl_mutex);
2220 if (ret)
2221 return ret;
2222 idr_for_each_entry(&loop_index_idr, lo, id) {
2223 /*
2224 * Hitting a race results in creating a new loop device
2225 * which is harmless.
2226 */
2227 if (lo->idr_visible &&
2228 data_race(READ_ONCE(lo->lo_state)) == Lo_unbound)
2229 goto found;
2230 }
2231 mutex_unlock(&loop_ctl_mutex);
2232 return loop_add(-1);
2233 found:
2234 mutex_unlock(&loop_ctl_mutex);
2235 return id;
2236 }
2237
loop_control_ioctl(struct file * file,unsigned int cmd,unsigned long parm)2238 static long loop_control_ioctl(struct file *file, unsigned int cmd,
2239 unsigned long parm)
2240 {
2241 switch (cmd) {
2242 case LOOP_CTL_ADD:
2243 return loop_add(parm);
2244 case LOOP_CTL_REMOVE:
2245 return loop_control_remove(parm);
2246 case LOOP_CTL_GET_FREE:
2247 return loop_control_get_free(parm);
2248 default:
2249 return -ENOSYS;
2250 }
2251 }
2252
2253 static const struct file_operations loop_ctl_fops = {
2254 .open = nonseekable_open,
2255 .unlocked_ioctl = loop_control_ioctl,
2256 .compat_ioctl = loop_control_ioctl,
2257 .owner = THIS_MODULE,
2258 .llseek = noop_llseek,
2259 };
2260
2261 static struct miscdevice loop_misc = {
2262 .minor = LOOP_CTRL_MINOR,
2263 .name = "loop-control",
2264 .fops = &loop_ctl_fops,
2265 };
2266
2267 MODULE_ALIAS_MISCDEV(LOOP_CTRL_MINOR);
2268 MODULE_ALIAS("devname:loop-control");
2269
loop_init(void)2270 static int __init loop_init(void)
2271 {
2272 int i;
2273 int err;
2274
2275 part_shift = 0;
2276 if (max_part > 0) {
2277 part_shift = fls(max_part);
2278
2279 /*
2280 * Adjust max_part according to part_shift as it is exported
2281 * to user space so that user can decide correct minor number
2282 * if [s]he want to create more devices.
2283 *
2284 * Note that -1 is required because partition 0 is reserved
2285 * for the whole disk.
2286 */
2287 max_part = (1UL << part_shift) - 1;
2288 }
2289
2290 if ((1UL << part_shift) > DISK_MAX_PARTS) {
2291 err = -EINVAL;
2292 goto err_out;
2293 }
2294
2295 if (max_loop > 1UL << (MINORBITS - part_shift)) {
2296 err = -EINVAL;
2297 goto err_out;
2298 }
2299
2300 err = misc_register(&loop_misc);
2301 if (err < 0)
2302 goto err_out;
2303
2304
2305 if (__register_blkdev(LOOP_MAJOR, "loop", loop_probe)) {
2306 err = -EIO;
2307 goto misc_out;
2308 }
2309
2310 /* pre-create number of devices given by config or max_loop */
2311 for (i = 0; i < max_loop; i++)
2312 loop_add(i);
2313
2314 printk(KERN_INFO "loop: module loaded\n");
2315 return 0;
2316
2317 misc_out:
2318 misc_deregister(&loop_misc);
2319 err_out:
2320 return err;
2321 }
2322
loop_exit(void)2323 static void __exit loop_exit(void)
2324 {
2325 struct loop_device *lo;
2326 int id;
2327
2328 unregister_blkdev(LOOP_MAJOR, "loop");
2329 misc_deregister(&loop_misc);
2330
2331 /*
2332 * There is no need to use loop_ctl_mutex here, for nobody else can
2333 * access loop_index_idr when this module is unloading (unless forced
2334 * module unloading is requested). If this is not a clean unloading,
2335 * we have no means to avoid kernel crash.
2336 */
2337 idr_for_each_entry(&loop_index_idr, lo, id)
2338 loop_remove(lo);
2339
2340 idr_destroy(&loop_index_idr);
2341 }
2342
2343 module_init(loop_init);
2344 module_exit(loop_exit);
2345
2346 #ifndef MODULE
max_loop_setup(char * str)2347 static int __init max_loop_setup(char *str)
2348 {
2349 max_loop = simple_strtol(str, NULL, 0);
2350 #ifdef CONFIG_BLOCK_LEGACY_AUTOLOAD
2351 max_loop_specified = true;
2352 #endif
2353 return 1;
2354 }
2355
2356 __setup("max_loop=", max_loop_setup);
2357 #endif
2358