xref: /linux/block/fops.c (revision 7d1f84f1b94e79cb95d974ab3ec54cb3a4d64f5b)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 1991, 1992  Linus Torvalds
4  * Copyright (C) 2001  Andrea Arcangeli <andrea@suse.de> SuSE
5  * Copyright (C) 2016 - 2020 Christoph Hellwig
6  */
7 #include <linux/init.h>
8 #include <linux/mm.h>
9 #include <linux/blkdev.h>
10 #include <linux/blk-integrity.h>
11 #include <linux/buffer_head.h>
12 #include <linux/mpage.h>
13 #include <linux/uio.h>
14 #include <linux/namei.h>
15 #include <linux/task_io_accounting_ops.h>
16 #include <linux/falloc.h>
17 #include <linux/suspend.h>
18 #include <linux/fs.h>
19 #include <linux/iomap.h>
20 #include <linux/module.h>
21 #include <linux/io_uring/cmd.h>
22 #include <linux/dma-buf-io.h>
23 #include "blk.h"
24 
25 static inline struct inode *bdev_file_inode(struct file *file)
26 {
27 	return file->f_mapping->host;
28 }
29 
30 static bool blkdev_dio_fua(struct kiocb *iocb, struct block_device *bdev)
31 {
32 	if (!iocb_is_dsync(iocb))
33 		return false;
34 	/*
35 	 * Async writes cannot fall back to generic_write_sync(), so they must
36 	 * use FUA (emulated if needed); sync writes only need it when the
37 	 * device supports FUA natively.
38 	 */
39 	if (!is_sync_kiocb(iocb))
40 		return true;
41 	return bdev_fua(bdev);
42 }
43 
44 static blk_opf_t dio_bio_write_op(struct kiocb *iocb, struct block_device *bdev)
45 {
46 	blk_opf_t opf = REQ_OP_WRITE | REQ_SYNC | REQ_IDLE;
47 
48 	/* avoid the need for a I/O completion work item */
49 	if (blkdev_dio_fua(iocb, bdev))
50 		opf |= REQ_FUA;
51 	return opf;
52 }
53 
54 static bool blkdev_dio_invalid(struct block_device *bdev, struct kiocb *iocb,
55 				struct iov_iter *iter)
56 {
57 	return (iocb->ki_pos | iov_iter_count(iter)) &
58 			(bdev_logical_block_size(bdev) - 1);
59 }
60 
61 static inline int blkdev_iov_iter_get_pages(struct bio *bio,
62 		struct iov_iter *iter, struct block_device *bdev)
63 {
64 	return bio_iov_iter_get_pages(bio, iter, BIO_MAX_SIZE,
65 			bdev_dma_alignment(bdev),
66 			bdev_logical_block_size(bdev) - 1);
67 }
68 
69 #define DIO_INLINE_BIO_VECS 4
70 
71 static ssize_t __blkdev_direct_IO_simple(struct kiocb *iocb,
72 		struct iov_iter *iter, struct block_device *bdev,
73 		unsigned int nr_pages)
74 {
75 	struct bio_vec inline_vecs[DIO_INLINE_BIO_VECS], *vecs;
76 	loff_t pos = iocb->ki_pos;
77 	bool should_dirty = false;
78 	struct bio bio;
79 	ssize_t ret;
80 
81 	if (nr_pages <= DIO_INLINE_BIO_VECS)
82 		vecs = inline_vecs;
83 	else {
84 		vecs = kmalloc_objs(struct bio_vec, nr_pages);
85 		if (!vecs)
86 			return -ENOMEM;
87 	}
88 
89 	if (iov_iter_rw(iter) == READ) {
90 		bio_init(&bio, bdev, vecs, nr_pages, REQ_OP_READ);
91 		if (user_backed_iter(iter))
92 			should_dirty = true;
93 	} else {
94 		bio_init(&bio, bdev, vecs, nr_pages,
95 			 dio_bio_write_op(iocb, bdev));
96 	}
97 	bio.bi_iter.bi_sector = pos >> SECTOR_SHIFT;
98 	bio.bi_write_hint = file_inode(iocb->ki_filp)->i_write_hint;
99 	bio.bi_write_stream = iocb->ki_write_stream;
100 	bio.bi_ioprio = iocb->ki_ioprio;
101 	if (iocb->ki_flags & IOCB_ATOMIC)
102 		bio.bi_opf |= REQ_ATOMIC;
103 
104 	ret = blkdev_iov_iter_get_pages(&bio, iter, bdev);
105 	if (unlikely(ret))
106 		goto out;
107 	ret = bio.bi_iter.bi_size;
108 
109 	if (iov_iter_rw(iter) == WRITE)
110 		task_io_account_write(ret);
111 
112 	if (iocb->ki_flags & IOCB_NOWAIT)
113 		bio.bi_opf |= REQ_NOWAIT;
114 
115 	submit_bio_wait(&bio);
116 
117 	bio_release_pages(&bio, should_dirty);
118 	if (unlikely(bio.bi_status))
119 		ret = blk_status_to_errno(bio.bi_status);
120 
121 out:
122 	if (vecs != inline_vecs)
123 		kfree(vecs);
124 
125 	bio_uninit(&bio);
126 
127 	return ret;
128 }
129 
130 enum {
131 	DIO_SHOULD_DIRTY	= 1,
132 	DIO_IS_SYNC		= 2,
133 };
134 
135 struct blkdev_dio {
136 	union {
137 		struct kiocb		*iocb;
138 		struct task_struct	*waiter;
139 	};
140 	size_t			size;
141 	atomic_t		ref;
142 	unsigned int		flags;
143 	struct bio		bio ____cacheline_aligned_in_smp;
144 };
145 
146 static struct bio_set blkdev_dio_pool;
147 
148 static void blkdev_bio_end_io(struct bio *bio)
149 {
150 	struct blkdev_dio *dio = bio->bi_private;
151 	bool should_dirty = dio->flags & DIO_SHOULD_DIRTY;
152 	bool is_sync = dio->flags & DIO_IS_SYNC;
153 
154 	if (bio->bi_status && !dio->bio.bi_status)
155 		dio->bio.bi_status = bio->bi_status;
156 
157 	if (bio_integrity(bio))
158 		bio_integrity_unmap_user(bio);
159 
160 	if (atomic_dec_and_test(&dio->ref)) {
161 		if (!is_sync) {
162 			struct kiocb *iocb = dio->iocb;
163 			ssize_t ret;
164 
165 			WRITE_ONCE(iocb->private, NULL);
166 
167 			if (likely(!dio->bio.bi_status)) {
168 				ret = dio->size;
169 				iocb->ki_pos += ret;
170 			} else {
171 				ret = blk_status_to_errno(dio->bio.bi_status);
172 			}
173 
174 			dio->iocb->ki_complete(iocb, ret);
175 			bio_put(&dio->bio);
176 		} else {
177 			struct task_struct *waiter = dio->waiter;
178 
179 			WRITE_ONCE(dio->waiter, NULL);
180 			blk_wake_io_task(waiter);
181 		}
182 	}
183 
184 	if (should_dirty) {
185 		bio_check_pages_dirty(bio);
186 	} else {
187 		bio_release_pages(bio, false);
188 		bio_put(bio);
189 	}
190 }
191 
192 static ssize_t __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter,
193 		struct block_device *bdev, unsigned int nr_pages)
194 {
195 	struct blk_plug plug;
196 	struct blkdev_dio *dio;
197 	struct bio *bio;
198 	bool is_read = (iov_iter_rw(iter) == READ), is_sync;
199 	blk_opf_t opf = is_read ? REQ_OP_READ : dio_bio_write_op(iocb, bdev);
200 	loff_t pos = iocb->ki_pos;
201 	int ret = 0;
202 
203 	bio = bio_alloc_bioset(bdev, nr_pages, opf, GFP_KERNEL,
204 			       &blkdev_dio_pool);
205 	dio = container_of(bio, struct blkdev_dio, bio);
206 	atomic_set(&dio->ref, 1);
207 	/*
208 	 * Grab an extra reference to ensure the dio structure which is embedded
209 	 * into the first bio stays around.
210 	 */
211 	bio_get(bio);
212 
213 	is_sync = is_sync_kiocb(iocb);
214 	if (is_sync) {
215 		dio->flags = DIO_IS_SYNC;
216 		dio->waiter = current;
217 	} else {
218 		dio->flags = 0;
219 		dio->iocb = iocb;
220 	}
221 
222 	dio->size = 0;
223 	if (is_read && user_backed_iter(iter))
224 		dio->flags |= DIO_SHOULD_DIRTY;
225 
226 	blk_start_plug(&plug);
227 
228 	for (;;) {
229 		bio->bi_iter.bi_sector = pos >> SECTOR_SHIFT;
230 		bio->bi_write_hint = file_inode(iocb->ki_filp)->i_write_hint;
231 		bio->bi_write_stream = iocb->ki_write_stream;
232 		bio->bi_private = dio;
233 		bio->bi_end_io = blkdev_bio_end_io;
234 		bio->bi_ioprio = iocb->ki_ioprio;
235 
236 		ret = blkdev_iov_iter_get_pages(bio, iter, bdev);
237 		if (unlikely(ret)) {
238 			bio_endio_status(bio, errno_to_blk_status(ret));
239 			break;
240 		}
241 		if (iocb->ki_flags & IOCB_NOWAIT) {
242 			/*
243 			 * This is nonblocking IO, and we need to allocate
244 			 * another bio if we have data left to map. As we
245 			 * cannot guarantee that one of the sub bios will not
246 			 * fail getting issued FOR NOWAIT and as error results
247 			 * are coalesced across all of them, be safe and ask for
248 			 * a retry of this from blocking context.
249 			 */
250 			if (unlikely(iov_iter_count(iter))) {
251 				ret = -EAGAIN;
252 				goto fail;
253 			}
254 			bio->bi_opf |= REQ_NOWAIT;
255 		}
256 		if (iocb->ki_flags & IOCB_HAS_METADATA) {
257 			ret = bio_integrity_map_iter(bio, iocb->private);
258 			if (unlikely(ret)) {
259 				bio_endio_status(bio, errno_to_blk_status(ret));
260 				break;
261 			}
262 		}
263 
264 		if (is_read) {
265 			if (dio->flags & DIO_SHOULD_DIRTY)
266 				bio_set_pages_dirty(bio);
267 		} else {
268 			task_io_account_write(bio->bi_iter.bi_size);
269 		}
270 		dio->size += bio->bi_iter.bi_size;
271 		pos += bio->bi_iter.bi_size;
272 
273 		nr_pages = bio_iov_vecs_to_alloc(iter, BIO_MAX_VECS);
274 		if (!nr_pages) {
275 			submit_bio(bio);
276 			break;
277 		}
278 		atomic_inc(&dio->ref);
279 		submit_bio(bio);
280 		bio = bio_alloc(bdev, nr_pages, opf, GFP_KERNEL);
281 	}
282 
283 	blk_finish_plug(&plug);
284 
285 	if (!is_sync)
286 		return -EIOCBQUEUED;
287 
288 	for (;;) {
289 		set_current_state(TASK_UNINTERRUPTIBLE);
290 		if (!READ_ONCE(dio->waiter))
291 			break;
292 		blk_io_schedule();
293 	}
294 	__set_current_state(TASK_RUNNING);
295 
296 	if (!ret)
297 		ret = blk_status_to_errno(dio->bio.bi_status);
298 	if (likely(!ret))
299 		ret = dio->size;
300 
301 	bio_put(&dio->bio);
302 	return ret;
303 fail:
304 	bio_release_pages(bio, false);
305 	bio_clear_flag(bio, BIO_REFFED);
306 	bio_put(bio);
307 	blk_finish_plug(&plug);
308 	return ret;
309 }
310 
311 static void blkdev_bio_end_io_async(struct bio *bio)
312 {
313 	struct blkdev_dio *dio = container_of(bio, struct blkdev_dio, bio);
314 	struct kiocb *iocb = dio->iocb;
315 	ssize_t ret;
316 
317 	WRITE_ONCE(iocb->private, NULL);
318 
319 	if (likely(!bio->bi_status)) {
320 		ret = dio->size;
321 		iocb->ki_pos += ret;
322 	} else {
323 		ret = blk_status_to_errno(bio->bi_status);
324 	}
325 
326 	iocb->ki_complete(iocb, ret);
327 
328 	if (dio->flags & DIO_SHOULD_DIRTY) {
329 		bio_check_pages_dirty(bio);
330 	} else {
331 		bio_release_pages(bio, false);
332 		bio_put(bio);
333 	}
334 }
335 
336 static ssize_t __blkdev_direct_IO_async(struct kiocb *iocb,
337 					struct iov_iter *iter,
338 					struct block_device *bdev,
339 					unsigned int nr_pages)
340 {
341 	bool is_read = iov_iter_rw(iter) == READ;
342 	blk_opf_t opf = is_read ? REQ_OP_READ : dio_bio_write_op(iocb, bdev);
343 	struct blkdev_dio *dio;
344 	struct bio *bio;
345 	loff_t pos = iocb->ki_pos;
346 	int ret = 0;
347 
348 	bio = bio_alloc_bioset(bdev, nr_pages, opf, GFP_KERNEL,
349 			       &blkdev_dio_pool);
350 	dio = container_of(bio, struct blkdev_dio, bio);
351 	dio->flags = 0;
352 	dio->iocb = iocb;
353 	bio->bi_iter.bi_sector = pos >> SECTOR_SHIFT;
354 	bio->bi_write_hint = file_inode(iocb->ki_filp)->i_write_hint;
355 	bio->bi_write_stream = iocb->ki_write_stream;
356 	bio->bi_end_io = blkdev_bio_end_io_async;
357 	bio->bi_ioprio = iocb->ki_ioprio;
358 
359 	if (iocb->ki_flags & IOCB_ATOMIC)
360 		bio->bi_opf |= REQ_ATOMIC;
361 
362 	if (iocb->ki_flags & IOCB_NOWAIT)
363 		bio->bi_opf |= REQ_NOWAIT;
364 
365 	/*
366 	 * Users don't rely on the iterator being in any particular
367 	 * state for async I/O returning -EIOCBQUEUED, hence we can
368 	 * avoid expensive iov_iter_advance(). Bypass
369 	 * bio_iov_iter_get_pages() and set the bvec/dmabuf directly.
370 	 */
371 	if (!bio_iov_iter_set(bio, iter)) {
372 		ret = blkdev_iov_iter_get_pages(bio, iter, bdev);
373 		if (unlikely(ret))
374 			goto out_bio_put;
375 	}
376 	dio->size = bio->bi_iter.bi_size;
377 
378 	if (is_read) {
379 		if (user_backed_iter(iter)) {
380 			dio->flags |= DIO_SHOULD_DIRTY;
381 			bio_set_pages_dirty(bio);
382 		}
383 	} else {
384 		task_io_account_write(bio->bi_iter.bi_size);
385 	}
386 
387 	if (iocb->ki_flags & IOCB_HIPRI) {
388 		bio->bi_opf |= REQ_POLLED;
389 		submit_bio(bio);
390 		WRITE_ONCE(iocb->private, bio);
391 	} else {
392 		submit_bio(bio);
393 	}
394 	return -EIOCBQUEUED;
395 
396 out_bio_put:
397 	bio_put(bio);
398 	return ret;
399 }
400 
401 static ssize_t blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
402 {
403 	struct block_device *bdev = I_BDEV(iocb->ki_filp->f_mapping->host);
404 	unsigned int nr_pages;
405 
406 	if (!iov_iter_count(iter))
407 		return 0;
408 
409 	if (blkdev_dio_invalid(bdev, iocb, iter))
410 		return -EINVAL;
411 
412 	/* HIPRI needs private as bio; HAS_METADATA keeps it as uio_meta */
413 	if ((iocb->ki_flags & IOCB_HIPRI) &&
414 	    (iocb->ki_flags & IOCB_HAS_METADATA))
415 		return -EOPNOTSUPP;
416 
417 	if (iov_iter_rw(iter) == WRITE) {
418 		u16 max_write_streams = bdev_max_write_streams(bdev);
419 
420 		if (iocb->ki_write_stream) {
421 			if (iocb->ki_write_stream > max_write_streams)
422 				return -EINVAL;
423 		} else if (max_write_streams) {
424 			enum rw_hint write_hint =
425 				file_inode(iocb->ki_filp)->i_write_hint;
426 
427 			/*
428 			 * Just use the write hint as write stream for block
429 			 * device writes.  This assumes no file system is
430 			 * mounted that would use the streams differently.
431 			 */
432 			if (write_hint <= max_write_streams)
433 				iocb->ki_write_stream = write_hint;
434 		}
435 	}
436 
437 	nr_pages = bio_iov_vecs_to_alloc(iter, BIO_MAX_VECS + 1);
438 	if (likely(nr_pages <= BIO_MAX_VECS &&
439 		   !(iocb->ki_flags & IOCB_HAS_METADATA))) {
440 		if (is_sync_kiocb(iocb))
441 			return __blkdev_direct_IO_simple(iocb, iter, bdev,
442 							nr_pages);
443 		return __blkdev_direct_IO_async(iocb, iter, bdev, nr_pages);
444 	} else if (iocb->ki_flags & IOCB_ATOMIC) {
445 		return -EINVAL;
446 	}
447 	return __blkdev_direct_IO(iocb, iter, bdev, bio_max_segs(nr_pages));
448 }
449 
450 static int blkdev_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
451 		unsigned int flags, struct iomap *iomap, struct iomap *srcmap)
452 {
453 	struct block_device *bdev = I_BDEV(inode);
454 	loff_t isize = i_size_read(inode);
455 
456 	if (offset >= isize)
457 		return -EIO;
458 
459 	iomap->bdev = bdev;
460 	iomap->offset = ALIGN_DOWN(offset, bdev_logical_block_size(bdev));
461 	iomap->type = IOMAP_MAPPED;
462 	iomap->addr = iomap->offset;
463 	iomap->length = isize - iomap->offset;
464 	iomap->flags |= IOMAP_F_BUFFER_HEAD; /* noop for !CONFIG_BUFFER_HEAD */
465 	return 0;
466 }
467 
468 static DEFINE_IOMAP_ITER_NEXT(blkdev_iomap_next, blkdev_iomap_begin);
469 
470 static const struct iomap_ops blkdev_iomap_ops = {
471 	.iomap_next		= blkdev_iomap_next,
472 };
473 
474 #ifdef CONFIG_BUFFER_HEAD
475 static int blkdev_get_block(struct inode *inode, sector_t iblock,
476 		struct buffer_head *bh, int create)
477 {
478 	bh->b_bdev = I_BDEV(inode);
479 	bh->b_blocknr = iblock;
480 	set_buffer_mapped(bh);
481 	return 0;
482 }
483 
484 /*
485  * We cannot call mpage_writepages() as it does not take the buffer lock.
486  * We must use block_write_full_folio() directly which holds the buffer
487  * lock.  The buffer lock provides the synchronisation with writeback
488  * that filesystems rely on when they use the blockdev's mapping.
489  */
490 static int blkdev_writepages(struct address_space *mapping,
491 		struct writeback_control *wbc)
492 {
493 	struct folio *folio = NULL;
494 	struct blk_plug plug;
495 	int err;
496 
497 	blk_start_plug(&plug);
498 	while ((folio = writeback_iter(mapping, wbc, folio, &err)))
499 		err = block_write_full_folio(folio, wbc, blkdev_get_block);
500 	blk_finish_plug(&plug);
501 
502 	return err;
503 }
504 
505 static int blkdev_read_folio(struct file *file, struct folio *folio)
506 {
507 	return block_read_full_folio(folio, blkdev_get_block);
508 }
509 
510 static void blkdev_readahead(struct readahead_control *rac)
511 {
512 	mpage_readahead(rac, blkdev_get_block);
513 }
514 
515 const struct address_space_operations def_blk_aops = {
516 	.dirty_folio	= block_dirty_folio,
517 	.invalidate_folio = block_invalidate_folio,
518 	.read_folio	= blkdev_read_folio,
519 	.readahead	= blkdev_readahead,
520 	.writepages	= blkdev_writepages,
521 	.migrate_folio	= buffer_migrate_folio_norefs,
522 	.is_dirty_writeback = buffer_check_dirty_writeback,
523 };
524 #else /* CONFIG_BUFFER_HEAD */
525 static int blkdev_read_folio(struct file *file, struct folio *folio)
526 {
527 	iomap_bio_read_folio(folio, &blkdev_iomap_ops);
528 	return 0;
529 }
530 
531 static void blkdev_readahead(struct readahead_control *rac)
532 {
533 	iomap_bio_readahead(rac, &blkdev_iomap_ops);
534 }
535 
536 static ssize_t blkdev_writeback_range(struct iomap_writepage_ctx *wpc,
537 		struct folio *folio, u64 offset, unsigned int len, u64 end_pos)
538 {
539 	loff_t isize = i_size_read(wpc->inode);
540 
541 	if (WARN_ON_ONCE(offset >= isize))
542 		return -EIO;
543 
544 	if (offset < wpc->iomap.offset ||
545 	    offset >= wpc->iomap.offset + wpc->iomap.length) {
546 		int error;
547 
548 		error = blkdev_iomap_begin(wpc->inode, offset, isize - offset,
549 				IOMAP_WRITE, &wpc->iomap, NULL);
550 		if (error)
551 			return error;
552 	}
553 
554 	return iomap_add_to_ioend(wpc, folio, offset, end_pos, len);
555 }
556 
557 static const struct iomap_writeback_ops blkdev_writeback_ops = {
558 	.writeback_range	= blkdev_writeback_range,
559 	.writeback_submit	= iomap_ioend_writeback_submit,
560 };
561 
562 static int blkdev_writepages(struct address_space *mapping,
563 		struct writeback_control *wbc)
564 {
565 	struct iomap_writepage_ctx wpc = {
566 		.inode		= mapping->host,
567 		.wbc		= wbc,
568 		.ops		= &blkdev_writeback_ops
569 	};
570 
571 	return iomap_writepages(&wpc);
572 }
573 
574 const struct address_space_operations def_blk_aops = {
575 	.dirty_folio		= iomap_dirty_folio,
576 	.release_folio		= iomap_release_folio,
577 	.invalidate_folio	= iomap_invalidate_folio,
578 	.read_folio		= blkdev_read_folio,
579 	.readahead		= blkdev_readahead,
580 	.writepages		= blkdev_writepages,
581 	.is_partially_uptodate  = iomap_is_partially_uptodate,
582 	.error_remove_folio	= generic_error_remove_folio,
583 	.migrate_folio		= filemap_migrate_folio,
584 };
585 #endif /* CONFIG_BUFFER_HEAD */
586 
587 /*
588  * for a block special file file_inode(file)->i_size is zero
589  * so we compute the size by hand (just as in block_read/write above)
590  */
591 static loff_t blkdev_llseek(struct file *file, loff_t offset, int whence)
592 {
593 	struct inode *bd_inode = bdev_file_inode(file);
594 	loff_t retval;
595 
596 	inode_lock(bd_inode);
597 	retval = fixed_size_llseek(file, offset, whence, i_size_read(bd_inode));
598 	inode_unlock(bd_inode);
599 	return retval;
600 }
601 
602 static int blkdev_fsync(struct file *filp, loff_t start, loff_t end,
603 		int datasync)
604 {
605 	struct block_device *bdev = I_BDEV(filp->f_mapping->host);
606 	int error;
607 
608 	error = file_write_and_wait_range(filp, start, end);
609 	if (error)
610 		return error;
611 
612 	/*
613 	 * There is no need to serialise calls to blkdev_issue_flush with
614 	 * i_mutex and doing so causes performance issues with concurrent
615 	 * O_SYNC writers to a block device.
616 	 */
617 	error = blkdev_issue_flush(bdev);
618 	if (error == -EOPNOTSUPP)
619 		error = 0;
620 
621 	return error;
622 }
623 
624 /**
625  * file_to_blk_mode - get block open flags from file flags
626  * @file: file whose open flags should be converted
627  *
628  * Look at file open flags and generate corresponding block open flags from
629  * them. The function works both for file just being open (e.g. during ->open
630  * callback) and for file that is already open. This is actually non-trivial
631  * (see comment in the function).
632  */
633 blk_mode_t file_to_blk_mode(struct file *file)
634 {
635 	blk_mode_t mode = 0;
636 
637 	if (file->f_mode & FMODE_READ)
638 		mode |= BLK_OPEN_READ;
639 	if (file->f_mode & FMODE_WRITE)
640 		mode |= BLK_OPEN_WRITE;
641 	/*
642 	 * do_dentry_open() clears O_EXCL from f_flags, use file->private_data
643 	 * to determine whether the open was exclusive for already open files.
644 	 */
645 	if (file->private_data)
646 		mode |= BLK_OPEN_EXCL;
647 	else if (file->f_flags & O_EXCL)
648 		mode |= BLK_OPEN_EXCL;
649 	if (file->f_flags & O_NDELAY)
650 		mode |= BLK_OPEN_NDELAY;
651 
652 	/*
653 	 * If all bits in O_ACCMODE set (aka O_RDWR | O_WRONLY), the floppy
654 	 * driver has historically allowed ioctls as if the file was opened for
655 	 * writing, but does not allow and actual reads or writes.
656 	 */
657 	if ((file->f_flags & O_ACCMODE) == (O_RDWR | O_WRONLY))
658 		mode |= BLK_OPEN_WRITE_IOCTL;
659 
660 	return mode;
661 }
662 
663 static int blkdev_open(struct inode *inode, struct file *filp)
664 {
665 	struct block_device *bdev;
666 	blk_mode_t mode;
667 	int ret;
668 
669 	mode = file_to_blk_mode(filp);
670 	/* Use the file as the holder. */
671 	if (mode & BLK_OPEN_EXCL)
672 		filp->private_data = filp;
673 	ret = bdev_permission(inode->i_rdev, mode, filp->private_data);
674 	if (ret)
675 		return ret;
676 
677 	bdev = blkdev_get_no_open(inode->i_rdev, true);
678 	if (!bdev)
679 		return -ENXIO;
680 
681 	if (bdev_can_atomic_write(bdev))
682 		filp->f_mode |= FMODE_CAN_ATOMIC_WRITE;
683 	if (blk_get_integrity(bdev->bd_disk))
684 		filp->f_mode |= FMODE_HAS_METADATA;
685 
686 	ret = bdev_open(bdev, mode, filp->private_data, NULL, filp);
687 	if (ret)
688 		blkdev_put_no_open(bdev);
689 	return ret;
690 }
691 
692 static int blkdev_release(struct inode *inode, struct file *filp)
693 {
694 	bdev_release(filp);
695 	return 0;
696 }
697 
698 static ssize_t
699 blkdev_direct_write(struct kiocb *iocb, struct iov_iter *from)
700 {
701 	size_t count = iov_iter_count(from);
702 	ssize_t written;
703 
704 	written = kiocb_invalidate_pages(iocb, count);
705 	if (written) {
706 		if (written == -EBUSY)
707 			return 0;
708 		return written;
709 	}
710 
711 	written = blkdev_direct_IO(iocb, from);
712 	if (written > 0) {
713 		kiocb_invalidate_post_direct_write(iocb, count);
714 		iocb->ki_pos += written;
715 		count -= written;
716 	}
717 	if (written != -EIOCBQUEUED)
718 		iov_iter_revert(from, count - iov_iter_count(from));
719 	return written;
720 }
721 
722 static ssize_t blkdev_buffered_write(struct kiocb *iocb, struct iov_iter *from)
723 {
724 	struct inode *inode = bdev_file_inode(iocb->ki_filp);
725 	ssize_t ret;
726 
727 	/* Buffered fallback waits for writeback, which NOWAIT cannot allow. */
728 	if (iocb->ki_flags & IOCB_NOWAIT)
729 		return -EAGAIN;
730 
731 	if (unlikely(iov_iter_is_dmabuf_map(from)))
732 		return -EOPNOTSUPP;
733 
734 	/*
735 	 * Take i_rwsem to avoid racing with set_blocksize() changing
736 	 * i_blkbits/folio order and truncating the page cache.
737 	 */
738 	inode_lock_shared(inode);
739 	ret = iomap_file_buffered_write(iocb, from, &blkdev_iomap_ops, NULL,
740 					NULL);
741 	inode_unlock_shared(inode);
742 	return ret;
743 }
744 
745 /*
746  * Write data to the block device.  Only intended for the block device itself
747  * and the raw driver which basically is a fake block device.
748  *
749  * Does not take i_mutex for the write and thus is not for general purpose
750  * use.
751  */
752 static ssize_t blkdev_write_iter(struct kiocb *iocb, struct iov_iter *from)
753 {
754 	struct file *file = iocb->ki_filp;
755 	struct inode *bd_inode = bdev_file_inode(file);
756 	struct block_device *bdev = I_BDEV(bd_inode);
757 	bool atomic = iocb->ki_flags & IOCB_ATOMIC;
758 	loff_t size = bdev_nr_bytes(bdev);
759 	size_t shorted = 0;
760 	bool need_sync = false;
761 	ssize_t ret;
762 
763 	if (bdev_read_only(bdev))
764 		return -EPERM;
765 
766 	if (IS_SWAPFILE(bd_inode) && !is_hibernate_resume_dev(bd_inode->i_rdev))
767 		return -ETXTBSY;
768 
769 	if (!iov_iter_count(from))
770 		return 0;
771 
772 	if (iocb->ki_pos >= size)
773 		return -ENOSPC;
774 
775 	if ((iocb->ki_flags & (IOCB_NOWAIT | IOCB_DIRECT)) == IOCB_NOWAIT)
776 		return -EOPNOTSUPP;
777 
778 	if (atomic) {
779 		ret = generic_atomic_write_valid(iocb, from);
780 		if (ret)
781 			return ret;
782 	}
783 
784 	size -= iocb->ki_pos;
785 	if (iov_iter_count(from) > size) {
786 		if (atomic)
787 			return -EINVAL;
788 		shorted = iov_iter_count(from) - size;
789 		iov_iter_truncate(from, size);
790 	}
791 
792 	ret = file_update_time(file);
793 	if (ret)
794 		return ret;
795 
796 	if (iocb->ki_flags & IOCB_DIRECT) {
797 		ret = blkdev_direct_write(iocb, from);
798 		if (ret >= 0 && iov_iter_count(from)) {
799 			if (iocb->ki_flags & (IOCB_NOWAIT | IOCB_ATOMIC)) {
800 				/*
801 				 * The buffered fallback blocks on i_rwsem and
802 				 * on writeback of the data it copied, and
803 				 * can't provide torn-write protection: return
804 				 * the short direct write instead and let the
805 				 * caller retry. The bytes already written
806 				 * still need a flush if REQ_FUA was not set.
807 				 */
808 				if (!ret)
809 					ret = -EAGAIN;
810 				else if (!blkdev_dio_fua(iocb, bdev))
811 					need_sync = true;
812 			} else {
813 				ssize_t ret2;
814 
815 				inode_lock_shared(bd_inode);
816 				ret2 = blkdev_buffered_write(iocb, from);
817 				inode_unlock_shared(bd_inode);
818 				ret = direct_write_fallback(iocb, from, ret,
819 							    ret2);
820 				need_sync = true;
821 			}
822 		} else if (ret > 0 && !blkdev_dio_fua(iocb, bdev)) {
823 			/*
824 			 * The device does not support FUA, so REQ_FUA was not
825 			 * set and the O_DSYNC direct write still needs a flush
826 			 * to be made durable.
827 			 */
828 			need_sync = true;
829 		}
830 	} else {
831 		ret = blkdev_buffered_write(iocb, from);
832 		need_sync = true;
833 	}
834 
835 	if (ret > 0 && need_sync)
836 		ret = generic_write_sync(iocb, ret);
837 	iov_iter_reexpand(from, iov_iter_count(from) + shorted);
838 	return ret;
839 }
840 
841 static ssize_t blkdev_read_iter(struct kiocb *iocb, struct iov_iter *to)
842 {
843 	struct inode *bd_inode = bdev_file_inode(iocb->ki_filp);
844 	struct block_device *bdev = I_BDEV(iocb->ki_filp->f_mapping->host);
845 	loff_t size = bdev_nr_bytes(bdev);
846 	loff_t pos = iocb->ki_pos;
847 	size_t shorted = 0;
848 	ssize_t ret = 0;
849 	size_t count;
850 
851 	if (unlikely(pos + iov_iter_count(to) > size)) {
852 		if (pos >= size)
853 			return 0;
854 		size -= pos;
855 		shorted = iov_iter_count(to) - size;
856 		iov_iter_truncate(to, size);
857 	}
858 
859 	count = iov_iter_count(to);
860 	if (!count)
861 		goto reexpand; /* skip atime */
862 
863 	if (iocb->ki_flags & IOCB_DIRECT) {
864 		ret = kiocb_write_and_wait(iocb, count);
865 		if (ret < 0)
866 			goto reexpand;
867 		file_accessed(iocb->ki_filp);
868 
869 		ret = blkdev_direct_IO(iocb, to);
870 		if (ret > 0) {
871 			iocb->ki_pos += ret;
872 			count -= ret;
873 		}
874 		if (ret != -EIOCBQUEUED)
875 			iov_iter_revert(to, count - iov_iter_count(to));
876 		if (ret < 0 || !count)
877 			goto reexpand;
878 	}
879 	if (unlikely(iov_iter_is_dmabuf_map(to))) {
880 		if (!ret)
881 			ret = -EOPNOTSUPP;
882 		goto reexpand;
883 	}
884 
885 	/*
886 	 * Take i_rwsem and invalidate_lock to avoid racing with set_blocksize
887 	 * changing i_blkbits/folio order and punching out the pagecache.
888 	 */
889 	if (iocb->ki_flags & IOCB_NOWAIT) {
890 		if (!inode_trylock_shared(bd_inode)) {
891 			if (!ret)
892 				ret = -EAGAIN;
893 			goto reexpand;
894 		}
895 	} else {
896 		inode_lock_shared(bd_inode);
897 	}
898 	ret = filemap_read(iocb, to, ret);
899 	inode_unlock_shared(bd_inode);
900 
901 reexpand:
902 	if (unlikely(shorted))
903 		iov_iter_reexpand(to, iov_iter_count(to) + shorted);
904 	return ret;
905 }
906 
907 /*
908  * Take i_rwsem to avoid racing with set_blocksize changing i_blkbits/folio
909  * order and punching out the pagecache.
910  */
911 static ssize_t blkdev_splice_read(struct file *in, loff_t *ppos,
912 		struct pipe_inode_info *pipe, size_t len, unsigned int flags)
913 {
914 	struct inode *bd_inode = bdev_file_inode(in);
915 	ssize_t ret;
916 
917 	inode_lock_shared(bd_inode);
918 	ret = filemap_splice_read(in, ppos, pipe, len, flags);
919 	inode_unlock_shared(bd_inode);
920 	return ret;
921 }
922 
923 #define	BLKDEV_FALLOC_FL_SUPPORTED					\
924 		(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE |		\
925 		 FALLOC_FL_ZERO_RANGE | FALLOC_FL_WRITE_ZEROES)
926 
927 static long blkdev_fallocate(struct file *file, int mode, loff_t start,
928 			     loff_t len)
929 {
930 	struct inode *inode = bdev_file_inode(file);
931 	struct block_device *bdev = I_BDEV(inode);
932 	loff_t end = start + len - 1;
933 	loff_t isize;
934 	unsigned int flags;
935 	int error;
936 
937 	/* Fail if we don't recognize the flags. */
938 	if (mode & ~BLKDEV_FALLOC_FL_SUPPORTED)
939 		return -EOPNOTSUPP;
940 	/*
941 	 * Don't allow writing zeroes if the device does not enable the
942 	 * unmap write zeroes operation.
943 	 */
944 	if ((mode & FALLOC_FL_WRITE_ZEROES) &&
945 	    !bdev_write_zeroes_unmap_sectors(bdev))
946 		return -EOPNOTSUPP;
947 
948 	/* Don't go off the end of the device. */
949 	isize = bdev_nr_bytes(bdev);
950 	if (start >= isize)
951 		return -EINVAL;
952 	if (end >= isize) {
953 		if (mode & FALLOC_FL_KEEP_SIZE) {
954 			len = isize - start;
955 			end = start + len - 1;
956 		} else
957 			return -EINVAL;
958 	}
959 
960 	/*
961 	 * Don't allow IO that isn't aligned to logical block size.
962 	 */
963 	if ((start | len) & (bdev_logical_block_size(bdev) - 1))
964 		return -EINVAL;
965 
966 	inode_lock(inode);
967 	filemap_invalidate_lock(inode->i_mapping);
968 
969 	switch (mode) {
970 	case FALLOC_FL_ZERO_RANGE:
971 	case FALLOC_FL_ZERO_RANGE | FALLOC_FL_KEEP_SIZE:
972 		flags = BLKDEV_ZERO_NOUNMAP;
973 		break;
974 	case FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE:
975 		flags = BLKDEV_ZERO_NOFALLBACK;
976 		break;
977 	case FALLOC_FL_WRITE_ZEROES:
978 		flags = 0;
979 		break;
980 	default:
981 		error = -EOPNOTSUPP;
982 		goto fail;
983 	}
984 
985 	/*
986 	 * Invalidate the page cache, including dirty pages, for valid
987 	 * de-allocate mode calls to fallocate().
988 	 */
989 	error = truncate_bdev_range(bdev, file_to_blk_mode(file), start, end);
990 	if (error)
991 		goto fail;
992 
993 	error = blkdev_issue_zeroout(bdev, start >> SECTOR_SHIFT,
994 				     len >> SECTOR_SHIFT, GFP_KERNEL, flags);
995  fail:
996 	filemap_invalidate_unlock(inode->i_mapping);
997 	inode_unlock(inode);
998 	return error;
999 }
1000 
1001 static int blkdev_mmap_prepare(struct vm_area_desc *desc)
1002 {
1003 	struct file *file = desc->file;
1004 
1005 	if (bdev_read_only(I_BDEV(bdev_file_inode(file))))
1006 		return generic_file_readonly_mmap_prepare(desc);
1007 
1008 	return generic_file_mmap_prepare(desc);
1009 }
1010 
1011 static int blkdev_init_dma_buf_io_ctx(struct file *file,
1012 				      struct dma_buf_io_ctx *ctx)
1013 {
1014 	struct block_device *bdev = file_bdev(file);
1015 	struct gendisk *disk = bdev->bd_disk;
1016 
1017 	if (!(file->f_flags & O_DIRECT))
1018 		return -EINVAL;
1019 	if (ctx->dmabuf->size > UINT_MAX)
1020 		return -EINVAL;
1021 	if (!disk->fops->init_dma_buf_io_ctx)
1022 		return -EOPNOTSUPP;
1023 	if (bdev_has_integrity_csum(bdev))
1024 		return -EOPNOTSUPP;
1025 	return disk->fops->init_dma_buf_io_ctx(bdev, ctx);
1026 }
1027 
1028 const struct file_operations def_blk_fops = {
1029 	.open		= blkdev_open,
1030 	.release	= blkdev_release,
1031 	.llseek		= blkdev_llseek,
1032 	.read_iter	= blkdev_read_iter,
1033 	.write_iter	= blkdev_write_iter,
1034 	.iopoll		= iocb_bio_iopoll,
1035 	.mmap_prepare	= blkdev_mmap_prepare,
1036 	.fsync		= blkdev_fsync,
1037 	.unlocked_ioctl	= blkdev_ioctl,
1038 #ifdef CONFIG_COMPAT
1039 	.compat_ioctl	= compat_blkdev_ioctl,
1040 #endif
1041 	.splice_read	= blkdev_splice_read,
1042 	.splice_write	= iter_file_splice_write,
1043 	.fallocate	= blkdev_fallocate,
1044 	.uring_cmd	= blkdev_uring_cmd,
1045 	.fop_flags	= FOP_BUFFER_RASYNC | FOP_DONTCACHE,
1046 	.init_dma_buf_io_ctx = blkdev_init_dma_buf_io_ctx,
1047 };
1048 
1049 static __init int blkdev_init(void)
1050 {
1051 	return bioset_init(&blkdev_dio_pool, 4,
1052 				offsetof(struct blkdev_dio, bio),
1053 				BIOSET_NEED_BVECS|BIOSET_PERCPU_CACHE);
1054 }
1055 module_init(blkdev_init);
1056