1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 1991, 1992 Linus Torvalds 4 * Copyright (C) 2001 Andrea Arcangeli <andrea@suse.de> SuSE 5 * Copyright (C) 2016 - 2020 Christoph Hellwig 6 */ 7 #include <linux/init.h> 8 #include <linux/mm.h> 9 #include <linux/blkdev.h> 10 #include <linux/blk-integrity.h> 11 #include <linux/buffer_head.h> 12 #include <linux/mpage.h> 13 #include <linux/uio.h> 14 #include <linux/namei.h> 15 #include <linux/task_io_accounting_ops.h> 16 #include <linux/falloc.h> 17 #include <linux/suspend.h> 18 #include <linux/fs.h> 19 #include <linux/iomap.h> 20 #include <linux/module.h> 21 #include <linux/io_uring/cmd.h> 22 #include <linux/dma-buf-io.h> 23 #include "blk.h" 24 25 static inline struct inode *bdev_file_inode(struct file *file) 26 { 27 return file->f_mapping->host; 28 } 29 30 static bool blkdev_dio_fua(struct kiocb *iocb, struct block_device *bdev) 31 { 32 if (!iocb_is_dsync(iocb)) 33 return false; 34 /* 35 * Async writes cannot fall back to generic_write_sync(), so they must 36 * use FUA (emulated if needed); sync writes only need it when the 37 * device supports FUA natively. 38 */ 39 if (!is_sync_kiocb(iocb)) 40 return true; 41 return bdev_fua(bdev); 42 } 43 44 static blk_opf_t dio_bio_write_op(struct kiocb *iocb, struct block_device *bdev) 45 { 46 blk_opf_t opf = REQ_OP_WRITE | REQ_SYNC | REQ_IDLE; 47 48 /* avoid the need for a I/O completion work item */ 49 if (blkdev_dio_fua(iocb, bdev)) 50 opf |= REQ_FUA; 51 return opf; 52 } 53 54 static bool blkdev_dio_invalid(struct block_device *bdev, struct kiocb *iocb, 55 struct iov_iter *iter) 56 { 57 return (iocb->ki_pos | iov_iter_count(iter)) & 58 (bdev_logical_block_size(bdev) - 1); 59 } 60 61 static inline int blkdev_iov_iter_get_pages(struct bio *bio, 62 struct iov_iter *iter, struct block_device *bdev) 63 { 64 return bio_iov_iter_get_pages(bio, iter, BIO_MAX_SIZE, 65 bdev_dma_alignment(bdev), 66 bdev_logical_block_size(bdev) - 1); 67 } 68 69 #define DIO_INLINE_BIO_VECS 4 70 71 static ssize_t __blkdev_direct_IO_simple(struct kiocb *iocb, 72 struct iov_iter *iter, struct block_device *bdev, 73 unsigned int nr_pages) 74 { 75 struct bio_vec inline_vecs[DIO_INLINE_BIO_VECS], *vecs; 76 loff_t pos = iocb->ki_pos; 77 bool should_dirty = false; 78 struct bio bio; 79 ssize_t ret; 80 81 if (nr_pages <= DIO_INLINE_BIO_VECS) 82 vecs = inline_vecs; 83 else { 84 vecs = kmalloc_objs(struct bio_vec, nr_pages); 85 if (!vecs) 86 return -ENOMEM; 87 } 88 89 if (iov_iter_rw(iter) == READ) { 90 bio_init(&bio, bdev, vecs, nr_pages, REQ_OP_READ); 91 if (user_backed_iter(iter)) 92 should_dirty = true; 93 } else { 94 bio_init(&bio, bdev, vecs, nr_pages, 95 dio_bio_write_op(iocb, bdev)); 96 } 97 bio.bi_iter.bi_sector = pos >> SECTOR_SHIFT; 98 bio.bi_write_hint = file_inode(iocb->ki_filp)->i_write_hint; 99 bio.bi_write_stream = iocb->ki_write_stream; 100 bio.bi_ioprio = iocb->ki_ioprio; 101 if (iocb->ki_flags & IOCB_ATOMIC) 102 bio.bi_opf |= REQ_ATOMIC; 103 104 ret = blkdev_iov_iter_get_pages(&bio, iter, bdev); 105 if (unlikely(ret)) 106 goto out; 107 ret = bio.bi_iter.bi_size; 108 109 if (iov_iter_rw(iter) == WRITE) 110 task_io_account_write(ret); 111 112 if (iocb->ki_flags & IOCB_NOWAIT) 113 bio.bi_opf |= REQ_NOWAIT; 114 115 submit_bio_wait(&bio); 116 117 bio_release_pages(&bio, should_dirty); 118 if (unlikely(bio.bi_status)) 119 ret = blk_status_to_errno(bio.bi_status); 120 121 out: 122 if (vecs != inline_vecs) 123 kfree(vecs); 124 125 bio_uninit(&bio); 126 127 return ret; 128 } 129 130 enum { 131 DIO_SHOULD_DIRTY = 1, 132 DIO_IS_SYNC = 2, 133 }; 134 135 struct blkdev_dio { 136 union { 137 struct kiocb *iocb; 138 struct task_struct *waiter; 139 }; 140 size_t size; 141 atomic_t ref; 142 unsigned int flags; 143 struct bio bio ____cacheline_aligned_in_smp; 144 }; 145 146 static struct bio_set blkdev_dio_pool; 147 148 static void blkdev_bio_end_io(struct bio *bio) 149 { 150 struct blkdev_dio *dio = bio->bi_private; 151 bool should_dirty = dio->flags & DIO_SHOULD_DIRTY; 152 bool is_sync = dio->flags & DIO_IS_SYNC; 153 154 if (bio->bi_status && !dio->bio.bi_status) 155 dio->bio.bi_status = bio->bi_status; 156 157 if (bio_integrity(bio)) 158 bio_integrity_unmap_user(bio); 159 160 if (atomic_dec_and_test(&dio->ref)) { 161 if (!is_sync) { 162 struct kiocb *iocb = dio->iocb; 163 ssize_t ret; 164 165 WRITE_ONCE(iocb->private, NULL); 166 167 if (likely(!dio->bio.bi_status)) { 168 ret = dio->size; 169 iocb->ki_pos += ret; 170 } else { 171 ret = blk_status_to_errno(dio->bio.bi_status); 172 } 173 174 dio->iocb->ki_complete(iocb, ret); 175 bio_put(&dio->bio); 176 } else { 177 struct task_struct *waiter = dio->waiter; 178 179 WRITE_ONCE(dio->waiter, NULL); 180 blk_wake_io_task(waiter); 181 } 182 } 183 184 if (should_dirty) { 185 bio_check_pages_dirty(bio); 186 } else { 187 bio_release_pages(bio, false); 188 bio_put(bio); 189 } 190 } 191 192 static ssize_t __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, 193 struct block_device *bdev, unsigned int nr_pages) 194 { 195 struct blk_plug plug; 196 struct blkdev_dio *dio; 197 struct bio *bio; 198 bool is_read = (iov_iter_rw(iter) == READ), is_sync; 199 blk_opf_t opf = is_read ? REQ_OP_READ : dio_bio_write_op(iocb, bdev); 200 loff_t pos = iocb->ki_pos; 201 int ret = 0; 202 203 bio = bio_alloc_bioset(bdev, nr_pages, opf, GFP_KERNEL, 204 &blkdev_dio_pool); 205 dio = container_of(bio, struct blkdev_dio, bio); 206 atomic_set(&dio->ref, 1); 207 /* 208 * Grab an extra reference to ensure the dio structure which is embedded 209 * into the first bio stays around. 210 */ 211 bio_get(bio); 212 213 is_sync = is_sync_kiocb(iocb); 214 if (is_sync) { 215 dio->flags = DIO_IS_SYNC; 216 dio->waiter = current; 217 } else { 218 dio->flags = 0; 219 dio->iocb = iocb; 220 } 221 222 dio->size = 0; 223 if (is_read && user_backed_iter(iter)) 224 dio->flags |= DIO_SHOULD_DIRTY; 225 226 blk_start_plug(&plug); 227 228 for (;;) { 229 bio->bi_iter.bi_sector = pos >> SECTOR_SHIFT; 230 bio->bi_write_hint = file_inode(iocb->ki_filp)->i_write_hint; 231 bio->bi_write_stream = iocb->ki_write_stream; 232 bio->bi_private = dio; 233 bio->bi_end_io = blkdev_bio_end_io; 234 bio->bi_ioprio = iocb->ki_ioprio; 235 236 ret = blkdev_iov_iter_get_pages(bio, iter, bdev); 237 if (unlikely(ret)) { 238 bio_endio_status(bio, errno_to_blk_status(ret)); 239 break; 240 } 241 if (iocb->ki_flags & IOCB_NOWAIT) { 242 /* 243 * This is nonblocking IO, and we need to allocate 244 * another bio if we have data left to map. As we 245 * cannot guarantee that one of the sub bios will not 246 * fail getting issued FOR NOWAIT and as error results 247 * are coalesced across all of them, be safe and ask for 248 * a retry of this from blocking context. 249 */ 250 if (unlikely(iov_iter_count(iter))) { 251 ret = -EAGAIN; 252 goto fail; 253 } 254 bio->bi_opf |= REQ_NOWAIT; 255 } 256 if (iocb->ki_flags & IOCB_HAS_METADATA) { 257 ret = bio_integrity_map_iter(bio, iocb->private); 258 if (unlikely(ret)) { 259 bio_endio_status(bio, errno_to_blk_status(ret)); 260 break; 261 } 262 } 263 264 if (is_read) { 265 if (dio->flags & DIO_SHOULD_DIRTY) 266 bio_set_pages_dirty(bio); 267 } else { 268 task_io_account_write(bio->bi_iter.bi_size); 269 } 270 dio->size += bio->bi_iter.bi_size; 271 pos += bio->bi_iter.bi_size; 272 273 nr_pages = bio_iov_vecs_to_alloc(iter, BIO_MAX_VECS); 274 if (!nr_pages) { 275 submit_bio(bio); 276 break; 277 } 278 atomic_inc(&dio->ref); 279 submit_bio(bio); 280 bio = bio_alloc(bdev, nr_pages, opf, GFP_KERNEL); 281 } 282 283 blk_finish_plug(&plug); 284 285 if (!is_sync) 286 return -EIOCBQUEUED; 287 288 for (;;) { 289 set_current_state(TASK_UNINTERRUPTIBLE); 290 if (!READ_ONCE(dio->waiter)) 291 break; 292 blk_io_schedule(); 293 } 294 __set_current_state(TASK_RUNNING); 295 296 if (!ret) 297 ret = blk_status_to_errno(dio->bio.bi_status); 298 if (likely(!ret)) 299 ret = dio->size; 300 301 bio_put(&dio->bio); 302 return ret; 303 fail: 304 bio_release_pages(bio, false); 305 bio_clear_flag(bio, BIO_REFFED); 306 bio_put(bio); 307 blk_finish_plug(&plug); 308 return ret; 309 } 310 311 static void blkdev_bio_end_io_async(struct bio *bio) 312 { 313 struct blkdev_dio *dio = container_of(bio, struct blkdev_dio, bio); 314 struct kiocb *iocb = dio->iocb; 315 ssize_t ret; 316 317 WRITE_ONCE(iocb->private, NULL); 318 319 if (likely(!bio->bi_status)) { 320 ret = dio->size; 321 iocb->ki_pos += ret; 322 } else { 323 ret = blk_status_to_errno(bio->bi_status); 324 } 325 326 iocb->ki_complete(iocb, ret); 327 328 if (dio->flags & DIO_SHOULD_DIRTY) { 329 bio_check_pages_dirty(bio); 330 } else { 331 bio_release_pages(bio, false); 332 bio_put(bio); 333 } 334 } 335 336 static ssize_t __blkdev_direct_IO_async(struct kiocb *iocb, 337 struct iov_iter *iter, 338 struct block_device *bdev, 339 unsigned int nr_pages) 340 { 341 bool is_read = iov_iter_rw(iter) == READ; 342 blk_opf_t opf = is_read ? REQ_OP_READ : dio_bio_write_op(iocb, bdev); 343 struct blkdev_dio *dio; 344 struct bio *bio; 345 loff_t pos = iocb->ki_pos; 346 int ret = 0; 347 348 bio = bio_alloc_bioset(bdev, nr_pages, opf, GFP_KERNEL, 349 &blkdev_dio_pool); 350 dio = container_of(bio, struct blkdev_dio, bio); 351 dio->flags = 0; 352 dio->iocb = iocb; 353 bio->bi_iter.bi_sector = pos >> SECTOR_SHIFT; 354 bio->bi_write_hint = file_inode(iocb->ki_filp)->i_write_hint; 355 bio->bi_write_stream = iocb->ki_write_stream; 356 bio->bi_end_io = blkdev_bio_end_io_async; 357 bio->bi_ioprio = iocb->ki_ioprio; 358 359 if (iocb->ki_flags & IOCB_ATOMIC) 360 bio->bi_opf |= REQ_ATOMIC; 361 362 if (iocb->ki_flags & IOCB_NOWAIT) 363 bio->bi_opf |= REQ_NOWAIT; 364 365 /* 366 * Users don't rely on the iterator being in any particular 367 * state for async I/O returning -EIOCBQUEUED, hence we can 368 * avoid expensive iov_iter_advance(). Bypass 369 * bio_iov_iter_get_pages() and set the bvec/dmabuf directly. 370 */ 371 if (!bio_iov_iter_set(bio, iter)) { 372 ret = blkdev_iov_iter_get_pages(bio, iter, bdev); 373 if (unlikely(ret)) 374 goto out_bio_put; 375 } 376 dio->size = bio->bi_iter.bi_size; 377 378 if (is_read) { 379 if (user_backed_iter(iter)) { 380 dio->flags |= DIO_SHOULD_DIRTY; 381 bio_set_pages_dirty(bio); 382 } 383 } else { 384 task_io_account_write(bio->bi_iter.bi_size); 385 } 386 387 if (iocb->ki_flags & IOCB_HIPRI) { 388 bio->bi_opf |= REQ_POLLED; 389 submit_bio(bio); 390 WRITE_ONCE(iocb->private, bio); 391 } else { 392 submit_bio(bio); 393 } 394 return -EIOCBQUEUED; 395 396 out_bio_put: 397 bio_put(bio); 398 return ret; 399 } 400 401 static ssize_t blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter) 402 { 403 struct block_device *bdev = I_BDEV(iocb->ki_filp->f_mapping->host); 404 unsigned int nr_pages; 405 406 if (!iov_iter_count(iter)) 407 return 0; 408 409 if (blkdev_dio_invalid(bdev, iocb, iter)) 410 return -EINVAL; 411 412 /* HIPRI needs private as bio; HAS_METADATA keeps it as uio_meta */ 413 if ((iocb->ki_flags & IOCB_HIPRI) && 414 (iocb->ki_flags & IOCB_HAS_METADATA)) 415 return -EOPNOTSUPP; 416 417 if (iov_iter_rw(iter) == WRITE) { 418 u16 max_write_streams = bdev_max_write_streams(bdev); 419 420 if (iocb->ki_write_stream) { 421 if (iocb->ki_write_stream > max_write_streams) 422 return -EINVAL; 423 } else if (max_write_streams) { 424 enum rw_hint write_hint = 425 file_inode(iocb->ki_filp)->i_write_hint; 426 427 /* 428 * Just use the write hint as write stream for block 429 * device writes. This assumes no file system is 430 * mounted that would use the streams differently. 431 */ 432 if (write_hint <= max_write_streams) 433 iocb->ki_write_stream = write_hint; 434 } 435 } 436 437 nr_pages = bio_iov_vecs_to_alloc(iter, BIO_MAX_VECS + 1); 438 if (likely(nr_pages <= BIO_MAX_VECS && 439 !(iocb->ki_flags & IOCB_HAS_METADATA))) { 440 if (is_sync_kiocb(iocb)) 441 return __blkdev_direct_IO_simple(iocb, iter, bdev, 442 nr_pages); 443 return __blkdev_direct_IO_async(iocb, iter, bdev, nr_pages); 444 } else if (iocb->ki_flags & IOCB_ATOMIC) { 445 return -EINVAL; 446 } 447 return __blkdev_direct_IO(iocb, iter, bdev, bio_max_segs(nr_pages)); 448 } 449 450 static int blkdev_iomap_begin(struct inode *inode, loff_t offset, loff_t length, 451 unsigned int flags, struct iomap *iomap, struct iomap *srcmap) 452 { 453 struct block_device *bdev = I_BDEV(inode); 454 loff_t isize = i_size_read(inode); 455 456 if (offset >= isize) 457 return -EIO; 458 459 iomap->bdev = bdev; 460 iomap->offset = ALIGN_DOWN(offset, bdev_logical_block_size(bdev)); 461 iomap->type = IOMAP_MAPPED; 462 iomap->addr = iomap->offset; 463 iomap->length = isize - iomap->offset; 464 iomap->flags |= IOMAP_F_BUFFER_HEAD; /* noop for !CONFIG_BUFFER_HEAD */ 465 return 0; 466 } 467 468 static DEFINE_IOMAP_ITER_NEXT(blkdev_iomap_next, blkdev_iomap_begin); 469 470 static const struct iomap_ops blkdev_iomap_ops = { 471 .iomap_next = blkdev_iomap_next, 472 }; 473 474 #ifdef CONFIG_BUFFER_HEAD 475 static int blkdev_get_block(struct inode *inode, sector_t iblock, 476 struct buffer_head *bh, int create) 477 { 478 bh->b_bdev = I_BDEV(inode); 479 bh->b_blocknr = iblock; 480 set_buffer_mapped(bh); 481 return 0; 482 } 483 484 /* 485 * We cannot call mpage_writepages() as it does not take the buffer lock. 486 * We must use block_write_full_folio() directly which holds the buffer 487 * lock. The buffer lock provides the synchronisation with writeback 488 * that filesystems rely on when they use the blockdev's mapping. 489 */ 490 static int blkdev_writepages(struct address_space *mapping, 491 struct writeback_control *wbc) 492 { 493 struct folio *folio = NULL; 494 struct blk_plug plug; 495 int err; 496 497 blk_start_plug(&plug); 498 while ((folio = writeback_iter(mapping, wbc, folio, &err))) 499 err = block_write_full_folio(folio, wbc, blkdev_get_block); 500 blk_finish_plug(&plug); 501 502 return err; 503 } 504 505 static int blkdev_read_folio(struct file *file, struct folio *folio) 506 { 507 return block_read_full_folio(folio, blkdev_get_block); 508 } 509 510 static void blkdev_readahead(struct readahead_control *rac) 511 { 512 mpage_readahead(rac, blkdev_get_block); 513 } 514 515 const struct address_space_operations def_blk_aops = { 516 .dirty_folio = block_dirty_folio, 517 .invalidate_folio = block_invalidate_folio, 518 .read_folio = blkdev_read_folio, 519 .readahead = blkdev_readahead, 520 .writepages = blkdev_writepages, 521 .migrate_folio = buffer_migrate_folio_norefs, 522 .is_dirty_writeback = buffer_check_dirty_writeback, 523 }; 524 #else /* CONFIG_BUFFER_HEAD */ 525 static int blkdev_read_folio(struct file *file, struct folio *folio) 526 { 527 iomap_bio_read_folio(folio, &blkdev_iomap_ops); 528 return 0; 529 } 530 531 static void blkdev_readahead(struct readahead_control *rac) 532 { 533 iomap_bio_readahead(rac, &blkdev_iomap_ops); 534 } 535 536 static ssize_t blkdev_writeback_range(struct iomap_writepage_ctx *wpc, 537 struct folio *folio, u64 offset, unsigned int len, u64 end_pos) 538 { 539 loff_t isize = i_size_read(wpc->inode); 540 541 if (WARN_ON_ONCE(offset >= isize)) 542 return -EIO; 543 544 if (offset < wpc->iomap.offset || 545 offset >= wpc->iomap.offset + wpc->iomap.length) { 546 int error; 547 548 error = blkdev_iomap_begin(wpc->inode, offset, isize - offset, 549 IOMAP_WRITE, &wpc->iomap, NULL); 550 if (error) 551 return error; 552 } 553 554 return iomap_add_to_ioend(wpc, folio, offset, end_pos, len); 555 } 556 557 static const struct iomap_writeback_ops blkdev_writeback_ops = { 558 .writeback_range = blkdev_writeback_range, 559 .writeback_submit = iomap_ioend_writeback_submit, 560 }; 561 562 static int blkdev_writepages(struct address_space *mapping, 563 struct writeback_control *wbc) 564 { 565 struct iomap_writepage_ctx wpc = { 566 .inode = mapping->host, 567 .wbc = wbc, 568 .ops = &blkdev_writeback_ops 569 }; 570 571 return iomap_writepages(&wpc); 572 } 573 574 const struct address_space_operations def_blk_aops = { 575 .dirty_folio = iomap_dirty_folio, 576 .release_folio = iomap_release_folio, 577 .invalidate_folio = iomap_invalidate_folio, 578 .read_folio = blkdev_read_folio, 579 .readahead = blkdev_readahead, 580 .writepages = blkdev_writepages, 581 .is_partially_uptodate = iomap_is_partially_uptodate, 582 .error_remove_folio = generic_error_remove_folio, 583 .migrate_folio = filemap_migrate_folio, 584 }; 585 #endif /* CONFIG_BUFFER_HEAD */ 586 587 /* 588 * for a block special file file_inode(file)->i_size is zero 589 * so we compute the size by hand (just as in block_read/write above) 590 */ 591 static loff_t blkdev_llseek(struct file *file, loff_t offset, int whence) 592 { 593 struct inode *bd_inode = bdev_file_inode(file); 594 loff_t retval; 595 596 inode_lock(bd_inode); 597 retval = fixed_size_llseek(file, offset, whence, i_size_read(bd_inode)); 598 inode_unlock(bd_inode); 599 return retval; 600 } 601 602 static int blkdev_fsync(struct file *filp, loff_t start, loff_t end, 603 int datasync) 604 { 605 struct block_device *bdev = I_BDEV(filp->f_mapping->host); 606 int error; 607 608 error = file_write_and_wait_range(filp, start, end); 609 if (error) 610 return error; 611 612 /* 613 * There is no need to serialise calls to blkdev_issue_flush with 614 * i_mutex and doing so causes performance issues with concurrent 615 * O_SYNC writers to a block device. 616 */ 617 error = blkdev_issue_flush(bdev); 618 if (error == -EOPNOTSUPP) 619 error = 0; 620 621 return error; 622 } 623 624 /** 625 * file_to_blk_mode - get block open flags from file flags 626 * @file: file whose open flags should be converted 627 * 628 * Look at file open flags and generate corresponding block open flags from 629 * them. The function works both for file just being open (e.g. during ->open 630 * callback) and for file that is already open. This is actually non-trivial 631 * (see comment in the function). 632 */ 633 blk_mode_t file_to_blk_mode(struct file *file) 634 { 635 blk_mode_t mode = 0; 636 637 if (file->f_mode & FMODE_READ) 638 mode |= BLK_OPEN_READ; 639 if (file->f_mode & FMODE_WRITE) 640 mode |= BLK_OPEN_WRITE; 641 /* 642 * do_dentry_open() clears O_EXCL from f_flags, use file->private_data 643 * to determine whether the open was exclusive for already open files. 644 */ 645 if (file->private_data) 646 mode |= BLK_OPEN_EXCL; 647 else if (file->f_flags & O_EXCL) 648 mode |= BLK_OPEN_EXCL; 649 if (file->f_flags & O_NDELAY) 650 mode |= BLK_OPEN_NDELAY; 651 652 /* 653 * If all bits in O_ACCMODE set (aka O_RDWR | O_WRONLY), the floppy 654 * driver has historically allowed ioctls as if the file was opened for 655 * writing, but does not allow and actual reads or writes. 656 */ 657 if ((file->f_flags & O_ACCMODE) == (O_RDWR | O_WRONLY)) 658 mode |= BLK_OPEN_WRITE_IOCTL; 659 660 return mode; 661 } 662 663 static int blkdev_open(struct inode *inode, struct file *filp) 664 { 665 struct block_device *bdev; 666 blk_mode_t mode; 667 int ret; 668 669 mode = file_to_blk_mode(filp); 670 /* Use the file as the holder. */ 671 if (mode & BLK_OPEN_EXCL) 672 filp->private_data = filp; 673 ret = bdev_permission(inode->i_rdev, mode, filp->private_data); 674 if (ret) 675 return ret; 676 677 bdev = blkdev_get_no_open(inode->i_rdev, true); 678 if (!bdev) 679 return -ENXIO; 680 681 if (bdev_can_atomic_write(bdev)) 682 filp->f_mode |= FMODE_CAN_ATOMIC_WRITE; 683 if (blk_get_integrity(bdev->bd_disk)) 684 filp->f_mode |= FMODE_HAS_METADATA; 685 686 ret = bdev_open(bdev, mode, filp->private_data, NULL, filp); 687 if (ret) 688 blkdev_put_no_open(bdev); 689 return ret; 690 } 691 692 static int blkdev_release(struct inode *inode, struct file *filp) 693 { 694 bdev_release(filp); 695 return 0; 696 } 697 698 static ssize_t 699 blkdev_direct_write(struct kiocb *iocb, struct iov_iter *from) 700 { 701 size_t count = iov_iter_count(from); 702 ssize_t written; 703 704 written = kiocb_invalidate_pages(iocb, count); 705 if (written) { 706 if (written == -EBUSY) 707 return 0; 708 return written; 709 } 710 711 written = blkdev_direct_IO(iocb, from); 712 if (written > 0) { 713 kiocb_invalidate_post_direct_write(iocb, count); 714 iocb->ki_pos += written; 715 count -= written; 716 } 717 if (written != -EIOCBQUEUED) 718 iov_iter_revert(from, count - iov_iter_count(from)); 719 return written; 720 } 721 722 static ssize_t blkdev_buffered_write(struct kiocb *iocb, struct iov_iter *from) 723 { 724 struct inode *inode = bdev_file_inode(iocb->ki_filp); 725 ssize_t ret; 726 727 /* Buffered fallback waits for writeback, which NOWAIT cannot allow. */ 728 if (iocb->ki_flags & IOCB_NOWAIT) 729 return -EAGAIN; 730 731 if (unlikely(iov_iter_is_dmabuf_map(from))) 732 return -EOPNOTSUPP; 733 734 /* 735 * Take i_rwsem to avoid racing with set_blocksize() changing 736 * i_blkbits/folio order and truncating the page cache. 737 */ 738 inode_lock_shared(inode); 739 ret = iomap_file_buffered_write(iocb, from, &blkdev_iomap_ops, NULL, 740 NULL); 741 inode_unlock_shared(inode); 742 return ret; 743 } 744 745 /* 746 * Write data to the block device. Only intended for the block device itself 747 * and the raw driver which basically is a fake block device. 748 * 749 * Does not take i_mutex for the write and thus is not for general purpose 750 * use. 751 */ 752 static ssize_t blkdev_write_iter(struct kiocb *iocb, struct iov_iter *from) 753 { 754 struct file *file = iocb->ki_filp; 755 struct inode *bd_inode = bdev_file_inode(file); 756 struct block_device *bdev = I_BDEV(bd_inode); 757 bool atomic = iocb->ki_flags & IOCB_ATOMIC; 758 loff_t size = bdev_nr_bytes(bdev); 759 size_t shorted = 0; 760 bool need_sync = false; 761 ssize_t ret; 762 763 if (bdev_read_only(bdev)) 764 return -EPERM; 765 766 if (IS_SWAPFILE(bd_inode) && !is_hibernate_resume_dev(bd_inode->i_rdev)) 767 return -ETXTBSY; 768 769 if (!iov_iter_count(from)) 770 return 0; 771 772 if (iocb->ki_pos >= size) 773 return -ENOSPC; 774 775 if ((iocb->ki_flags & (IOCB_NOWAIT | IOCB_DIRECT)) == IOCB_NOWAIT) 776 return -EOPNOTSUPP; 777 778 if (atomic) { 779 ret = generic_atomic_write_valid(iocb, from); 780 if (ret) 781 return ret; 782 } 783 784 size -= iocb->ki_pos; 785 if (iov_iter_count(from) > size) { 786 if (atomic) 787 return -EINVAL; 788 shorted = iov_iter_count(from) - size; 789 iov_iter_truncate(from, size); 790 } 791 792 ret = file_update_time(file); 793 if (ret) 794 return ret; 795 796 if (iocb->ki_flags & IOCB_DIRECT) { 797 ret = blkdev_direct_write(iocb, from); 798 if (ret >= 0 && iov_iter_count(from)) { 799 if (iocb->ki_flags & (IOCB_NOWAIT | IOCB_ATOMIC)) { 800 /* 801 * The buffered fallback blocks on i_rwsem and 802 * on writeback of the data it copied, and 803 * can't provide torn-write protection: return 804 * the short direct write instead and let the 805 * caller retry. The bytes already written 806 * still need a flush if REQ_FUA was not set. 807 */ 808 if (!ret) 809 ret = -EAGAIN; 810 else if (!blkdev_dio_fua(iocb, bdev)) 811 need_sync = true; 812 } else { 813 ssize_t ret2; 814 815 inode_lock_shared(bd_inode); 816 ret2 = blkdev_buffered_write(iocb, from); 817 inode_unlock_shared(bd_inode); 818 ret = direct_write_fallback(iocb, from, ret, 819 ret2); 820 need_sync = true; 821 } 822 } else if (ret > 0 && !blkdev_dio_fua(iocb, bdev)) { 823 /* 824 * The device does not support FUA, so REQ_FUA was not 825 * set and the O_DSYNC direct write still needs a flush 826 * to be made durable. 827 */ 828 need_sync = true; 829 } 830 } else { 831 ret = blkdev_buffered_write(iocb, from); 832 need_sync = true; 833 } 834 835 if (ret > 0 && need_sync) 836 ret = generic_write_sync(iocb, ret); 837 iov_iter_reexpand(from, iov_iter_count(from) + shorted); 838 return ret; 839 } 840 841 static ssize_t blkdev_read_iter(struct kiocb *iocb, struct iov_iter *to) 842 { 843 struct inode *bd_inode = bdev_file_inode(iocb->ki_filp); 844 struct block_device *bdev = I_BDEV(iocb->ki_filp->f_mapping->host); 845 loff_t size = bdev_nr_bytes(bdev); 846 loff_t pos = iocb->ki_pos; 847 size_t shorted = 0; 848 ssize_t ret = 0; 849 size_t count; 850 851 if (unlikely(pos + iov_iter_count(to) > size)) { 852 if (pos >= size) 853 return 0; 854 size -= pos; 855 shorted = iov_iter_count(to) - size; 856 iov_iter_truncate(to, size); 857 } 858 859 count = iov_iter_count(to); 860 if (!count) 861 goto reexpand; /* skip atime */ 862 863 if (iocb->ki_flags & IOCB_DIRECT) { 864 ret = kiocb_write_and_wait(iocb, count); 865 if (ret < 0) 866 goto reexpand; 867 file_accessed(iocb->ki_filp); 868 869 ret = blkdev_direct_IO(iocb, to); 870 if (ret > 0) { 871 iocb->ki_pos += ret; 872 count -= ret; 873 } 874 if (ret != -EIOCBQUEUED) 875 iov_iter_revert(to, count - iov_iter_count(to)); 876 if (ret < 0 || !count) 877 goto reexpand; 878 } 879 if (unlikely(iov_iter_is_dmabuf_map(to))) { 880 if (!ret) 881 ret = -EOPNOTSUPP; 882 goto reexpand; 883 } 884 885 /* 886 * Take i_rwsem and invalidate_lock to avoid racing with set_blocksize 887 * changing i_blkbits/folio order and punching out the pagecache. 888 */ 889 if (iocb->ki_flags & IOCB_NOWAIT) { 890 if (!inode_trylock_shared(bd_inode)) { 891 if (!ret) 892 ret = -EAGAIN; 893 goto reexpand; 894 } 895 } else { 896 inode_lock_shared(bd_inode); 897 } 898 ret = filemap_read(iocb, to, ret); 899 inode_unlock_shared(bd_inode); 900 901 reexpand: 902 if (unlikely(shorted)) 903 iov_iter_reexpand(to, iov_iter_count(to) + shorted); 904 return ret; 905 } 906 907 /* 908 * Take i_rwsem to avoid racing with set_blocksize changing i_blkbits/folio 909 * order and punching out the pagecache. 910 */ 911 static ssize_t blkdev_splice_read(struct file *in, loff_t *ppos, 912 struct pipe_inode_info *pipe, size_t len, unsigned int flags) 913 { 914 struct inode *bd_inode = bdev_file_inode(in); 915 ssize_t ret; 916 917 inode_lock_shared(bd_inode); 918 ret = filemap_splice_read(in, ppos, pipe, len, flags); 919 inode_unlock_shared(bd_inode); 920 return ret; 921 } 922 923 #define BLKDEV_FALLOC_FL_SUPPORTED \ 924 (FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE | \ 925 FALLOC_FL_ZERO_RANGE | FALLOC_FL_WRITE_ZEROES) 926 927 static long blkdev_fallocate(struct file *file, int mode, loff_t start, 928 loff_t len) 929 { 930 struct inode *inode = bdev_file_inode(file); 931 struct block_device *bdev = I_BDEV(inode); 932 loff_t end = start + len - 1; 933 loff_t isize; 934 unsigned int flags; 935 int error; 936 937 /* Fail if we don't recognize the flags. */ 938 if (mode & ~BLKDEV_FALLOC_FL_SUPPORTED) 939 return -EOPNOTSUPP; 940 /* 941 * Don't allow writing zeroes if the device does not enable the 942 * unmap write zeroes operation. 943 */ 944 if ((mode & FALLOC_FL_WRITE_ZEROES) && 945 !bdev_write_zeroes_unmap_sectors(bdev)) 946 return -EOPNOTSUPP; 947 948 /* Don't go off the end of the device. */ 949 isize = bdev_nr_bytes(bdev); 950 if (start >= isize) 951 return -EINVAL; 952 if (end >= isize) { 953 if (mode & FALLOC_FL_KEEP_SIZE) { 954 len = isize - start; 955 end = start + len - 1; 956 } else 957 return -EINVAL; 958 } 959 960 /* 961 * Don't allow IO that isn't aligned to logical block size. 962 */ 963 if ((start | len) & (bdev_logical_block_size(bdev) - 1)) 964 return -EINVAL; 965 966 inode_lock(inode); 967 filemap_invalidate_lock(inode->i_mapping); 968 969 switch (mode) { 970 case FALLOC_FL_ZERO_RANGE: 971 case FALLOC_FL_ZERO_RANGE | FALLOC_FL_KEEP_SIZE: 972 flags = BLKDEV_ZERO_NOUNMAP; 973 break; 974 case FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE: 975 flags = BLKDEV_ZERO_NOFALLBACK; 976 break; 977 case FALLOC_FL_WRITE_ZEROES: 978 flags = 0; 979 break; 980 default: 981 error = -EOPNOTSUPP; 982 goto fail; 983 } 984 985 /* 986 * Invalidate the page cache, including dirty pages, for valid 987 * de-allocate mode calls to fallocate(). 988 */ 989 error = truncate_bdev_range(bdev, file_to_blk_mode(file), start, end); 990 if (error) 991 goto fail; 992 993 error = blkdev_issue_zeroout(bdev, start >> SECTOR_SHIFT, 994 len >> SECTOR_SHIFT, GFP_KERNEL, flags); 995 fail: 996 filemap_invalidate_unlock(inode->i_mapping); 997 inode_unlock(inode); 998 return error; 999 } 1000 1001 static int blkdev_mmap_prepare(struct vm_area_desc *desc) 1002 { 1003 struct file *file = desc->file; 1004 1005 if (bdev_read_only(I_BDEV(bdev_file_inode(file)))) 1006 return generic_file_readonly_mmap_prepare(desc); 1007 1008 return generic_file_mmap_prepare(desc); 1009 } 1010 1011 static int blkdev_init_dma_buf_io_ctx(struct file *file, 1012 struct dma_buf_io_ctx *ctx) 1013 { 1014 struct block_device *bdev = file_bdev(file); 1015 struct gendisk *disk = bdev->bd_disk; 1016 1017 if (!(file->f_flags & O_DIRECT)) 1018 return -EINVAL; 1019 if (ctx->dmabuf->size > UINT_MAX) 1020 return -EINVAL; 1021 if (!disk->fops->init_dma_buf_io_ctx) 1022 return -EOPNOTSUPP; 1023 if (bdev_has_integrity_csum(bdev)) 1024 return -EOPNOTSUPP; 1025 return disk->fops->init_dma_buf_io_ctx(bdev, ctx); 1026 } 1027 1028 const struct file_operations def_blk_fops = { 1029 .open = blkdev_open, 1030 .release = blkdev_release, 1031 .llseek = blkdev_llseek, 1032 .read_iter = blkdev_read_iter, 1033 .write_iter = blkdev_write_iter, 1034 .iopoll = iocb_bio_iopoll, 1035 .mmap_prepare = blkdev_mmap_prepare, 1036 .fsync = blkdev_fsync, 1037 .unlocked_ioctl = blkdev_ioctl, 1038 #ifdef CONFIG_COMPAT 1039 .compat_ioctl = compat_blkdev_ioctl, 1040 #endif 1041 .splice_read = blkdev_splice_read, 1042 .splice_write = iter_file_splice_write, 1043 .fallocate = blkdev_fallocate, 1044 .uring_cmd = blkdev_uring_cmd, 1045 .fop_flags = FOP_BUFFER_RASYNC | FOP_DONTCACHE, 1046 .init_dma_buf_io_ctx = blkdev_init_dma_buf_io_ctx, 1047 }; 1048 1049 static __init int blkdev_init(void) 1050 { 1051 return bioset_init(&blkdev_dio_pool, 4, 1052 offsetof(struct blkdev_dio, bio), 1053 BIOSET_NEED_BVECS|BIOSET_PERCPU_CACHE); 1054 } 1055 module_init(blkdev_init); 1056