1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * linux/fs/ext4/file.c 4 * 5 * Copyright (C) 1992, 1993, 1994, 1995 6 * Remy Card (card@masi.ibp.fr) 7 * Laboratoire MASI - Institut Blaise Pascal 8 * Universite Pierre et Marie Curie (Paris VI) 9 * 10 * from 11 * 12 * linux/fs/minix/file.c 13 * 14 * Copyright (C) 1991, 1992 Linus Torvalds 15 * 16 * ext4 fs regular file handling primitives 17 * 18 * 64-bit file support on 64-bit platforms by Jakub Jelinek 19 * (jj@sunsite.ms.mff.cuni.cz) 20 */ 21 22 #include <linux/time.h> 23 #include <linux/fs.h> 24 #include <linux/iomap.h> 25 #include <linux/mount.h> 26 #include <linux/path.h> 27 #include <linux/dax.h> 28 #include <linux/filelock.h> 29 #include <linux/quotaops.h> 30 #include <linux/uio.h> 31 #include <linux/mman.h> 32 #include <linux/backing-dev.h> 33 #include "ext4.h" 34 #include "ext4_jbd2.h" 35 #include "xattr.h" 36 #include "acl.h" 37 #include "truncate.h" 38 39 /* 40 * Returns %true if the given DIO request should be attempted with DIO, or 41 * %false if it should fall back to buffered I/O. 42 * 43 * DIO isn't well specified; when it's unsupported (either due to the request 44 * being misaligned, or due to the file not supporting DIO at all), filesystems 45 * either fall back to buffered I/O or return EINVAL. For files that don't use 46 * any special features like encryption or verity, ext4 has traditionally 47 * returned EINVAL for misaligned DIO. iomap_dio_rw() uses this convention too. 48 * In this case, we should attempt the DIO, *not* fall back to buffered I/O. 49 * 50 * In contrast, in cases where DIO is unsupported due to ext4 features, ext4 51 * traditionally falls back to buffered I/O. 52 * 53 * This function implements the traditional ext4 behavior in all these cases. 54 */ 55 static bool ext4_should_use_dio(struct kiocb *iocb, struct iov_iter *iter) 56 { 57 struct inode *inode = file_inode(iocb->ki_filp); 58 u32 dio_align = ext4_dio_alignment(inode); 59 60 if (dio_align == 0) 61 return false; 62 63 if (dio_align == 1) 64 return true; 65 66 return IS_ALIGNED(iocb->ki_pos | iov_iter_alignment(iter), dio_align); 67 } 68 69 static ssize_t ext4_dio_read_iter(struct kiocb *iocb, struct iov_iter *to) 70 { 71 ssize_t ret; 72 struct inode *inode = file_inode(iocb->ki_filp); 73 74 if (iocb->ki_flags & IOCB_NOWAIT) { 75 if (!inode_trylock_shared(inode)) 76 return -EAGAIN; 77 } else { 78 inode_lock_shared(inode); 79 } 80 81 if (!ext4_should_use_dio(iocb, to)) { 82 inode_unlock_shared(inode); 83 /* 84 * Fallback to buffered I/O if the operation being performed on 85 * the inode is not supported by direct I/O. The IOCB_DIRECT 86 * flag needs to be cleared here in order to ensure that the 87 * direct I/O path within generic_file_read_iter() is not 88 * taken. 89 */ 90 iocb->ki_flags &= ~IOCB_DIRECT; 91 return generic_file_read_iter(iocb, to); 92 } 93 94 ret = iomap_dio_read_simple(iocb, to, ext4_iomap_begin); 95 if (ret == -ENOTBLK) 96 ret = iomap_dio_rw(iocb, to, &ext4_iomap_ops, NULL, 0, NULL, 0); 97 inode_unlock_shared(inode); 98 99 file_accessed(iocb->ki_filp); 100 return ret; 101 } 102 103 #ifdef CONFIG_FS_DAX 104 static ssize_t ext4_dax_read_iter(struct kiocb *iocb, struct iov_iter *to) 105 { 106 struct inode *inode = file_inode(iocb->ki_filp); 107 ssize_t ret; 108 109 if (iocb->ki_flags & IOCB_NOWAIT) { 110 if (!inode_trylock_shared(inode)) 111 return -EAGAIN; 112 } else { 113 inode_lock_shared(inode); 114 } 115 /* 116 * Recheck under inode lock - at this point we are sure it cannot 117 * change anymore 118 */ 119 if (!IS_DAX(inode)) { 120 inode_unlock_shared(inode); 121 /* Fallback to buffered IO in case we cannot support DAX */ 122 return generic_file_read_iter(iocb, to); 123 } 124 ret = dax_iomap_rw(iocb, to, &ext4_iomap_ops); 125 inode_unlock_shared(inode); 126 127 file_accessed(iocb->ki_filp); 128 return ret; 129 } 130 #endif 131 132 static ssize_t ext4_file_read_iter(struct kiocb *iocb, struct iov_iter *to) 133 { 134 struct inode *inode = file_inode(iocb->ki_filp); 135 136 if (unlikely(ext4_forced_shutdown(inode->i_sb))) 137 return -EIO; 138 139 if (!iov_iter_count(to)) 140 return 0; /* skip atime */ 141 142 #ifdef CONFIG_FS_DAX 143 if (IS_DAX(inode)) 144 return ext4_dax_read_iter(iocb, to); 145 #endif 146 if (iocb->ki_flags & IOCB_DIRECT) 147 return ext4_dio_read_iter(iocb, to); 148 149 return generic_file_read_iter(iocb, to); 150 } 151 152 static ssize_t ext4_file_splice_read(struct file *in, loff_t *ppos, 153 struct pipe_inode_info *pipe, 154 size_t len, unsigned int flags) 155 { 156 struct inode *inode = file_inode(in); 157 158 if (unlikely(ext4_forced_shutdown(inode->i_sb))) 159 return -EIO; 160 return filemap_splice_read(in, ppos, pipe, len, flags); 161 } 162 163 /* 164 * Called when an inode is released. Note that this is different 165 * from ext4_file_open: open gets called at every open, but release 166 * gets called only when /all/ the files are closed. 167 */ 168 static int ext4_release_file(struct inode *inode, struct file *filp) 169 { 170 if (ext4_test_inode_state(inode, EXT4_STATE_DA_ALLOC_CLOSE)) { 171 ext4_alloc_da_blocks(inode); 172 ext4_clear_inode_state(inode, EXT4_STATE_DA_ALLOC_CLOSE); 173 } 174 /* if we are the last writer on the inode, drop the block reservation */ 175 if ((filp->f_mode & FMODE_WRITE) && 176 (atomic_read(&inode->i_writecount) == 1) && 177 !EXT4_I(inode)->i_reserved_data_blocks) { 178 down_write(&EXT4_I(inode)->i_data_sem); 179 ext4_discard_preallocations(inode); 180 up_write(&EXT4_I(inode)->i_data_sem); 181 } 182 if (is_dx(inode) && filp->private_data) 183 ext4_htree_free_dir_info(filp->private_data); 184 185 return 0; 186 } 187 188 /* 189 * This tests whether the IO in question is block-aligned or not. 190 * Ext4 utilizes unwritten extents when hole-filling during direct IO, and they 191 * are converted to written only after the IO is complete. Until they are 192 * mapped, these blocks appear as holes, so dio_zero_block() will assume that 193 * it needs to zero out portions of the start and/or end block. If 2 AIO 194 * threads are at work on the same unwritten block, they must be synchronized 195 * or one thread will zero the other's data, causing corruption. 196 */ 197 static bool 198 ext4_unaligned_io(struct inode *inode, struct iov_iter *from, loff_t pos) 199 { 200 struct super_block *sb = inode->i_sb; 201 unsigned long blockmask = sb->s_blocksize - 1; 202 203 if ((pos | iov_iter_alignment(from)) & blockmask) 204 return true; 205 206 return false; 207 } 208 209 static bool 210 ext4_extending_io(struct inode *inode, loff_t offset, size_t len) 211 { 212 if (offset + len > i_size_read(inode) || 213 offset + len > EXT4_I(inode)->i_disksize) 214 return true; 215 return false; 216 } 217 218 /* Is IO overwriting allocated or initialized blocks? */ 219 static bool ext4_overwrite_io(struct inode *inode, 220 loff_t pos, loff_t len, bool *unwritten) 221 { 222 struct ext4_map_blocks map; 223 unsigned int blkbits = inode->i_blkbits; 224 int err, blklen; 225 226 if (pos + len > i_size_read(inode)) 227 return false; 228 229 map.m_lblk = pos >> blkbits; 230 map.m_len = EXT4_MAX_BLOCKS(len, pos, blkbits); 231 blklen = map.m_len; 232 233 err = ext4_map_blocks(NULL, inode, &map, 0); 234 if (err != blklen) 235 return false; 236 /* 237 * 'err==len' means that all of the blocks have been preallocated, 238 * regardless of whether they have been initialized or not. We need to 239 * check m_flags to distinguish the unwritten extents. 240 */ 241 *unwritten = !(map.m_flags & EXT4_MAP_MAPPED); 242 return true; 243 } 244 245 static ssize_t ext4_generic_write_checks(struct kiocb *iocb, 246 struct iov_iter *from) 247 { 248 struct inode *inode = file_inode(iocb->ki_filp); 249 ssize_t ret; 250 251 if (unlikely(IS_IMMUTABLE(inode))) 252 return -EPERM; 253 254 ret = generic_write_checks(iocb, from); 255 if (ret <= 0) 256 return ret; 257 258 /* 259 * If we have encountered a bitmap-format file, the size limit 260 * is smaller than s_maxbytes, which is for extent-mapped files. 261 */ 262 if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) { 263 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); 264 265 if (iocb->ki_pos >= sbi->s_bitmap_maxbytes) 266 return -EFBIG; 267 iov_iter_truncate(from, sbi->s_bitmap_maxbytes - iocb->ki_pos); 268 } 269 270 return iov_iter_count(from); 271 } 272 273 static ssize_t ext4_write_checks(struct kiocb *iocb, struct iov_iter *from) 274 { 275 struct inode *inode = file_inode(iocb->ki_filp); 276 loff_t old_size = i_size_read(inode); 277 ssize_t ret, count; 278 279 count = ext4_generic_write_checks(iocb, from); 280 if (count <= 0) 281 return count; 282 283 ret = file_modified(iocb->ki_filp); 284 if (ret) 285 return ret; 286 287 /* 288 * If the position is beyond the EOF, it is necessary to zero out the 289 * partial block that beyond the existing EOF, as it may contains 290 * stale data written through mmap. 291 */ 292 if (iocb->ki_pos > old_size && !ext4_verity_in_progress(inode)) { 293 if (iocb->ki_flags & IOCB_NOWAIT) 294 return -EAGAIN; 295 296 ret = ext4_block_zero_eof(inode, old_size, iocb->ki_pos); 297 if (ret) 298 return ret; 299 } 300 301 return count; 302 } 303 304 static ssize_t ext4_buffered_write_iter(struct kiocb *iocb, 305 struct iov_iter *from) 306 { 307 ssize_t ret; 308 struct inode *inode = file_inode(iocb->ki_filp); 309 310 if (iocb->ki_flags & IOCB_NOWAIT) 311 return -EOPNOTSUPP; 312 313 inode_lock(inode); 314 ret = ext4_write_checks(iocb, from); 315 if (ret <= 0) 316 goto out; 317 318 ret = generic_perform_write(iocb, from); 319 320 out: 321 inode_unlock(inode); 322 if (unlikely(ret <= 0)) 323 return ret; 324 return generic_write_sync(iocb, ret); 325 } 326 327 static ssize_t ext4_handle_inode_extension(struct inode *inode, loff_t offset, 328 ssize_t written, ssize_t count) 329 { 330 handle_t *handle; 331 332 lockdep_assert_held_write(&inode->i_rwsem); 333 handle = ext4_journal_start(inode, EXT4_HT_INODE, 2); 334 if (IS_ERR(handle)) 335 return PTR_ERR(handle); 336 337 if (ext4_update_inode_size(inode, offset + written)) { 338 int ret = ext4_mark_inode_dirty(handle, inode); 339 if (unlikely(ret)) { 340 ext4_journal_stop(handle); 341 return ret; 342 } 343 } 344 345 if ((written == count) && inode->i_nlink) 346 ext4_orphan_del(handle, inode); 347 ext4_journal_stop(handle); 348 349 return written; 350 } 351 352 /* 353 * Clean up the inode after DIO or DAX extending write has completed and the 354 * inode size has been updated using ext4_handle_inode_extension(). 355 */ 356 static void ext4_inode_extension_cleanup(struct inode *inode, bool need_trunc) 357 { 358 lockdep_assert_held_write(&inode->i_rwsem); 359 if (need_trunc) { 360 ext4_truncate_failed_write(inode); 361 /* 362 * If the truncate operation failed early, then the inode may 363 * still be on the orphan list. In that case, we need to try 364 * remove the inode from the in-memory linked list. 365 */ 366 if (inode->i_nlink) 367 ext4_orphan_del(NULL, inode); 368 return; 369 } 370 /* 371 * If i_disksize got extended either due to writeback of delalloc 372 * blocks or extending truncate while the DIO was running we could fail 373 * to cleanup the orphan list in ext4_handle_inode_extension(). Do it 374 * now. 375 */ 376 if (ext4_inode_orphan_tracked(inode) && inode->i_nlink) { 377 handle_t *handle = ext4_journal_start(inode, EXT4_HT_INODE, 2); 378 379 if (IS_ERR(handle)) { 380 /* 381 * The write has successfully completed. Not much to 382 * do with the error here so just cleanup the orphan 383 * list and hope for the best. 384 */ 385 ext4_orphan_del(NULL, inode); 386 return; 387 } 388 ext4_orphan_del(handle, inode); 389 ext4_journal_stop(handle); 390 } 391 } 392 393 static int ext4_dio_write_end_io(struct kiocb *iocb, ssize_t size, 394 int error, unsigned int flags) 395 { 396 loff_t pos = iocb->ki_pos; 397 struct inode *inode = file_inode(iocb->ki_filp); 398 399 400 if (!error && size && (flags & IOMAP_DIO_UNWRITTEN) && 401 (iocb->ki_flags & IOCB_ATOMIC)) 402 error = ext4_convert_unwritten_extents_atomic(NULL, inode, pos, 403 size); 404 else if (!error && size && flags & IOMAP_DIO_UNWRITTEN) 405 error = ext4_convert_unwritten_extents(NULL, inode, pos, size); 406 if (error) 407 return error; 408 /* 409 * Note that EXT4_I(inode)->i_disksize can get extended up to 410 * inode->i_size while the I/O was running due to writeback of delalloc 411 * blocks. But the code in ext4_iomap_alloc() is careful to use 412 * zeroed/unwritten extents if this is possible; thus we won't leave 413 * uninitialized blocks in a file even if we didn't succeed in writing 414 * as much as we intended. Also we can race with truncate or write 415 * expanding the file so we have to be a bit careful here. 416 */ 417 if (pos + size <= READ_ONCE(EXT4_I(inode)->i_disksize) && 418 pos + size <= i_size_read(inode)) 419 return 0; 420 error = ext4_handle_inode_extension(inode, pos, size, size); 421 return error < 0 ? error : 0; 422 } 423 424 static const struct iomap_dio_ops ext4_dio_write_ops = { 425 .end_io = ext4_dio_write_end_io, 426 }; 427 428 /* 429 * The intention here is to start with shared lock acquired then see if any 430 * condition requires an exclusive inode lock. If yes, then we restart the 431 * whole operation by releasing the shared lock and acquiring exclusive lock. 432 * 433 * - For unaligned_io we never take shared lock as it may cause data corruption 434 * when two unaligned IO tries to modify the same block e.g. while zeroing. 435 * 436 * - For extending writes case we don't take the shared lock, since it requires 437 * updating inode i_disksize and/or orphan handling with exclusive lock. 438 * 439 * - shared locking will only be true mostly with overwrites, including 440 * initialized blocks and unwritten blocks. 441 * 442 * - Otherwise we will switch to exclusive i_rwsem lock. 443 */ 444 static ssize_t ext4_dio_write_checks(struct kiocb *iocb, struct iov_iter *from, 445 bool *ilock_shared, bool *extend, 446 int *dio_flags) 447 { 448 struct file *file = iocb->ki_filp; 449 struct inode *inode = file_inode(file); 450 loff_t offset; 451 size_t count; 452 ssize_t ret; 453 bool overwrite, unaligned_io, unwritten; 454 455 restart: 456 ret = ext4_generic_write_checks(iocb, from); 457 if (ret <= 0) 458 goto out; 459 460 offset = iocb->ki_pos; 461 count = ret; 462 463 unaligned_io = ext4_unaligned_io(inode, from, offset); 464 *extend = ext4_extending_io(inode, offset, count); 465 overwrite = ext4_overwrite_io(inode, offset, count, &unwritten); 466 467 /* 468 * Determine whether we need to upgrade to an exclusive lock. This is 469 * required to change security info in file_modified(), for extending 470 * I/O, any form of non-overwrite I/O, and unaligned I/O to unwritten 471 * extents (as partial block zeroing may be required). 472 * 473 * Note that unaligned writes are allowed under shared lock so long as 474 * they are pure overwrites. Otherwise, concurrent unaligned writes risk 475 * data corruption due to partial block zeroing in the dio layer, and so 476 * the I/O must occur exclusively. 477 */ 478 if (*ilock_shared && 479 ((!IS_NOSEC(inode) || *extend || !overwrite || 480 (unaligned_io && unwritten)))) { 481 if (iocb->ki_flags & IOCB_NOWAIT) { 482 ret = -EAGAIN; 483 goto out; 484 } 485 inode_unlock_shared(inode); 486 *ilock_shared = false; 487 inode_lock(inode); 488 goto restart; 489 } 490 491 /* 492 * Now that locking is settled, determine dio flags and exclusivity 493 * requirements. We don't use DIO_OVERWRITE_ONLY because we enforce 494 * behavior already. The inode lock is already held exclusive if the 495 * write is non-overwrite or extending, so drain all outstanding dio and 496 * set the force wait dio flag. 497 */ 498 if (!*ilock_shared && (unaligned_io || *extend)) { 499 if (iocb->ki_flags & IOCB_NOWAIT) { 500 ret = -EAGAIN; 501 goto out; 502 } 503 if (unaligned_io && (!overwrite || unwritten)) 504 inode_dio_wait(inode); 505 *dio_flags = IOMAP_DIO_FORCE_WAIT; 506 } 507 508 ret = file_modified(file); 509 if (ret < 0) 510 goto out; 511 512 return count; 513 out: 514 if (*ilock_shared) 515 inode_unlock_shared(inode); 516 else 517 inode_unlock(inode); 518 return ret; 519 } 520 521 static ssize_t ext4_dio_write_iter(struct kiocb *iocb, struct iov_iter *from) 522 { 523 ssize_t ret; 524 handle_t *handle; 525 struct inode *inode = file_inode(iocb->ki_filp); 526 loff_t offset = iocb->ki_pos; 527 size_t count = iov_iter_count(from); 528 bool extend = false; 529 bool ilock_shared = true; 530 int dio_flags = 0; 531 532 /* 533 * Quick check here without any i_rwsem lock to see if it is extending 534 * IO. A more reliable check is done in ext4_dio_write_checks() with 535 * proper locking in place. 536 */ 537 if (offset + count > i_size_read(inode)) 538 ilock_shared = false; 539 540 if (iocb->ki_flags & IOCB_NOWAIT) { 541 if (ilock_shared) { 542 if (!inode_trylock_shared(inode)) 543 return -EAGAIN; 544 } else { 545 if (!inode_trylock(inode)) 546 return -EAGAIN; 547 } 548 } else { 549 if (ilock_shared) 550 inode_lock_shared(inode); 551 else 552 inode_lock(inode); 553 } 554 555 /* Fallback to buffered I/O if the inode does not support direct I/O. */ 556 if (!ext4_should_use_dio(iocb, from)) { 557 if (ilock_shared) 558 inode_unlock_shared(inode); 559 else 560 inode_unlock(inode); 561 return ext4_buffered_write_iter(iocb, from); 562 } 563 564 /* 565 * Prevent inline data from being created since we are going to allocate 566 * blocks for DIO. We know the inode does not currently have inline data 567 * because ext4_should_use_dio() checked for it, but we have to clear 568 * the state flag before the write checks because a lock cycle could 569 * introduce races with other writers. 570 */ 571 ext4_clear_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA); 572 573 ret = ext4_dio_write_checks(iocb, from, &ilock_shared, &extend, 574 &dio_flags); 575 if (ret <= 0) 576 return ret; 577 578 offset = iocb->ki_pos; 579 count = ret; 580 581 if (extend) { 582 handle = ext4_journal_start(inode, EXT4_HT_INODE, 2); 583 if (IS_ERR(handle)) { 584 ret = PTR_ERR(handle); 585 goto out; 586 } 587 588 ret = ext4_orphan_add(handle, inode); 589 ext4_journal_stop(handle); 590 if (ret) 591 goto out; 592 } 593 594 ret = iomap_dio_rw(iocb, from, &ext4_iomap_ops, &ext4_dio_write_ops, 595 dio_flags, NULL, 0); 596 if (ret == -ENOTBLK) 597 ret = 0; 598 if (extend) { 599 /* 600 * We always perform extending DIO write synchronously so by 601 * now the IO is completed and ext4_handle_inode_extension() 602 * was called. Cleanup the inode in case of error or race with 603 * writeback of delalloc blocks. 604 */ 605 WARN_ON_ONCE(ret == -EIOCBQUEUED); 606 ext4_inode_extension_cleanup(inode, ret < 0); 607 } 608 609 out: 610 if (ilock_shared) 611 inode_unlock_shared(inode); 612 else 613 inode_unlock(inode); 614 615 if (ret >= 0 && iov_iter_count(from)) { 616 ssize_t err; 617 loff_t endbyte; 618 619 /* 620 * There is no support for atomic writes on buffered-io yet, 621 * we should never fallback to buffered-io for DIO atomic 622 * writes. 623 */ 624 WARN_ON_ONCE(iocb->ki_flags & IOCB_ATOMIC); 625 626 offset = iocb->ki_pos; 627 err = ext4_buffered_write_iter(iocb, from); 628 if (err < 0) 629 return err; 630 631 /* 632 * We need to ensure that the pages within the page cache for 633 * the range covered by this I/O are written to disk and 634 * invalidated. This is in attempt to preserve the expected 635 * direct I/O semantics in the case we fallback to buffered I/O 636 * to complete off the I/O request. 637 */ 638 ret += err; 639 endbyte = offset + err - 1; 640 err = filemap_write_and_wait_range(iocb->ki_filp->f_mapping, 641 offset, endbyte); 642 if (!err) 643 invalidate_mapping_pages(iocb->ki_filp->f_mapping, 644 offset >> PAGE_SHIFT, 645 endbyte >> PAGE_SHIFT); 646 } 647 648 return ret; 649 } 650 651 #ifdef CONFIG_FS_DAX 652 static ssize_t 653 ext4_dax_write_iter(struct kiocb *iocb, struct iov_iter *from) 654 { 655 ssize_t ret; 656 size_t count; 657 loff_t offset; 658 handle_t *handle; 659 bool extend = false; 660 struct inode *inode = file_inode(iocb->ki_filp); 661 662 if (iocb->ki_flags & IOCB_NOWAIT) { 663 if (!inode_trylock(inode)) 664 return -EAGAIN; 665 } else { 666 inode_lock(inode); 667 } 668 669 ret = ext4_write_checks(iocb, from); 670 if (ret <= 0) 671 goto out; 672 673 offset = iocb->ki_pos; 674 count = iov_iter_count(from); 675 676 if (offset + count > EXT4_I(inode)->i_disksize) { 677 handle = ext4_journal_start(inode, EXT4_HT_INODE, 2); 678 if (IS_ERR(handle)) { 679 ret = PTR_ERR(handle); 680 goto out; 681 } 682 683 ret = ext4_orphan_add(handle, inode); 684 if (ret) { 685 ext4_journal_stop(handle); 686 goto out; 687 } 688 689 extend = true; 690 ext4_journal_stop(handle); 691 } 692 693 ret = dax_iomap_rw(iocb, from, &ext4_iomap_ops); 694 695 if (extend) { 696 ret = ext4_handle_inode_extension(inode, offset, ret, count); 697 ext4_inode_extension_cleanup(inode, ret < (ssize_t)count); 698 } 699 out: 700 inode_unlock(inode); 701 if (ret > 0) 702 ret = generic_write_sync(iocb, ret); 703 return ret; 704 } 705 #endif 706 707 static ssize_t 708 ext4_file_write_iter(struct kiocb *iocb, struct iov_iter *from) 709 { 710 int ret; 711 struct inode *inode = file_inode(iocb->ki_filp); 712 713 ret = ext4_emergency_state(inode->i_sb); 714 if (unlikely(ret)) 715 return ret; 716 717 #ifdef CONFIG_FS_DAX 718 if (IS_DAX(inode)) 719 return ext4_dax_write_iter(iocb, from); 720 #endif 721 722 if (iocb->ki_flags & IOCB_ATOMIC) { 723 size_t len = iov_iter_count(from); 724 725 if (len < EXT4_SB(inode->i_sb)->s_awu_min || 726 len > EXT4_SB(inode->i_sb)->s_awu_max) 727 return -EINVAL; 728 729 ret = generic_atomic_write_valid(iocb, from); 730 if (ret) 731 return ret; 732 } 733 734 if (iocb->ki_flags & IOCB_DIRECT) 735 return ext4_dio_write_iter(iocb, from); 736 else 737 return ext4_buffered_write_iter(iocb, from); 738 } 739 740 #ifdef CONFIG_FS_DAX 741 static vm_fault_t ext4_dax_huge_fault(struct vm_fault *vmf, unsigned int order) 742 { 743 int error = 0; 744 vm_fault_t result; 745 int retries = 0; 746 handle_t *handle = NULL; 747 struct inode *inode = file_inode(vmf->vma->vm_file); 748 struct super_block *sb = inode->i_sb; 749 750 /* 751 * We have to distinguish real writes from writes which will result in a 752 * COW page; COW writes should *not* poke the journal (the file will not 753 * be changed). Doing so would cause unintended failures when mounted 754 * read-only. 755 * 756 * We check for VM_SHARED rather than vmf->cow_page since the latter is 757 * unset for order != 0 (i.e. only in do_cow_fault); for 758 * other sizes, dax_iomap_fault will handle splitting / fallback so that 759 * we eventually come back with a COW page. 760 */ 761 bool write = (vmf->flags & FAULT_FLAG_WRITE) && 762 (vmf->vma->vm_flags & VM_SHARED); 763 struct address_space *mapping = vmf->vma->vm_file->f_mapping; 764 unsigned long pfn; 765 766 if (write) { 767 sb_start_pagefault(sb); 768 file_update_time(vmf->vma->vm_file); 769 filemap_invalidate_lock_shared(mapping); 770 retry: 771 handle = ext4_journal_start_sb(sb, EXT4_HT_WRITE_PAGE, 772 EXT4_DATA_TRANS_BLOCKS(sb)); 773 if (IS_ERR(handle)) { 774 filemap_invalidate_unlock_shared(mapping); 775 sb_end_pagefault(sb); 776 return VM_FAULT_SIGBUS; 777 } 778 } else { 779 filemap_invalidate_lock_shared(mapping); 780 } 781 result = dax_iomap_fault(vmf, order, &pfn, &error, &ext4_iomap_ops); 782 if (write) { 783 ext4_journal_stop(handle); 784 785 if ((result & VM_FAULT_ERROR) && error == -ENOSPC && 786 ext4_should_retry_alloc(sb, &retries)) 787 goto retry; 788 /* Handling synchronous page fault? */ 789 if (result & VM_FAULT_NEEDDSYNC) 790 result = dax_finish_sync_fault(vmf, order, pfn); 791 filemap_invalidate_unlock_shared(mapping); 792 sb_end_pagefault(sb); 793 } else { 794 filemap_invalidate_unlock_shared(mapping); 795 } 796 797 return result; 798 } 799 800 static vm_fault_t ext4_dax_fault(struct vm_fault *vmf) 801 { 802 return ext4_dax_huge_fault(vmf, 0); 803 } 804 805 static const struct vm_operations_struct ext4_dax_vm_ops = { 806 .fault = ext4_dax_fault, 807 .huge_fault = ext4_dax_huge_fault, 808 .page_mkwrite = ext4_dax_fault, 809 .pfn_mkwrite = ext4_dax_fault, 810 }; 811 #else 812 #define ext4_dax_vm_ops ext4_file_vm_ops 813 #endif 814 815 static const struct vm_operations_struct ext4_file_vm_ops = { 816 .fault = filemap_fault, 817 .map_pages = filemap_map_pages, 818 .page_mkwrite = ext4_page_mkwrite, 819 }; 820 821 static int ext4_file_mmap_prepare(struct vm_area_desc *desc) 822 { 823 int ret; 824 struct file *file = desc->file; 825 struct inode *inode = file->f_mapping->host; 826 struct dax_device *dax_dev = EXT4_SB(inode->i_sb)->s_daxdev; 827 828 if (file->f_mode & FMODE_WRITE) 829 ret = ext4_emergency_state(inode->i_sb); 830 else 831 ret = ext4_forced_shutdown(inode->i_sb) ? -EIO : 0; 832 if (unlikely(ret)) 833 return ret; 834 835 /* 836 * We don't support synchronous mappings for non-DAX files and 837 * for DAX files if underneath dax_device is not synchronous. 838 */ 839 if (!daxdev_mapping_supported(desc, file_inode(file), dax_dev)) 840 return -EOPNOTSUPP; 841 842 file_accessed(file); 843 if (IS_DAX(file_inode(file))) { 844 desc->vm_ops = &ext4_dax_vm_ops; 845 vma_desc_set_flags(desc, VMA_HUGEPAGE_BIT); 846 } else { 847 desc->vm_ops = &ext4_file_vm_ops; 848 } 849 return 0; 850 } 851 852 static int ext4_sample_last_mounted(struct super_block *sb, 853 struct vfsmount *mnt) 854 { 855 struct ext4_sb_info *sbi = EXT4_SB(sb); 856 struct path path; 857 char buf[64], *cp; 858 handle_t *handle; 859 int err; 860 861 if (likely(ext4_test_mount_flag(sb, EXT4_MF_MNTDIR_SAMPLED))) 862 return 0; 863 864 if (ext4_emergency_state(sb) || sb_rdonly(sb) || 865 !sb_start_intwrite_trylock(sb)) 866 return 0; 867 868 ext4_set_mount_flag(sb, EXT4_MF_MNTDIR_SAMPLED); 869 /* 870 * Sample where the filesystem has been mounted and 871 * store it in the superblock for sysadmin convenience 872 * when trying to sort through large numbers of block 873 * devices or filesystem images. 874 */ 875 path.mnt = mnt; 876 path.dentry = mnt->mnt_root; 877 cp = d_path(&path, buf, sizeof(buf)); 878 err = 0; 879 if (IS_ERR(cp)) 880 goto out; 881 882 handle = ext4_journal_start_sb(sb, EXT4_HT_MISC, 1); 883 err = PTR_ERR(handle); 884 if (IS_ERR(handle)) 885 goto out; 886 BUFFER_TRACE(sbi->s_sbh, "get_write_access"); 887 err = ext4_journal_get_write_access(handle, sb, sbi->s_sbh, 888 EXT4_JTR_NONE); 889 if (err) 890 goto out_journal; 891 lock_buffer(sbi->s_sbh); 892 strtomem_pad(sbi->s_es->s_last_mounted, cp, 0); 893 ext4_superblock_csum_set(sb); 894 unlock_buffer(sbi->s_sbh); 895 ext4_handle_dirty_metadata(handle, NULL, sbi->s_sbh); 896 out_journal: 897 ext4_journal_stop(handle); 898 out: 899 sb_end_intwrite(sb); 900 return err; 901 } 902 903 static int ext4_file_open(struct inode *inode, struct file *filp) 904 { 905 int ret; 906 907 if (filp->f_mode & FMODE_WRITE) 908 ret = ext4_emergency_state(inode->i_sb); 909 else 910 ret = ext4_forced_shutdown(inode->i_sb) ? -EIO : 0; 911 if (unlikely(ret)) 912 return ret; 913 914 ret = ext4_sample_last_mounted(inode->i_sb, filp->f_path.mnt); 915 if (ret) 916 return ret; 917 918 ret = fscrypt_file_open(inode, filp); 919 if (ret) 920 return ret; 921 922 ret = fsverity_file_open(inode, filp); 923 if (ret) 924 return ret; 925 926 /* 927 * Set up the jbd2_inode if we are opening the inode for 928 * writing and the journal is present 929 */ 930 if (filp->f_mode & FMODE_WRITE) { 931 ret = ext4_inode_attach_jinode(inode); 932 if (ret < 0) 933 return ret; 934 } 935 936 if (ext4_inode_can_atomic_write(inode)) 937 filp->f_mode |= FMODE_CAN_ATOMIC_WRITE; 938 939 filp->f_mode |= FMODE_NOWAIT | FMODE_CAN_ODIRECT; 940 return dquot_file_open(inode, filp); 941 } 942 943 /* 944 * ext4_llseek() handles both block-mapped and extent-mapped maxbytes values 945 * by calling generic_file_llseek_size() with the appropriate maxbytes 946 * value for each. 947 */ 948 loff_t ext4_llseek(struct file *file, loff_t offset, int whence) 949 { 950 struct inode *inode = file->f_mapping->host; 951 loff_t maxbytes = ext4_get_maxbytes(inode); 952 953 switch (whence) { 954 default: 955 return generic_file_llseek_size(file, offset, whence, 956 maxbytes, i_size_read(inode)); 957 case SEEK_HOLE: 958 inode_lock_shared(inode); 959 offset = iomap_seek_hole(inode, offset, 960 &ext4_iomap_report_ops); 961 inode_unlock_shared(inode); 962 break; 963 case SEEK_DATA: 964 inode_lock_shared(inode); 965 offset = iomap_seek_data(inode, offset, 966 &ext4_iomap_report_ops); 967 inode_unlock_shared(inode); 968 break; 969 } 970 971 if (offset < 0) 972 return offset; 973 return vfs_setpos(file, offset, maxbytes); 974 } 975 976 const struct file_operations ext4_file_operations = { 977 .llseek = ext4_llseek, 978 .read_iter = ext4_file_read_iter, 979 .write_iter = ext4_file_write_iter, 980 .iopoll = iocb_bio_iopoll, 981 .unlocked_ioctl = ext4_ioctl, 982 #ifdef CONFIG_COMPAT 983 .compat_ioctl = ext4_compat_ioctl, 984 #endif 985 .mmap_prepare = ext4_file_mmap_prepare, 986 .open = ext4_file_open, 987 .release = ext4_release_file, 988 .fsync = ext4_sync_file, 989 .get_unmapped_area = thp_get_unmapped_area, 990 .splice_read = ext4_file_splice_read, 991 .splice_write = iter_file_splice_write, 992 .fallocate = ext4_fallocate, 993 .fop_flags = FOP_MMAP_SYNC | FOP_BUFFER_RASYNC | 994 FOP_DIO_PARALLEL_WRITE | 995 FOP_DONTCACHE, 996 .setlease = generic_setlease, 997 }; 998 999 const struct inode_operations ext4_file_inode_operations = { 1000 .setattr = ext4_setattr, 1001 .getattr = ext4_file_getattr, 1002 .listxattr = ext4_listxattr, 1003 .get_inode_acl = ext4_get_acl, 1004 .set_acl = ext4_set_acl, 1005 .fiemap = ext4_fiemap, 1006 .fileattr_get = ext4_fileattr_get, 1007 .fileattr_set = ext4_fileattr_set, 1008 }; 1009 1010