1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * linux/fs/ext4/inode.c 4 * 5 * Copyright (C) 1992, 1993, 1994, 1995 6 * Remy Card (card@masi.ibp.fr) 7 * Laboratoire MASI - Institut Blaise Pascal 8 * Universite Pierre et Marie Curie (Paris VI) 9 * 10 * from 11 * 12 * linux/fs/minix/inode.c 13 * 14 * Copyright (C) 1991, 1992 Linus Torvalds 15 * 16 * 64-bit file support on 64-bit platforms by Jakub Jelinek 17 * (jj@sunsite.ms.mff.cuni.cz) 18 * 19 * Assorted race fixes, rewrite of ext4_get_block() by Al Viro, 2000 20 */ 21 22 #include <linux/fs.h> 23 #include <linux/mount.h> 24 #include <linux/time.h> 25 #include <linux/highuid.h> 26 #include <linux/pagemap.h> 27 #include <linux/dax.h> 28 #include <linux/quotaops.h> 29 #include <linux/string.h> 30 #include <linux/buffer_head.h> 31 #include <linux/writeback.h> 32 #include <linux/folio_batch.h> 33 #include <linux/mpage.h> 34 #include <linux/rmap.h> 35 #include <linux/namei.h> 36 #include <linux/uio.h> 37 #include <linux/bio.h> 38 #include <linux/workqueue.h> 39 #include <linux/kernel.h> 40 #include <linux/printk.h> 41 #include <linux/slab.h> 42 #include <linux/bitops.h> 43 #include <linux/iomap.h> 44 #include <linux/iversion.h> 45 46 #include "ext4_jbd2.h" 47 #include "ext4_extents.h" 48 #include "xattr.h" 49 #include "acl.h" 50 #include "truncate.h" 51 52 #include <kunit/static_stub.h> 53 54 #include <trace/events/ext4.h> 55 56 static void ext4_journalled_zero_new_buffers(handle_t *handle, 57 struct inode *inode, 58 struct folio *folio, 59 unsigned from, unsigned to); 60 61 static __u32 ext4_inode_csum(struct inode *inode, struct ext4_inode *raw, 62 struct ext4_inode_info *ei) 63 { 64 __u32 csum; 65 __u16 dummy_csum = 0; 66 int offset = offsetof(struct ext4_inode, i_checksum_lo); 67 unsigned int csum_size = sizeof(dummy_csum); 68 69 csum = ext4_chksum(ei->i_csum_seed, (__u8 *)raw, offset); 70 csum = ext4_chksum(csum, (__u8 *)&dummy_csum, csum_size); 71 offset += csum_size; 72 csum = ext4_chksum(csum, (__u8 *)raw + offset, 73 EXT4_GOOD_OLD_INODE_SIZE - offset); 74 75 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) { 76 offset = offsetof(struct ext4_inode, i_checksum_hi); 77 csum = ext4_chksum(csum, (__u8 *)raw + EXT4_GOOD_OLD_INODE_SIZE, 78 offset - EXT4_GOOD_OLD_INODE_SIZE); 79 if (EXT4_FITS_IN_INODE(raw, ei, i_checksum_hi)) { 80 csum = ext4_chksum(csum, (__u8 *)&dummy_csum, 81 csum_size); 82 offset += csum_size; 83 } 84 csum = ext4_chksum(csum, (__u8 *)raw + offset, 85 EXT4_INODE_SIZE(inode->i_sb) - offset); 86 } 87 88 return csum; 89 } 90 91 static int ext4_inode_csum_verify(struct inode *inode, struct ext4_inode *raw, 92 struct ext4_inode_info *ei) 93 { 94 __u32 provided, calculated; 95 96 if (EXT4_SB(inode->i_sb)->s_es->s_creator_os != 97 cpu_to_le32(EXT4_OS_LINUX) || 98 !ext4_has_feature_metadata_csum(inode->i_sb)) 99 return 1; 100 101 provided = le16_to_cpu(raw->i_checksum_lo); 102 calculated = ext4_inode_csum(inode, raw, ei); 103 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE && 104 EXT4_FITS_IN_INODE(raw, ei, i_checksum_hi)) 105 provided |= ((__u32)le16_to_cpu(raw->i_checksum_hi)) << 16; 106 else 107 calculated &= 0xFFFF; 108 109 return provided == calculated; 110 } 111 112 void ext4_inode_csum_set(struct inode *inode, struct ext4_inode *raw, 113 struct ext4_inode_info *ei) 114 { 115 __u32 csum; 116 117 if (EXT4_SB(inode->i_sb)->s_es->s_creator_os != 118 cpu_to_le32(EXT4_OS_LINUX) || 119 !ext4_has_feature_metadata_csum(inode->i_sb)) 120 return; 121 122 csum = ext4_inode_csum(inode, raw, ei); 123 raw->i_checksum_lo = cpu_to_le16(csum & 0xFFFF); 124 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE && 125 EXT4_FITS_IN_INODE(raw, ei, i_checksum_hi)) 126 raw->i_checksum_hi = cpu_to_le16(csum >> 16); 127 } 128 129 /* 130 * Clear the disksize-grow-pending state and wake up all waiters. 131 * Called when the pending zeroed EOF block which straddles i_disksize 132 * has completed writeback or its folio is discarded. 133 */ 134 void ext4_iomap_clear_disksize_pending(struct inode *inode) 135 { 136 if (!test_and_clear_wake_up_bit( 137 ext4_inode_state_wait_bit(EXT4_STATE_DISKSIZE_GROW_PENDING), 138 ext4_inode_state_wait_word(inode))) 139 return; 140 141 trace_ext4_iomap_clear_disksize_pending(inode); 142 } 143 144 /* 145 * Wait for the disksize-grow-pending zeroed EOF block which straddles 146 * i_disksize to be written back or cleared. 147 */ 148 void ext4_iomap_wait_disksize_pending(struct inode *inode) 149 { 150 /* Only emit the trace when the bit is actually set */ 151 if (ext4_test_inode_state(inode, EXT4_STATE_DISKSIZE_GROW_PENDING)) 152 trace_ext4_iomap_wait_disksize_pending(inode); 153 154 wait_on_bit(ext4_inode_state_wait_word(inode), 155 ext4_inode_state_wait_bit(EXT4_STATE_DISKSIZE_GROW_PENDING), 156 TASK_UNINTERRUPTIBLE); 157 } 158 159 /* 160 * Get the range of the disksize-grow-pending zeroed EOF block range 161 * which straddles i_disksize if the EXT4_STATE_DISKSIZE_GROW_PENDING 162 * bit is set. 163 * 164 * Return the pending range, or zero if the BIT has already been cleared. 165 */ 166 unsigned int ext4_iomap_get_disksize_pending_range(struct inode *inode, 167 loff_t *start) 168 { 169 unsigned int blocksize = i_blocksize(inode); 170 loff_t disksize; 171 172 if (!ext4_test_inode_state(inode, EXT4_STATE_DISKSIZE_GROW_PENDING)) 173 return 0; 174 175 /* 176 * The pending bit should be set only when i_disksize is not 177 * block-size aligned. While set, i_disksize must not be advanced, 178 * and the bit must be cleared when i_disksize is shrunk. 179 */ 180 down_read(&EXT4_I(inode)->i_data_sem); 181 disksize = READ_ONCE(EXT4_I(inode)->i_disksize); 182 if (!ext4_test_inode_state(inode, EXT4_STATE_DISKSIZE_GROW_PENDING) || 183 WARN_ON_ONCE(IS_ALIGNED(disksize, blocksize))) { 184 up_read(&EXT4_I(inode)->i_data_sem); 185 return 0; 186 } 187 188 up_read(&EXT4_I(inode)->i_data_sem); 189 *start = disksize; 190 return blocksize - (disksize & (blocksize - 1)); 191 } 192 193 static inline int ext4_begin_ordered_truncate(struct inode *inode, 194 loff_t new_size) 195 { 196 struct jbd2_inode *jinode = READ_ONCE(EXT4_I(inode)->jinode); 197 198 trace_ext4_begin_ordered_truncate(inode, new_size); 199 /* 200 * If jinode is zero, then we never opened the file for 201 * writing, so there's no need to call 202 * jbd2_journal_begin_ordered_truncate() since there's no 203 * outstanding writes we need to flush. 204 */ 205 if (!jinode) 206 return 0; 207 return jbd2_journal_begin_ordered_truncate(EXT4_JOURNAL(inode), 208 jinode, 209 new_size); 210 } 211 212 /* 213 * Test whether an inode is a fast symlink. 214 * A fast symlink has its symlink data stored in ext4_inode_info->i_data. 215 */ 216 int ext4_inode_is_fast_symlink(struct inode *inode) 217 { 218 if (!ext4_has_feature_ea_inode(inode->i_sb)) { 219 int ea_blocks = EXT4_I(inode)->i_file_acl ? 220 EXT4_CLUSTER_SIZE(inode->i_sb) >> 9 : 0; 221 222 if (ext4_has_inline_data(inode)) 223 return 0; 224 225 return (S_ISLNK(inode->i_mode) && inode->i_blocks - ea_blocks == 0); 226 } 227 return S_ISLNK(inode->i_mode) && inode->i_size && 228 (inode->i_size < EXT4_N_BLOCKS * 4); 229 } 230 231 /* 232 * Called at the last iput() if i_nlink is zero. 233 */ 234 void ext4_evict_inode(struct inode *inode) 235 { 236 handle_t *handle; 237 int err; 238 /* 239 * Credits for final inode cleanup and freeing: 240 * sb + inode (ext4_orphan_del()), block bitmap, group descriptor 241 * (xattr block freeing), bitmap, group descriptor (inode freeing) 242 */ 243 int extra_credits = 6; 244 bool freeze_protected = false; 245 246 trace_ext4_evict_inode(inode); 247 248 dax_break_layout_final(inode); 249 250 if (EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL) 251 ext4_evict_ea_inode(inode); 252 if (inode->i_nlink) { 253 /* 254 * If there's dirty page will lead to data loss, user 255 * could see stale data. 256 */ 257 if (unlikely(!ext4_emergency_state(inode->i_sb) && 258 mapping_tagged(&inode->i_data, PAGECACHE_TAG_DIRTY))) 259 ext4_warning_inode(inode, "data will be lost"); 260 261 truncate_inode_pages_final(&inode->i_data); 262 goto no_delete; 263 } 264 265 if (is_bad_inode(inode)) 266 goto no_delete; 267 dquot_initialize(inode); 268 269 if (ext4_should_order_data(inode)) 270 ext4_begin_ordered_truncate(inode, 0); 271 if (ext4_inode_buffered_iomap(inode)) 272 ext4_iomap_clear_disksize_pending(inode); 273 truncate_inode_pages_final(&inode->i_data); 274 275 /* 276 * For inodes with journalled data, transaction commit could have 277 * dirtied the inode. And for inodes with dioread_nolock, unwritten 278 * extents converting worker could merge extents and also have dirtied 279 * the inode. Flush worker is ignoring it because of I_FREEING flag but 280 * we still need to remove the inode from the writeback lists. 281 */ 282 inode_io_list_del(inode); 283 284 /* 285 * Protect us against freezing - iput() caller didn't have to have any 286 * protection against it. When we are in a running transaction though, 287 * we are already protected against freezing and we cannot grab further 288 * protection due to lock ordering constraints. 289 */ 290 if (!ext4_journal_current_handle()) { 291 sb_start_intwrite(inode->i_sb); 292 freeze_protected = true; 293 } 294 295 if (!IS_NOQUOTA(inode)) 296 extra_credits += EXT4_MAXQUOTAS_DEL_BLOCKS(inode->i_sb); 297 298 /* 299 * Block bitmap, group descriptor, and inode are accounted in both 300 * ext4_blocks_for_truncate() and extra_credits. So subtract 3. 301 */ 302 handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE, 303 ext4_blocks_for_truncate(inode) + extra_credits - 3); 304 if (IS_ERR(handle)) { 305 ext4_std_error(inode->i_sb, PTR_ERR(handle)); 306 /* 307 * If we're going to skip the normal cleanup, we still need to 308 * make sure that the in-core orphan linked list is properly 309 * cleaned up. 310 */ 311 ext4_orphan_del(NULL, inode); 312 if (freeze_protected) 313 sb_end_intwrite(inode->i_sb); 314 goto no_delete; 315 } 316 317 if (IS_SYNC(inode)) 318 ext4_handle_sync(handle); 319 320 /* 321 * Set inode->i_size to 0 before calling ext4_truncate(). We need 322 * special handling of symlinks here because i_size is used to 323 * determine whether ext4_inode_info->i_data contains symlink data or 324 * block mappings. Setting i_size to 0 will remove its fast symlink 325 * status. Erase i_data so that it becomes a valid empty block map. 326 */ 327 if (ext4_inode_is_fast_symlink(inode)) 328 memset(EXT4_I(inode)->i_data, 0, sizeof(EXT4_I(inode)->i_data)); 329 inode->i_size = 0; 330 ext4_set_inode_state(inode, EXT4_STATE_NO_EXPAND); 331 err = ext4_mark_inode_dirty(handle, inode); 332 if (err) { 333 ext4_warning(inode->i_sb, 334 "couldn't mark inode dirty (err %d)", err); 335 goto stop_handle; 336 } 337 if (inode->i_blocks) { 338 err = ext4_truncate(inode); 339 if (err) { 340 ext4_error_err(inode->i_sb, -err, 341 "couldn't truncate inode %llu (err %d)", 342 inode->i_ino, err); 343 goto stop_handle; 344 } 345 } 346 347 /* Remove xattr references. */ 348 err = ext4_xattr_delete_inode(handle, inode, extra_credits); 349 if (err) { 350 ext4_warning(inode->i_sb, "xattr delete (err %d)", err); 351 stop_handle: 352 ext4_journal_stop(handle); 353 ext4_orphan_del(NULL, inode); 354 if (freeze_protected) 355 sb_end_intwrite(inode->i_sb); 356 goto no_delete; 357 } 358 359 /* 360 * Kill off the orphan record which ext4_truncate created. 361 * AKPM: I think this can be inside the above `if'. 362 * Note that ext4_orphan_del() has to be able to cope with the 363 * deletion of a non-existent orphan - this is because we don't 364 * know if ext4_truncate() actually created an orphan record. 365 * (Well, we could do this if we need to, but heck - it works) 366 */ 367 ext4_orphan_del(handle, inode); 368 EXT4_I(inode)->i_dtime = (__u32)ktime_get_real_seconds(); 369 370 /* 371 * One subtle ordering requirement: if anything has gone wrong 372 * (transaction abort, IO errors, whatever), then we can still 373 * do these next steps (the fs will already have been marked as 374 * having errors), but we can't free the inode if the mark_dirty 375 * fails. 376 */ 377 if (ext4_mark_inode_dirty(handle, inode)) 378 /* If that failed, just do the required in-core inode clear. */ 379 ext4_clear_inode(inode); 380 else 381 ext4_free_inode(handle, inode); 382 ext4_journal_stop(handle); 383 if (freeze_protected) 384 sb_end_intwrite(inode->i_sb); 385 return; 386 no_delete: 387 /* 388 * Check out some where else accidentally dirty the evicting inode, 389 * which may probably cause inode use-after-free issues later. 390 */ 391 WARN_ON_ONCE(!list_empty_careful(&inode->i_io_list)); 392 393 if (!list_empty(&EXT4_I(inode)->i_fc_list)) 394 ext4_fc_mark_ineligible(inode->i_sb, EXT4_FC_REASON_NOMEM, NULL); 395 ext4_clear_inode(inode); /* We must guarantee clearing of inode... */ 396 } 397 398 #ifdef CONFIG_QUOTA 399 qsize_t *ext4_get_reserved_space(struct inode *inode) 400 { 401 return &EXT4_I(inode)->i_reserved_quota; 402 } 403 #endif 404 405 /* 406 * Called with i_data_sem down, which is important since we can call 407 * ext4_discard_preallocations() from here. 408 */ 409 void ext4_da_update_reserve_space(struct inode *inode, 410 int used, int quota_claim) 411 { 412 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); 413 struct ext4_inode_info *ei = EXT4_I(inode); 414 415 spin_lock(&ei->i_block_reservation_lock); 416 trace_ext4_da_update_reserve_space(inode, used, quota_claim); 417 if (unlikely(used > ei->i_reserved_data_blocks)) { 418 ext4_warning(inode->i_sb, "%s: ino %llu, used %d " 419 "with only %d reserved data blocks", 420 __func__, inode->i_ino, used, 421 ei->i_reserved_data_blocks); 422 WARN_ON(1); 423 used = ei->i_reserved_data_blocks; 424 } 425 426 /* Update per-inode reservations */ 427 ei->i_reserved_data_blocks -= used; 428 percpu_counter_sub(&sbi->s_dirtyclusters_counter, used); 429 430 spin_unlock(&ei->i_block_reservation_lock); 431 432 /* Update quota subsystem for data blocks */ 433 if (quota_claim) 434 dquot_claim_block(inode, EXT4_C2B(sbi, used)); 435 else { 436 /* 437 * We did fallocate with an offset that is already delayed 438 * allocated. So on delayed allocated writeback we should 439 * not re-claim the quota for fallocated blocks. 440 */ 441 dquot_release_reservation_block(inode, EXT4_C2B(sbi, used)); 442 } 443 444 /* 445 * If we have done all the pending block allocations and if 446 * there aren't any writers on the inode, we can discard the 447 * inode's preallocations. 448 */ 449 if ((ei->i_reserved_data_blocks == 0) && 450 !inode_is_open_for_write(inode)) 451 ext4_discard_preallocations(inode); 452 } 453 454 static int __check_block_validity(struct inode *inode, const char *func, 455 unsigned int line, 456 struct ext4_map_blocks *map) 457 { 458 journal_t *journal = EXT4_SB(inode->i_sb)->s_journal; 459 460 if (journal && inode == journal->j_inode) 461 return 0; 462 463 if (!ext4_inode_block_valid(inode, map->m_pblk, map->m_len)) { 464 ext4_error_inode(inode, func, line, map->m_pblk, 465 "lblock %lu mapped to illegal pblock %llu " 466 "(length %d)", (unsigned long) map->m_lblk, 467 map->m_pblk, map->m_len); 468 return -EFSCORRUPTED; 469 } 470 return 0; 471 } 472 473 int ext4_issue_zeroout(struct inode *inode, ext4_lblk_t lblk, ext4_fsblk_t pblk, 474 ext4_lblk_t len) 475 { 476 int ret; 477 478 KUNIT_STATIC_STUB_REDIRECT(ext4_issue_zeroout, inode, lblk, pblk, len); 479 480 if (IS_ENCRYPTED(inode) && S_ISREG(inode->i_mode)) 481 return fscrypt_zeroout_range(inode, 482 (loff_t)lblk << inode->i_blkbits, 483 pblk << (inode->i_blkbits - SECTOR_SHIFT), 484 (u64)len << inode->i_blkbits); 485 486 ret = sb_issue_zeroout(inode->i_sb, pblk, len, GFP_NOFS); 487 if (ret > 0) 488 ret = 0; 489 490 return ret; 491 } 492 493 /* 494 * For generic regular files, when updating the extent tree, Ext4 should 495 * hold the i_rwsem and invalidate_lock exclusively. This ensures 496 * exclusion against concurrent page faults, as well as reads and writes. 497 */ 498 #ifdef CONFIG_EXT4_DEBUG 499 void ext4_check_map_extents_env(struct inode *inode) 500 { 501 if (EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY) 502 return; 503 504 if (!S_ISREG(inode->i_mode) || 505 IS_NOQUOTA(inode) || IS_VERITY(inode) || 506 is_special_ino(inode->i_sb, inode->i_ino) || 507 (inode_state_read_once(inode) & (I_FREEING | I_WILL_FREE | I_NEW)) || 508 ext4_test_inode_flag(inode, EXT4_INODE_EA_INODE) || 509 ext4_verity_in_progress(inode)) 510 return; 511 512 WARN_ON_ONCE(!inode_is_locked(inode) && 513 !rwsem_is_locked(&inode->i_mapping->invalidate_lock)); 514 } 515 #else 516 void ext4_check_map_extents_env(struct inode *inode) {} 517 #endif 518 519 #define check_block_validity(inode, map) \ 520 __check_block_validity((inode), __func__, __LINE__, (map)) 521 522 #ifdef ES_AGGRESSIVE_TEST 523 static void ext4_map_blocks_es_recheck(handle_t *handle, 524 struct inode *inode, 525 struct ext4_map_blocks *es_map, 526 struct ext4_map_blocks *map, 527 int flags) 528 { 529 int retval; 530 531 map->m_flags = 0; 532 /* 533 * There is a race window that the result is not the same. 534 * e.g. xfstests #223 when dioread_nolock enables. The reason 535 * is that we lookup a block mapping in extent status tree with 536 * out taking i_data_sem. So at the time the unwritten extent 537 * could be converted. 538 */ 539 down_read(&EXT4_I(inode)->i_data_sem); 540 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) { 541 retval = ext4_ext_map_blocks(handle, inode, map, 0); 542 } else { 543 retval = ext4_ind_map_blocks(handle, inode, map, 0); 544 } 545 up_read((&EXT4_I(inode)->i_data_sem)); 546 547 /* 548 * We don't check m_len because extent will be collpased in status 549 * tree. So the m_len might not equal. 550 */ 551 if (es_map->m_lblk != map->m_lblk || 552 es_map->m_flags != map->m_flags || 553 es_map->m_pblk != map->m_pblk) { 554 printk("ES cache assertion failed for inode: %llu " 555 "es_cached ex [%d/%d/%llu/%x] != " 556 "found ex [%d/%d/%llu/%x] retval %d flags %x\n", 557 inode->i_ino, es_map->m_lblk, es_map->m_len, 558 es_map->m_pblk, es_map->m_flags, map->m_lblk, 559 map->m_len, map->m_pblk, map->m_flags, 560 retval, flags); 561 } 562 } 563 #endif /* ES_AGGRESSIVE_TEST */ 564 565 static int ext4_map_query_blocks_next_in_leaf(handle_t *handle, 566 struct inode *inode, struct ext4_map_blocks *map, 567 unsigned int orig_mlen) 568 { 569 struct ext4_map_blocks map2; 570 unsigned int status, status2; 571 int retval; 572 573 status = map->m_flags & EXT4_MAP_UNWRITTEN ? 574 EXTENT_STATUS_UNWRITTEN : EXTENT_STATUS_WRITTEN; 575 576 WARN_ON_ONCE(!(map->m_flags & EXT4_MAP_QUERY_LAST_IN_LEAF)); 577 WARN_ON_ONCE(orig_mlen <= map->m_len); 578 579 /* Prepare map2 for lookup in next leaf block */ 580 map2.m_lblk = map->m_lblk + map->m_len; 581 map2.m_len = orig_mlen - map->m_len; 582 map2.m_flags = 0; 583 retval = ext4_ext_map_blocks(handle, inode, &map2, 0); 584 585 if (retval <= 0) { 586 ext4_es_cache_extent(inode, map->m_lblk, map->m_len, 587 map->m_pblk, status); 588 return map->m_len; 589 } 590 591 if (unlikely(retval != map2.m_len)) { 592 ext4_warning(inode->i_sb, 593 "ES len assertion failed for inode " 594 "%llu: retval %d != map->m_len %d", 595 inode->i_ino, retval, map2.m_len); 596 WARN_ON(1); 597 } 598 599 status2 = map2.m_flags & EXT4_MAP_UNWRITTEN ? 600 EXTENT_STATUS_UNWRITTEN : EXTENT_STATUS_WRITTEN; 601 602 /* 603 * If map2 is contiguous with map, then let's insert it as a single 604 * extent in es cache and return the combined length of both the maps. 605 */ 606 if (map->m_pblk + map->m_len == map2.m_pblk && 607 status == status2) { 608 ext4_es_cache_extent(inode, map->m_lblk, 609 map->m_len + map2.m_len, map->m_pblk, 610 status); 611 map->m_len += map2.m_len; 612 } else { 613 ext4_es_cache_extent(inode, map->m_lblk, map->m_len, 614 map->m_pblk, status); 615 } 616 617 return map->m_len; 618 } 619 620 int ext4_map_query_blocks(handle_t *handle, struct inode *inode, 621 struct ext4_map_blocks *map, int flags) 622 { 623 unsigned int status; 624 int retval; 625 unsigned int orig_mlen = map->m_len; 626 627 flags &= EXT4_EX_QUERY_FILTER; 628 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) 629 retval = ext4_ext_map_blocks(handle, inode, map, flags); 630 else 631 retval = ext4_ind_map_blocks(handle, inode, map, flags); 632 if (retval < 0) 633 return retval; 634 635 /* A hole? */ 636 if (retval == 0) 637 goto out; 638 639 if (unlikely(retval != map->m_len)) { 640 ext4_warning(inode->i_sb, 641 "ES len assertion failed for inode " 642 "%llu: retval %d != map->m_len %d", 643 inode->i_ino, retval, map->m_len); 644 WARN_ON(1); 645 } 646 647 /* 648 * No need to query next in leaf: 649 * - if returned extent is not last in leaf or 650 * - if the last in leaf is the full requested range 651 */ 652 if (!(map->m_flags & EXT4_MAP_QUERY_LAST_IN_LEAF) || 653 map->m_len == orig_mlen) { 654 status = map->m_flags & EXT4_MAP_UNWRITTEN ? 655 EXTENT_STATUS_UNWRITTEN : EXTENT_STATUS_WRITTEN; 656 ext4_es_cache_extent(inode, map->m_lblk, map->m_len, 657 map->m_pblk, status); 658 } else { 659 retval = ext4_map_query_blocks_next_in_leaf(handle, inode, map, 660 orig_mlen); 661 } 662 out: 663 map->m_seq = READ_ONCE(EXT4_I(inode)->i_es_seq); 664 return retval; 665 } 666 667 int ext4_map_create_blocks(handle_t *handle, struct inode *inode, 668 struct ext4_map_blocks *map, int flags) 669 { 670 unsigned int status; 671 int err, retval = 0; 672 673 /* 674 * We pass in the magic EXT4_GET_BLOCKS_DELALLOC_RESERVE 675 * indicates that the blocks and quotas has already been 676 * checked when the data was copied into the page cache. 677 */ 678 if (map->m_flags & EXT4_MAP_DELAYED) 679 flags |= EXT4_GET_BLOCKS_DELALLOC_RESERVE; 680 681 /* 682 * Here we clear m_flags because after allocating an new extent, 683 * it will be set again. 684 */ 685 map->m_flags &= ~EXT4_MAP_FLAGS; 686 687 /* 688 * We need to check for EXT4 here because migrate could have 689 * changed the inode type in between. 690 */ 691 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) { 692 retval = ext4_ext_map_blocks(handle, inode, map, flags); 693 } else { 694 retval = ext4_ind_map_blocks(handle, inode, map, flags); 695 696 /* 697 * We allocated new blocks which will result in i_data's 698 * format changing. Force the migrate to fail by clearing 699 * migrate flags. 700 */ 701 if (retval > 0 && map->m_flags & EXT4_MAP_NEW) 702 ext4_clear_inode_state(inode, EXT4_STATE_EXT_MIGRATE); 703 } 704 if (retval <= 0) 705 return retval; 706 707 if (unlikely(retval != map->m_len)) { 708 ext4_warning(inode->i_sb, 709 "ES len assertion failed for inode %llu: " 710 "retval %d != map->m_len %d", 711 inode->i_ino, retval, map->m_len); 712 WARN_ON(1); 713 } 714 715 /* 716 * We have to zeroout blocks before inserting them into extent 717 * status tree. Otherwise someone could look them up there and 718 * use them before they are really zeroed. We also have to 719 * unmap metadata before zeroing as otherwise writeback can 720 * overwrite zeros with stale data from block device. 721 */ 722 if (flags & EXT4_GET_BLOCKS_ZERO && 723 map->m_flags & EXT4_MAP_MAPPED && map->m_flags & EXT4_MAP_NEW) { 724 err = ext4_issue_zeroout(inode, map->m_lblk, map->m_pblk, 725 map->m_len); 726 if (err) 727 return err; 728 } 729 730 status = map->m_flags & EXT4_MAP_UNWRITTEN ? 731 EXTENT_STATUS_UNWRITTEN : EXTENT_STATUS_WRITTEN; 732 ext4_es_insert_extent(inode, map->m_lblk, map->m_len, map->m_pblk, 733 status, flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE); 734 map->m_seq = READ_ONCE(EXT4_I(inode)->i_es_seq); 735 736 return retval; 737 } 738 739 /* 740 * The ext4_map_blocks() function tries to look up the requested blocks, 741 * and returns if the blocks are already mapped. 742 * 743 * Otherwise it takes the write lock of the i_data_sem and allocate blocks 744 * and store the allocated blocks in the result buffer head and mark it 745 * mapped. 746 * 747 * If file type is extents based, it will call ext4_ext_map_blocks(), 748 * Otherwise, call with ext4_ind_map_blocks() to handle indirect mapping 749 * based files 750 * 751 * ext4_map_blocks() may also be called with a NULL @handle. In that case 752 * it starts its own transaction handle only when block allocation is 753 * actually needed, i.e. after the lookup confirms the blocks are not yet 754 * mapped, and stops it before returning. 755 * 756 * On success, it returns the number of blocks being mapped or allocated. 757 * If flags doesn't contain EXT4_GET_BLOCKS_CREATE the blocks are 758 * pre-allocated and unwritten, the resulting @map is marked as unwritten. 759 * If the flags contain EXT4_GET_BLOCKS_CREATE, it will mark @map as mapped. 760 * 761 * It returns 0 if plain look up failed (blocks have not been allocated), in 762 * that case, @map is returned as unmapped but we still do fill map->m_len to 763 * indicate the length of a hole starting at map->m_lblk. 764 * 765 * It returns the error in case of allocation failure. 766 */ 767 int ext4_map_blocks(handle_t *handle, struct inode *inode, 768 struct ext4_map_blocks *map, int flags) 769 { 770 struct extent_status es; 771 int retval; 772 int ret = 0; 773 bool internal_handle = false; 774 unsigned int orig_mlen; 775 #ifdef ES_AGGRESSIVE_TEST 776 struct ext4_map_blocks orig_map; 777 778 memcpy(&orig_map, map, sizeof(*map)); 779 #endif 780 781 map->m_flags = 0; 782 ext_debug(inode, "flag 0x%x, max_blocks %u, logical block %lu\n", 783 flags, map->m_len, (unsigned long) map->m_lblk); 784 785 /* 786 * ext4_map_blocks returns an int, and m_len is an unsigned int 787 */ 788 if (unlikely(map->m_len > INT_MAX)) 789 map->m_len = INT_MAX; 790 orig_mlen = map->m_len; 791 792 /* We can handle the block number less than EXT_MAX_BLOCKS */ 793 if (unlikely(map->m_lblk >= EXT_MAX_BLOCKS)) 794 return -EFSCORRUPTED; 795 796 /* 797 * Callers from the context of data submission are the only exceptions 798 * for regular files that do not hold the i_rwsem or invalidate_lock. 799 * However, caching unrelated ranges is not permitted. 800 */ 801 if (flags & EXT4_GET_BLOCKS_IO_SUBMIT) 802 WARN_ON_ONCE(!(flags & EXT4_EX_NOCACHE)); 803 else 804 ext4_check_map_extents_env(inode); 805 806 create_retry: 807 /* Lookup extent status tree firstly */ 808 if (ext4_es_lookup_extent(inode, map->m_lblk, NULL, &es, &map->m_seq)) { 809 if (ext4_es_is_written(&es) || ext4_es_is_unwritten(&es)) { 810 map->m_pblk = ext4_es_pblock(&es) + 811 map->m_lblk - es.es_lblk; 812 map->m_flags |= ext4_es_is_written(&es) ? 813 EXT4_MAP_MAPPED : EXT4_MAP_UNWRITTEN; 814 retval = es.es_len - (map->m_lblk - es.es_lblk); 815 if (retval > map->m_len) 816 retval = map->m_len; 817 map->m_len = retval; 818 } else if (ext4_es_is_delayed(&es) || ext4_es_is_hole(&es)) { 819 map->m_pblk = 0; 820 map->m_flags |= ext4_es_is_delayed(&es) ? 821 EXT4_MAP_DELAYED : 0; 822 retval = es.es_len - (map->m_lblk - es.es_lblk); 823 if (retval > map->m_len) 824 retval = map->m_len; 825 map->m_len = retval; 826 retval = 0; 827 } else { 828 BUG(); 829 } 830 831 if (flags & EXT4_GET_BLOCKS_CACHED_NOWAIT) 832 return retval; 833 #ifdef ES_AGGRESSIVE_TEST 834 ext4_map_blocks_es_recheck(handle, inode, map, 835 &orig_map, flags); 836 #endif 837 if (!(flags & EXT4_GET_BLOCKS_QUERY_LAST_IN_LEAF) || 838 orig_mlen == map->m_len) 839 goto found; 840 841 map->m_len = orig_mlen; 842 } 843 /* 844 * In the query cache no-wait mode, nothing we can do more if we 845 * cannot find extent in the cache. 846 */ 847 if (flags & EXT4_GET_BLOCKS_CACHED_NOWAIT) 848 return 0; 849 850 /* 851 * Try to see if we can get the block without requesting a new 852 * file system block. 853 */ 854 down_read(&EXT4_I(inode)->i_data_sem); 855 retval = ext4_map_query_blocks(handle, inode, map, flags); 856 up_read((&EXT4_I(inode)->i_data_sem)); 857 if (retval < 0) 858 goto out; 859 860 found: 861 if (retval > 0 && map->m_flags & EXT4_MAP_MAPPED) { 862 ret = check_block_validity(inode, map); 863 if (ret != 0) 864 goto out; 865 } 866 867 /* If it is only a block(s) look up */ 868 if ((flags & EXT4_GET_BLOCKS_CREATE) == 0) 869 return retval; 870 871 /* 872 * Returns if the blocks have already allocated 873 * 874 * Note that if blocks have been preallocated 875 * ext4_ext_map_blocks() returns with buffer head unmapped 876 */ 877 if (retval > 0) { 878 /* 879 * If we need to convert written extent to unwritten or 880 * convert unwritten extent to written, continue and do 881 * the actual work in ext4_ext_map_blocks(). 882 */ 883 if (map->m_flags & EXT4_MAP_MAPPED && 884 !(flags & EXT4_GET_BLOCKS_CONVERT_UNWRITTEN)) 885 goto out; 886 if (map->m_flags & EXT4_MAP_UNWRITTEN && 887 (flags & EXT4_GET_BLOCKS_UNWRIT_EXT) && 888 !(flags & EXT4_GET_BLOCKS_CONVERT)) { 889 /* Contains EXT4_GET_BLOCKS_CREATE - mark mapped. */ 890 ret = check_block_validity(inode, map); 891 if (!ret) 892 map->m_flags |= EXT4_MAP_MAPPED; 893 goto out; 894 } 895 } else if (retval == 0) { 896 /* 897 * Do not allocate blocks for holes in the context of 898 * data submission path. 899 */ 900 if (!map->m_flags && (flags & EXT4_GET_BLOCKS_IO_SUBMIT)) 901 goto out; 902 } 903 904 if (!handle) { 905 handle = ext4_journal_start(inode, EXT4_HT_MAP_BLOCKS, 906 ext4_chunk_trans_blocks(inode, orig_mlen)); 907 if (IS_ERR(handle)) { 908 ret = PTR_ERR(handle); 909 goto out; 910 } 911 internal_handle = true; 912 } 913 914 ext4_fc_track_inode(handle, inode); 915 /* 916 * New blocks allocate and/or writing to unwritten extent 917 * will possibly result in updating i_data, so we take 918 * the write lock of i_data_sem, and call get_block() 919 * with create == 1 flag. 920 */ 921 down_write(&EXT4_I(inode)->i_data_sem); 922 923 /* 924 * Check the validity of the mapping found via the extent status 925 * tree or the disk query. A racing truncate may have changed the 926 * extent, since writeback may not hold i_rwsem or the folio locks 927 * covering the full extent(e.g., the iomap writeback path may 928 * allocate an extent that extends beyond the length of the locked 929 * folio in advance). 930 */ 931 if (map->m_seq != READ_ONCE(EXT4_I(inode)->i_es_seq)) { 932 up_write(&EXT4_I(inode)->i_data_sem); 933 map->m_flags = 0; 934 map->m_len = orig_mlen; 935 goto create_retry; 936 } 937 retval = ext4_map_create_blocks(handle, inode, map, flags); 938 up_write((&EXT4_I(inode)->i_data_sem)); 939 940 if (retval < 0) 941 ext_debug(inode, "failed with err %d\n", retval); 942 if (retval <= 0) 943 goto out; 944 945 if (map->m_flags & EXT4_MAP_MAPPED) { 946 ret = check_block_validity(inode, map); 947 if (ret != 0) 948 goto out; 949 950 /* 951 * Inodes with freshly allocated blocks where contents will be 952 * visible after transaction commit must be on transaction's 953 * ordered data list. 954 */ 955 if (map->m_flags & EXT4_MAP_NEW && 956 !(map->m_flags & EXT4_MAP_UNWRITTEN) && 957 !(flags & EXT4_GET_BLOCKS_ZERO) && 958 !ext4_is_quota_file(inode) && 959 ext4_should_order_data(inode)) { 960 loff_t start_byte = EXT4_LBLK_TO_B(inode, map->m_lblk); 961 loff_t length = EXT4_LBLK_TO_B(inode, map->m_len); 962 963 if (flags & EXT4_GET_BLOCKS_IO_SUBMIT) 964 ret = ext4_jbd2_inode_add_wait(handle, inode, 965 start_byte, length); 966 else 967 ret = ext4_jbd2_inode_add_write(handle, inode, 968 start_byte, length); 969 if (ret) 970 goto out; 971 } 972 } 973 ext4_fc_track_range(handle, inode, map->m_lblk, map->m_lblk + 974 map->m_len - 1); 975 out: 976 if (internal_handle) 977 ext4_journal_stop(handle); 978 979 return ret ? ret : retval; 980 } 981 982 /* 983 * Update EXT4_MAP_FLAGS in bh->b_state. For buffer heads attached to pages 984 * we have to be careful as someone else may be manipulating b_state as well. 985 */ 986 static void ext4_update_bh_state(struct buffer_head *bh, unsigned long flags) 987 { 988 unsigned long old_state; 989 unsigned long new_state; 990 991 flags &= EXT4_MAP_FLAGS; 992 993 /* Dummy buffer_head? Set non-atomically. */ 994 if (!bh->b_folio) { 995 bh->b_state = (bh->b_state & ~EXT4_MAP_FLAGS) | flags; 996 return; 997 } 998 /* 999 * Someone else may be modifying b_state. Be careful! This is ugly but 1000 * once we get rid of using bh as a container for mapping information 1001 * to pass to / from get_block functions, this can go away. 1002 */ 1003 old_state = READ_ONCE(bh->b_state); 1004 do { 1005 new_state = (old_state & ~EXT4_MAP_FLAGS) | flags; 1006 } while (unlikely(!try_cmpxchg(&bh->b_state, &old_state, new_state))); 1007 } 1008 1009 /* 1010 * Make sure that the current journal transaction has enough credits to map 1011 * one extent. Return -EAGAIN if it cannot extend the current running 1012 * transaction. 1013 */ 1014 static inline int ext4_journal_ensure_extent_credits(handle_t *handle, 1015 struct inode *inode) 1016 { 1017 int credits; 1018 int ret; 1019 1020 /* Called from ext4_da_write_begin() which has no handle started? */ 1021 if (!handle) 1022 return 0; 1023 1024 credits = ext4_chunk_trans_blocks(inode, 1); 1025 ret = __ext4_journal_ensure_credits(handle, credits, credits, 0); 1026 return ret <= 0 ? ret : -EAGAIN; 1027 } 1028 1029 static int _ext4_get_block(struct inode *inode, sector_t iblock, 1030 struct buffer_head *bh, int flags) 1031 { 1032 struct ext4_map_blocks map; 1033 int ret = 0; 1034 1035 if (ext4_has_inline_data(inode)) 1036 return -ERANGE; 1037 /* inode using the iomap buffered I/O path should not go here. */ 1038 if (WARN_ON_ONCE(ext4_inode_buffered_iomap(inode))) 1039 return -EINVAL; 1040 1041 map.m_lblk = iblock; 1042 map.m_len = bh->b_size >> inode->i_blkbits; 1043 1044 ret = ext4_map_blocks(ext4_journal_current_handle(), inode, &map, 1045 flags); 1046 if (ret > 0) { 1047 map_bh(bh, inode->i_sb, map.m_pblk); 1048 ext4_update_bh_state(bh, map.m_flags); 1049 bh->b_size = inode->i_sb->s_blocksize * map.m_len; 1050 ret = 0; 1051 } else if (ret == 0) { 1052 /* hole case, need to fill in bh->b_size */ 1053 bh->b_size = inode->i_sb->s_blocksize * map.m_len; 1054 } 1055 return ret; 1056 } 1057 1058 int ext4_get_block(struct inode *inode, sector_t iblock, 1059 struct buffer_head *bh, int create) 1060 { 1061 return _ext4_get_block(inode, iblock, bh, 1062 create ? EXT4_GET_BLOCKS_CREATE : 0); 1063 } 1064 1065 /* 1066 * Get block function used when preparing for buffered write if we require 1067 * creating an unwritten extent if blocks haven't been allocated. The extent 1068 * will be converted to written after the IO is complete. 1069 */ 1070 int ext4_get_block_unwritten(struct inode *inode, sector_t iblock, 1071 struct buffer_head *bh_result, int create) 1072 { 1073 int ret = 0; 1074 1075 ext4_debug("ext4_get_block_unwritten: inode %llu, create flag %d\n", 1076 inode->i_ino, create); 1077 ret = _ext4_get_block(inode, iblock, bh_result, 1078 EXT4_GET_BLOCKS_CREATE_UNWRIT_EXT); 1079 1080 /* 1081 * If the buffer is marked unwritten, mark it as new to make sure it is 1082 * zeroed out correctly in case of partial writes. Otherwise, there is 1083 * a chance of stale data getting exposed. 1084 */ 1085 if (ret == 0 && buffer_unwritten(bh_result)) 1086 set_buffer_new(bh_result); 1087 1088 return ret; 1089 } 1090 1091 /* Maximum number of blocks we map for direct IO at once. */ 1092 #define DIO_MAX_BLOCKS 4096 1093 1094 /* 1095 * `handle' can be NULL if create is zero 1096 */ 1097 struct buffer_head *ext4_getblk(handle_t *handle, struct inode *inode, 1098 ext4_lblk_t block, int map_flags) 1099 { 1100 struct ext4_map_blocks map; 1101 struct buffer_head *bh; 1102 int create = map_flags & EXT4_GET_BLOCKS_CREATE; 1103 bool nowait = map_flags & EXT4_GET_BLOCKS_CACHED_NOWAIT; 1104 int err; 1105 1106 ASSERT((EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY) 1107 || handle != NULL || create == 0); 1108 ASSERT(create == 0 || !nowait); 1109 1110 map.m_lblk = block; 1111 map.m_len = 1; 1112 err = ext4_map_blocks(handle, inode, &map, map_flags); 1113 1114 if (err == 0) 1115 return create ? ERR_PTR(-ENOSPC) : NULL; 1116 if (err < 0) 1117 return ERR_PTR(err); 1118 1119 if (nowait) 1120 return sb_find_get_block(inode->i_sb, map.m_pblk); 1121 1122 /* 1123 * Since bh could introduce extra ref count such as referred by 1124 * journal_head etc. Try to avoid using __GFP_MOVABLE here 1125 * as it may fail the migration when journal_head remains. 1126 */ 1127 bh = getblk_unmovable(inode->i_sb->s_bdev, map.m_pblk, 1128 inode->i_sb->s_blocksize); 1129 1130 if (unlikely(!bh)) 1131 return ERR_PTR(-ENOMEM); 1132 if (map.m_flags & EXT4_MAP_NEW) { 1133 ASSERT(create != 0); 1134 ASSERT((EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY) 1135 || (handle != NULL)); 1136 1137 /* 1138 * Now that we do not always journal data, we should 1139 * keep in mind whether this should always journal the 1140 * new buffer as metadata. For now, regular file 1141 * writes use ext4_get_block instead, so it's not a 1142 * problem. 1143 */ 1144 lock_buffer(bh); 1145 BUFFER_TRACE(bh, "call get_create_access"); 1146 err = ext4_journal_get_create_access(handle, inode->i_sb, bh, 1147 EXT4_JTR_NONE); 1148 if (unlikely(err)) { 1149 unlock_buffer(bh); 1150 goto errout; 1151 } 1152 if (!buffer_uptodate(bh)) { 1153 memset(bh->b_data, 0, inode->i_sb->s_blocksize); 1154 set_buffer_uptodate(bh); 1155 } 1156 unlock_buffer(bh); 1157 BUFFER_TRACE(bh, "call ext4_handle_dirty_metadata"); 1158 err = ext4_handle_dirty_metadata(handle, inode, bh); 1159 if (unlikely(err)) 1160 goto errout; 1161 } else 1162 BUFFER_TRACE(bh, "not a new buffer"); 1163 return bh; 1164 errout: 1165 brelse(bh); 1166 return ERR_PTR(err); 1167 } 1168 1169 struct buffer_head *ext4_bread(handle_t *handle, struct inode *inode, 1170 ext4_lblk_t block, int map_flags) 1171 { 1172 struct buffer_head *bh; 1173 int ret; 1174 1175 bh = ext4_getblk(handle, inode, block, map_flags); 1176 if (IS_ERR(bh)) 1177 return bh; 1178 if (!bh || ext4_buffer_uptodate(bh)) 1179 return bh; 1180 1181 ret = ext4_read_bh_lock(bh, REQ_META | REQ_PRIO, true); 1182 if (ret) { 1183 put_bh(bh); 1184 return ERR_PTR(ret); 1185 } 1186 return bh; 1187 } 1188 1189 /* Read a contiguous batch of blocks. */ 1190 int ext4_bread_batch(struct inode *inode, ext4_lblk_t block, int bh_count, 1191 bool wait, struct buffer_head **bhs) 1192 { 1193 int i, err; 1194 1195 for (i = 0; i < bh_count; i++) { 1196 bhs[i] = ext4_getblk(NULL, inode, block + i, 0 /* map_flags */); 1197 if (IS_ERR(bhs[i])) { 1198 err = PTR_ERR(bhs[i]); 1199 bh_count = i; 1200 goto out_brelse; 1201 } 1202 } 1203 1204 for (i = 0; i < bh_count; i++) 1205 /* Note that NULL bhs[i] is valid because of holes. */ 1206 if (bhs[i] && !ext4_buffer_uptodate(bhs[i])) 1207 ext4_read_bh_lock(bhs[i], REQ_META | REQ_PRIO, false); 1208 1209 if (!wait) 1210 return 0; 1211 1212 for (i = 0; i < bh_count; i++) 1213 if (bhs[i]) 1214 wait_on_buffer(bhs[i]); 1215 1216 for (i = 0; i < bh_count; i++) { 1217 if (bhs[i] && !buffer_uptodate(bhs[i])) { 1218 err = -EIO; 1219 goto out_brelse; 1220 } 1221 } 1222 return 0; 1223 1224 out_brelse: 1225 for (i = 0; i < bh_count; i++) { 1226 brelse(bhs[i]); 1227 bhs[i] = NULL; 1228 } 1229 return err; 1230 } 1231 1232 int ext4_walk_page_buffers(handle_t *handle, struct inode *inode, 1233 struct buffer_head *head, 1234 unsigned from, 1235 unsigned to, 1236 int *partial, 1237 int (*fn)(handle_t *handle, struct inode *inode, 1238 struct buffer_head *bh)) 1239 { 1240 struct buffer_head *bh; 1241 unsigned block_start, block_end; 1242 unsigned blocksize = head->b_size; 1243 int err, ret = 0; 1244 struct buffer_head *next; 1245 1246 for (bh = head, block_start = 0; 1247 ret == 0 && (bh != head || !block_start); 1248 block_start = block_end, bh = next) { 1249 next = bh->b_this_page; 1250 block_end = block_start + blocksize; 1251 if (block_end <= from || block_start >= to) { 1252 if (partial && !buffer_uptodate(bh)) 1253 *partial = 1; 1254 continue; 1255 } 1256 err = (*fn)(handle, inode, bh); 1257 if (!ret) 1258 ret = err; 1259 } 1260 return ret; 1261 } 1262 1263 /* 1264 * Helper for handling dirtying of journalled data. We also mark the folio as 1265 * dirty so that writeback code knows about this page (and inode) contains 1266 * dirty data. ext4_writepages() then commits appropriate transaction to 1267 * make data stable. 1268 */ 1269 static int ext4_dirty_journalled_data(handle_t *handle, struct buffer_head *bh) 1270 { 1271 struct folio *folio = bh->b_folio; 1272 struct inode *inode = folio->mapping->host; 1273 1274 /* only regular files have a_ops */ 1275 if (S_ISREG(inode->i_mode)) 1276 folio_mark_dirty(folio); 1277 return ext4_handle_dirty_metadata(handle, NULL, bh); 1278 } 1279 1280 int do_journal_get_write_access(handle_t *handle, struct inode *inode, 1281 struct buffer_head *bh) 1282 { 1283 if (!buffer_mapped(bh) || buffer_freed(bh)) 1284 return 0; 1285 BUFFER_TRACE(bh, "get write access"); 1286 return ext4_journal_get_write_access(handle, inode->i_sb, bh, 1287 EXT4_JTR_NONE); 1288 } 1289 1290 int ext4_block_write_begin(handle_t *handle, struct folio *folio, 1291 loff_t pos, unsigned len, 1292 get_block_t *get_block) 1293 { 1294 unsigned int from = offset_in_folio(folio, pos); 1295 unsigned to = from + len; 1296 struct inode *inode = folio->mapping->host; 1297 unsigned block_start, block_end; 1298 sector_t block; 1299 int err = 0; 1300 unsigned int blocksize = i_blocksize(inode); 1301 struct buffer_head *bh, *head, *wait[2]; 1302 int nr_wait = 0; 1303 int i; 1304 bool should_journal_data = ext4_should_journal_data(inode); 1305 bool folio_uptodate = folio_test_uptodate(folio); 1306 1307 BUG_ON(!folio_test_locked(folio)); 1308 BUG_ON(to > folio_size(folio)); 1309 BUG_ON(from > to); 1310 WARN_ON_ONCE(blocksize > folio_size(folio)); 1311 1312 head = folio_buffers(folio); 1313 if (!head) 1314 head = create_empty_buffers(folio, blocksize, 0); 1315 block = EXT4_PG_TO_LBLK(inode, folio->index); 1316 1317 for (bh = head, block_start = 0; 1318 block_start < to || (!folio_uptodate && bh != head); 1319 block++, block_start = block_end, bh = bh->b_this_page) { 1320 block_end = block_start + blocksize; 1321 if (block_end <= from || block_start >= to) { 1322 if (folio_uptodate) 1323 set_buffer_uptodate(bh); 1324 continue; 1325 } 1326 if (WARN_ON_ONCE(buffer_new(bh))) 1327 clear_buffer_new(bh); 1328 if (!buffer_mapped(bh)) { 1329 WARN_ON(bh->b_size != blocksize); 1330 err = ext4_journal_ensure_extent_credits(handle, inode); 1331 if (!err) 1332 err = get_block(inode, block, bh, 1); 1333 if (err) 1334 break; 1335 if (buffer_new(bh)) { 1336 /* 1337 * We may be zeroing partial buffers or all new 1338 * buffers in case of failure. Prepare JBD2 for 1339 * that. 1340 */ 1341 if (should_journal_data) 1342 do_journal_get_write_access(handle, 1343 inode, bh); 1344 if (folio_uptodate) { 1345 /* 1346 * Unlike __block_write_begin() we leave 1347 * dirtying of new uptodate buffers to 1348 * ->write_end() time or 1349 * folio_zero_new_buffers(). 1350 */ 1351 set_buffer_uptodate(bh); 1352 continue; 1353 } 1354 if (block_end > to || block_start < from) 1355 folio_zero_segments(folio, to, 1356 block_end, 1357 block_start, from); 1358 continue; 1359 } 1360 } 1361 if (folio_uptodate) { 1362 set_buffer_uptodate(bh); 1363 continue; 1364 } 1365 if (!buffer_uptodate(bh) && !buffer_delay(bh) && 1366 !buffer_unwritten(bh) && 1367 (block_start < from || block_end > to)) { 1368 ext4_read_bh_lock(bh, 0, false); 1369 wait[nr_wait++] = bh; 1370 } 1371 } 1372 /* 1373 * If we issued read requests, let them complete. 1374 */ 1375 for (i = 0; i < nr_wait; i++) { 1376 wait_on_buffer(wait[i]); 1377 if (!buffer_uptodate(wait[i])) 1378 err = -EIO; 1379 } 1380 if (unlikely(err)) { 1381 if (should_journal_data) 1382 ext4_journalled_zero_new_buffers(handle, inode, folio, 1383 from, to); 1384 else 1385 folio_zero_new_buffers(folio, from, to); 1386 } 1387 1388 return err; 1389 } 1390 1391 /* 1392 * To preserve ordering, it is essential that the hole instantiation and 1393 * the data write be encapsulated in a single transaction. We cannot 1394 * close off a transaction and start a new one between the ext4_get_block() 1395 * and the ext4_write_end(). So doing the jbd2_journal_start at the start of 1396 * ext4_write_begin() is the right place. 1397 */ 1398 static int ext4_write_begin(const struct kiocb *iocb, 1399 struct address_space *mapping, 1400 loff_t pos, unsigned len, 1401 struct folio **foliop, void **fsdata) 1402 { 1403 struct inode *inode = mapping->host; 1404 int ret, needed_blocks; 1405 handle_t *handle; 1406 int retries = 0; 1407 struct folio *folio; 1408 pgoff_t index; 1409 unsigned from, to; 1410 1411 ret = ext4_emergency_state(inode->i_sb); 1412 if (unlikely(ret)) 1413 return ret; 1414 1415 *fsdata = (void *)((unsigned long)*fsdata & ~EXT4_WRITE_DATA_INLINE); 1416 1417 trace_ext4_write_begin(inode, pos, len); 1418 /* 1419 * Reserve one block more for addition to orphan list in case 1420 * we allocate blocks but write fails for some reason 1421 */ 1422 needed_blocks = ext4_chunk_trans_extent(inode, 1423 ext4_journal_blocks_per_folio(inode)) + 1; 1424 index = pos >> PAGE_SHIFT; 1425 1426 if (ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) { 1427 ret = ext4_try_to_write_inline_data(mapping, inode, pos, len, 1428 foliop); 1429 if (ret < 0) 1430 return ret; 1431 if (ret == 1) { 1432 *fsdata = (void *)((unsigned long)*fsdata | EXT4_WRITE_DATA_INLINE); 1433 return 0; 1434 } 1435 } 1436 1437 /* 1438 * write_begin_get_folio() can take a long time if the 1439 * system is thrashing due to memory pressure, or if the folio 1440 * is being written back. So grab it first before we start 1441 * the transaction handle. This also allows us to allocate 1442 * the folio (if needed) without using GFP_NOFS. 1443 */ 1444 retry_grab: 1445 folio = write_begin_get_folio(iocb, mapping, index, len); 1446 if (IS_ERR(folio)) 1447 return PTR_ERR(folio); 1448 1449 if (len > folio_next_pos(folio) - pos) 1450 len = folio_next_pos(folio) - pos; 1451 1452 from = offset_in_folio(folio, pos); 1453 to = from + len; 1454 1455 /* 1456 * The same as page allocation, we prealloc buffer heads before 1457 * starting the handle. 1458 */ 1459 if (!folio_buffers(folio)) 1460 create_empty_buffers(folio, inode->i_sb->s_blocksize, 0); 1461 1462 folio_unlock(folio); 1463 1464 retry_journal: 1465 handle = ext4_journal_start(inode, EXT4_HT_WRITE_PAGE, needed_blocks); 1466 if (IS_ERR(handle)) { 1467 folio_put(folio); 1468 return PTR_ERR(handle); 1469 } 1470 1471 folio_lock(folio); 1472 if (folio->mapping != mapping) { 1473 /* The folio got truncated from under us */ 1474 folio_unlock(folio); 1475 folio_put(folio); 1476 ext4_journal_stop(handle); 1477 goto retry_grab; 1478 } 1479 /* In case writeback began while the folio was unlocked */ 1480 folio_wait_stable(folio); 1481 1482 if (ext4_should_dioread_nolock(inode)) 1483 ret = ext4_block_write_begin(handle, folio, pos, len, 1484 ext4_get_block_unwritten); 1485 else 1486 ret = ext4_block_write_begin(handle, folio, pos, len, 1487 ext4_get_block); 1488 if (!ret && ext4_should_journal_data(inode)) { 1489 ret = ext4_walk_page_buffers(handle, inode, 1490 folio_buffers(folio), from, to, 1491 NULL, do_journal_get_write_access); 1492 } 1493 1494 if (ret) { 1495 bool extended = (pos + len > inode->i_size) && 1496 !ext4_verity_in_progress(inode); 1497 1498 folio_unlock(folio); 1499 /* 1500 * ext4_block_write_begin may have instantiated a few blocks 1501 * outside i_size. Trim these off again. Don't need 1502 * i_size_read because we hold i_rwsem. 1503 * 1504 * Add inode to orphan list in case we crash before 1505 * truncate finishes 1506 */ 1507 if (extended && ext4_can_truncate(inode)) 1508 ext4_orphan_add(handle, inode); 1509 1510 ext4_journal_stop(handle); 1511 if (extended) { 1512 ext4_truncate_failed_write(inode); 1513 /* 1514 * If truncate failed early the inode might 1515 * still be on the orphan list; we need to 1516 * make sure the inode is removed from the 1517 * orphan list in that case. 1518 */ 1519 if (inode->i_nlink) 1520 ext4_orphan_del(NULL, inode); 1521 } 1522 1523 if (ret == -EAGAIN || 1524 (ret == -ENOSPC && 1525 ext4_should_retry_alloc(inode->i_sb, &retries))) 1526 goto retry_journal; 1527 folio_put(folio); 1528 return ret; 1529 } 1530 *foliop = folio; 1531 return ret; 1532 } 1533 1534 /* For write_end() in data=journal mode */ 1535 static int write_end_fn(handle_t *handle, struct inode *inode, 1536 struct buffer_head *bh) 1537 { 1538 int ret; 1539 if (!buffer_mapped(bh) || buffer_freed(bh)) 1540 return 0; 1541 set_buffer_uptodate(bh); 1542 ret = ext4_dirty_journalled_data(handle, bh); 1543 clear_buffer_meta(bh); 1544 clear_buffer_prio(bh); 1545 clear_buffer_new(bh); 1546 return ret; 1547 } 1548 1549 /* 1550 * We need to pick up the new inode size which generic_commit_write gave us 1551 * `iocb` can be NULL - eg, when called from page_symlink(). 1552 */ 1553 static int ext4_write_end(const struct kiocb *iocb, 1554 struct address_space *mapping, 1555 loff_t pos, unsigned len, unsigned copied, 1556 struct folio *folio, void *fsdata) 1557 { 1558 handle_t *handle = ext4_journal_current_handle(); 1559 struct inode *inode = mapping->host; 1560 loff_t old_size = inode->i_size; 1561 int ret = 0, ret2; 1562 int i_size_changed = 0; 1563 bool verity = ext4_verity_in_progress(inode); 1564 1565 trace_ext4_write_end(inode, pos, len, copied); 1566 1567 if ((unsigned long)fsdata & EXT4_WRITE_DATA_INLINE) 1568 return ext4_write_inline_data_end(inode, pos, len, copied, 1569 folio); 1570 1571 copied = block_write_end(pos, len, copied, folio); 1572 /* 1573 * it's important to update i_size while still holding folio lock: 1574 * page writeout could otherwise come in and zero beyond i_size. 1575 * 1576 * If FS_IOC_ENABLE_VERITY is running on this inode, then Merkle tree 1577 * blocks are being written past EOF, so skip the i_size update. 1578 */ 1579 if (!verity) 1580 i_size_changed = ext4_update_inode_size(inode, pos + copied); 1581 folio_unlock(folio); 1582 folio_put(folio); 1583 1584 if (old_size < pos && !verity) 1585 pagecache_isize_extended(inode, old_size, pos); 1586 1587 /* 1588 * Don't mark the inode dirty under folio lock. First, it unnecessarily 1589 * makes the holding time of folio lock longer. Second, it forces lock 1590 * ordering of folio lock and transaction start for journaling 1591 * filesystems. 1592 */ 1593 if (i_size_changed) 1594 ret = ext4_mark_inode_dirty(handle, inode); 1595 1596 if (pos + len > inode->i_size && !verity && ext4_can_truncate(inode)) 1597 /* if we have allocated more blocks and copied 1598 * less. We will have blocks allocated outside 1599 * inode->i_size. So truncate them 1600 */ 1601 ext4_orphan_add(handle, inode); 1602 1603 ret2 = ext4_journal_stop(handle); 1604 if (!ret) 1605 ret = ret2; 1606 1607 if (pos + len > inode->i_size && !verity) { 1608 ext4_truncate_failed_write(inode); 1609 /* 1610 * If truncate failed early the inode might still be 1611 * on the orphan list; we need to make sure the inode 1612 * is removed from the orphan list in that case. 1613 */ 1614 if (inode->i_nlink) 1615 ext4_orphan_del(NULL, inode); 1616 } 1617 1618 return ret ? ret : copied; 1619 } 1620 1621 /* 1622 * This is a private version of folio_zero_new_buffers() which doesn't 1623 * set the buffer to be dirty, since in data=journalled mode we need 1624 * to call ext4_dirty_journalled_data() instead. 1625 */ 1626 static void ext4_journalled_zero_new_buffers(handle_t *handle, 1627 struct inode *inode, 1628 struct folio *folio, 1629 unsigned from, unsigned to) 1630 { 1631 unsigned int block_start = 0, block_end; 1632 struct buffer_head *head, *bh; 1633 1634 bh = head = folio_buffers(folio); 1635 do { 1636 block_end = block_start + bh->b_size; 1637 if (buffer_new(bh)) { 1638 if (block_end > from && block_start < to) { 1639 if (!folio_test_uptodate(folio)) { 1640 unsigned start, size; 1641 1642 start = max(from, block_start); 1643 size = min(to, block_end) - start; 1644 1645 folio_zero_range(folio, start, size); 1646 } 1647 clear_buffer_new(bh); 1648 write_end_fn(handle, inode, bh); 1649 } 1650 } 1651 block_start = block_end; 1652 bh = bh->b_this_page; 1653 } while (bh != head); 1654 } 1655 1656 static int ext4_journalled_write_end(const struct kiocb *iocb, 1657 struct address_space *mapping, 1658 loff_t pos, unsigned len, unsigned copied, 1659 struct folio *folio, void *fsdata) 1660 { 1661 handle_t *handle = ext4_journal_current_handle(); 1662 struct inode *inode = mapping->host; 1663 loff_t old_size = inode->i_size; 1664 int ret = 0, ret2; 1665 int partial = 0; 1666 unsigned from, to; 1667 int size_changed = 0; 1668 bool verity = ext4_verity_in_progress(inode); 1669 1670 trace_ext4_journalled_write_end(inode, pos, len, copied); 1671 from = pos & (PAGE_SIZE - 1); 1672 to = from + len; 1673 1674 BUG_ON(!ext4_handle_valid(handle)); 1675 1676 if ((unsigned long)fsdata & EXT4_WRITE_DATA_INLINE) 1677 return ext4_write_inline_data_end(inode, pos, len, copied, 1678 folio); 1679 1680 if (unlikely(copied < len) && !folio_test_uptodate(folio)) { 1681 copied = 0; 1682 ext4_journalled_zero_new_buffers(handle, inode, folio, 1683 from, to); 1684 } else { 1685 if (unlikely(copied < len)) 1686 ext4_journalled_zero_new_buffers(handle, inode, folio, 1687 from + copied, to); 1688 ret = ext4_walk_page_buffers(handle, inode, 1689 folio_buffers(folio), 1690 from, from + copied, &partial, 1691 write_end_fn); 1692 if (!partial) 1693 folio_mark_uptodate(folio); 1694 } 1695 if (!verity) 1696 size_changed = ext4_update_inode_size(inode, pos + copied); 1697 EXT4_I(inode)->i_datasync_tid = handle->h_transaction->t_tid; 1698 folio_unlock(folio); 1699 folio_put(folio); 1700 1701 if (old_size < pos && !verity) 1702 pagecache_isize_extended(inode, old_size, pos); 1703 1704 if (size_changed) { 1705 ret2 = ext4_mark_inode_dirty(handle, inode); 1706 if (!ret) 1707 ret = ret2; 1708 } 1709 1710 if (pos + len > inode->i_size && !verity && ext4_can_truncate(inode)) 1711 /* if we have allocated more blocks and copied 1712 * less. We will have blocks allocated outside 1713 * inode->i_size. So truncate them 1714 */ 1715 ext4_orphan_add(handle, inode); 1716 1717 ret2 = ext4_journal_stop(handle); 1718 if (!ret) 1719 ret = ret2; 1720 if (pos + len > inode->i_size && !verity) { 1721 ext4_truncate_failed_write(inode); 1722 /* 1723 * If truncate failed early the inode might still be 1724 * on the orphan list; we need to make sure the inode 1725 * is removed from the orphan list in that case. 1726 */ 1727 if (inode->i_nlink) 1728 ext4_orphan_del(NULL, inode); 1729 } 1730 1731 return ret ? ret : copied; 1732 } 1733 1734 /* 1735 * Reserve space for 'nr_resv' clusters 1736 */ 1737 static int ext4_da_reserve_space(struct inode *inode, int nr_resv) 1738 { 1739 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); 1740 struct ext4_inode_info *ei = EXT4_I(inode); 1741 int ret; 1742 1743 /* 1744 * We will charge metadata quota at writeout time; this saves 1745 * us from metadata over-estimation, though we may go over by 1746 * a small amount in the end. Here we just reserve for data. 1747 */ 1748 ret = dquot_reserve_block(inode, EXT4_C2B(sbi, nr_resv)); 1749 if (ret) 1750 return ret; 1751 1752 spin_lock(&ei->i_block_reservation_lock); 1753 if (ext4_claim_free_clusters(sbi, nr_resv, 0)) { 1754 spin_unlock(&ei->i_block_reservation_lock); 1755 dquot_release_reservation_block(inode, EXT4_C2B(sbi, nr_resv)); 1756 return -ENOSPC; 1757 } 1758 ei->i_reserved_data_blocks += nr_resv; 1759 trace_ext4_da_reserve_space(inode, nr_resv); 1760 spin_unlock(&ei->i_block_reservation_lock); 1761 1762 return 0; /* success */ 1763 } 1764 1765 void ext4_da_release_space(struct inode *inode, int to_free) 1766 { 1767 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); 1768 struct ext4_inode_info *ei = EXT4_I(inode); 1769 1770 if (!to_free) 1771 return; /* Nothing to release, exit */ 1772 1773 spin_lock(&EXT4_I(inode)->i_block_reservation_lock); 1774 1775 trace_ext4_da_release_space(inode, to_free); 1776 if (unlikely(to_free > ei->i_reserved_data_blocks)) { 1777 /* 1778 * if there aren't enough reserved blocks, then the 1779 * counter is messed up somewhere. Since this 1780 * function is called from invalidate page, it's 1781 * harmless to return without any action. 1782 */ 1783 ext4_warning(inode->i_sb, "ext4_da_release_space: " 1784 "ino %llu, to_free %d with only %d reserved " 1785 "data blocks", inode->i_ino, to_free, 1786 ei->i_reserved_data_blocks); 1787 WARN_ON(1); 1788 to_free = ei->i_reserved_data_blocks; 1789 } 1790 ei->i_reserved_data_blocks -= to_free; 1791 1792 /* update fs dirty data blocks counter */ 1793 percpu_counter_sub(&sbi->s_dirtyclusters_counter, to_free); 1794 1795 spin_unlock(&EXT4_I(inode)->i_block_reservation_lock); 1796 1797 dquot_release_reservation_block(inode, EXT4_C2B(sbi, to_free)); 1798 } 1799 1800 /* 1801 * Delayed allocation stuff 1802 */ 1803 1804 struct mpage_da_data { 1805 /* These are input fields for ext4_do_writepages() */ 1806 struct inode *inode; 1807 struct writeback_control *wbc; 1808 unsigned int can_map:1; /* Can writepages call map blocks? */ 1809 1810 /* These are internal state of ext4_do_writepages() */ 1811 loff_t start_pos; /* The start pos to write */ 1812 loff_t next_pos; /* Current pos to examine */ 1813 loff_t end_pos; /* Last pos to examine */ 1814 1815 /* 1816 * Extent to map - this can be after start_pos because that can be 1817 * fully mapped. We somewhat abuse m_flags to store whether the extent 1818 * is delalloc or unwritten. 1819 */ 1820 struct ext4_map_blocks map; 1821 struct ext4_io_submit io_submit; /* IO submission data */ 1822 unsigned int do_map:1; 1823 unsigned int scanned_until_end:1; 1824 unsigned int journalled_more_data:1; 1825 }; 1826 1827 static void mpage_release_unused_pages(struct mpage_da_data *mpd, 1828 bool invalidate) 1829 { 1830 unsigned nr, i; 1831 pgoff_t index, end; 1832 struct folio_batch fbatch; 1833 struct inode *inode = mpd->inode; 1834 struct address_space *mapping = inode->i_mapping; 1835 1836 /* This is necessary when next_pos == 0. */ 1837 if (mpd->start_pos >= mpd->next_pos) 1838 return; 1839 1840 mpd->scanned_until_end = 0; 1841 if (invalidate) { 1842 ext4_lblk_t start, last; 1843 start = EXT4_B_TO_LBLK(inode, mpd->start_pos); 1844 last = mpd->next_pos >> inode->i_blkbits; 1845 1846 /* 1847 * avoid racing with extent status tree scans made by 1848 * ext4_insert_delayed_block() 1849 */ 1850 down_write(&EXT4_I(inode)->i_data_sem); 1851 ext4_es_remove_extent(inode, start, last - start); 1852 up_write(&EXT4_I(inode)->i_data_sem); 1853 } 1854 1855 folio_batch_init(&fbatch); 1856 index = mpd->start_pos >> PAGE_SHIFT; 1857 end = mpd->next_pos >> PAGE_SHIFT; 1858 while (index < end) { 1859 nr = filemap_get_folios(mapping, &index, end - 1, &fbatch); 1860 if (nr == 0) 1861 break; 1862 for (i = 0; i < nr; i++) { 1863 struct folio *folio = fbatch.folios[i]; 1864 1865 if (folio_pos(folio) < mpd->start_pos) 1866 continue; 1867 if (folio_next_index(folio) > end) 1868 continue; 1869 BUG_ON(!folio_test_locked(folio)); 1870 BUG_ON(folio_test_writeback(folio)); 1871 if (invalidate) { 1872 if (folio_mapped(folio)) { 1873 folio_clear_dirty_for_io(folio); 1874 /* 1875 * Unmap folio from page 1876 * tables to prevent 1877 * subsequent accesses through 1878 * stale PTEs. This ensures 1879 * future accesses trigger new 1880 * page faults rather than 1881 * reusing the invalidated 1882 * folio. 1883 */ 1884 unmap_mapping_pages(folio->mapping, 1885 folio->index, 1886 folio_nr_pages(folio), false); 1887 } 1888 block_invalidate_folio(folio, 0, 1889 folio_size(folio)); 1890 folio_clear_uptodate(folio); 1891 } 1892 folio_unlock(folio); 1893 } 1894 folio_batch_release(&fbatch); 1895 } 1896 } 1897 1898 static void ext4_print_free_blocks(struct inode *inode) 1899 { 1900 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); 1901 struct super_block *sb = inode->i_sb; 1902 struct ext4_inode_info *ei = EXT4_I(inode); 1903 1904 ext4_msg(sb, KERN_CRIT, "Total free blocks count %lld", 1905 EXT4_C2B(EXT4_SB(inode->i_sb), 1906 ext4_count_free_clusters(sb))); 1907 ext4_msg(sb, KERN_CRIT, "Free/Dirty block details"); 1908 ext4_msg(sb, KERN_CRIT, "free_blocks=%lld", 1909 (long long) EXT4_C2B(EXT4_SB(sb), 1910 percpu_counter_sum(&sbi->s_freeclusters_counter))); 1911 ext4_msg(sb, KERN_CRIT, "dirty_blocks=%lld", 1912 (long long) EXT4_C2B(EXT4_SB(sb), 1913 percpu_counter_sum(&sbi->s_dirtyclusters_counter))); 1914 ext4_msg(sb, KERN_CRIT, "Block reservation details"); 1915 ext4_msg(sb, KERN_CRIT, "i_reserved_data_blocks=%u", 1916 ei->i_reserved_data_blocks); 1917 return; 1918 } 1919 1920 /* 1921 * Check whether the cluster containing lblk has been allocated or has 1922 * delalloc reservation. 1923 * 1924 * Returns 0 if the cluster doesn't have either, 1 if it has delalloc 1925 * reservation, 2 if it's already been allocated, negative error code on 1926 * failure. 1927 */ 1928 static int ext4_clu_alloc_state(struct inode *inode, ext4_lblk_t lblk) 1929 { 1930 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); 1931 int ret; 1932 1933 /* Has delalloc reservation? */ 1934 if (ext4_es_scan_clu(inode, &ext4_es_is_delayed, lblk)) 1935 return 1; 1936 1937 /* Already been allocated? */ 1938 if (ext4_es_scan_clu(inode, &ext4_es_is_mapped, lblk)) 1939 return 2; 1940 ret = ext4_clu_mapped(inode, EXT4_B2C(sbi, lblk)); 1941 if (ret < 0) 1942 return ret; 1943 if (ret > 0) 1944 return 2; 1945 1946 return 0; 1947 } 1948 1949 /* 1950 * ext4_insert_delayed_blocks - adds a multiple delayed blocks to the extents 1951 * status tree, incrementing the reserved 1952 * cluster/block count or making pending 1953 * reservations where needed 1954 * 1955 * @inode - file containing the newly added block 1956 * @lblk - start logical block to be added 1957 * @len - length of blocks to be added 1958 * 1959 * Returns 0 on success, negative error code on failure. 1960 */ 1961 static int ext4_insert_delayed_blocks(struct inode *inode, ext4_lblk_t lblk, 1962 ext4_lblk_t len) 1963 { 1964 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); 1965 int ret; 1966 bool lclu_allocated = false; 1967 bool end_allocated = false; 1968 ext4_lblk_t resv_clu; 1969 ext4_lblk_t end = lblk + len - 1; 1970 1971 /* 1972 * If the cluster containing lblk or end is shared with a delayed, 1973 * written, or unwritten extent in a bigalloc file system, it's 1974 * already been accounted for and does not need to be reserved. 1975 * A pending reservation must be made for the cluster if it's 1976 * shared with a written or unwritten extent and doesn't already 1977 * have one. Written and unwritten extents can be purged from the 1978 * extents status tree if the system is under memory pressure, so 1979 * it's necessary to examine the extent tree if a search of the 1980 * extents status tree doesn't get a match. 1981 */ 1982 if (sbi->s_cluster_ratio == 1) { 1983 ret = ext4_da_reserve_space(inode, len); 1984 if (ret != 0) /* ENOSPC */ 1985 return ret; 1986 } else { /* bigalloc */ 1987 resv_clu = EXT4_B2C(sbi, end) - EXT4_B2C(sbi, lblk) + 1; 1988 1989 ret = ext4_clu_alloc_state(inode, lblk); 1990 if (ret < 0) 1991 return ret; 1992 if (ret > 0) { 1993 resv_clu--; 1994 lclu_allocated = (ret == 2); 1995 } 1996 1997 if (EXT4_B2C(sbi, lblk) != EXT4_B2C(sbi, end)) { 1998 ret = ext4_clu_alloc_state(inode, end); 1999 if (ret < 0) 2000 return ret; 2001 if (ret > 0) { 2002 resv_clu--; 2003 end_allocated = (ret == 2); 2004 } 2005 } 2006 2007 if (resv_clu) { 2008 ret = ext4_da_reserve_space(inode, resv_clu); 2009 if (ret != 0) /* ENOSPC */ 2010 return ret; 2011 } 2012 } 2013 2014 ext4_es_insert_delayed_extent(inode, lblk, len, lclu_allocated, 2015 end_allocated); 2016 return 0; 2017 } 2018 2019 /* 2020 * Looks up the requested blocks and sets the delalloc extent map. 2021 * First try to look up for the extent entry that contains the requested 2022 * blocks in the extent status tree without i_data_sem, then try to look 2023 * up for the ondisk extent mapping with i_data_sem in read mode, 2024 * finally hold i_data_sem in write mode, looks up again and add a 2025 * delalloc extent entry if it still couldn't find any extent. Pass out 2026 * the mapped extent through @map and return 0 on success. 2027 */ 2028 static int ext4_da_map_blocks(struct inode *inode, struct ext4_map_blocks *map) 2029 { 2030 struct extent_status es; 2031 int retval; 2032 #ifdef ES_AGGRESSIVE_TEST 2033 struct ext4_map_blocks orig_map; 2034 2035 memcpy(&orig_map, map, sizeof(*map)); 2036 #endif 2037 2038 map->m_flags = 0; 2039 ext_debug(inode, "max_blocks %u, logical block %lu\n", map->m_len, 2040 (unsigned long) map->m_lblk); 2041 2042 ext4_check_map_extents_env(inode); 2043 2044 /* Lookup extent status tree firstly */ 2045 if (ext4_es_lookup_extent(inode, map->m_lblk, NULL, &es, &map->m_seq)) { 2046 map->m_len = min_t(unsigned int, map->m_len, 2047 es.es_len - (map->m_lblk - es.es_lblk)); 2048 2049 if (ext4_es_is_hole(&es)) 2050 goto add_delayed; 2051 2052 found: 2053 /* 2054 * Delayed extent could be allocated by fallocate. 2055 * So we need to check it. 2056 */ 2057 if (ext4_es_is_delayed(&es)) { 2058 map->m_flags |= EXT4_MAP_DELAYED; 2059 return 0; 2060 } 2061 2062 map->m_pblk = ext4_es_pblock(&es) + map->m_lblk - es.es_lblk; 2063 if (ext4_es_is_written(&es)) 2064 map->m_flags |= EXT4_MAP_MAPPED; 2065 else if (ext4_es_is_unwritten(&es)) 2066 map->m_flags |= EXT4_MAP_UNWRITTEN; 2067 else 2068 BUG(); 2069 2070 #ifdef ES_AGGRESSIVE_TEST 2071 ext4_map_blocks_es_recheck(NULL, inode, map, &orig_map, 0); 2072 #endif 2073 return 0; 2074 } 2075 2076 /* 2077 * Try to see if we can get the block without requesting a new 2078 * file system block. 2079 */ 2080 down_read(&EXT4_I(inode)->i_data_sem); 2081 if (ext4_has_inline_data(inode)) 2082 retval = 0; 2083 else 2084 retval = ext4_map_query_blocks(NULL, inode, map, 0); 2085 up_read(&EXT4_I(inode)->i_data_sem); 2086 if (retval) 2087 return retval < 0 ? retval : 0; 2088 2089 add_delayed: 2090 down_write(&EXT4_I(inode)->i_data_sem); 2091 /* 2092 * Page fault path (ext4_page_mkwrite does not take i_rwsem) 2093 * and fallocate path (no folio lock) can race. Make sure we 2094 * lookup the extent status tree here again while i_data_sem 2095 * is held in write mode, before inserting a new da entry in 2096 * the extent status tree. 2097 */ 2098 if (ext4_es_lookup_extent(inode, map->m_lblk, NULL, &es, &map->m_seq)) { 2099 map->m_len = min_t(unsigned int, map->m_len, 2100 es.es_len - (map->m_lblk - es.es_lblk)); 2101 2102 if (!ext4_es_is_hole(&es)) { 2103 up_write(&EXT4_I(inode)->i_data_sem); 2104 goto found; 2105 } 2106 } else if (!ext4_has_inline_data(inode)) { 2107 retval = ext4_map_query_blocks(NULL, inode, map, 0); 2108 if (retval) { 2109 up_write(&EXT4_I(inode)->i_data_sem); 2110 return retval < 0 ? retval : 0; 2111 } 2112 } 2113 2114 map->m_flags |= EXT4_MAP_DELAYED | EXT4_MAP_NEW; 2115 retval = ext4_insert_delayed_blocks(inode, map->m_lblk, map->m_len); 2116 if (!retval) 2117 map->m_seq = READ_ONCE(EXT4_I(inode)->i_es_seq); 2118 up_write(&EXT4_I(inode)->i_data_sem); 2119 2120 return retval; 2121 } 2122 2123 /* 2124 * This is a special get_block_t callback which is used by 2125 * ext4_da_write_begin(). It will either return mapped block or 2126 * reserve space for a single block. 2127 * 2128 * For delayed buffer_head we have BH_Mapped, BH_New, BH_Delay set. 2129 * We also have b_blocknr = -1 and b_bdev initialized properly 2130 * 2131 * For unwritten buffer_head we have BH_Mapped, BH_New, BH_Unwritten set. 2132 * We also have b_blocknr = physicalblock mapping unwritten extent and b_bdev 2133 * initialized properly. 2134 */ 2135 int ext4_da_get_block_prep(struct inode *inode, sector_t iblock, 2136 struct buffer_head *bh, int create) 2137 { 2138 struct ext4_map_blocks map; 2139 sector_t invalid_block = ~((sector_t) 0xffff); 2140 int ret = 0; 2141 2142 BUG_ON(create == 0); 2143 BUG_ON(bh->b_size != inode->i_sb->s_blocksize); 2144 2145 if (invalid_block < ext4_blocks_count(EXT4_SB(inode->i_sb)->s_es)) 2146 invalid_block = ~0; 2147 2148 map.m_lblk = iblock; 2149 map.m_len = 1; 2150 2151 /* 2152 * first, we need to know whether the block is allocated already 2153 * preallocated blocks are unmapped but should treated 2154 * the same as allocated blocks. 2155 */ 2156 ret = ext4_da_map_blocks(inode, &map); 2157 if (ret < 0) 2158 return ret; 2159 2160 if (map.m_flags & EXT4_MAP_DELAYED) { 2161 map_bh(bh, inode->i_sb, invalid_block); 2162 set_buffer_new(bh); 2163 set_buffer_delay(bh); 2164 return 0; 2165 } 2166 2167 map_bh(bh, inode->i_sb, map.m_pblk); 2168 ext4_update_bh_state(bh, map.m_flags); 2169 2170 if (buffer_unwritten(bh)) { 2171 /* A delayed write to unwritten bh should be marked 2172 * new and mapped. Mapped ensures that we don't do 2173 * get_block multiple times when we write to the same 2174 * offset and new ensures that we do proper zero out 2175 * for partial write. 2176 */ 2177 set_buffer_new(bh); 2178 set_buffer_mapped(bh); 2179 } 2180 return 0; 2181 } 2182 2183 static void mpage_folio_done(struct mpage_da_data *mpd, struct folio *folio) 2184 { 2185 mpd->start_pos += folio_size(folio); 2186 mpd->wbc->nr_to_write -= folio_nr_pages(folio); 2187 folio_unlock(folio); 2188 } 2189 2190 static void mpage_submit_folio(struct mpage_da_data *mpd, struct folio *folio) 2191 { 2192 size_t len; 2193 loff_t size; 2194 2195 WARN_ON_ONCE(folio_pos(folio) != mpd->start_pos); 2196 folio_clear_dirty_for_io(folio); 2197 /* 2198 * We have to be very careful here! Nothing protects writeback path 2199 * against i_size changes and the page can be writeably mapped into 2200 * page tables. So an application can be growing i_size and writing 2201 * data through mmap while writeback runs. folio_clear_dirty_for_io() 2202 * write-protects our page in page tables and the page cannot get 2203 * written to again until we release folio lock. So only after 2204 * folio_clear_dirty_for_io() we are safe to sample i_size for 2205 * ext4_bio_write_folio() to zero-out tail of the written page. We rely 2206 * on the barrier provided by folio_test_clear_dirty() in 2207 * folio_clear_dirty_for_io() to make sure i_size is really sampled only 2208 * after page tables are updated. 2209 */ 2210 size = i_size_read(mpd->inode); 2211 len = folio_size(folio); 2212 if (folio_pos(folio) + len > size && 2213 !ext4_verity_in_progress(mpd->inode)) 2214 len = size & (len - 1); 2215 ext4_bio_write_folio(&mpd->io_submit, folio, len); 2216 } 2217 2218 #define BH_FLAGS (BIT(BH_Unwritten) | BIT(BH_Delay)) 2219 2220 /* 2221 * mballoc gives us at most this number of blocks... 2222 * XXX: That seems to be only a limitation of ext4_mb_normalize_request(). 2223 * The rest of mballoc seems to handle chunks up to full group size. 2224 */ 2225 #define MAX_WRITEPAGES_EXTENT_LEN 2048 2226 2227 /* 2228 * mpage_add_bh_to_extent - try to add bh to extent of blocks to map 2229 * 2230 * @mpd - extent of blocks 2231 * @lblk - logical number of the block in the file 2232 * @bh - buffer head we want to add to the extent 2233 * 2234 * The function is used to collect contig. blocks in the same state. If the 2235 * buffer doesn't require mapping for writeback and we haven't started the 2236 * extent of buffers to map yet, the function returns 'true' immediately - the 2237 * caller can write the buffer right away. Otherwise the function returns true 2238 * if the block has been added to the extent, false if the block couldn't be 2239 * added. 2240 */ 2241 static bool mpage_add_bh_to_extent(struct mpage_da_data *mpd, ext4_lblk_t lblk, 2242 struct buffer_head *bh) 2243 { 2244 struct ext4_map_blocks *map = &mpd->map; 2245 2246 /* Buffer that doesn't need mapping for writeback? */ 2247 if (!buffer_dirty(bh) || !buffer_mapped(bh) || 2248 (!buffer_delay(bh) && !buffer_unwritten(bh))) { 2249 /* So far no extent to map => we write the buffer right away */ 2250 if (map->m_len == 0) 2251 return true; 2252 return false; 2253 } 2254 2255 /* First block in the extent? */ 2256 if (map->m_len == 0) { 2257 /* We cannot map unless handle is started... */ 2258 if (!mpd->do_map) 2259 return false; 2260 map->m_lblk = lblk; 2261 map->m_len = 1; 2262 map->m_flags = bh->b_state & BH_FLAGS; 2263 return true; 2264 } 2265 2266 /* Don't go larger than mballoc is willing to allocate */ 2267 if (map->m_len >= MAX_WRITEPAGES_EXTENT_LEN) 2268 return false; 2269 2270 /* Can we merge the block to our big extent? */ 2271 if (lblk == map->m_lblk + map->m_len && 2272 (bh->b_state & BH_FLAGS) == map->m_flags) { 2273 map->m_len++; 2274 return true; 2275 } 2276 return false; 2277 } 2278 2279 /* 2280 * mpage_process_page_bufs - submit page buffers for IO or add them to extent 2281 * 2282 * @mpd - extent of blocks for mapping 2283 * @head - the first buffer in the page 2284 * @bh - buffer we should start processing from 2285 * @lblk - logical number of the block in the file corresponding to @bh 2286 * 2287 * Walk through page buffers from @bh upto @head (exclusive) and either submit 2288 * the page for IO if all buffers in this page were mapped and there's no 2289 * accumulated extent of buffers to map or add buffers in the page to the 2290 * extent of buffers to map. The function returns 1 if the caller can continue 2291 * by processing the next page, 0 if it should stop adding buffers to the 2292 * extent to map because we cannot extend it anymore. 2293 */ 2294 static int mpage_process_page_bufs(struct mpage_da_data *mpd, 2295 struct buffer_head *head, 2296 struct buffer_head *bh, 2297 ext4_lblk_t lblk) 2298 { 2299 struct inode *inode = mpd->inode; 2300 ext4_lblk_t blocks = (i_size_read(inode) + i_blocksize(inode) - 1) 2301 >> inode->i_blkbits; 2302 2303 if (ext4_verity_in_progress(inode)) 2304 blocks = EXT_MAX_BLOCKS; 2305 2306 do { 2307 BUG_ON(buffer_locked(bh)); 2308 2309 if (lblk >= blocks || !mpage_add_bh_to_extent(mpd, lblk, bh)) { 2310 /* Found extent to map? */ 2311 if (mpd->map.m_len) 2312 return 0; 2313 /* Buffer needs mapping and handle is not started? */ 2314 if (!mpd->do_map) 2315 return 0; 2316 /* Everything mapped so far and we hit EOF */ 2317 break; 2318 } 2319 } while (lblk++, (bh = bh->b_this_page) != head); 2320 /* So far everything mapped? Submit the page for IO. */ 2321 if (mpd->map.m_len == 0) { 2322 mpage_submit_folio(mpd, head->b_folio); 2323 mpage_folio_done(mpd, head->b_folio); 2324 } 2325 if (lblk >= blocks) { 2326 mpd->scanned_until_end = 1; 2327 return 0; 2328 } 2329 return 1; 2330 } 2331 2332 /* 2333 * mpage_process_folio - update folio buffers corresponding to changed extent 2334 * and may submit fully mapped page for IO 2335 * @mpd: description of extent to map, on return next extent to map 2336 * @folio: Contains these buffers. 2337 * @m_lblk: logical block mapping. 2338 * @m_pblk: corresponding physical mapping. 2339 * @map_bh: determines on return whether this page requires any further 2340 * mapping or not. 2341 * 2342 * Scan given folio buffers corresponding to changed extent and update buffer 2343 * state according to new extent state. 2344 * We map delalloc buffers to their physical location, clear unwritten bits. 2345 * If the given folio is not fully mapped, we update @mpd to the next extent in 2346 * the given folio that needs mapping & return @map_bh as true. 2347 */ 2348 static int mpage_process_folio(struct mpage_da_data *mpd, struct folio *folio, 2349 ext4_lblk_t *m_lblk, ext4_fsblk_t *m_pblk, 2350 bool *map_bh) 2351 { 2352 struct buffer_head *head, *bh; 2353 ext4_io_end_t *io_end = mpd->io_submit.io_end; 2354 ext4_lblk_t lblk = *m_lblk; 2355 ext4_fsblk_t pblock = *m_pblk; 2356 int err = 0; 2357 ssize_t io_end_size = 0; 2358 struct ext4_io_end_vec *io_end_vec = ext4_last_io_end_vec(io_end); 2359 2360 bh = head = folio_buffers(folio); 2361 do { 2362 if (lblk < mpd->map.m_lblk) 2363 continue; 2364 if (lblk >= mpd->map.m_lblk + mpd->map.m_len) { 2365 /* 2366 * Buffer after end of mapped extent. 2367 * Find next buffer in the folio to map. 2368 */ 2369 mpd->map.m_len = 0; 2370 mpd->map.m_flags = 0; 2371 io_end_vec->size += io_end_size; 2372 2373 err = mpage_process_page_bufs(mpd, head, bh, lblk); 2374 if (err > 0) 2375 err = 0; 2376 if (!err && mpd->map.m_len && mpd->map.m_lblk > lblk) { 2377 io_end_vec = ext4_alloc_io_end_vec(io_end); 2378 if (IS_ERR(io_end_vec)) { 2379 err = PTR_ERR(io_end_vec); 2380 goto out; 2381 } 2382 io_end_vec->offset = EXT4_LBLK_TO_B(mpd->inode, 2383 mpd->map.m_lblk); 2384 } 2385 *map_bh = true; 2386 goto out; 2387 } 2388 if (buffer_delay(bh)) { 2389 clear_buffer_delay(bh); 2390 bh->b_blocknr = pblock++; 2391 } 2392 clear_buffer_unwritten(bh); 2393 io_end_size += i_blocksize(mpd->inode); 2394 } while (lblk++, (bh = bh->b_this_page) != head); 2395 2396 io_end_vec->size += io_end_size; 2397 *map_bh = false; 2398 out: 2399 *m_lblk = lblk; 2400 *m_pblk = pblock; 2401 return err; 2402 } 2403 2404 /* 2405 * mpage_map_buffers - update buffers corresponding to changed extent and 2406 * submit fully mapped pages for IO 2407 * 2408 * @mpd - description of extent to map, on return next extent to map 2409 * 2410 * Scan buffers corresponding to changed extent (we expect corresponding pages 2411 * to be already locked) and update buffer state according to new extent state. 2412 * We map delalloc buffers to their physical location, clear unwritten bits, 2413 * and mark buffers as uninit when we perform writes to unwritten extents 2414 * and do extent conversion after IO is finished. If the last page is not fully 2415 * mapped, we update @map to the next extent in the last page that needs 2416 * mapping. Otherwise we submit the page for IO. 2417 */ 2418 static int mpage_map_and_submit_buffers(struct mpage_da_data *mpd) 2419 { 2420 struct folio_batch fbatch; 2421 unsigned nr, i; 2422 struct inode *inode = mpd->inode; 2423 pgoff_t start, end; 2424 ext4_lblk_t lblk; 2425 ext4_fsblk_t pblock; 2426 int err; 2427 bool map_bh = false; 2428 2429 start = EXT4_LBLK_TO_PG(inode, mpd->map.m_lblk); 2430 end = EXT4_LBLK_TO_PG(inode, mpd->map.m_lblk + mpd->map.m_len - 1); 2431 pblock = mpd->map.m_pblk; 2432 2433 folio_batch_init(&fbatch); 2434 while (start <= end) { 2435 nr = filemap_get_folios(inode->i_mapping, &start, end, &fbatch); 2436 if (nr == 0) 2437 break; 2438 for (i = 0; i < nr; i++) { 2439 struct folio *folio = fbatch.folios[i]; 2440 2441 lblk = EXT4_PG_TO_LBLK(inode, folio->index); 2442 err = mpage_process_folio(mpd, folio, &lblk, &pblock, 2443 &map_bh); 2444 /* 2445 * If map_bh is true, means page may require further bh 2446 * mapping, or maybe the page was submitted for IO. 2447 * So we return to call further extent mapping. 2448 */ 2449 if (err < 0 || map_bh) 2450 goto out; 2451 /* Page fully mapped - let IO run! */ 2452 mpage_submit_folio(mpd, folio); 2453 mpage_folio_done(mpd, folio); 2454 } 2455 folio_batch_release(&fbatch); 2456 } 2457 /* Extent fully mapped and matches with page boundary. We are done. */ 2458 mpd->map.m_len = 0; 2459 mpd->map.m_flags = 0; 2460 return 0; 2461 out: 2462 folio_batch_release(&fbatch); 2463 return err; 2464 } 2465 2466 static int mpage_map_one_extent(handle_t *handle, struct mpage_da_data *mpd) 2467 { 2468 struct inode *inode = mpd->inode; 2469 struct ext4_map_blocks *map = &mpd->map; 2470 int get_blocks_flags; 2471 int err, dioread_nolock; 2472 2473 /* Make sure transaction has enough credits for this extent */ 2474 err = ext4_journal_ensure_extent_credits(handle, inode); 2475 if (err < 0) 2476 return err; 2477 2478 trace_ext4_da_write_pages_extent(inode, map); 2479 /* 2480 * Call ext4_map_blocks() to allocate any delayed allocation blocks, or 2481 * to convert an unwritten extent to be initialized (in the case 2482 * where we have written into one or more preallocated blocks). It is 2483 * possible that we're going to need more metadata blocks than 2484 * previously reserved. However we must not fail because we're in 2485 * writeback and there is nothing we can do about it so it might result 2486 * in data loss. So use reserved blocks to allocate metadata if 2487 * possible. In addition, do not cache any unrelated extents, as it 2488 * only holds the folio lock but does not hold the i_rwsem or 2489 * invalidate_lock, which could corrupt the extent status tree. 2490 */ 2491 get_blocks_flags = EXT4_GET_BLOCKS_CREATE | 2492 EXT4_GET_BLOCKS_METADATA_NOFAIL | 2493 EXT4_GET_BLOCKS_IO_SUBMIT | 2494 EXT4_EX_NOCACHE; 2495 2496 dioread_nolock = ext4_should_dioread_nolock(inode); 2497 if (dioread_nolock) 2498 get_blocks_flags |= EXT4_GET_BLOCKS_UNWRIT_EXT; 2499 2500 err = ext4_map_blocks(handle, inode, map, get_blocks_flags); 2501 if (err < 0) 2502 return err; 2503 /* 2504 * A hole? This should never happen since mpage_add_bh_to_extent() 2505 * has filtered it out. 2506 */ 2507 if (WARN_ON_ONCE(!err)) 2508 return -EINVAL; 2509 if (dioread_nolock && (map->m_flags & EXT4_MAP_UNWRITTEN)) { 2510 if (!mpd->io_submit.io_end->handle && 2511 ext4_handle_valid(handle)) { 2512 mpd->io_submit.io_end->handle = handle->h_rsv_handle; 2513 handle->h_rsv_handle = NULL; 2514 } 2515 ext4_set_io_unwritten_flag(mpd->io_submit.io_end); 2516 } 2517 2518 BUG_ON(map->m_len == 0); 2519 return 0; 2520 } 2521 2522 /* 2523 * This is used to submit mapped buffers in a single folio that is not fully 2524 * mapped for various reasons, such as insufficient space or journal credits. 2525 */ 2526 static int mpage_submit_partial_folio(struct mpage_da_data *mpd) 2527 { 2528 struct inode *inode = mpd->inode; 2529 struct folio *folio; 2530 loff_t pos; 2531 2532 folio = filemap_get_folio(inode->i_mapping, 2533 mpd->start_pos >> PAGE_SHIFT); 2534 if (IS_ERR(folio)) 2535 return PTR_ERR(folio); 2536 /* 2537 * The mapped position should be within the current processing folio 2538 * but must not be the folio start position. 2539 */ 2540 pos = ((loff_t)mpd->map.m_lblk) << inode->i_blkbits; 2541 if (WARN_ON_ONCE((folio_pos(folio) == pos) || 2542 !folio_contains(folio, pos >> PAGE_SHIFT))) 2543 return -EINVAL; 2544 2545 mpage_submit_folio(mpd, folio); 2546 /* 2547 * Update start_pos to prevent this folio from being released in 2548 * mpage_release_unused_pages(), it will be reset to the aligned folio 2549 * pos when this folio is written again in the next round. Additionally, 2550 * do not update wbc->nr_to_write here, as it will be updated once the 2551 * entire folio has finished processing. 2552 */ 2553 mpd->start_pos = pos; 2554 folio_unlock(folio); 2555 folio_put(folio); 2556 return 0; 2557 } 2558 2559 /* 2560 * mpage_map_and_submit_extent - map extent starting at mpd->lblk of length 2561 * mpd->len and submit pages underlying it for IO 2562 * 2563 * @handle - handle for journal operations 2564 * @mpd - extent to map 2565 * @give_up_on_write - we set this to true iff there is a fatal error and there 2566 * is no hope of writing the data. The caller should discard 2567 * dirty pages to avoid infinite loops. 2568 * 2569 * The function maps extent starting at mpd->lblk of length mpd->len. If it is 2570 * delayed, blocks are allocated, if it is unwritten, we may need to convert 2571 * them to initialized or split the described range from larger unwritten 2572 * extent. Note that we need not map all the described range since allocation 2573 * can return less blocks or the range is covered by more unwritten extents. We 2574 * cannot map more because we are limited by reserved transaction credits. On 2575 * the other hand we always make sure that the last touched page is fully 2576 * mapped so that it can be written out (and thus forward progress is 2577 * guaranteed). After mapping we submit all mapped pages for IO. 2578 */ 2579 static int mpage_map_and_submit_extent(handle_t *handle, 2580 struct mpage_da_data *mpd, 2581 bool *give_up_on_write) 2582 { 2583 struct inode *inode = mpd->inode; 2584 struct ext4_map_blocks *map = &mpd->map; 2585 int err; 2586 loff_t disksize; 2587 int progress = 0; 2588 ext4_io_end_t *io_end = mpd->io_submit.io_end; 2589 struct ext4_io_end_vec *io_end_vec; 2590 2591 io_end_vec = ext4_alloc_io_end_vec(io_end); 2592 if (IS_ERR(io_end_vec)) 2593 return PTR_ERR(io_end_vec); 2594 io_end_vec->offset = EXT4_LBLK_TO_B(inode, map->m_lblk); 2595 do { 2596 err = mpage_map_one_extent(handle, mpd); 2597 if (err < 0) { 2598 struct super_block *sb = inode->i_sb; 2599 2600 if (ext4_emergency_state(sb)) 2601 goto invalidate_dirty_pages; 2602 /* 2603 * Let the uper layers retry transient errors. 2604 * In the case of ENOSPC, if ext4_count_free_blocks() 2605 * is non-zero, a commit should free up blocks. 2606 */ 2607 if ((err == -ENOMEM) || (err == -EAGAIN) || 2608 (err == -ENOSPC && ext4_count_free_clusters(sb))) { 2609 /* 2610 * We may have already allocated extents for 2611 * some bhs inside the folio, issue the 2612 * corresponding data to prevent stale data. 2613 */ 2614 if (progress) { 2615 if (mpage_submit_partial_folio(mpd)) 2616 goto invalidate_dirty_pages; 2617 goto update_disksize; 2618 } 2619 return err; 2620 } 2621 ext4_msg(sb, KERN_CRIT, 2622 "Delayed block allocation failed for " 2623 "inode %llu at logical offset %llu with" 2624 " max blocks %u with error %d", 2625 inode->i_ino, 2626 (unsigned long long)map->m_lblk, 2627 (unsigned)map->m_len, -err); 2628 ext4_msg(sb, KERN_CRIT, 2629 "This should not happen!! Data will " 2630 "be lost\n"); 2631 if (err == -ENOSPC) 2632 ext4_print_free_blocks(inode); 2633 invalidate_dirty_pages: 2634 *give_up_on_write = true; 2635 return err; 2636 } 2637 progress = 1; 2638 /* 2639 * Update buffer state, submit mapped pages, and get us new 2640 * extent to map 2641 */ 2642 err = mpage_map_and_submit_buffers(mpd); 2643 if (err < 0) 2644 goto update_disksize; 2645 } while (map->m_len); 2646 2647 update_disksize: 2648 /* 2649 * Update on-disk size after IO is submitted. Races with 2650 * truncate are avoided by checking i_size under i_data_sem. 2651 */ 2652 disksize = mpd->start_pos; 2653 if (disksize > READ_ONCE(EXT4_I(inode)->i_disksize)) { 2654 int err2; 2655 loff_t i_size; 2656 2657 down_write(&EXT4_I(inode)->i_data_sem); 2658 i_size = i_size_read(inode); 2659 if (disksize > i_size) 2660 disksize = i_size; 2661 if (disksize > EXT4_I(inode)->i_disksize) 2662 EXT4_I(inode)->i_disksize = disksize; 2663 up_write(&EXT4_I(inode)->i_data_sem); 2664 err2 = ext4_mark_inode_dirty(handle, inode); 2665 if (err2) { 2666 ext4_error_err(inode->i_sb, -err2, 2667 "Failed to mark inode %llu dirty", 2668 inode->i_ino); 2669 } 2670 if (!err) 2671 err = err2; 2672 } 2673 return err; 2674 } 2675 2676 static int ext4_journal_folio_buffers(handle_t *handle, struct folio *folio, 2677 size_t len) 2678 { 2679 struct buffer_head *page_bufs = folio_buffers(folio); 2680 struct inode *inode = folio->mapping->host; 2681 int ret, err; 2682 2683 ret = ext4_walk_page_buffers(handle, inode, page_bufs, 0, len, 2684 NULL, do_journal_get_write_access); 2685 err = ext4_walk_page_buffers(handle, inode, page_bufs, 0, len, 2686 NULL, write_end_fn); 2687 if (ret == 0) 2688 ret = err; 2689 err = ext4_jbd2_inode_add_write(handle, inode, folio_pos(folio), len); 2690 if (ret == 0) 2691 ret = err; 2692 EXT4_I(inode)->i_datasync_tid = handle->h_transaction->t_tid; 2693 2694 return ret; 2695 } 2696 2697 static int mpage_journal_page_buffers(handle_t *handle, 2698 struct mpage_da_data *mpd, 2699 struct folio *folio) 2700 { 2701 struct inode *inode = mpd->inode; 2702 loff_t size = i_size_read(inode); 2703 size_t len = folio_size(folio); 2704 2705 folio_clear_checked(folio); 2706 mpd->wbc->nr_to_write -= folio_nr_pages(folio); 2707 2708 if (folio_pos(folio) + len > size && 2709 !ext4_verity_in_progress(inode)) 2710 len = size & (len - 1); 2711 2712 return ext4_journal_folio_buffers(handle, folio, len); 2713 } 2714 2715 /* 2716 * mpage_prepare_extent_to_map - find & lock contiguous range of dirty pages 2717 * needing mapping, submit mapped pages 2718 * 2719 * @mpd - where to look for pages 2720 * 2721 * Walk dirty pages in the mapping. If they are fully mapped, submit them for 2722 * IO immediately. If we cannot map blocks, we submit just already mapped 2723 * buffers in the page for IO and keep page dirty. When we can map blocks and 2724 * we find a page which isn't mapped we start accumulating extent of buffers 2725 * underlying these pages that needs mapping (formed by either delayed or 2726 * unwritten buffers). We also lock the pages containing these buffers. The 2727 * extent found is returned in @mpd structure (starting at mpd->lblk with 2728 * length mpd->len blocks). 2729 * 2730 * Note that this function can attach bios to one io_end structure which are 2731 * neither logically nor physically contiguous. Although it may seem as an 2732 * unnecessary complication, it is actually inevitable in blocksize < pagesize 2733 * case as we need to track IO to all buffers underlying a page in one io_end. 2734 */ 2735 static int mpage_prepare_extent_to_map(struct mpage_da_data *mpd) 2736 { 2737 struct address_space *mapping = mpd->inode->i_mapping; 2738 struct folio_batch fbatch; 2739 unsigned int nr_folios; 2740 pgoff_t index = mpd->start_pos >> PAGE_SHIFT; 2741 pgoff_t end = mpd->end_pos >> PAGE_SHIFT; 2742 xa_mark_t tag; 2743 int i, err = 0; 2744 ext4_lblk_t lblk; 2745 struct buffer_head *head; 2746 handle_t *handle = NULL; 2747 int bpp = ext4_journal_blocks_per_folio(mpd->inode); 2748 2749 tag = wbc_to_tag(mpd->wbc); 2750 2751 mpd->map.m_len = 0; 2752 mpd->next_pos = mpd->start_pos; 2753 if (ext4_should_journal_data(mpd->inode)) { 2754 handle = ext4_journal_start(mpd->inode, EXT4_HT_WRITE_PAGE, 2755 bpp); 2756 if (IS_ERR(handle)) 2757 return PTR_ERR(handle); 2758 } 2759 folio_batch_init(&fbatch); 2760 while (index <= end) { 2761 nr_folios = filemap_get_folios_tag(mapping, &index, end, 2762 tag, &fbatch); 2763 if (nr_folios == 0) 2764 break; 2765 2766 for (i = 0; i < nr_folios; i++) { 2767 struct folio *folio = fbatch.folios[i]; 2768 2769 /* 2770 * Accumulated enough dirty pages? This doesn't apply 2771 * to WB_SYNC_ALL mode. For integrity sync we have to 2772 * keep going because someone may be concurrently 2773 * dirtying pages, and we might have synced a lot of 2774 * newly appeared dirty pages, but have not synced all 2775 * of the old dirty pages. 2776 */ 2777 if (mpd->wbc->sync_mode == WB_SYNC_NONE && 2778 mpd->wbc->nr_to_write <= 2779 EXT4_LBLK_TO_PG(mpd->inode, mpd->map.m_len)) 2780 goto out; 2781 2782 /* If we can't merge this page, we are done. */ 2783 if (mpd->map.m_len > 0 && 2784 mpd->next_pos != folio_pos(folio)) 2785 goto out; 2786 2787 if (handle) { 2788 err = ext4_journal_ensure_credits(handle, bpp, 2789 0); 2790 if (err < 0) 2791 goto out; 2792 } 2793 2794 folio_lock(folio); 2795 /* 2796 * If the page is no longer dirty, or its mapping no 2797 * longer corresponds to inode we are writing (which 2798 * means it has been truncated or invalidated), or the 2799 * page is already under writeback and we are not doing 2800 * a data integrity writeback, skip the page 2801 */ 2802 if ((folio_test_writeback(folio) && 2803 mpd->wbc->sync_mode == WB_SYNC_NONE) || 2804 unlikely(folio->mapping != mapping)) { 2805 folio_unlock(folio); 2806 continue; 2807 } 2808 /* 2809 * If the folio is clean, skip writing it back. 2810 * Cycle the folio through the writeback state 2811 * though, to clear stale xarray tags. 2812 */ 2813 if (!folio_test_dirty(folio)) { 2814 if (!folio_test_writeback(folio)) { 2815 __folio_start_writeback(folio, false); 2816 folio_end_writeback(folio); 2817 } 2818 folio_unlock(folio); 2819 continue; 2820 } 2821 2822 folio_wait_writeback(folio); 2823 BUG_ON(folio_test_writeback(folio)); 2824 2825 /* 2826 * Should never happen but for buggy code in 2827 * other subsystems that call 2828 * set_page_dirty() without properly warning 2829 * the file system first. See [1] for more 2830 * information. 2831 * 2832 * [1] https://lore.kernel.org/linux-mm/20180103100430.GE4911@quack2.suse.cz 2833 */ 2834 if (!folio_buffers(folio)) { 2835 ext4_warning_inode(mpd->inode, "page %lu does not have buffers attached", folio->index); 2836 folio_clear_dirty(folio); 2837 folio_unlock(folio); 2838 continue; 2839 } 2840 2841 if (mpd->map.m_len == 0) 2842 mpd->start_pos = folio_pos(folio); 2843 mpd->next_pos = folio_next_pos(folio); 2844 /* 2845 * Writeout when we cannot modify metadata is simple. 2846 * Just submit the page. For data=journal mode we 2847 * first handle writeout of the page for checkpoint and 2848 * only after that handle delayed page dirtying. This 2849 * makes sure current data is checkpointed to the final 2850 * location before possibly journalling it again which 2851 * is desirable when the page is frequently dirtied 2852 * through a pin. 2853 */ 2854 if (!mpd->can_map) { 2855 mpage_submit_folio(mpd, folio); 2856 err = 0; 2857 /* Pending dirtying of journalled data? */ 2858 if (folio_test_checked(folio)) { 2859 err = mpage_journal_page_buffers(handle, 2860 mpd, folio); 2861 if (err < 0) 2862 goto out; 2863 mpd->journalled_more_data = 1; 2864 } 2865 mpage_folio_done(mpd, folio); 2866 } else { 2867 /* Add all dirty buffers to mpd */ 2868 lblk = EXT4_PG_TO_LBLK(mpd->inode, folio->index); 2869 head = folio_buffers(folio); 2870 err = mpage_process_page_bufs(mpd, head, head, 2871 lblk); 2872 if (err <= 0) 2873 goto out; 2874 err = 0; 2875 } 2876 } 2877 folio_batch_release(&fbatch); 2878 cond_resched(); 2879 } 2880 mpd->scanned_until_end = 1; 2881 if (handle) 2882 ext4_journal_stop(handle); 2883 return 0; 2884 out: 2885 folio_batch_release(&fbatch); 2886 if (handle) 2887 ext4_journal_stop(handle); 2888 return err; 2889 } 2890 2891 static int ext4_do_writepages(struct mpage_da_data *mpd) 2892 { 2893 struct writeback_control *wbc = mpd->wbc; 2894 pgoff_t writeback_index = 0; 2895 long nr_to_write = wbc->nr_to_write; 2896 int range_whole = 0; 2897 int cycled = 1; 2898 handle_t *handle = NULL; 2899 struct inode *inode = mpd->inode; 2900 struct address_space *mapping = inode->i_mapping; 2901 int needed_blocks, rsv_blocks = 0, ret = 0; 2902 struct ext4_sb_info *sbi = EXT4_SB(mapping->host->i_sb); 2903 struct blk_plug plug; 2904 bool give_up_on_write = false; 2905 2906 trace_ext4_writepages(inode, wbc); 2907 2908 /* 2909 * No pages to write? This is mainly a kludge to avoid starting 2910 * a transaction for special inodes like journal inode on last iput() 2911 * because that could violate lock ordering on umount 2912 */ 2913 if (!mapping->nrpages || !mapping_tagged(mapping, PAGECACHE_TAG_DIRTY)) 2914 goto out_writepages; 2915 2916 /* 2917 * Does ext4_change_inode_journal_flag() change the inode's 2918 * buffered I/O path? 2919 */ 2920 if (ext4_inode_buffered_iomap(inode)) 2921 goto out_writepages; 2922 2923 /* 2924 * If the filesystem has aborted, it is read-only, so return 2925 * right away instead of dumping stack traces later on that 2926 * will obscure the real source of the problem. We test 2927 * fs shutdown state instead of sb->s_flag's SB_RDONLY because 2928 * the latter could be true if the filesystem is mounted 2929 * read-only, and in that case, ext4_writepages should 2930 * *never* be called, so if that ever happens, we would want 2931 * the stack trace. 2932 */ 2933 ret = ext4_emergency_state(mapping->host->i_sb); 2934 if (unlikely(ret)) 2935 goto out_writepages; 2936 2937 /* 2938 * If we have inline data and arrive here, it means that 2939 * we will soon create the block for the 1st page, so 2940 * we'd better clear the inline data here. 2941 */ 2942 if (ext4_has_inline_data(inode)) { 2943 /* Just inode will be modified... */ 2944 handle = ext4_journal_start(inode, EXT4_HT_INODE, 1); 2945 if (IS_ERR(handle)) { 2946 ret = PTR_ERR(handle); 2947 goto out_writepages; 2948 } 2949 BUG_ON(ext4_test_inode_state(inode, 2950 EXT4_STATE_MAY_INLINE_DATA)); 2951 ext4_destroy_inline_data(handle, inode); 2952 ext4_journal_stop(handle); 2953 } 2954 2955 /* 2956 * data=journal mode does not do delalloc so we just need to writeout / 2957 * journal already mapped buffers. On the other hand we need to commit 2958 * transaction to make data stable. We expect all the data to be 2959 * already in the journal (the only exception are DMA pinned pages 2960 * dirtied behind our back) so we commit transaction here and run the 2961 * writeback loop to checkpoint them. The checkpointing is not actually 2962 * necessary to make data persistent *but* quite a few places (extent 2963 * shifting operations, fsverity, ...) depend on being able to drop 2964 * pagecache pages after calling filemap_write_and_wait() and for that 2965 * checkpointing needs to happen. 2966 */ 2967 if (ext4_should_journal_data(inode)) { 2968 mpd->can_map = 0; 2969 if (wbc->sync_mode == WB_SYNC_ALL) 2970 ext4_fc_commit(sbi->s_journal, 2971 EXT4_I(inode)->i_datasync_tid); 2972 } 2973 mpd->journalled_more_data = 0; 2974 2975 if (ext4_should_dioread_nolock(inode)) { 2976 int bpf = ext4_journal_blocks_per_folio(inode); 2977 /* 2978 * We may need to convert up to one extent per block in the 2979 * folio. 2980 */ 2981 rsv_blocks = ext4_meta_trans_blocks(inode, bpf, bpf, 0); 2982 } 2983 2984 if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX) 2985 range_whole = 1; 2986 2987 if (wbc->range_cyclic) { 2988 writeback_index = mapping->writeback_index; 2989 if (writeback_index) 2990 cycled = 0; 2991 mpd->start_pos = writeback_index << PAGE_SHIFT; 2992 mpd->end_pos = LLONG_MAX; 2993 } else { 2994 mpd->start_pos = wbc->range_start; 2995 mpd->end_pos = wbc->range_end; 2996 } 2997 2998 ext4_io_submit_init(&mpd->io_submit, wbc); 2999 retry: 3000 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages) 3001 tag_pages_for_writeback(mapping, mpd->start_pos >> PAGE_SHIFT, 3002 mpd->end_pos >> PAGE_SHIFT); 3003 blk_start_plug(&plug); 3004 3005 /* 3006 * First writeback pages that don't need mapping - we can avoid 3007 * starting a transaction unnecessarily and also avoid being blocked 3008 * in the block layer on device congestion while having transaction 3009 * started. 3010 */ 3011 mpd->do_map = 0; 3012 mpd->scanned_until_end = 0; 3013 mpd->io_submit.io_end = ext4_init_io_end(inode, GFP_KERNEL); 3014 if (!mpd->io_submit.io_end) { 3015 ret = -ENOMEM; 3016 goto unplug; 3017 } 3018 ret = mpage_prepare_extent_to_map(mpd); 3019 /* Unlock pages we didn't use */ 3020 mpage_release_unused_pages(mpd, false); 3021 /* Submit prepared bio */ 3022 ext4_io_submit(&mpd->io_submit); 3023 ext4_put_io_end_defer(mpd->io_submit.io_end); 3024 mpd->io_submit.io_end = NULL; 3025 if (ret < 0) 3026 goto unplug; 3027 3028 while (!mpd->scanned_until_end && wbc->nr_to_write > 0) { 3029 /* For each extent of pages we use new io_end */ 3030 mpd->io_submit.io_end = ext4_init_io_end(inode, GFP_KERNEL); 3031 if (!mpd->io_submit.io_end) { 3032 ret = -ENOMEM; 3033 break; 3034 } 3035 3036 WARN_ON_ONCE(!mpd->can_map); 3037 /* 3038 * We have two constraints: We find one extent to map and we 3039 * must always write out whole page (makes a difference when 3040 * blocksize < pagesize) so that we don't block on IO when we 3041 * try to write out the rest of the page. Journalled mode is 3042 * not supported by delalloc. 3043 */ 3044 BUG_ON(ext4_should_journal_data(inode)); 3045 /* 3046 * Calculate the number of credits needed to reserve for one 3047 * extent of up to MAX_WRITEPAGES_EXTENT_LEN blocks. It will 3048 * attempt to extend the transaction or start a new iteration 3049 * if the reserved credits are insufficient. 3050 */ 3051 needed_blocks = ext4_chunk_trans_blocks(inode, 3052 MAX_WRITEPAGES_EXTENT_LEN); 3053 /* start a new transaction */ 3054 handle = ext4_journal_start_with_reserve(inode, 3055 EXT4_HT_WRITE_PAGE, needed_blocks, rsv_blocks); 3056 if (IS_ERR(handle)) { 3057 ret = PTR_ERR(handle); 3058 ext4_msg(inode->i_sb, KERN_CRIT, "%s: jbd2_start: " 3059 "%ld pages, ino %llu; err %d", __func__, 3060 wbc->nr_to_write, inode->i_ino, ret); 3061 /* Release allocated io_end */ 3062 ext4_put_io_end(mpd->io_submit.io_end); 3063 mpd->io_submit.io_end = NULL; 3064 break; 3065 } 3066 mpd->do_map = 1; 3067 3068 trace_ext4_da_write_folios_start(inode, mpd->start_pos, 3069 mpd->next_pos, wbc); 3070 ret = mpage_prepare_extent_to_map(mpd); 3071 if (!ret && mpd->map.m_len) 3072 ret = mpage_map_and_submit_extent(handle, mpd, 3073 &give_up_on_write); 3074 /* 3075 * Caution: If the handle is synchronous, 3076 * ext4_journal_stop() can wait for transaction commit 3077 * to finish which may depend on writeback of pages to 3078 * complete or on page lock to be released. In that 3079 * case, we have to wait until after we have 3080 * submitted all the IO, released page locks we hold, 3081 * and dropped io_end reference (for extent conversion 3082 * to be able to complete) before stopping the handle. 3083 */ 3084 if (!ext4_handle_valid(handle) || handle->h_sync == 0) { 3085 ext4_journal_stop(handle); 3086 handle = NULL; 3087 mpd->do_map = 0; 3088 } 3089 /* Unlock pages we didn't use */ 3090 mpage_release_unused_pages(mpd, give_up_on_write); 3091 /* Submit prepared bio */ 3092 ext4_io_submit(&mpd->io_submit); 3093 3094 /* 3095 * Drop our io_end reference we got from init. We have 3096 * to be careful and use deferred io_end finishing if 3097 * we are still holding the transaction as we can 3098 * release the last reference to io_end which may end 3099 * up doing unwritten extent conversion. 3100 */ 3101 if (handle) { 3102 ext4_put_io_end_defer(mpd->io_submit.io_end); 3103 ext4_journal_stop(handle); 3104 } else 3105 ext4_put_io_end(mpd->io_submit.io_end); 3106 mpd->io_submit.io_end = NULL; 3107 trace_ext4_da_write_folios_end(inode, mpd->start_pos, 3108 mpd->next_pos, wbc, ret); 3109 3110 if (ret == -ENOSPC && sbi->s_journal) { 3111 /* 3112 * Commit the transaction which would 3113 * free blocks released in the transaction 3114 * and try again 3115 */ 3116 jbd2_journal_force_commit_nested(sbi->s_journal); 3117 ret = 0; 3118 continue; 3119 } 3120 if (ret == -EAGAIN) 3121 ret = 0; 3122 /* Fatal error - ENOMEM, EIO... */ 3123 if (ret) 3124 break; 3125 } 3126 unplug: 3127 blk_finish_plug(&plug); 3128 if (!ret && !cycled && wbc->nr_to_write > 0) { 3129 cycled = 1; 3130 mpd->end_pos = (writeback_index << PAGE_SHIFT) - 1; 3131 mpd->start_pos = 0; 3132 goto retry; 3133 } 3134 3135 /* Update index */ 3136 if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0)) 3137 /* 3138 * Set the writeback_index so that range_cyclic 3139 * mode will write it back later 3140 */ 3141 mapping->writeback_index = mpd->start_pos >> PAGE_SHIFT; 3142 3143 out_writepages: 3144 trace_ext4_writepages_result(inode, wbc, ret, 3145 nr_to_write - wbc->nr_to_write); 3146 return ret; 3147 } 3148 3149 static int ext4_writepages(struct address_space *mapping, 3150 struct writeback_control *wbc) 3151 { 3152 struct super_block *sb = mapping->host->i_sb; 3153 struct mpage_da_data mpd = { 3154 .inode = mapping->host, 3155 .wbc = wbc, 3156 .can_map = 1, 3157 }; 3158 int ret; 3159 int alloc_ctx; 3160 3161 ret = ext4_emergency_state(sb); 3162 if (unlikely(ret)) 3163 return ret; 3164 3165 alloc_ctx = ext4_writepages_down_read(sb); 3166 ret = ext4_do_writepages(&mpd); 3167 /* 3168 * For data=journal writeback we could have come across pages marked 3169 * for delayed dirtying (PageChecked) which were just added to the 3170 * running transaction. Try once more to get them to stable storage. 3171 */ 3172 if (!ret && mpd.journalled_more_data) 3173 ret = ext4_do_writepages(&mpd); 3174 ext4_writepages_up_read(sb, alloc_ctx); 3175 3176 return ret; 3177 } 3178 3179 int ext4_normal_submit_inode_data_buffers(struct jbd2_inode *jinode) 3180 { 3181 loff_t range_start, range_end; 3182 struct writeback_control wbc = { 3183 .sync_mode = WB_SYNC_ALL, 3184 .nr_to_write = LONG_MAX, 3185 }; 3186 struct mpage_da_data mpd = { 3187 .inode = jinode->i_vfs_inode, 3188 .wbc = &wbc, 3189 .can_map = 0, 3190 }; 3191 3192 if (!jbd2_jinode_get_dirty_range(jinode, &range_start, &range_end)) 3193 return 0; 3194 3195 wbc.range_start = range_start; 3196 wbc.range_end = range_end; 3197 3198 return ext4_do_writepages(&mpd); 3199 } 3200 3201 static int ext4_dax_writepages(struct address_space *mapping, 3202 struct writeback_control *wbc) 3203 { 3204 int ret; 3205 long nr_to_write = wbc->nr_to_write; 3206 struct inode *inode = mapping->host; 3207 int alloc_ctx; 3208 3209 ret = ext4_emergency_state(inode->i_sb); 3210 if (unlikely(ret)) 3211 return ret; 3212 3213 alloc_ctx = ext4_writepages_down_read(inode->i_sb); 3214 trace_ext4_writepages(inode, wbc); 3215 3216 ret = dax_writeback_mapping_range(mapping, 3217 EXT4_SB(inode->i_sb)->s_daxdev, wbc); 3218 trace_ext4_writepages_result(inode, wbc, ret, 3219 nr_to_write - wbc->nr_to_write); 3220 ext4_writepages_up_read(inode->i_sb, alloc_ctx); 3221 return ret; 3222 } 3223 3224 int ext4_nonda_switch(struct super_block *sb) 3225 { 3226 s64 free_clusters, dirty_clusters; 3227 struct ext4_sb_info *sbi = EXT4_SB(sb); 3228 3229 /* 3230 * switch to non delalloc mode if we are running low 3231 * on free block. The free block accounting via percpu 3232 * counters can get slightly wrong with percpu_counter_batch getting 3233 * accumulated on each CPU without updating global counters 3234 * Delalloc need an accurate free block accounting. So switch 3235 * to non delalloc when we are near to error range. 3236 */ 3237 free_clusters = 3238 percpu_counter_read_positive(&sbi->s_freeclusters_counter); 3239 dirty_clusters = 3240 percpu_counter_read_positive(&sbi->s_dirtyclusters_counter); 3241 /* 3242 * Start pushing delalloc when 1/2 of free blocks are dirty. 3243 */ 3244 if (dirty_clusters && (free_clusters < 2 * dirty_clusters)) 3245 try_to_writeback_inodes_sb(sb, WB_REASON_FS_FREE_SPACE); 3246 3247 if (2 * free_clusters < 3 * dirty_clusters || 3248 free_clusters < (dirty_clusters + EXT4_FREECLUSTERS_WATERMARK)) { 3249 /* 3250 * free block count is less than 150% of dirty blocks 3251 * or free blocks is less than watermark 3252 */ 3253 return 1; 3254 } 3255 return 0; 3256 } 3257 3258 static int ext4_da_write_begin(const struct kiocb *iocb, 3259 struct address_space *mapping, 3260 loff_t pos, unsigned len, 3261 struct folio **foliop, void **fsdata) 3262 { 3263 int ret, retries = 0; 3264 struct folio *folio; 3265 pgoff_t index; 3266 struct inode *inode = mapping->host; 3267 3268 ret = ext4_emergency_state(inode->i_sb); 3269 if (unlikely(ret)) 3270 return ret; 3271 3272 index = pos >> PAGE_SHIFT; 3273 3274 if (ext4_nonda_switch(inode->i_sb) || ext4_verity_in_progress(inode)) { 3275 *fsdata = (void *)FALL_BACK_TO_NONDELALLOC; 3276 return ext4_write_begin(iocb, mapping, pos, 3277 len, foliop, fsdata); 3278 } 3279 *fsdata = (void *)0; 3280 trace_ext4_da_write_begin(inode, pos, len); 3281 3282 if (ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) { 3283 ret = ext4_generic_write_inline_data(mapping, inode, pos, len, 3284 foliop, true); 3285 if (ret < 0) 3286 return ret; 3287 if (ret == 1) { 3288 *fsdata = (void *)((unsigned long)*fsdata | EXT4_WRITE_DATA_INLINE); 3289 return 0; 3290 } 3291 } 3292 3293 retry: 3294 folio = write_begin_get_folio(iocb, mapping, index, len); 3295 if (IS_ERR(folio)) 3296 return PTR_ERR(folio); 3297 3298 if (len > folio_next_pos(folio) - pos) 3299 len = folio_next_pos(folio) - pos; 3300 3301 ret = ext4_block_write_begin(NULL, folio, pos, len, 3302 ext4_da_get_block_prep); 3303 if (ret < 0) { 3304 folio_unlock(folio); 3305 folio_put(folio); 3306 /* 3307 * ext4_block_write_begin may have instantiated a few blocks 3308 * outside i_size. Trim these off again. Don't need 3309 * i_size_read because we hold inode lock. 3310 */ 3311 if (pos + len > inode->i_size) 3312 ext4_truncate_failed_write(inode); 3313 3314 if (ret == -ENOSPC && 3315 ext4_should_retry_alloc(inode->i_sb, &retries)) 3316 goto retry; 3317 return ret; 3318 } 3319 3320 *foliop = folio; 3321 return ret; 3322 } 3323 3324 /* 3325 * Check if we should update i_disksize 3326 * when write to the end of file but not require block allocation 3327 */ 3328 static int ext4_da_should_update_i_disksize(struct folio *folio, 3329 unsigned long offset) 3330 { 3331 struct buffer_head *bh; 3332 struct inode *inode = folio->mapping->host; 3333 unsigned int idx; 3334 int i; 3335 3336 bh = folio_buffers(folio); 3337 idx = offset >> inode->i_blkbits; 3338 3339 for (i = 0; i < idx; i++) 3340 bh = bh->b_this_page; 3341 3342 if (!buffer_mapped(bh) || (buffer_delay(bh)) || buffer_unwritten(bh)) 3343 return 0; 3344 return 1; 3345 } 3346 3347 static int ext4_da_do_write_end(struct address_space *mapping, 3348 loff_t pos, unsigned len, unsigned copied, 3349 struct folio *folio) 3350 { 3351 struct inode *inode = mapping->host; 3352 loff_t old_size = inode->i_size; 3353 bool disksize_changed = false; 3354 loff_t new_i_size; 3355 handle_t *handle; 3356 3357 if (unlikely(!folio_buffers(folio))) { 3358 folio_unlock(folio); 3359 folio_put(folio); 3360 return -EIO; 3361 } 3362 /* 3363 * block_write_end() will mark the inode as dirty with I_DIRTY_PAGES 3364 * flag, which all that's needed to trigger page writeback. 3365 */ 3366 copied = block_write_end(pos, len, copied, folio); 3367 new_i_size = pos + copied; 3368 3369 /* 3370 * It's important to update i_size while still holding folio lock, 3371 * because folio writeout could otherwise come in and zero beyond 3372 * i_size. 3373 * 3374 * Since we are holding inode lock, we are sure i_disksize <= 3375 * i_size. We also know that if i_disksize < i_size, there are 3376 * delalloc writes pending in the range up to i_size. If the end of 3377 * the current write is <= i_size, there's no need to touch 3378 * i_disksize since writeback will push i_disksize up to i_size 3379 * eventually. If the end of the current write is > i_size and 3380 * inside an allocated block which ext4_da_should_update_i_disksize() 3381 * checked, we need to update i_disksize here as certain 3382 * ext4_writepages() paths not allocating blocks and update i_disksize. 3383 */ 3384 if (new_i_size > inode->i_size) { 3385 unsigned long end; 3386 3387 i_size_write(inode, new_i_size); 3388 end = offset_in_folio(folio, new_i_size - 1); 3389 if (copied && ext4_da_should_update_i_disksize(folio, end)) { 3390 ext4_update_i_disksize(inode, new_i_size); 3391 disksize_changed = true; 3392 } 3393 } 3394 3395 folio_unlock(folio); 3396 folio_put(folio); 3397 3398 if (pos > old_size) 3399 pagecache_isize_extended(inode, old_size, pos); 3400 3401 if (!disksize_changed) 3402 return copied; 3403 3404 handle = ext4_journal_start(inode, EXT4_HT_INODE, 1); 3405 if (IS_ERR(handle)) 3406 return PTR_ERR(handle); 3407 ext4_mark_inode_dirty(handle, inode); 3408 ext4_journal_stop(handle); 3409 3410 return copied; 3411 } 3412 3413 static int ext4_da_write_end(const struct kiocb *iocb, 3414 struct address_space *mapping, 3415 loff_t pos, unsigned len, unsigned copied, 3416 struct folio *folio, void *fsdata) 3417 { 3418 struct inode *inode = mapping->host; 3419 unsigned long write_mode = (unsigned long)fsdata; 3420 3421 if (write_mode & FALL_BACK_TO_NONDELALLOC) 3422 return ext4_write_end(iocb, mapping, pos, 3423 len, copied, folio, fsdata); 3424 3425 trace_ext4_da_write_end(inode, pos, len, copied); 3426 3427 if (write_mode & EXT4_WRITE_DATA_INLINE) 3428 return ext4_write_inline_data_end(inode, pos, len, copied, 3429 folio); 3430 3431 if (unlikely(copied < len) && !folio_test_uptodate(folio)) 3432 copied = 0; 3433 3434 return ext4_da_do_write_end(mapping, pos, len, copied, folio); 3435 } 3436 3437 /* 3438 * Force all delayed allocation blocks to be allocated for a given inode. 3439 */ 3440 int ext4_alloc_da_blocks(struct inode *inode) 3441 { 3442 trace_ext4_alloc_da_blocks(inode); 3443 3444 if (!EXT4_I(inode)->i_reserved_data_blocks) 3445 return 0; 3446 3447 /* 3448 * We do something simple for now. The filemap_flush() will 3449 * also start triggering a write of the data blocks, which is 3450 * not strictly speaking necessary. However, to do otherwise 3451 * would require replicating code paths in: 3452 * 3453 * ext4_writepages() -> 3454 * write_cache_pages() ---> (via passed in callback function) 3455 * __mpage_da_writepage() --> 3456 * mpage_add_bh_to_extent() 3457 * mpage_da_map_blocks() 3458 * 3459 * The problem is that write_cache_pages(), located in 3460 * mm/page-writeback.c, marks pages clean in preparation for 3461 * doing I/O, which is not desirable if we're not planning on 3462 * doing I/O at all. 3463 * 3464 * We could call write_cache_pages(), and then redirty all of 3465 * the pages by calling redirty_page_for_writepage() but that 3466 * would be ugly in the extreme. So instead we would need to 3467 * replicate parts of the code in the above functions, 3468 * simplifying them because we wouldn't actually intend to 3469 * write out the pages, but rather only collect contiguous 3470 * logical block extents, call the multi-block allocator, and 3471 * then update the buffer heads with the block allocations. 3472 * 3473 * For now, though, we'll cheat by calling filemap_flush(), 3474 * which will map the blocks, and start the I/O, but not 3475 * actually wait for the I/O to complete. 3476 */ 3477 return filemap_flush(inode->i_mapping); 3478 } 3479 3480 /* 3481 * bmap() is special. It gets used by applications such as lilo and by 3482 * the swapper to find the on-disk block of a specific piece of data. 3483 * 3484 * Naturally, this is dangerous if the block concerned is still in the 3485 * journal. If somebody makes a swapfile on an ext4 data-journaling 3486 * filesystem and enables swap, then they may get a nasty shock when the 3487 * data getting swapped to that swapfile suddenly gets overwritten by 3488 * the original zero's written out previously to the journal and 3489 * awaiting writeback in the kernel's buffer cache. 3490 * 3491 * So, if we see any bmap calls here on a modified, data-journaled file, 3492 * take extra steps to flush any blocks which might be in the cache. 3493 */ 3494 static sector_t ext4_bmap(struct address_space *mapping, sector_t block) 3495 { 3496 struct inode *inode = mapping->host; 3497 sector_t ret = 0; 3498 3499 inode_lock_shared(inode); 3500 /* 3501 * We can get here for an inline file via the FIBMAP ioctl 3502 */ 3503 if (ext4_has_inline_data(inode)) 3504 goto out; 3505 3506 if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY) && 3507 (test_opt(inode->i_sb, DELALLOC) || 3508 ext4_should_journal_data(inode))) { 3509 /* 3510 * With delalloc or journalled data we want to sync the file so 3511 * that we can make sure we allocate blocks for file and data 3512 * is in place for the user to see it 3513 */ 3514 filemap_write_and_wait(mapping); 3515 } 3516 3517 ret = iomap_bmap(mapping, block, ext4_iomap_next); 3518 3519 out: 3520 inode_unlock_shared(inode); 3521 return ret; 3522 } 3523 3524 static void ext4_invalidate_folio(struct folio *folio, size_t offset, 3525 size_t length) 3526 { 3527 trace_ext4_invalidate_folio(folio, offset, length); 3528 3529 /* No journalling happens on data buffers when this function is used */ 3530 WARN_ON(folio_buffers(folio) && buffer_jbd(folio_buffers(folio))); 3531 3532 block_invalidate_folio(folio, offset, length); 3533 } 3534 3535 static int __ext4_journalled_invalidate_folio(struct folio *folio, 3536 size_t offset, size_t length) 3537 { 3538 journal_t *journal = EXT4_JOURNAL(folio->mapping->host); 3539 3540 trace_ext4_journalled_invalidate_folio(folio, offset, length); 3541 3542 /* 3543 * If it's a full truncate we just forget about the pending dirtying 3544 */ 3545 if (offset == 0 && length == folio_size(folio)) 3546 folio_clear_checked(folio); 3547 3548 return jbd2_journal_invalidate_folio(journal, folio, offset, length); 3549 } 3550 3551 /* Wrapper for aops... */ 3552 static void ext4_journalled_invalidate_folio(struct folio *folio, 3553 size_t offset, 3554 size_t length) 3555 { 3556 WARN_ON(__ext4_journalled_invalidate_folio(folio, offset, length) < 0); 3557 } 3558 3559 static bool ext4_release_folio(struct folio *folio, gfp_t wait) 3560 { 3561 struct inode *inode = folio->mapping->host; 3562 journal_t *journal = EXT4_JOURNAL(inode); 3563 3564 trace_ext4_release_folio(inode, folio); 3565 3566 /* Page has dirty journalled data -> cannot release */ 3567 if (folio_test_checked(folio)) 3568 return false; 3569 if (journal) 3570 return jbd2_journal_try_to_free_buffers(journal, folio); 3571 else 3572 return try_to_free_buffers(folio); 3573 } 3574 3575 static bool ext4_inode_datasync_dirty(struct inode *inode) 3576 { 3577 journal_t *journal = EXT4_SB(inode->i_sb)->s_journal; 3578 struct mapping_metadata_bhs *mmb; 3579 3580 if (journal) { 3581 if (jbd2_transaction_committed(journal, 3582 EXT4_I(inode)->i_datasync_tid)) 3583 return false; 3584 if (test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT)) 3585 return !list_empty(&EXT4_I(inode)->i_fc_list); 3586 return true; 3587 } 3588 3589 mmb = ext4_i_metadata_bhs(inode); 3590 /* Any metadata buffers to write? */ 3591 if (mmb && mmb_has_buffers(mmb)) 3592 return true; 3593 return inode_state_read_once(inode) & I_DIRTY_DATASYNC; 3594 } 3595 3596 static bool ext4_iomap_valid(struct inode *inode, const struct iomap *iomap) 3597 { 3598 return iomap->validity_cookie == READ_ONCE(EXT4_I(inode)->i_es_seq); 3599 } 3600 3601 const struct iomap_write_ops ext4_iomap_write_ops = { 3602 .iomap_valid = ext4_iomap_valid, 3603 }; 3604 3605 static void ext4_set_iomap(struct inode *inode, struct iomap *iomap, 3606 struct ext4_map_blocks *map, loff_t offset, 3607 loff_t length, unsigned int flags) 3608 { 3609 u8 blkbits = inode->i_blkbits; 3610 3611 /* 3612 * Writes that span EOF might trigger an I/O size update on completion, 3613 * so consider them to be dirty for the purpose of O_DSYNC, even if 3614 * there is no other metadata changes being made or are pending. 3615 */ 3616 iomap->flags = 0; 3617 if (ext4_inode_datasync_dirty(inode) || 3618 offset + length > i_size_read(inode)) 3619 iomap->flags |= IOMAP_F_DIRTY; 3620 3621 if (map->m_flags & EXT4_MAP_NEW) 3622 iomap->flags |= IOMAP_F_NEW; 3623 3624 /* HW-offload atomics are always used */ 3625 if (flags & IOMAP_ATOMIC) 3626 iomap->flags |= IOMAP_F_ATOMIC_BIO; 3627 3628 if (flags & IOMAP_DAX) 3629 iomap->dax_dev = EXT4_SB(inode->i_sb)->s_daxdev; 3630 else 3631 iomap->bdev = inode->i_sb->s_bdev; 3632 iomap->offset = EXT4_LBLK_TO_B(inode, map->m_lblk); 3633 iomap->length = EXT4_LBLK_TO_B(inode, map->m_len); 3634 3635 if ((map->m_flags & EXT4_MAP_MAPPED) && 3636 !ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) 3637 iomap->flags |= IOMAP_F_MERGED; 3638 3639 iomap->validity_cookie = map->m_seq; 3640 3641 /* 3642 * Flags passed to ext4_map_blocks() for direct I/O writes can result 3643 * in m_flags having both EXT4_MAP_MAPPED and EXT4_MAP_UNWRITTEN bits 3644 * set. In order for any allocated unwritten extents to be converted 3645 * into written extents correctly within the ->end_io() handler, we 3646 * need to ensure that the iomap->type is set appropriately. Hence, the 3647 * reason why we need to check whether the EXT4_MAP_UNWRITTEN bit has 3648 * been set first. 3649 */ 3650 if (map->m_flags & EXT4_MAP_UNWRITTEN) { 3651 iomap->type = IOMAP_UNWRITTEN; 3652 iomap->addr = (u64) map->m_pblk << blkbits; 3653 if (flags & IOMAP_DAX) 3654 iomap->addr += EXT4_SB(inode->i_sb)->s_dax_part_off; 3655 } else if (map->m_flags & EXT4_MAP_MAPPED) { 3656 iomap->type = IOMAP_MAPPED; 3657 iomap->addr = (u64) map->m_pblk << blkbits; 3658 if (flags & IOMAP_DAX) 3659 iomap->addr += EXT4_SB(inode->i_sb)->s_dax_part_off; 3660 } else if (map->m_flags & EXT4_MAP_DELAYED) { 3661 iomap->type = IOMAP_DELALLOC; 3662 iomap->addr = IOMAP_NULL_ADDR; 3663 } else { 3664 iomap->type = IOMAP_HOLE; 3665 iomap->addr = IOMAP_NULL_ADDR; 3666 } 3667 } 3668 3669 static int ext4_map_blocks_atomic_write_slow(handle_t *handle, 3670 struct inode *inode, struct ext4_map_blocks *map) 3671 { 3672 ext4_lblk_t m_lblk = map->m_lblk; 3673 unsigned int m_len = map->m_len; 3674 unsigned int mapped_len = 0, m_flags = 0; 3675 ext4_fsblk_t next_pblk = 0; 3676 bool check_next_pblk = false; 3677 int ret = 0; 3678 3679 WARN_ON_ONCE(!ext4_has_feature_bigalloc(inode->i_sb)); 3680 3681 /* 3682 * This is a slow path in case of mixed mapping. We use 3683 * EXT4_GET_BLOCKS_CREATE_ZERO flag here to make sure we get a single 3684 * contiguous mapped mapping. This will ensure any unwritten or hole 3685 * regions within the requested range is zeroed out and we return 3686 * a single contiguous mapped extent. 3687 */ 3688 m_flags = EXT4_GET_BLOCKS_CREATE_ZERO; 3689 3690 do { 3691 ret = ext4_map_blocks(handle, inode, map, m_flags); 3692 if (ret < 0 && ret != -ENOSPC) 3693 goto out_err; 3694 /* 3695 * This should never happen, but let's return an error code to 3696 * avoid an infinite loop in here. 3697 */ 3698 if (ret == 0) { 3699 ret = -EFSCORRUPTED; 3700 ext4_warning_inode(inode, 3701 "ext4_map_blocks() couldn't allocate blocks m_flags: 0x%x, ret:%d", 3702 m_flags, ret); 3703 goto out_err; 3704 } 3705 /* 3706 * With bigalloc we should never get ENOSPC nor discontiguous 3707 * physical extents. 3708 */ 3709 if ((check_next_pblk && next_pblk != map->m_pblk) || 3710 ret == -ENOSPC) { 3711 ext4_warning_inode(inode, 3712 "Non-contiguous allocation detected: expected %llu, got %llu, " 3713 "or ext4_map_blocks() returned out of space ret: %d", 3714 next_pblk, map->m_pblk, ret); 3715 ret = -EFSCORRUPTED; 3716 goto out_err; 3717 } 3718 next_pblk = map->m_pblk + map->m_len; 3719 check_next_pblk = true; 3720 3721 mapped_len += map->m_len; 3722 map->m_lblk += map->m_len; 3723 map->m_len = m_len - mapped_len; 3724 } while (mapped_len < m_len); 3725 3726 /* 3727 * We might have done some work in above loop, so we need to query the 3728 * start of the physical extent, based on the origin m_lblk and m_len. 3729 * Let's also ensure we were able to allocate the required range for 3730 * mixed mapping case. 3731 */ 3732 map->m_lblk = m_lblk; 3733 map->m_len = m_len; 3734 map->m_flags = 0; 3735 3736 ret = ext4_map_blocks(handle, inode, map, 3737 EXT4_GET_BLOCKS_QUERY_LAST_IN_LEAF); 3738 if (ret != m_len) { 3739 ext4_warning_inode(inode, 3740 "allocation failed for atomic write request m_lblk:%u, m_len:%u, ret:%d\n", 3741 m_lblk, m_len, ret); 3742 ret = -EINVAL; 3743 } 3744 return ret; 3745 3746 out_err: 3747 /* reset map before returning an error */ 3748 map->m_lblk = m_lblk; 3749 map->m_len = m_len; 3750 map->m_flags = 0; 3751 return ret; 3752 } 3753 3754 /* 3755 * ext4_map_blocks_atomic: Helper routine to ensure the entire requested 3756 * range in @map [lblk, lblk + len) is one single contiguous extent with no 3757 * mixed mappings. 3758 * 3759 * We first use m_flags passed to us by our caller (ext4_iomap_alloc()). 3760 * We only call EXT4_GET_BLOCKS_ZERO in the slow path, when the underlying 3761 * physical extent for the requested range does not have a single contiguous 3762 * mapping type i.e. (Hole, Mapped, or Unwritten) throughout. 3763 * In that case we will loop over the requested range to allocate and zero out 3764 * the unwritten / holes in between, to get a single mapped extent from 3765 * [m_lblk, m_lblk + m_len). Note that this is only possible because we know 3766 * this can be called only with bigalloc enabled filesystem where the underlying 3767 * cluster is already allocated. This avoids allocating discontiguous extents 3768 * in the slow path due to multiple calls to ext4_map_blocks(). 3769 * The slow path is mostly non-performance critical path, so it should be ok to 3770 * loop using ext4_map_blocks() with appropriate flags to allocate & zero the 3771 * underlying short holes/unwritten extents within the requested range. 3772 */ 3773 static int ext4_map_blocks_atomic_write(handle_t *handle, struct inode *inode, 3774 struct ext4_map_blocks *map, int m_flags, 3775 bool *force_commit) 3776 { 3777 ext4_lblk_t m_lblk = map->m_lblk; 3778 unsigned int m_len = map->m_len; 3779 int ret = 0; 3780 3781 WARN_ON_ONCE(m_len > 1 && !ext4_has_feature_bigalloc(inode->i_sb)); 3782 3783 ret = ext4_map_blocks(handle, inode, map, m_flags); 3784 if (ret < 0 || ret == m_len) 3785 goto out; 3786 /* 3787 * This is a mixed mapping case where we were not able to allocate 3788 * a single contiguous extent. In that case let's reset requested 3789 * mapping and call the slow path. 3790 */ 3791 map->m_lblk = m_lblk; 3792 map->m_len = m_len; 3793 map->m_flags = 0; 3794 3795 /* 3796 * slow path means we have mixed mapping, that means we will need 3797 * to force txn commit. 3798 */ 3799 *force_commit = true; 3800 return ext4_map_blocks_atomic_write_slow(handle, inode, map); 3801 out: 3802 return ret; 3803 } 3804 3805 static int ext4_iomap_alloc(struct inode *inode, struct ext4_map_blocks *map, 3806 unsigned int flags) 3807 { 3808 handle_t *handle; 3809 int ret, dio_credits, m_flags = 0, retries = 0; 3810 bool force_commit = false; 3811 3812 if (flags & IOMAP_NOWAIT) 3813 return -EAGAIN; 3814 3815 /* 3816 * Trim the mapping request to the maximum value that we can map at 3817 * once for direct I/O. 3818 */ 3819 if (map->m_len > DIO_MAX_BLOCKS) 3820 map->m_len = DIO_MAX_BLOCKS; 3821 3822 /* 3823 * journal credits estimation for atomic writes. We call 3824 * ext4_map_blocks(), to find if there could be a mixed mapping. If yes, 3825 * then let's assume the no. of pextents required can be m_len i.e. 3826 * every alternate block can be unwritten and hole. 3827 */ 3828 if (flags & IOMAP_ATOMIC) { 3829 unsigned int orig_mlen = map->m_len; 3830 3831 ret = ext4_map_blocks(NULL, inode, map, 0); 3832 if (ret < 0) 3833 return ret; 3834 if (map->m_len < orig_mlen) { 3835 map->m_len = orig_mlen; 3836 dio_credits = ext4_meta_trans_blocks(inode, map->m_len, 3837 map->m_len, 0); 3838 } else { 3839 dio_credits = ext4_chunk_trans_blocks(inode, 3840 map->m_len); 3841 } 3842 } else { 3843 dio_credits = ext4_chunk_trans_blocks(inode, map->m_len); 3844 } 3845 3846 retry: 3847 /* 3848 * Either we allocate blocks and then don't get an unwritten extent, so 3849 * in that case we have reserved enough credits. Or, the blocks are 3850 * already allocated and unwritten. In that case, the extent conversion 3851 * fits into the credits as well. 3852 */ 3853 handle = ext4_journal_start(inode, EXT4_HT_MAP_BLOCKS, dio_credits); 3854 if (IS_ERR(handle)) 3855 return PTR_ERR(handle); 3856 3857 /* 3858 * DAX and direct I/O are the only two operations that are currently 3859 * supported with IOMAP_WRITE. 3860 */ 3861 WARN_ON(!(flags & (IOMAP_DAX | IOMAP_DIRECT))); 3862 if (flags & IOMAP_DAX) 3863 m_flags = EXT4_GET_BLOCKS_CREATE_ZERO; 3864 /* 3865 * We use i_size instead of i_disksize here because delalloc writeback 3866 * can complete at any point during the I/O and subsequently push the 3867 * i_disksize out to i_size. This could be beyond where direct I/O is 3868 * happening and thus expose allocated blocks to direct I/O reads. 3869 */ 3870 else if (EXT4_LBLK_TO_B(inode, map->m_lblk) >= i_size_read(inode)) 3871 m_flags = EXT4_GET_BLOCKS_CREATE; 3872 else if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) 3873 m_flags = EXT4_GET_BLOCKS_CREATE_UNWRIT_EXT; 3874 3875 if (flags & IOMAP_ATOMIC) 3876 ret = ext4_map_blocks_atomic_write(handle, inode, map, m_flags, 3877 &force_commit); 3878 else 3879 ret = ext4_map_blocks(handle, inode, map, m_flags); 3880 3881 /* 3882 * We cannot fill holes in indirect tree based inodes as that could 3883 * expose stale data in the case of a crash. Use the magic error code 3884 * to fallback to buffered I/O. 3885 */ 3886 if (!m_flags && !ret) 3887 ret = -ENOTBLK; 3888 3889 ext4_journal_stop(handle); 3890 if (ret == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries)) 3891 goto retry; 3892 3893 /* 3894 * Force commit the current transaction if the allocation spans a mixed 3895 * mapping range. This ensures any pending metadata updates (like 3896 * unwritten to written extents conversion) in this range are in 3897 * consistent state with the file data blocks, before performing the 3898 * actual write I/O. If the commit fails, the whole I/O must be aborted 3899 * to prevent any possible torn writes. 3900 */ 3901 if (ret > 0 && force_commit) { 3902 int ret2; 3903 3904 ret2 = ext4_force_commit(inode->i_sb); 3905 if (ret2) 3906 return ret2; 3907 } 3908 3909 return ret; 3910 } 3911 3912 3913 int ext4_iomap_begin(struct inode *inode, loff_t offset, loff_t length, 3914 unsigned flags, struct iomap *iomap, struct iomap *srcmap) 3915 { 3916 int ret; 3917 struct ext4_map_blocks map; 3918 u8 blkbits = inode->i_blkbits; 3919 unsigned int orig_mlen; 3920 3921 if ((offset >> blkbits) > EXT4_MAX_LOGICAL_BLOCK) 3922 return -EINVAL; 3923 3924 if (WARN_ON_ONCE(ext4_has_inline_data(inode))) 3925 return -ERANGE; 3926 3927 /* 3928 * Calculate the first and last logical blocks respectively. 3929 */ 3930 map.m_lblk = offset >> blkbits; 3931 map.m_len = min_t(loff_t, (offset + length - 1) >> blkbits, 3932 EXT4_MAX_LOGICAL_BLOCK) - map.m_lblk + 1; 3933 orig_mlen = map.m_len; 3934 3935 if (flags & IOMAP_WRITE) { 3936 /* 3937 * We check here if the blocks are already allocated, then we 3938 * don't need to start a journal txn and we can directly return 3939 * the mapping information. This could boost performance 3940 * especially in multi-threaded overwrite requests. 3941 */ 3942 if (offset + length <= i_size_read(inode)) { 3943 ret = ext4_map_blocks(NULL, inode, &map, 0); 3944 /* 3945 * For DAX we convert extents to initialized ones before 3946 * copying the data, otherwise we do it after I/O so 3947 * there's no need to call into ext4_iomap_alloc(). 3948 */ 3949 if ((map.m_flags & EXT4_MAP_MAPPED) || 3950 (!(flags & IOMAP_DAX) && 3951 (map.m_flags & EXT4_MAP_UNWRITTEN))) { 3952 /* 3953 * For atomic writes the entire requested 3954 * length should be mapped. 3955 */ 3956 if (ret == orig_mlen || 3957 (!(flags & IOMAP_ATOMIC) && ret > 0)) 3958 goto out; 3959 } 3960 map.m_len = orig_mlen; 3961 } 3962 ret = ext4_iomap_alloc(inode, &map, flags); 3963 } else { 3964 ret = ext4_map_blocks(NULL, inode, &map, 0); 3965 } 3966 3967 if (ret < 0) 3968 return ret; 3969 out: 3970 /* 3971 * Sometimes I/O to an encrypted file has to be broken up to guarantee 3972 * DUN contiguity. Handle this by limiting the length of the mapping 3973 * returned. 3974 */ 3975 map.m_len = fscrypt_limit_io_blocks(inode, map.m_lblk, map.m_len); 3976 3977 /* 3978 * Before returning to iomap, let's ensure the allocated mapping 3979 * covers the entire requested length for atomic writes. 3980 */ 3981 if (flags & IOMAP_ATOMIC) { 3982 if (map.m_len < (length >> blkbits)) { 3983 WARN_ON_ONCE(1); 3984 return -EINVAL; 3985 } 3986 } 3987 ext4_set_iomap(inode, iomap, &map, offset, length, flags); 3988 3989 return 0; 3990 } 3991 3992 DEFINE_IOMAP_ITER_NEXT(ext4_iomap_next, ext4_iomap_begin); 3993 3994 static int ext4_iomap_begin_report(struct inode *inode, loff_t offset, 3995 loff_t length, unsigned int flags, 3996 struct iomap *iomap, struct iomap *srcmap) 3997 { 3998 int ret; 3999 struct ext4_map_blocks map; 4000 u8 blkbits = inode->i_blkbits; 4001 4002 if ((offset >> blkbits) > EXT4_MAX_LOGICAL_BLOCK) 4003 return -EINVAL; 4004 4005 if (ext4_has_inline_data(inode)) { 4006 ret = ext4_inline_data_iomap(inode, iomap); 4007 if (ret != -EAGAIN) { 4008 if (ret == 0 && offset >= iomap->length) 4009 ret = -ENOENT; 4010 return ret; 4011 } 4012 } 4013 4014 /* 4015 * Calculate the first and last logical block respectively. 4016 */ 4017 map.m_lblk = offset >> blkbits; 4018 map.m_len = min_t(loff_t, (offset + length - 1) >> blkbits, 4019 EXT4_MAX_LOGICAL_BLOCK) - map.m_lblk + 1; 4020 4021 /* 4022 * Fiemap callers may call for offset beyond s_bitmap_maxbytes. 4023 * So handle it here itself instead of querying ext4_map_blocks(). 4024 * Since ext4_map_blocks() will warn about it and will return 4025 * -EIO error. 4026 */ 4027 if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) { 4028 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); 4029 4030 if (offset >= sbi->s_bitmap_maxbytes) { 4031 map.m_flags = 0; 4032 goto set_iomap; 4033 } 4034 } 4035 4036 ret = ext4_map_blocks(NULL, inode, &map, 0); 4037 if (ret < 0) 4038 return ret; 4039 set_iomap: 4040 ext4_set_iomap(inode, iomap, &map, offset, length, flags); 4041 4042 return 0; 4043 } 4044 4045 DEFINE_IOMAP_ITER_NEXT(ext4_iomap_next_report, ext4_iomap_begin_report); 4046 4047 static int ext4_iomap_map_blocks(struct inode *inode, loff_t offset, 4048 loff_t length, struct ext4_map_blocks *map, 4049 int flags) 4050 { 4051 u8 blkbits = inode->i_blkbits; 4052 4053 /* inode using the buffer_head buffered I/O path should not go here. */ 4054 if (WARN_ON_ONCE(!ext4_inode_buffered_iomap(inode))) 4055 return -EINVAL; 4056 if ((offset >> blkbits) > EXT4_MAX_LOGICAL_BLOCK) 4057 return -EINVAL; 4058 4059 /* Calculate the first and last logical blocks respectively. */ 4060 map->m_lblk = offset >> blkbits; 4061 map->m_len = min_t(loff_t, (offset + length - 1) >> blkbits, 4062 EXT4_MAX_LOGICAL_BLOCK) - map->m_lblk + 1; 4063 4064 if (flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE) 4065 return ext4_da_map_blocks(inode, map); 4066 4067 return ext4_map_blocks(NULL, inode, map, flags); 4068 } 4069 4070 static int ext4_iomap_buffered_read_begin(struct inode *inode, loff_t offset, 4071 loff_t length, unsigned int flags, struct iomap *iomap, 4072 struct iomap *srcmap) 4073 { 4074 struct ext4_map_blocks map; 4075 int ret; 4076 4077 if (unlikely(ext4_forced_shutdown(inode->i_sb))) 4078 return -EIO; 4079 4080 /* Inline data support is not yet available. */ 4081 if (WARN_ON_ONCE(ext4_has_inline_data(inode))) 4082 return -ERANGE; 4083 4084 ret = ext4_iomap_map_blocks(inode, offset, length, &map, 0); 4085 if (ret < 0) 4086 return ret; 4087 4088 trace_ext4_iomap_buffered_read_begin(inode, &map, offset, length, 4089 flags); 4090 ext4_set_iomap(inode, iomap, &map, offset, length, flags); 4091 return 0; 4092 } 4093 4094 static int ext4_iomap_buffered_do_write_begin(struct inode *inode, 4095 loff_t offset, loff_t length, unsigned int flags, 4096 struct iomap *iomap, struct iomap *srcmap, bool delalloc) 4097 { 4098 int ret, retries = 0; 4099 struct ext4_map_blocks map; 4100 int map_flags; 4101 4102 ret = ext4_emergency_state(inode->i_sb); 4103 if (unlikely(ret)) 4104 return ret; 4105 4106 /* Inline data and non-extent are not supported. */ 4107 if (WARN_ON_ONCE(ext4_has_inline_data(inode))) 4108 return -ERANGE; 4109 if (WARN_ON_ONCE(!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) 4110 return -EINVAL; 4111 if (WARN_ON_ONCE(!(flags & (IOMAP_WRITE | IOMAP_FAULT)))) 4112 return -EINVAL; 4113 4114 map_flags = delalloc ? EXT4_GET_BLOCKS_DELALLOC_RESERVE : 4115 EXT4_GET_BLOCKS_CREATE_UNWRIT_EXT; 4116 retry: 4117 ret = ext4_iomap_map_blocks(inode, offset, length, &map, map_flags); 4118 if (ret == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries)) 4119 goto retry; 4120 if (ret < 0) 4121 return ret; 4122 4123 trace_ext4_iomap_buffered_write_begin(inode, &map, offset, length, 4124 flags); 4125 ext4_set_iomap(inode, iomap, &map, offset, length, flags); 4126 return 0; 4127 } 4128 4129 static int ext4_iomap_buffered_write_begin(struct inode *inode, 4130 loff_t offset, loff_t length, unsigned int flags, 4131 struct iomap *iomap, struct iomap *srcmap) 4132 { 4133 return ext4_iomap_buffered_do_write_begin(inode, offset, length, flags, 4134 iomap, srcmap, false); 4135 } 4136 4137 static int ext4_iomap_buffered_da_write_begin(struct inode *inode, 4138 loff_t offset, loff_t length, unsigned int flags, 4139 struct iomap *iomap, struct iomap *srcmap) 4140 { 4141 return ext4_iomap_buffered_do_write_begin(inode, offset, length, flags, 4142 iomap, srcmap, true); 4143 } 4144 4145 /* 4146 * On write failure, drop the stale delayed allocation range and release 4147 * its reserved space for both start and end blocks. Otherwise, we may 4148 * leave a range of delayed extents covered by a clean folio, which can 4149 * result in inaccurate space reservation accounting. 4150 */ 4151 static void ext4_iomap_punch_delalloc(struct inode *inode, loff_t offset, 4152 loff_t length, struct iomap *iomap) 4153 { 4154 down_write(&EXT4_I(inode)->i_data_sem); 4155 ext4_es_remove_extent(inode, offset >> inode->i_blkbits, 4156 DIV_ROUND_UP_ULL(length, EXT4_BLOCK_SIZE(inode->i_sb))); 4157 up_write(&EXT4_I(inode)->i_data_sem); 4158 } 4159 4160 static int ext4_iomap_buffered_da_write_end(struct inode *inode, loff_t offset, 4161 loff_t length, ssize_t written, 4162 unsigned int flags, 4163 struct iomap *iomap) 4164 { 4165 loff_t start_byte, end_byte; 4166 4167 /* If we didn't reserve the blocks, we're not allowed to punch them. */ 4168 if (iomap->type != IOMAP_DELALLOC || !(iomap->flags & IOMAP_F_NEW)) 4169 return 0; 4170 4171 /* 4172 * iomap_page_mkwrite() will never fail in a way that requires delalloc 4173 * extents that it allocated to be revoked. Hence never try to release 4174 * them here. 4175 */ 4176 if (flags & IOMAP_FAULT) 4177 return 0; 4178 4179 /* Nothing to do if we've written the entire delalloc extent */ 4180 start_byte = iomap_last_written_block(inode, offset, written); 4181 end_byte = round_up(offset + length, i_blocksize(inode)); 4182 if (start_byte >= end_byte) 4183 return 0; 4184 4185 filemap_invalidate_lock(inode->i_mapping); 4186 iomap_write_delalloc_release(inode, start_byte, end_byte, flags, 4187 iomap, ext4_iomap_punch_delalloc); 4188 filemap_invalidate_unlock(inode->i_mapping); 4189 return 0; 4190 } 4191 4192 static int ext4_iomap_zero_begin(struct inode *inode, 4193 loff_t offset, loff_t length, unsigned int flags, 4194 struct iomap *iomap, struct iomap *srcmap) 4195 { 4196 struct iomap_iter *iter = container_of(iomap, struct iomap_iter, iomap); 4197 struct ext4_map_blocks map; 4198 u8 blkbits = inode->i_blkbits; 4199 unsigned int iomap_flags; 4200 int ret; 4201 4202 ret = ext4_emergency_state(inode->i_sb); 4203 if (unlikely(ret)) 4204 return ret; 4205 4206 if (WARN_ON_ONCE(!(flags & IOMAP_ZERO))) 4207 return -EINVAL; 4208 4209 again: 4210 ret = ext4_iomap_map_blocks(inode, offset, length, &map, 0); 4211 if (ret < 0) 4212 return ret; 4213 4214 /* 4215 * Look up dirty folios for unwritten mappings within EOF. Providing 4216 * this bypasses the flush iomap uses to trigger extent conversion 4217 * when unwritten mappings have dirty pagecache in need of zeroing. 4218 */ 4219 iomap_flags = 0; 4220 if (map.m_flags & EXT4_MAP_UNWRITTEN) { 4221 loff_t start = ((loff_t)map.m_lblk) << blkbits; 4222 loff_t end = ((loff_t)map.m_lblk + map.m_len) << blkbits; 4223 unsigned int count; 4224 4225 count = iomap_fill_dirty_folios(iter, &start, end, 4226 &iomap_flags); 4227 if ((start >> blkbits) < map.m_lblk + map.m_len) 4228 map.m_len = (start >> blkbits) - map.m_lblk; 4229 4230 /* 4231 * This can be raced by a concurrent writeback that cleans 4232 * the folio and converts the unwritten extent to written. 4233 * Recheck the mapping after a folio lock round in 4234 * iomap_fill_dirty_folios(). 4235 */ 4236 if (count == 0 && 4237 map.m_seq != READ_ONCE(EXT4_I(inode)->i_es_seq)) 4238 goto again; 4239 } 4240 4241 trace_ext4_iomap_zero_begin(inode, &map, offset, length, flags); 4242 ext4_set_iomap(inode, iomap, &map, offset, length, flags); 4243 iomap->flags |= iomap_flags; 4244 4245 return 0; 4246 } 4247 4248 static DEFINE_IOMAP_ITER_NEXT(ext4_iomap_zero_next, ext4_iomap_zero_begin); 4249 4250 /* 4251 * Since we always allocate unwritten extents, there is no need for 4252 * iomap_end to clean up allocated blocks on a short write. 4253 */ 4254 DEFINE_IOMAP_ITER_NEXT(ext4_iomap_buffered_write_next, 4255 ext4_iomap_buffered_write_begin); 4256 4257 DEFINE_IOMAP_ITER_NEXT_END(ext4_iomap_buffered_da_write_next, 4258 ext4_iomap_buffered_da_write_begin, 4259 ext4_iomap_buffered_da_write_end); 4260 4261 static DEFINE_IOMAP_ITER_NEXT(ext4_iomap_buffered_read_next, 4262 ext4_iomap_buffered_read_begin); 4263 4264 static int ext4_iomap_read_folio(struct file *file, struct folio *folio) 4265 { 4266 iomap_bio_read_folio(folio, ext4_iomap_buffered_read_next); 4267 return 0; 4268 } 4269 4270 static void ext4_iomap_readahead(struct readahead_control *rac) 4271 { 4272 iomap_bio_readahead(rac, ext4_iomap_buffered_read_next); 4273 } 4274 4275 4276 static int ext4_iomap_map_writeback_range(struct iomap_writepage_ctx *wpc, 4277 loff_t offset, unsigned int dirty_len) 4278 { 4279 struct inode *inode = wpc->inode; 4280 struct super_block *sb = inode->i_sb; 4281 struct journal_s *journal = EXT4_SB(sb)->s_journal; 4282 struct ext4_map_blocks map; 4283 ext4_lblk_t index = offset >> inode->i_blkbits; 4284 unsigned int blk_len, blk_end; 4285 int retries = 0; 4286 int ret; 4287 4288 ret = ext4_emergency_state(sb); 4289 if (unlikely(ret)) 4290 return ret; 4291 4292 /* Check validity of the cached writeback mapping. */ 4293 if (offset >= wpc->iomap.offset && 4294 offset < wpc->iomap.offset + wpc->iomap.length && 4295 ext4_iomap_valid(inode, &wpc->iomap)) 4296 return 0; 4297 4298 blk_len = dirty_len >> inode->i_blkbits; 4299 blk_end = umin(wpc->wbc->range_end >> inode->i_blkbits, UINT_MAX - 1); 4300 if (blk_end > index + blk_len) 4301 blk_len = blk_end - index + 1; 4302 4303 retry: 4304 map.m_lblk = index; 4305 map.m_len = min_t(unsigned int, MAX_WRITEPAGES_EXTENT_LEN, blk_len); 4306 ret = ext4_map_blocks(NULL, inode, &map, 4307 EXT4_GET_BLOCKS_CREATE_UNWRIT_EXT | 4308 EXT4_GET_BLOCKS_METADATA_NOFAIL | 4309 EXT4_GET_BLOCKS_IO_SUBMIT | 4310 EXT4_EX_NOCACHE); 4311 if (ret < 0) { 4312 if (ext4_emergency_state(sb)) 4313 return ret; 4314 4315 /* 4316 * Retry transient ENOSPC errors, if 4317 * ext4_count_free_blocks() is non-zero, a commit 4318 * should free up blocks. 4319 */ 4320 if (ret == -ENOSPC && journal && ext4_count_free_clusters(sb) && 4321 ++retries <= 3) { 4322 jbd2_journal_force_commit_nested(journal); 4323 goto retry; 4324 } 4325 4326 ext4_msg(sb, KERN_CRIT, 4327 "Delayed block allocation failed for inode %llu at logical offset %llu with max blocks %u with error %d", 4328 inode->i_ino, (unsigned long long)map.m_lblk, 4329 (unsigned int)map.m_len, -ret); 4330 ext4_msg(sb, KERN_CRIT, 4331 "This should not happen!! Data will be lost\n"); 4332 if (ret == -ENOSPC) 4333 ext4_print_free_blocks(inode); 4334 return ret; 4335 } 4336 4337 trace_ext4_iomap_map_writeback_range(inode, &map, offset, dirty_len, 0); 4338 ext4_set_iomap(inode, &wpc->iomap, &map, offset, dirty_len, 0); 4339 return 0; 4340 } 4341 4342 static void ext4_iomap_discard_folio(struct folio *folio, loff_t pos) 4343 { 4344 struct inode *inode = folio->mapping->host; 4345 loff_t length = folio_pos(folio) + folio_size(folio) - pos; 4346 loff_t pstart, plen; 4347 4348 ext4_iomap_punch_delalloc(inode, pos, length, NULL); 4349 4350 /* 4351 * Clear the disksize-grow-pending state if the zeroed EOF block 4352 * fails to write back and is discarded. 4353 */ 4354 plen = ext4_iomap_get_disksize_pending_range(inode, &pstart); 4355 if (plen && pos <= pstart && folio_next_pos(folio) >= pstart + plen) 4356 ext4_iomap_clear_disksize_pending(inode); 4357 } 4358 4359 static ssize_t ext4_iomap_writeback_range(struct iomap_writepage_ctx *wpc, 4360 struct folio *folio, u64 offset, 4361 unsigned int len, u64 end_pos) 4362 { 4363 ssize_t ret; 4364 4365 ret = ext4_iomap_map_writeback_range(wpc, offset, len); 4366 if (!ret) 4367 ret = iomap_add_to_ioend(wpc, folio, offset, end_pos, len); 4368 if (ret < 0) 4369 ext4_iomap_discard_folio(folio, offset); 4370 return ret; 4371 } 4372 4373 static int ext4_iomap_writeback_submit(struct iomap_writepage_ctx *wpc, 4374 int error) 4375 { 4376 struct iomap_ioend *ioend = wpc->wb_ctx; 4377 struct inode *inode = ioend->io_inode; 4378 struct ext4_inode_info *ei = EXT4_I(inode); 4379 unsigned int blocksize = i_blocksize(inode); 4380 loff_t pstart, plen; 4381 4382 /* 4383 * After I/O completion, a worker needs to be scheduled when: 4384 * 1) Unwritten extents require conversion. 4385 * 2) The file size needs to be extended. 4386 * 3) The journal needs to be aborted due to an I/O error. 4387 */ 4388 if ((ioend->io_flags & IOMAP_IOEND_UNWRITTEN) || 4389 (ioend->io_offset + ioend->io_size > READ_ONCE(ei->i_disksize)) || 4390 test_opt(ioend->io_inode->i_sb, DATA_ERR_ABORT)) 4391 ioend->io_bio.bi_end_io = ext4_iomap_end_bio; 4392 4393 /* 4394 * Mark the I/O as DISKSIZE_GROW_IO by setting io_private to 4395 * EXT4_IOMAP_IOEND_DISKSIZE_GROW_IO if it covers the pending range. 4396 * Such I/O will allow or trigger i_disksize advancement in the 4397 * ioend worker. 4398 */ 4399 plen = ext4_iomap_get_disksize_pending_range(inode, &pstart); 4400 if (plen && 4401 round_down(ioend->io_offset, blocksize) <= pstart && 4402 round_up(ioend->io_offset + ioend->io_size, blocksize) >= 4403 pstart + plen) { 4404 trace_ext4_iomap_wb_disksize_pending_submit(inode, 4405 ioend->io_offset, ioend->io_size); 4406 ioend->io_bio.bi_end_io = ext4_iomap_end_bio; 4407 ioend->io_private = (void *)EXT4_IOMAP_IOEND_DISKSIZE_GROW_IO; 4408 } 4409 4410 /* 4411 * ext4_iomap_end_bio() always defers endio processing, disable 4412 * generic BIO in task to avoid double deferral since we will use 4413 * a private defer endio handler in process context. 4414 * 4415 * TODO: Switch all defer handlers to the generic bio complete 4416 * in task framework. 4417 */ 4418 if (ioend->io_bio.bi_end_io) 4419 bio_clear_flag(&ioend->io_bio, BIO_COMPLETE_IN_TASK); 4420 4421 return iomap_ioend_writeback_submit(wpc, error); 4422 } 4423 4424 static const struct iomap_writeback_ops ext4_writeback_ops = { 4425 .writeback_range = ext4_iomap_writeback_range, 4426 .writeback_submit = ext4_iomap_writeback_submit, 4427 }; 4428 4429 /* 4430 * If the current writeback range begins after the pending zeroed EOF 4431 * block range which straddles i_disksize, issue a separate writeback to 4432 * flush it first, so as to avoid prolonged waiting. 4433 */ 4434 static void ext4_iomap_wb_submit_zeroed_eof(struct inode *inode, 4435 struct writeback_control *wbc) 4436 { 4437 struct address_space *mapping = inode->i_mapping; 4438 loff_t pstart, plen, range_start; 4439 4440 if (wbc->range_cyclic) 4441 range_start = (loff_t)mapping->writeback_index << PAGE_SHIFT; 4442 else 4443 range_start = wbc->range_start; 4444 4445 plen = ext4_iomap_get_disksize_pending_range(inode, &pstart); 4446 if (!plen) 4447 return; 4448 if (round_down(range_start, mapping_min_folio_nrbytes(mapping)) < 4449 pstart + plen) 4450 return; 4451 4452 /* Keep the caller's sync mode to avoid stalling the background flusher. */ 4453 if (wbc->sync_mode == WB_SYNC_ALL) 4454 filemap_fdatawrite_range(mapping, pstart, pstart + plen - 1); 4455 else 4456 filemap_flush_range(mapping, pstart, pstart + plen - 1); 4457 } 4458 4459 static int ext4_iomap_writepages(struct address_space *mapping, 4460 struct writeback_control *wbc) 4461 { 4462 struct inode *inode = mapping->host; 4463 struct super_block *sb = inode->i_sb; 4464 long nr = wbc->nr_to_write; 4465 int alloc_ctx, ret; 4466 struct iomap_writepage_ctx wpc = { 4467 .inode = inode, 4468 .wbc = wbc, 4469 .ops = &ext4_writeback_ops, 4470 }; 4471 4472 ret = ext4_emergency_state(sb); 4473 if (unlikely(ret)) 4474 return ret; 4475 4476 /* 4477 * Submit the pending zeroed EOF block range if the entire 4478 * writeback range lies beyond it. 4479 */ 4480 ext4_iomap_wb_submit_zeroed_eof(inode, wbc); 4481 4482 alloc_ctx = ext4_writepages_down_read(sb); 4483 /* 4484 * Does ext4_change_inode_journal_flag() change the inode's 4485 * buffered I/O path? 4486 */ 4487 if (!ext4_inode_buffered_iomap(inode)) { 4488 ext4_writepages_up_read(sb, alloc_ctx); 4489 return ret; 4490 } 4491 4492 trace_ext4_writepages(inode, wbc); 4493 ret = iomap_writepages(&wpc); 4494 trace_ext4_writepages_result(inode, wbc, ret, nr - wbc->nr_to_write); 4495 ext4_writepages_up_read(sb, alloc_ctx); 4496 4497 return ret; 4498 } 4499 4500 /* 4501 * For data=journal mode, folio should be marked dirty only when it was 4502 * writeably mapped. When that happens, it was already attached to the 4503 * transaction and marked as jbddirty (we take care of this in 4504 * ext4_page_mkwrite()). On transaction commit, we writeprotect page mappings 4505 * so we should have nothing to do here, except for the case when someone 4506 * had the page pinned and dirtied the page through this pin (e.g. by doing 4507 * direct IO to it). In that case we'd need to attach buffers here to the 4508 * transaction but we cannot due to lock ordering. We cannot just dirty the 4509 * folio and leave attached buffers clean, because the buffers' dirty state is 4510 * "definitive". We cannot just set the buffers dirty or jbddirty because all 4511 * the journalling code will explode. So what we do is to mark the folio 4512 * "pending dirty" and next time ext4_writepages() is called, attach buffers 4513 * to the transaction appropriately. 4514 */ 4515 static bool ext4_journalled_dirty_folio(struct address_space *mapping, 4516 struct folio *folio) 4517 { 4518 WARN_ON_ONCE(!folio_buffers(folio)); 4519 if (folio_maybe_dma_pinned(folio)) 4520 folio_set_checked(folio); 4521 return filemap_dirty_folio(mapping, folio); 4522 } 4523 4524 static bool ext4_dirty_folio(struct address_space *mapping, struct folio *folio) 4525 { 4526 WARN_ON_ONCE(!folio_test_locked(folio) && !folio_test_dirty(folio)); 4527 WARN_ON_ONCE(!folio_buffers(folio)); 4528 return block_dirty_folio(mapping, folio); 4529 } 4530 4531 static int ext4_iomap_swap_activate(struct swap_info_struct *sis, 4532 struct file *file, sector_t *span) 4533 { 4534 return iomap_swapfile_activate(sis, file, span, 4535 ext4_iomap_next_report); 4536 } 4537 4538 static const struct address_space_operations ext4_aops = { 4539 .read_folio = ext4_read_folio, 4540 .readahead = ext4_readahead, 4541 .writepages = ext4_writepages, 4542 .write_begin = ext4_write_begin, 4543 .write_end = ext4_write_end, 4544 .dirty_folio = ext4_dirty_folio, 4545 .bmap = ext4_bmap, 4546 .invalidate_folio = ext4_invalidate_folio, 4547 .release_folio = ext4_release_folio, 4548 .migrate_folio = buffer_migrate_folio, 4549 .is_partially_uptodate = block_is_partially_uptodate, 4550 .error_remove_folio = generic_error_remove_folio, 4551 .swap_activate = ext4_iomap_swap_activate, 4552 }; 4553 4554 static const struct address_space_operations ext4_journalled_aops = { 4555 .read_folio = ext4_read_folio, 4556 .readahead = ext4_readahead, 4557 .writepages = ext4_writepages, 4558 .write_begin = ext4_write_begin, 4559 .write_end = ext4_journalled_write_end, 4560 .dirty_folio = ext4_journalled_dirty_folio, 4561 .bmap = ext4_bmap, 4562 .invalidate_folio = ext4_journalled_invalidate_folio, 4563 .release_folio = ext4_release_folio, 4564 .migrate_folio = buffer_migrate_folio_norefs, 4565 .is_partially_uptodate = block_is_partially_uptodate, 4566 .error_remove_folio = generic_error_remove_folio, 4567 .swap_activate = ext4_iomap_swap_activate, 4568 }; 4569 4570 static const struct address_space_operations ext4_da_aops = { 4571 .read_folio = ext4_read_folio, 4572 .readahead = ext4_readahead, 4573 .writepages = ext4_writepages, 4574 .write_begin = ext4_da_write_begin, 4575 .write_end = ext4_da_write_end, 4576 .dirty_folio = ext4_dirty_folio, 4577 .bmap = ext4_bmap, 4578 .invalidate_folio = ext4_invalidate_folio, 4579 .release_folio = ext4_release_folio, 4580 .migrate_folio = buffer_migrate_folio, 4581 .is_partially_uptodate = block_is_partially_uptodate, 4582 .error_remove_folio = generic_error_remove_folio, 4583 .swap_activate = ext4_iomap_swap_activate, 4584 }; 4585 4586 static const struct address_space_operations ext4_iomap_aops = { 4587 .read_folio = ext4_iomap_read_folio, 4588 .readahead = ext4_iomap_readahead, 4589 .writepages = ext4_iomap_writepages, 4590 .dirty_folio = iomap_dirty_folio, 4591 .bmap = ext4_bmap, 4592 .invalidate_folio = iomap_invalidate_folio, 4593 .release_folio = iomap_release_folio, 4594 .migrate_folio = filemap_migrate_folio, 4595 .is_partially_uptodate = iomap_is_partially_uptodate, 4596 .error_remove_folio = generic_error_remove_folio, 4597 .swap_activate = ext4_iomap_swap_activate, 4598 }; 4599 4600 static const struct address_space_operations ext4_dax_aops = { 4601 .writepages = ext4_dax_writepages, 4602 .dirty_folio = noop_dirty_folio, 4603 .bmap = ext4_bmap, 4604 .swap_activate = ext4_iomap_swap_activate, 4605 }; 4606 4607 void ext4_set_aops(struct inode *inode) 4608 { 4609 switch (ext4_inode_journal_mode(inode)) { 4610 case EXT4_INODE_ORDERED_DATA_MODE: 4611 case EXT4_INODE_WRITEBACK_DATA_MODE: 4612 break; 4613 case EXT4_INODE_JOURNAL_DATA_MODE: 4614 inode->i_mapping->a_ops = &ext4_journalled_aops; 4615 return; 4616 default: 4617 BUG(); 4618 } 4619 if (IS_DAX(inode)) 4620 inode->i_mapping->a_ops = &ext4_dax_aops; 4621 else if (ext4_inode_buffered_iomap(inode)) 4622 inode->i_mapping->a_ops = &ext4_iomap_aops; 4623 else if (test_opt(inode->i_sb, DELALLOC)) 4624 inode->i_mapping->a_ops = &ext4_da_aops; 4625 else 4626 inode->i_mapping->a_ops = &ext4_aops; 4627 } 4628 4629 /* 4630 * Here we can't skip an unwritten buffer even though it usually reads zero 4631 * because it might have data in pagecache (eg, if called from ext4_zero_range, 4632 * ext4_punch_hole, etc) which needs to be properly zeroed out. Otherwise a 4633 * racing writeback can come later and flush the stale pagecache to disk. 4634 * 4635 * Return the loaded bh if it actually needs zeroing - in written, dirty 4636 * unwritten, or delalloc state. Return NULL if it's clean (i.e., a hole or 4637 * a clean unwritten block). 4638 */ 4639 static struct buffer_head *ext4_load_tail_bh(struct inode *inode, loff_t from) 4640 { 4641 unsigned int offset, blocksize, pos; 4642 ext4_lblk_t iblock; 4643 struct address_space *mapping = inode->i_mapping; 4644 struct buffer_head *bh; 4645 struct folio *folio; 4646 int err = 0; 4647 4648 folio = __filemap_get_folio(mapping, from >> PAGE_SHIFT, 4649 FGP_WRITEBEGIN | FGP_ACCESSED, 4650 mapping_gfp_constraint(mapping, ~__GFP_FS)); 4651 if (IS_ERR(folio)) 4652 return ERR_CAST(folio); 4653 4654 blocksize = inode->i_sb->s_blocksize; 4655 4656 iblock = EXT4_PG_TO_LBLK(inode, folio->index); 4657 4658 bh = folio_buffers(folio); 4659 if (!bh) 4660 bh = create_empty_buffers(folio, blocksize, 0); 4661 4662 /* Find the buffer that contains "offset" */ 4663 offset = offset_in_folio(folio, from); 4664 pos = blocksize; 4665 while (offset >= pos) { 4666 bh = bh->b_this_page; 4667 iblock++; 4668 pos += blocksize; 4669 } 4670 if (buffer_freed(bh)) { 4671 BUFFER_TRACE(bh, "freed: skip"); 4672 goto unlock; 4673 } 4674 if (!buffer_mapped(bh)) { 4675 BUFFER_TRACE(bh, "unmapped"); 4676 err = ext4_get_block(inode, iblock, bh, 0); 4677 if (err < 0) 4678 goto unlock; 4679 /* 4680 * It's a hole or a clean unwritten block - nothing to do. 4681 * Note that a lookup-only get_block (without 4682 * EXT4_GET_BLOCKS_CREATE) never sets BH_Mapped for clean 4683 * unwritten extents. 4684 */ 4685 if (!buffer_mapped(bh)) { 4686 BUFFER_TRACE(bh, "still unmapped"); 4687 goto unlock; 4688 } 4689 } 4690 4691 /* Ok, it's mapped. Make sure it's up-to-date */ 4692 if (folio_test_uptodate(folio)) 4693 set_buffer_uptodate(bh); 4694 4695 if (!buffer_uptodate(bh)) { 4696 err = ext4_read_bh_lock(bh, 0, true); 4697 if (err) 4698 goto unlock; 4699 } 4700 return bh; 4701 4702 unlock: 4703 folio_unlock(folio); 4704 folio_put(folio); 4705 return err ? ERR_PTR(err) : NULL; 4706 } 4707 4708 static int ext4_block_do_zero_range(struct inode *inode, loff_t from, 4709 loff_t length, bool *did_zero, 4710 bool *zero_written) 4711 { 4712 struct buffer_head *bh; 4713 struct folio *folio; 4714 4715 bh = ext4_load_tail_bh(inode, from); 4716 if (IS_ERR_OR_NULL(bh)) 4717 return PTR_ERR_OR_ZERO(bh); 4718 4719 folio = bh->b_folio; 4720 folio_zero_range(folio, offset_in_folio(folio, from), length); 4721 BUFFER_TRACE(bh, "zeroed end of block"); 4722 4723 mark_buffer_dirty(bh); 4724 if (did_zero) 4725 *did_zero = true; 4726 if (zero_written && !buffer_unwritten(bh) && !buffer_delay(bh)) 4727 *zero_written = true; 4728 4729 folio_unlock(folio); 4730 folio_put(folio); 4731 return 0; 4732 } 4733 4734 static int ext4_block_journalled_zero_range(struct inode *inode, loff_t from, 4735 loff_t length, bool *did_zero) 4736 { 4737 struct buffer_head *bh; 4738 struct folio *folio; 4739 handle_t *handle; 4740 int err; 4741 4742 handle = ext4_journal_start(inode, EXT4_HT_MISC, 1); 4743 if (IS_ERR(handle)) 4744 return PTR_ERR(handle); 4745 4746 bh = ext4_load_tail_bh(inode, from); 4747 if (IS_ERR_OR_NULL(bh)) { 4748 err = PTR_ERR_OR_ZERO(bh); 4749 goto out_handle; 4750 } 4751 folio = bh->b_folio; 4752 4753 BUFFER_TRACE(bh, "get write access"); 4754 err = ext4_journal_get_write_access(handle, inode->i_sb, bh, 4755 EXT4_JTR_NONE); 4756 if (err) 4757 goto out; 4758 4759 folio_zero_range(folio, offset_in_folio(folio, from), length); 4760 BUFFER_TRACE(bh, "zeroed end of block"); 4761 4762 err = ext4_dirty_journalled_data(handle, bh); 4763 if (err) 4764 goto out; 4765 4766 if (did_zero) 4767 *did_zero = true; 4768 out: 4769 folio_unlock(folio); 4770 folio_put(folio); 4771 out_handle: 4772 ext4_journal_stop(handle); 4773 return err; 4774 } 4775 4776 static int ext4_block_iomap_zero_range(struct inode *inode, loff_t from, 4777 loff_t length, bool *did_zero, 4778 bool *zero_written) 4779 { 4780 int ret; 4781 4782 /* 4783 * Zeroing out under an active handle can cause deadlock since 4784 * the order of acquiring the folio lock and starting a handle is 4785 * inconsistent with the iomap writeback procedure. 4786 */ 4787 if (WARN_ON_ONCE(ext4_handle_valid(journal_current_handle()))) 4788 return -EINVAL; 4789 4790 /* The zeroing scope should not extend across a block. */ 4791 if (WARN_ON_ONCE((from >> inode->i_blkbits) != 4792 ((from + length - 1) >> inode->i_blkbits))) 4793 return -EINVAL; 4794 4795 if (!(EXT4_SB(inode->i_sb)->s_mount_state & EXT4_ORPHAN_FS) && 4796 !(inode_state_read_once(inode) & (I_NEW | I_FREEING))) 4797 WARN_ON_ONCE(!inode_is_locked(inode) && 4798 !rwsem_is_locked(&inode->i_mapping->invalidate_lock)); 4799 4800 ret = iomap_zero_range(inode, from, length, did_zero, 4801 ext4_iomap_zero_next, &ext4_iomap_write_ops, 4802 NULL); 4803 if (ret) 4804 return ret; 4805 4806 /* 4807 * TODO: The iomap does not distinguish between different types 4808 * of zeroing operations. So we always set zero_written whenever 4809 * zeroing is performed, which may cause unnecessary folio 4810 * flushing when zeroing occurs on delayed-allocated blocks. 4811 */ 4812 if (did_zero && zero_written) 4813 *zero_written = *did_zero; 4814 4815 return 0; 4816 } 4817 4818 /* 4819 * Zeros out a mapping of length 'length' starting from file offset 4820 * 'from'. The range to be zero'd must be contained with in one block. 4821 * If the specified range exceeds the end of the block it will be 4822 * shortened to end of the block that corresponds to 'from'. 4823 */ 4824 static int ext4_block_zero_range(struct inode *inode, 4825 loff_t from, loff_t length, bool *did_zero, 4826 bool *zero_written) 4827 { 4828 unsigned blocksize = inode->i_sb->s_blocksize; 4829 unsigned int max = blocksize - (from & (blocksize - 1)); 4830 4831 /* 4832 * correct length if it does not fall between 4833 * 'from' and the end of the block 4834 */ 4835 if (length > max || length < 0) 4836 length = max; 4837 4838 if (IS_DAX(inode)) { 4839 return dax_zero_range(inode, from, length, did_zero, 4840 ext4_iomap_next); 4841 } else if (ext4_should_journal_data(inode)) { 4842 return ext4_block_journalled_zero_range(inode, from, length, 4843 did_zero); 4844 } else if (ext4_inode_buffered_iomap(inode)) { 4845 return ext4_block_iomap_zero_range(inode, from, length, 4846 did_zero, zero_written); 4847 } 4848 return ext4_block_do_zero_range(inode, from, length, did_zero, 4849 zero_written); 4850 } 4851 4852 /* 4853 * Inodes using the iomap buffered I/O path do not use data=ordered mode. 4854 * Therefore, we mark the inode as disksize-grow-pending after zeroing the 4855 * EOF block. The zeroed block will be submitted before any subsequent 4856 * data. 4857 * 4858 * In the I/O completion path, ext4_iomap_wb_disksize_pending_wait() will 4859 * wait for I/O completion before advancing i_disksize if the write 4860 * extends beyond the zeroed boundary. 4861 * 4862 * When zeroed I/O is in progress, operations that extend i_disksize are 4863 * handled as follows: 4864 * 4865 * - Truncate up, append fallocate and zero_range: 4866 * Defer the update. The file size will be updated to i_size by the 4867 * end_io handler once the ongoing pending I/O completes. 4868 * 4869 * - Insert range and collapse range operations: 4870 * Wait synchronously for the relevant I/O to complete before updating 4871 * i_disksize. 4872 */ 4873 static int ext4_iomap_mark_disksize_pending(struct inode *inode, loff_t from) 4874 { 4875 struct folio *folio; 4876 4877 folio = filemap_lock_folio(inode->i_mapping, from >> PAGE_SHIFT); 4878 if (IS_ERR(folio)) 4879 /* Already in writeback and cleared? */ 4880 return PTR_ERR(folio) == -ENOENT ? 0 : PTR_ERR(folio); 4881 4882 /* 4883 * Ensure that in-flight writeback, possibly started after 4884 * iomap_zero_range() unlocked the folio, has completed. Without 4885 * this wait folio_test_dirty() below may miss the zeroed data 4886 * (writeback clears PG_dirty), causing us to skip the 4887 * disksize-grow-pending tracking and potentially expose stale 4888 * on-disk data. 4889 */ 4890 folio_wait_writeback(folio); 4891 WARN_ON_ONCE(folio_test_writeback(folio)); 4892 4893 /* 4894 * If the zeroed range does not overlap the existing on-disk tail 4895 * block, the zeroed data lies beyond the currently on-disk data 4896 * and will be written back before i_disksize is advanced past it, 4897 * so no stale data can be exposed. 4898 * 4899 * Checking i_disksize here (after folio_wait_writeback()) is 4900 * necessary and safe. If a delalloc writeback of this folio was 4901 * in-flight, it could be raced by a concurrent mmap write which 4902 * corrupts the tail block beyond i_size but the i_disksize is not 4903 * advanced. folio_wait_writeback() has waited for its completion 4904 * and the ioend has advanced i_disksize accordingly. If writeback 4905 * had not started, we don't need to mark any pending state because 4906 * any future writeback will carry the pagecache content that now 4907 * includes the zeroed data, so no stale data can appear on disk 4908 * even without the pending tracking. 4909 */ 4910 if (from >= round_up(READ_ONCE(EXT4_I(inode)->i_disksize), 4911 i_blocksize(inode))) 4912 goto out; 4913 4914 /* 4915 * Mark the inode as disksize-grow-pending. The zeroed block will 4916 * be written out by the generic writepages cycle or any other 4917 * syncing operation. 4918 * 4919 * Multiple overlapping unaligned EOF writes should not happen, 4920 * because we only mark the pending state after zeroing the on-disk 4921 * EOF block, and i_disksize can only be updated after the previous 4922 * zeroed pending block has been written back or the dirty folio 4923 * has been discarded. 4924 */ 4925 if (likely(folio_test_dirty(folio) && 4926 !ext4_test_inode_state(inode, 4927 EXT4_STATE_DISKSIZE_GROW_PENDING))) { 4928 trace_ext4_iomap_mark_disksize_pending(inode); 4929 ext4_set_inode_state(inode, EXT4_STATE_DISKSIZE_GROW_PENDING); 4930 } 4931 out: 4932 folio_unlock(folio); 4933 folio_put(folio); 4934 return 0; 4935 } 4936 4937 /* 4938 * Submit and wait for the pending zeroed EOF block range to complete 4939 * if the given range [@offset, @end) fully covers it. Must be called 4940 * outside the context of an active journal handle and hold the i_rwsem. 4941 */ 4942 int ext4_iomap_sync_zeroed_eof(struct inode *inode, loff_t offset, loff_t end) 4943 { 4944 loff_t pstart, plen; 4945 int ret; 4946 4947 if (!ext4_inode_buffered_iomap(inode)) 4948 return 0; 4949 4950 plen = ext4_iomap_get_disksize_pending_range(inode, &pstart); 4951 if (!plen || offset > pstart || end < pstart + plen) 4952 return 0; 4953 4954 ret = filemap_write_and_wait_range(inode->i_mapping, pstart, 4955 pstart + plen - 1); 4956 if (ret) 4957 return ret; 4958 4959 /* 4960 * The pending bit should have been cleared by the ioend worker, 4961 * which runs before the folio writeback flag is cleared. The 4962 * caller holds i_rwsem, so no concurrent ext4_block_zero_eof() 4963 * can re-set it. 4964 */ 4965 WARN_ON_ONCE(ext4_test_inode_state(inode, 4966 EXT4_STATE_DISKSIZE_GROW_PENDING)); 4967 return 0; 4968 } 4969 4970 /* 4971 * Zero out a mapping from file offset 'from' up to the end of the block 4972 * which corresponds to 'from' or to the given 'end' inside this block. 4973 * This required during truncate up and performing append writes. We need 4974 * to physically zero the tail end of that block so it doesn't yield old 4975 * data if the file is grown. 4976 */ 4977 int ext4_block_zero_eof(struct inode *inode, loff_t from, loff_t end) 4978 { 4979 unsigned int blocksize = i_blocksize(inode); 4980 unsigned int offset; 4981 loff_t length = end - from; 4982 bool did_zero = false; 4983 bool zero_written = false; 4984 int err; 4985 4986 offset = from & (blocksize - 1); 4987 if (!offset || from >= end) 4988 return 0; 4989 /* 4990 * Inline data has no tail block to zero out. Note that a race with 4991 * ext4_page_mkwrite() converting inline data to an extent without 4992 * holding i_rwsem is safe, as that path zeroes the full block before 4993 * copying in the inline data. 4994 */ 4995 if (ext4_has_inline_data(inode)) 4996 return 0; 4997 /* If we are processing an encrypted inode during orphan list handling */ 4998 if (IS_ENCRYPTED(inode) && !fscrypt_has_encryption_key(inode)) 4999 return 0; 5000 5001 if (length > blocksize - offset) 5002 length = blocksize - offset; 5003 5004 err = ext4_block_zero_range(inode, from, length, 5005 &did_zero, &zero_written); 5006 if (err) 5007 return err; 5008 /* 5009 * It's necessary to order zeroed data before update i_disksize when 5010 * truncating up or performing an append write, because there might be 5011 * exposing stale on-disk data which may caused by concurrent post-EOF 5012 * mmap write during folio writeback. 5013 */ 5014 if (did_zero && zero_written && !IS_DAX(inode)) { 5015 if (ext4_should_order_data(inode)) { 5016 handle_t *handle; 5017 5018 handle = ext4_journal_start(inode, EXT4_HT_MISC, 1); 5019 if (IS_ERR(handle)) 5020 return PTR_ERR(handle); 5021 5022 err = ext4_jbd2_inode_add_write(handle, inode, from, 5023 length); 5024 ext4_journal_stop(handle); 5025 } else if (ext4_inode_buffered_iomap(inode)) 5026 err = ext4_iomap_mark_disksize_pending(inode, from); 5027 if (err) 5028 return err; 5029 } 5030 5031 trace_ext4_block_zero_eof(inode, from, length, did_zero, zero_written); 5032 return 0; 5033 } 5034 5035 /* 5036 * Zero out the unaligned head and tail of the [lstart, lstart+length) 5037 * range. 5038 * 5039 * On return, @partial_zeroed records which edges actually got 5040 * partial-zeroed. Set EXT4_PARTIAL_ZERO_START/EXT4_PARTIAL_ZERO_END if 5041 * the head/tail block got actually partially zeroed (in written, dirty 5042 * unwritten or delalloc state). Cleared if the head/tail block is a 5043 * hole or a clean unwritten block, in which case there is nothing that 5044 * needs zeroing. When the head and tail land in the same block, both 5045 * bits are set together on a successful zeroing. 5046 */ 5047 int ext4_zero_partial_blocks(struct inode *inode, loff_t lstart, loff_t length, 5048 unsigned int *partial_zeroed) 5049 { 5050 struct super_block *sb = inode->i_sb; 5051 unsigned partial_start, partial_end; 5052 ext4_fsblk_t start, end; 5053 loff_t byte_end = (lstart + length - 1); 5054 bool did_zero = false; 5055 int err = 0; 5056 5057 partial_start = lstart & (sb->s_blocksize - 1); 5058 partial_end = byte_end & (sb->s_blocksize - 1); 5059 5060 start = lstart >> sb->s_blocksize_bits; 5061 end = byte_end >> sb->s_blocksize_bits; 5062 5063 /* Handle partial zero within the single block */ 5064 if (start == end && 5065 (partial_start || (partial_end != sb->s_blocksize - 1))) { 5066 err = ext4_block_zero_range(inode, lstart, length, &did_zero, 5067 NULL); 5068 if (did_zero) 5069 *partial_zeroed |= (EXT4_PARTIAL_ZERO_START | 5070 EXT4_PARTIAL_ZERO_END); 5071 return err; 5072 } 5073 /* Handle partial zero out on the start of the range */ 5074 if (partial_start) { 5075 err = ext4_block_zero_range(inode, lstart, sb->s_blocksize, 5076 &did_zero, NULL); 5077 if (err) 5078 return err; 5079 if (did_zero) 5080 *partial_zeroed |= EXT4_PARTIAL_ZERO_START; 5081 } 5082 /* Handle partial zero out on the end of the range */ 5083 if (partial_end != sb->s_blocksize - 1) { 5084 did_zero = false; 5085 err = ext4_block_zero_range(inode, byte_end - partial_end, 5086 partial_end + 1, &did_zero, NULL); 5087 if (err) 5088 return err; 5089 if (did_zero) 5090 *partial_zeroed |= EXT4_PARTIAL_ZERO_END; 5091 } 5092 return err; 5093 } 5094 5095 int ext4_can_truncate(struct inode *inode) 5096 { 5097 if (S_ISREG(inode->i_mode)) 5098 return 1; 5099 if (S_ISDIR(inode->i_mode)) 5100 return 1; 5101 if (S_ISLNK(inode->i_mode)) 5102 return !ext4_inode_is_fast_symlink(inode); 5103 return 0; 5104 } 5105 5106 /* 5107 * We have to make sure i_disksize gets properly updated before we truncate 5108 * page cache due to hole punching or zero range. Otherwise i_disksize update 5109 * can get lost as it may have been postponed to submission of writeback but 5110 * that will never happen if we remove the folio containing i_size from the 5111 * page cache. Also if we punch hole within i_size but above i_disksize, 5112 * following ext4_page_mkwrite() may mistakenly allocate written blocks over 5113 * the hole and thus introduce allocated blocks beyond i_disksize which is 5114 * not allowed (e2fsck would complain in case of crash). 5115 */ 5116 int ext4_update_disksize_before_punch(struct inode *inode, loff_t offset, 5117 loff_t len) 5118 { 5119 handle_t *handle; 5120 int ret; 5121 5122 loff_t size = i_size_read(inode); 5123 5124 WARN_ON(!inode_is_locked(inode)); 5125 if (offset > size) 5126 return 0; 5127 5128 /* 5129 * We are going to punch the pending zeroed EOF block, persist 5130 * it to ensure i_disksize can be safely updated thereafter. 5131 */ 5132 ret = ext4_iomap_sync_zeroed_eof(inode, offset, offset + len); 5133 if (ret) 5134 return ret; 5135 5136 if (offset + len < size) 5137 size = offset + len; 5138 if (EXT4_I(inode)->i_disksize >= size) 5139 return 0; 5140 5141 handle = ext4_journal_start(inode, EXT4_HT_MISC, 1); 5142 if (IS_ERR(handle)) 5143 return PTR_ERR(handle); 5144 ext4_update_i_disksize(inode, size); 5145 ret = ext4_mark_inode_dirty(handle, inode); 5146 ext4_journal_stop(handle); 5147 5148 return ret; 5149 } 5150 5151 static inline void ext4_truncate_folio(struct inode *inode, 5152 loff_t start, loff_t end) 5153 { 5154 unsigned long blocksize = i_blocksize(inode); 5155 struct folio *folio; 5156 5157 /* Nothing to be done if no complete block needs to be truncated. */ 5158 if (round_up(start, blocksize) >= round_down(end, blocksize)) 5159 return; 5160 5161 folio = filemap_lock_folio(inode->i_mapping, start >> PAGE_SHIFT); 5162 if (IS_ERR(folio)) 5163 return; 5164 5165 if (folio_mkclean(folio)) 5166 folio_mark_dirty(folio); 5167 folio_unlock(folio); 5168 folio_put(folio); 5169 } 5170 5171 int ext4_truncate_page_cache_block_range(struct inode *inode, 5172 loff_t start, loff_t end) 5173 { 5174 unsigned long blocksize = i_blocksize(inode); 5175 int ret; 5176 5177 /* 5178 * For journalled data we need to write (and checkpoint) pages 5179 * before discarding page cache to avoid inconsitent data on disk 5180 * in case of crash before freeing or unwritten converting trans 5181 * is committed. 5182 */ 5183 if (ext4_should_journal_data(inode)) { 5184 ret = filemap_write_and_wait_range(inode->i_mapping, start, 5185 end - 1); 5186 if (ret) 5187 return ret; 5188 goto truncate_pagecache; 5189 } 5190 5191 /* 5192 * If the block size is less than the page size, the file's mapped 5193 * blocks within one page could be freed or converted to unwritten. 5194 * So it's necessary to remove writable userspace mappings, and then 5195 * ext4_page_mkwrite() can be called during subsequent write access 5196 * to these partial folios. 5197 */ 5198 if (!IS_ALIGNED(start | end, PAGE_SIZE) && 5199 blocksize < PAGE_SIZE && start < inode->i_size) { 5200 loff_t page_boundary = round_up(start, PAGE_SIZE); 5201 5202 ext4_truncate_folio(inode, start, min(page_boundary, end)); 5203 if (end > page_boundary) 5204 ext4_truncate_folio(inode, 5205 round_down(end, PAGE_SIZE), end); 5206 } 5207 5208 truncate_pagecache: 5209 truncate_pagecache_range(inode, start, end - 1); 5210 return 0; 5211 } 5212 5213 static void ext4_wait_dax_page(struct inode *inode) 5214 { 5215 filemap_invalidate_unlock(inode->i_mapping); 5216 schedule(); 5217 filemap_invalidate_lock(inode->i_mapping); 5218 } 5219 5220 int ext4_break_layouts(struct inode *inode) 5221 { 5222 if (WARN_ON_ONCE(!rwsem_is_locked(&inode->i_mapping->invalidate_lock))) 5223 return -EINVAL; 5224 5225 return dax_break_layout_inode(inode, ext4_wait_dax_page); 5226 } 5227 5228 /* 5229 * ext4_punch_hole: punches a hole in a file by releasing the blocks 5230 * associated with the given offset and length 5231 * 5232 * @inode: File inode 5233 * @offset: The offset where the hole will begin 5234 * @len: The length of the hole 5235 * 5236 * Returns: 0 on success or negative on failure 5237 */ 5238 5239 int ext4_punch_hole(struct file *file, loff_t offset, loff_t length) 5240 { 5241 struct inode *inode = file_inode(file); 5242 struct super_block *sb = inode->i_sb; 5243 ext4_lblk_t start_lblk, end_lblk; 5244 loff_t max_end = sb->s_maxbytes; 5245 loff_t end = offset + length; 5246 handle_t *handle; 5247 unsigned int credits; 5248 unsigned int partial_zeroed = 0; 5249 int ret; 5250 5251 trace_ext4_punch_hole(inode, offset, length, 0); 5252 WARN_ON_ONCE(!inode_is_locked(inode)); 5253 5254 /* 5255 * For indirect-block based inodes, make sure that the hole within 5256 * one block before last range. 5257 */ 5258 if (!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) 5259 max_end = EXT4_SB(sb)->s_bitmap_maxbytes - sb->s_blocksize; 5260 5261 /* No need to punch hole beyond i_size */ 5262 if (offset >= inode->i_size || offset >= max_end) 5263 return 0; 5264 5265 /* 5266 * If the hole extends beyond i_size, set the hole to end after 5267 * the block that contains i_size to save pointless tail block zeroing. 5268 */ 5269 if (end >= inode->i_size) 5270 end = round_up(inode->i_size, sb->s_blocksize); 5271 if (end > max_end) 5272 end = max_end; 5273 length = end - offset; 5274 5275 ret = ext4_update_disksize_before_punch(inode, offset, length); 5276 if (ret) 5277 return ret; 5278 5279 /* Now release the pages and zero block aligned part of pages*/ 5280 ret = ext4_truncate_page_cache_block_range(inode, offset, end); 5281 if (ret) 5282 return ret; 5283 5284 ret = ext4_zero_partial_blocks(inode, offset, length, &partial_zeroed); 5285 if (ret) 5286 return ret; 5287 /* 5288 * On the iomap path, partial invalidate of a folio without an ifs 5289 * leaves sub-block dirty bits uncleared. Drain writeback before 5290 * removing extents so that any bio in flight on a partial folio 5291 * completes against blocks still owned by this inode. 5292 */ 5293 if (ext4_inode_buffered_iomap(inode) || 5294 (((file->f_flags & O_SYNC) || IS_SYNC(inode)) && partial_zeroed)) { 5295 ret = filemap_write_and_wait_range(inode->i_mapping, offset, 5296 end - 1); 5297 if (ret) 5298 return ret; 5299 } 5300 5301 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) 5302 credits = ext4_chunk_trans_extent(inode, 0); 5303 else 5304 credits = ext4_blocks_for_truncate(inode); 5305 handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE, credits); 5306 if (IS_ERR(handle)) { 5307 ret = PTR_ERR(handle); 5308 ext4_std_error(sb, ret); 5309 return ret; 5310 } 5311 5312 /* If there are blocks to remove, do it */ 5313 start_lblk = EXT4_B_TO_LBLK(inode, offset); 5314 end_lblk = end >> inode->i_blkbits; 5315 5316 if (end_lblk > start_lblk) { 5317 ext4_lblk_t hole_len = end_lblk - start_lblk; 5318 5319 ext4_fc_track_inode(handle, inode); 5320 ext4_check_map_extents_env(inode); 5321 down_write(&EXT4_I(inode)->i_data_sem); 5322 ext4_discard_preallocations(inode); 5323 5324 ext4_es_remove_extent(inode, start_lblk, hole_len); 5325 5326 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) 5327 ret = ext4_ext_remove_space(inode, start_lblk, 5328 end_lblk - 1); 5329 else 5330 ret = ext4_ind_remove_space(handle, inode, start_lblk, 5331 end_lblk); 5332 if (ret) { 5333 up_write(&EXT4_I(inode)->i_data_sem); 5334 goto out_handle; 5335 } 5336 5337 ext4_es_insert_extent(inode, start_lblk, hole_len, ~0, 5338 EXTENT_STATUS_HOLE, 0); 5339 up_write(&EXT4_I(inode)->i_data_sem); 5340 } 5341 ext4_fc_track_range(handle, inode, start_lblk, end_lblk); 5342 5343 ret = ext4_mark_inode_dirty(handle, inode); 5344 if (unlikely(ret)) 5345 goto out_handle; 5346 5347 ext4_update_inode_fsync_trans(handle, inode, 1); 5348 if ((file->f_flags & O_SYNC) || IS_SYNC(inode)) 5349 ext4_handle_sync(handle); 5350 out_handle: 5351 ext4_journal_stop(handle); 5352 return ret; 5353 } 5354 5355 int ext4_inode_attach_jinode(struct inode *inode) 5356 { 5357 struct ext4_inode_info *ei = EXT4_I(inode); 5358 struct jbd2_inode *jinode; 5359 5360 if (ei->jinode || !EXT4_SB(inode->i_sb)->s_journal) 5361 return 0; 5362 5363 jinode = jbd2_alloc_inode(GFP_KERNEL); 5364 spin_lock(&inode->i_lock); 5365 if (!ei->jinode) { 5366 if (!jinode) { 5367 spin_unlock(&inode->i_lock); 5368 return -ENOMEM; 5369 } 5370 jbd2_journal_init_jbd_inode(jinode, inode); 5371 /* 5372 * Publish ->jinode only after it is fully initialized so that 5373 * readers never observe a partially initialized jbd2_inode. 5374 */ 5375 smp_wmb(); 5376 WRITE_ONCE(ei->jinode, jinode); 5377 jinode = NULL; 5378 } 5379 spin_unlock(&inode->i_lock); 5380 if (unlikely(jinode != NULL)) 5381 jbd2_free_inode(jinode); 5382 return 0; 5383 } 5384 5385 /* 5386 * ext4_truncate() 5387 * 5388 * We block out ext4_get_block() block instantiations across the entire 5389 * transaction, and VFS/VM ensures that ext4_truncate() cannot run 5390 * simultaneously on behalf of the same inode. 5391 * 5392 * As we work through the truncate and commit bits of it to the journal there 5393 * is one core, guiding principle: the file's tree must always be consistent on 5394 * disk. We must be able to restart the truncate after a crash. 5395 * 5396 * The file's tree may be transiently inconsistent in memory (although it 5397 * probably isn't), but whenever we close off and commit a journal transaction, 5398 * the contents of (the filesystem + the journal) must be consistent and 5399 * restartable. It's pretty simple, really: bottom up, right to left (although 5400 * left-to-right works OK too). 5401 * 5402 * Note that at recovery time, journal replay occurs *before* the restart of 5403 * truncate against the orphan inode list. 5404 * 5405 * The committed inode has the new, desired i_size (which is the same as 5406 * i_disksize in this case). After a crash, ext4_orphan_cleanup() will see 5407 * that this inode's truncate did not complete and it will again call 5408 * ext4_truncate() to have another go. So there will be instantiated blocks 5409 * to the right of the truncation point in a crashed ext4 filesystem. But 5410 * that's fine - as long as they are linked from the inode, the post-crash 5411 * ext4_truncate() run will find them and release them. 5412 */ 5413 int ext4_truncate(struct inode *inode) 5414 { 5415 struct ext4_inode_info *ei = EXT4_I(inode); 5416 unsigned int credits; 5417 int err = 0, err2; 5418 handle_t *handle; 5419 5420 /* 5421 * There is a possibility that we're either freeing the inode 5422 * or it's a completely new inode. In those cases we might not 5423 * have i_rwsem locked because it's not necessary. 5424 */ 5425 if (!(inode_state_read_once(inode) & (I_NEW | I_FREEING))) 5426 WARN_ON(!inode_is_locked(inode)); 5427 trace_ext4_truncate_enter(inode); 5428 5429 if (!ext4_can_truncate(inode)) 5430 goto out_trace; 5431 5432 if (inode->i_size == 0 && !test_opt(inode->i_sb, NO_AUTO_DA_ALLOC)) 5433 ext4_set_inode_state(inode, EXT4_STATE_DA_ALLOC_CLOSE); 5434 5435 if (ext4_has_inline_data(inode)) { 5436 int has_inline = 1; 5437 5438 err = ext4_inline_data_truncate(inode, &has_inline); 5439 if (err || has_inline) 5440 goto out_trace; 5441 } 5442 5443 /* If we zero-out tail of the page, we have to create jinode for jbd2 */ 5444 if (inode->i_size & (inode->i_sb->s_blocksize - 1)) { 5445 err = ext4_inode_attach_jinode(inode); 5446 if (err) 5447 goto out_trace; 5448 5449 /* Zero to the end of the block containing i_size */ 5450 err = ext4_block_zero_eof(inode, inode->i_size, LLONG_MAX); 5451 if (err) 5452 goto out_trace; 5453 } 5454 5455 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) 5456 credits = ext4_chunk_trans_extent(inode, 1); 5457 else 5458 credits = ext4_blocks_for_truncate(inode); 5459 5460 handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE, credits); 5461 if (IS_ERR(handle)) { 5462 err = PTR_ERR(handle); 5463 goto out_trace; 5464 } 5465 5466 /* 5467 * We add the inode to the orphan list, so that if this 5468 * truncate spans multiple transactions, and we crash, we will 5469 * resume the truncate when the filesystem recovers. It also 5470 * marks the inode dirty, to catch the new size. 5471 * 5472 * Implication: the file must always be in a sane, consistent 5473 * truncatable state while each transaction commits. 5474 */ 5475 err = ext4_orphan_add(handle, inode); 5476 if (err) 5477 goto out_stop; 5478 5479 ext4_fc_track_inode(handle, inode); 5480 ext4_check_map_extents_env(inode); 5481 5482 down_write(&EXT4_I(inode)->i_data_sem); 5483 ext4_discard_preallocations(inode); 5484 5485 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) 5486 err = ext4_ext_truncate(handle, inode); 5487 else 5488 ext4_ind_truncate(handle, inode); 5489 5490 up_write(&ei->i_data_sem); 5491 if (err) 5492 goto out_stop; 5493 5494 if (IS_SYNC(inode)) 5495 ext4_handle_sync(handle); 5496 5497 out_stop: 5498 /* 5499 * If this was a simple ftruncate() and the file will remain alive, 5500 * then we need to clear up the orphan record which we created above. 5501 * However, if this was a real unlink then we were called by 5502 * ext4_evict_inode(), and we allow that function to clean up the 5503 * orphan info for us. 5504 */ 5505 if (inode->i_nlink) 5506 ext4_orphan_del(handle, inode); 5507 5508 inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); 5509 err2 = ext4_mark_inode_dirty(handle, inode); 5510 if (unlikely(err2 && !err)) 5511 err = err2; 5512 ext4_journal_stop(handle); 5513 5514 out_trace: 5515 trace_ext4_truncate_exit(inode); 5516 return err; 5517 } 5518 5519 static inline u64 ext4_inode_peek_iversion(const struct inode *inode) 5520 { 5521 if (unlikely(EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL)) 5522 return inode_peek_iversion_raw(inode); 5523 else 5524 return inode_peek_iversion(inode); 5525 } 5526 5527 static int ext4_inode_blocks_set(struct ext4_inode *raw_inode, 5528 struct ext4_inode_info *ei) 5529 { 5530 struct inode *inode = &(ei->vfs_inode); 5531 u64 i_blocks = READ_ONCE(inode->i_blocks); 5532 struct super_block *sb = inode->i_sb; 5533 5534 if (i_blocks <= ~0U) { 5535 /* 5536 * i_blocks can be represented in a 32 bit variable 5537 * as multiple of 512 bytes 5538 */ 5539 raw_inode->i_blocks_lo = cpu_to_le32(i_blocks); 5540 raw_inode->i_blocks_high = 0; 5541 ext4_clear_inode_flag(inode, EXT4_INODE_HUGE_FILE); 5542 return 0; 5543 } 5544 5545 /* 5546 * This should never happen since sb->s_maxbytes should not have 5547 * allowed this, sb->s_maxbytes was set according to the huge_file 5548 * feature in ext4_fill_super(). 5549 */ 5550 if (!ext4_has_feature_huge_file(sb)) 5551 return -EFSCORRUPTED; 5552 5553 if (i_blocks <= 0xffffffffffffULL) { 5554 /* 5555 * i_blocks can be represented in a 48 bit variable 5556 * as multiple of 512 bytes 5557 */ 5558 raw_inode->i_blocks_lo = cpu_to_le32(i_blocks); 5559 raw_inode->i_blocks_high = cpu_to_le16(i_blocks >> 32); 5560 ext4_clear_inode_flag(inode, EXT4_INODE_HUGE_FILE); 5561 } else { 5562 ext4_set_inode_flag(inode, EXT4_INODE_HUGE_FILE); 5563 /* i_block is stored in file system block size */ 5564 i_blocks = i_blocks >> (inode->i_blkbits - 9); 5565 raw_inode->i_blocks_lo = cpu_to_le32(i_blocks); 5566 raw_inode->i_blocks_high = cpu_to_le16(i_blocks >> 32); 5567 } 5568 return 0; 5569 } 5570 5571 static int ext4_fill_raw_inode(struct inode *inode, struct ext4_inode *raw_inode) 5572 { 5573 struct ext4_inode_info *ei = EXT4_I(inode); 5574 uid_t i_uid; 5575 gid_t i_gid; 5576 projid_t i_projid; 5577 int block; 5578 int err; 5579 5580 err = ext4_inode_blocks_set(raw_inode, ei); 5581 5582 raw_inode->i_mode = cpu_to_le16(inode->i_mode); 5583 i_uid = i_uid_read(inode); 5584 i_gid = i_gid_read(inode); 5585 i_projid = from_kprojid(&init_user_ns, ei->i_projid); 5586 if (!(test_opt(inode->i_sb, NO_UID32))) { 5587 raw_inode->i_uid_low = cpu_to_le16(low_16_bits(i_uid)); 5588 raw_inode->i_gid_low = cpu_to_le16(low_16_bits(i_gid)); 5589 /* 5590 * Fix up interoperability with old kernels. Otherwise, 5591 * old inodes get re-used with the upper 16 bits of the 5592 * uid/gid intact. 5593 */ 5594 if (ei->i_dtime && !ext4_inode_orphan_tracked(inode)) { 5595 raw_inode->i_uid_high = 0; 5596 raw_inode->i_gid_high = 0; 5597 } else { 5598 raw_inode->i_uid_high = 5599 cpu_to_le16(high_16_bits(i_uid)); 5600 raw_inode->i_gid_high = 5601 cpu_to_le16(high_16_bits(i_gid)); 5602 } 5603 } else { 5604 raw_inode->i_uid_low = cpu_to_le16(fs_high2lowuid(i_uid)); 5605 raw_inode->i_gid_low = cpu_to_le16(fs_high2lowgid(i_gid)); 5606 raw_inode->i_uid_high = 0; 5607 raw_inode->i_gid_high = 0; 5608 } 5609 raw_inode->i_links_count = cpu_to_le16(inode->i_nlink); 5610 5611 EXT4_INODE_SET_CTIME(inode, raw_inode); 5612 EXT4_INODE_SET_MTIME(inode, raw_inode); 5613 EXT4_INODE_SET_ATIME(inode, raw_inode); 5614 EXT4_EINODE_SET_XTIME(i_crtime, ei, raw_inode); 5615 5616 raw_inode->i_dtime = cpu_to_le32(ei->i_dtime); 5617 raw_inode->i_flags = cpu_to_le32(ei->i_flags & 0xFFFFFFFF); 5618 if (likely(!test_opt2(inode->i_sb, HURD_COMPAT))) 5619 raw_inode->i_file_acl_high = 5620 cpu_to_le16(ei->i_file_acl >> 32); 5621 raw_inode->i_file_acl_lo = cpu_to_le32(ei->i_file_acl); 5622 ext4_isize_set(raw_inode, ei->i_disksize); 5623 5624 raw_inode->i_generation = cpu_to_le32(inode->i_generation); 5625 if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode)) { 5626 if (old_valid_dev(inode->i_rdev)) { 5627 raw_inode->i_block[0] = 5628 cpu_to_le32(old_encode_dev(inode->i_rdev)); 5629 raw_inode->i_block[1] = 0; 5630 } else { 5631 raw_inode->i_block[0] = 0; 5632 raw_inode->i_block[1] = 5633 cpu_to_le32(new_encode_dev(inode->i_rdev)); 5634 raw_inode->i_block[2] = 0; 5635 } 5636 } else if (!ext4_has_inline_data(inode)) { 5637 for (block = 0; block < EXT4_N_BLOCKS; block++) 5638 raw_inode->i_block[block] = ei->i_data[block]; 5639 } 5640 5641 if (likely(!test_opt2(inode->i_sb, HURD_COMPAT))) { 5642 u64 ivers = ext4_inode_peek_iversion(inode); 5643 5644 raw_inode->i_disk_version = cpu_to_le32(ivers); 5645 if (ei->i_extra_isize) { 5646 if (EXT4_FITS_IN_INODE(raw_inode, ei, i_version_hi)) 5647 raw_inode->i_version_hi = 5648 cpu_to_le32(ivers >> 32); 5649 raw_inode->i_extra_isize = 5650 cpu_to_le16(ei->i_extra_isize); 5651 } 5652 } 5653 5654 if (i_projid != EXT4_DEF_PROJID && 5655 !ext4_has_feature_project(inode->i_sb)) 5656 err = err ?: -EFSCORRUPTED; 5657 5658 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE && 5659 EXT4_FITS_IN_INODE(raw_inode, ei, i_projid)) 5660 raw_inode->i_projid = cpu_to_le32(i_projid); 5661 5662 ext4_inode_csum_set(inode, raw_inode, ei); 5663 return err; 5664 } 5665 5666 /* 5667 * ext4_get_inode_loc returns with an extra refcount against the inode's 5668 * underlying buffer_head on success. If we pass 'inode' and it does not 5669 * have in-inode xattr, we have all inode data in memory that is needed 5670 * to recreate the on-disk version of this inode. 5671 */ 5672 static int __ext4_get_inode_loc(struct super_block *sb, unsigned long ino, 5673 struct inode *inode, struct ext4_iloc *iloc, 5674 ext4_fsblk_t *ret_block) 5675 { 5676 struct ext4_group_desc *gdp; 5677 struct buffer_head *bh; 5678 ext4_fsblk_t block; 5679 struct blk_plug plug; 5680 int inodes_per_block, inode_offset; 5681 5682 iloc->bh = NULL; 5683 if (ino < EXT4_ROOT_INO || 5684 ino > le32_to_cpu(EXT4_SB(sb)->s_es->s_inodes_count)) 5685 return -EFSCORRUPTED; 5686 5687 iloc->block_group = (ino - 1) / EXT4_INODES_PER_GROUP(sb); 5688 gdp = ext4_get_group_desc(sb, iloc->block_group, NULL); 5689 if (!gdp) 5690 return -EIO; 5691 5692 /* 5693 * Figure out the offset within the block group inode table 5694 */ 5695 inodes_per_block = EXT4_SB(sb)->s_inodes_per_block; 5696 inode_offset = ((ino - 1) % 5697 EXT4_INODES_PER_GROUP(sb)); 5698 iloc->offset = (inode_offset % inodes_per_block) * EXT4_INODE_SIZE(sb); 5699 5700 block = ext4_inode_table(sb, gdp); 5701 if ((block <= le32_to_cpu(EXT4_SB(sb)->s_es->s_first_data_block)) || 5702 (block >= ext4_blocks_count(EXT4_SB(sb)->s_es))) { 5703 ext4_error(sb, "Invalid inode table block %llu in " 5704 "block_group %u", block, iloc->block_group); 5705 return -EFSCORRUPTED; 5706 } 5707 block += (inode_offset / inodes_per_block); 5708 5709 bh = sb_getblk(sb, block); 5710 if (unlikely(!bh)) 5711 return -ENOMEM; 5712 if (ext4_buffer_uptodate(bh)) 5713 goto has_buffer; 5714 5715 lock_buffer(bh); 5716 if (ext4_buffer_uptodate(bh)) { 5717 /* Someone brought it uptodate while we waited */ 5718 unlock_buffer(bh); 5719 goto has_buffer; 5720 } 5721 5722 /* 5723 * If we have all information of the inode in memory and this 5724 * is the only valid inode in the block, we need not read the 5725 * block. 5726 */ 5727 if (inode && !ext4_test_inode_state(inode, EXT4_STATE_XATTR)) { 5728 struct buffer_head *bitmap_bh; 5729 int i, start; 5730 5731 start = inode_offset & ~(inodes_per_block - 1); 5732 5733 /* Is the inode bitmap in cache? */ 5734 bitmap_bh = sb_getblk(sb, ext4_inode_bitmap(sb, gdp)); 5735 if (unlikely(!bitmap_bh)) 5736 goto make_io; 5737 5738 /* 5739 * If the inode bitmap isn't in cache then the 5740 * optimisation may end up performing two reads instead 5741 * of one, so skip it. 5742 */ 5743 if (!buffer_uptodate(bitmap_bh)) { 5744 brelse(bitmap_bh); 5745 goto make_io; 5746 } 5747 for (i = start; i < start + inodes_per_block; i++) { 5748 if (i == inode_offset) 5749 continue; 5750 if (ext4_test_bit(i, bitmap_bh->b_data)) 5751 break; 5752 } 5753 brelse(bitmap_bh); 5754 if (i == start + inodes_per_block) { 5755 struct ext4_inode *raw_inode = 5756 (struct ext4_inode *) (bh->b_data + iloc->offset); 5757 5758 /* all other inodes are free, so skip I/O */ 5759 memset(bh->b_data, 0, bh->b_size); 5760 if (!ext4_test_inode_state(inode, EXT4_STATE_NEW)) 5761 ext4_fill_raw_inode(inode, raw_inode); 5762 set_buffer_uptodate(bh); 5763 unlock_buffer(bh); 5764 goto has_buffer; 5765 } 5766 } 5767 5768 make_io: 5769 /* 5770 * If we need to do any I/O, try to pre-readahead extra 5771 * blocks from the inode table. 5772 */ 5773 blk_start_plug(&plug); 5774 if (EXT4_SB(sb)->s_inode_readahead_blks) { 5775 ext4_fsblk_t b, end, table; 5776 unsigned num; 5777 __u32 ra_blks = EXT4_SB(sb)->s_inode_readahead_blks; 5778 5779 table = ext4_inode_table(sb, gdp); 5780 /* s_inode_readahead_blks is always a power of 2 */ 5781 b = block & ~((ext4_fsblk_t) ra_blks - 1); 5782 if (table > b) 5783 b = table; 5784 end = b + ra_blks; 5785 num = EXT4_INODES_PER_GROUP(sb); 5786 if (ext4_has_group_desc_csum(sb)) 5787 num -= ext4_itable_unused_count(sb, gdp); 5788 table += num / inodes_per_block; 5789 if (end > table) 5790 end = table; 5791 while (b <= end) 5792 ext4_sb_breadahead_unmovable(sb, b++); 5793 } 5794 5795 /* 5796 * There are other valid inodes in the buffer, this inode 5797 * has in-inode xattrs, or we don't have this inode in memory. 5798 * Read the block from disk. 5799 */ 5800 trace_ext4_load_inode(sb, ino); 5801 ext4_read_bh_nowait(bh, REQ_META | REQ_PRIO, NULL, 5802 ext4_simulate_fail(sb, EXT4_SIM_INODE_EIO)); 5803 blk_finish_plug(&plug); 5804 wait_on_buffer(bh); 5805 if (!buffer_uptodate(bh)) { 5806 if (ret_block) 5807 *ret_block = block; 5808 brelse(bh); 5809 return -EIO; 5810 } 5811 has_buffer: 5812 iloc->bh = bh; 5813 return 0; 5814 } 5815 5816 static int __ext4_get_inode_loc_noinmem(struct inode *inode, 5817 struct ext4_iloc *iloc) 5818 { 5819 ext4_fsblk_t err_blk = 0; 5820 int ret; 5821 5822 ret = __ext4_get_inode_loc(inode->i_sb, inode->i_ino, NULL, iloc, 5823 &err_blk); 5824 5825 if (ret == -EIO) 5826 ext4_error_inode_block(inode, err_blk, EIO, 5827 "unable to read itable block"); 5828 5829 return ret; 5830 } 5831 5832 int ext4_get_inode_loc(struct inode *inode, struct ext4_iloc *iloc) 5833 { 5834 ext4_fsblk_t err_blk = 0; 5835 int ret; 5836 5837 ret = __ext4_get_inode_loc(inode->i_sb, inode->i_ino, inode, iloc, 5838 &err_blk); 5839 5840 if (ret == -EIO) 5841 ext4_error_inode_block(inode, err_blk, EIO, 5842 "unable to read itable block"); 5843 5844 return ret; 5845 } 5846 5847 /* 5848 * ext4_get_inode_loc_noio() is a best-effort variant of ext4_get_inode_loc(). 5849 * It looks up the inode table block in the buffer cache and returns -EAGAIN if 5850 * the block is not present or not uptodate, without starting any I/O. It may 5851 * sleep on the folio lock of the block. 5852 */ 5853 int ext4_get_inode_loc_noio(struct inode *inode, struct ext4_iloc *iloc) 5854 { 5855 struct super_block *sb = inode->i_sb; 5856 struct ext4_group_desc *gdp; 5857 struct buffer_head *bh; 5858 ext4_fsblk_t block; 5859 int inodes_per_block, inode_offset; 5860 unsigned long ino = inode->i_ino; 5861 5862 iloc->bh = NULL; 5863 if (ino < EXT4_ROOT_INO || 5864 ino > le32_to_cpu(EXT4_SB(sb)->s_es->s_inodes_count)) 5865 return -EFSCORRUPTED; 5866 5867 iloc->block_group = (ino - 1) / EXT4_INODES_PER_GROUP(sb); 5868 gdp = ext4_get_group_desc(sb, iloc->block_group, NULL); 5869 if (!gdp) 5870 return -EIO; 5871 5872 /* Figure out the offset within the block group inode table. */ 5873 inodes_per_block = EXT4_SB(sb)->s_inodes_per_block; 5874 inode_offset = ((ino - 1) % EXT4_INODES_PER_GROUP(sb)); 5875 iloc->offset = (inode_offset % inodes_per_block) * EXT4_INODE_SIZE(sb); 5876 5877 block = ext4_inode_table(sb, gdp); 5878 if (block <= le32_to_cpu(EXT4_SB(sb)->s_es->s_first_data_block) || 5879 block >= ext4_blocks_count(EXT4_SB(sb)->s_es)) { 5880 ext4_error(sb, 5881 "Invalid inode table block %llu in block_group %u", 5882 block, iloc->block_group); 5883 return -EFSCORRUPTED; 5884 } 5885 block += inode_offset / inodes_per_block; 5886 5887 bh = sb_find_get_block_nonatomic(sb, block); 5888 if (!bh) 5889 return -EAGAIN; 5890 if (!ext4_buffer_uptodate(bh)) { 5891 brelse(bh); 5892 return -EAGAIN; 5893 } 5894 5895 iloc->bh = bh; 5896 return 0; 5897 } 5898 5899 5900 int ext4_get_fc_inode_loc(struct super_block *sb, unsigned long ino, 5901 struct ext4_iloc *iloc) 5902 { 5903 return __ext4_get_inode_loc(sb, ino, NULL, iloc, NULL); 5904 } 5905 5906 static bool ext4_should_enable_dax(struct inode *inode) 5907 { 5908 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); 5909 5910 if (test_opt2(inode->i_sb, DAX_NEVER)) 5911 return false; 5912 if (!S_ISREG(inode->i_mode)) 5913 return false; 5914 if (ext4_should_journal_data(inode)) 5915 return false; 5916 if (ext4_has_inline_data(inode)) 5917 return false; 5918 if (ext4_test_inode_flag(inode, EXT4_INODE_ENCRYPT)) 5919 return false; 5920 if (ext4_test_inode_flag(inode, EXT4_INODE_VERITY)) 5921 return false; 5922 if (!test_bit(EXT4_FLAGS_BDEV_IS_DAX, &sbi->s_ext4_flags)) 5923 return false; 5924 if (test_opt(inode->i_sb, DAX_ALWAYS)) 5925 return true; 5926 5927 return ext4_test_inode_flag(inode, EXT4_INODE_DAX); 5928 } 5929 5930 void ext4_set_inode_flags(struct inode *inode, bool init) 5931 { 5932 unsigned int flags = EXT4_I(inode)->i_flags; 5933 unsigned int new_fl = 0; 5934 5935 WARN_ON_ONCE(IS_DAX(inode) && init); 5936 5937 if (flags & EXT4_SYNC_FL) 5938 new_fl |= S_SYNC; 5939 if (flags & EXT4_APPEND_FL) 5940 new_fl |= S_APPEND; 5941 if (flags & EXT4_IMMUTABLE_FL) 5942 new_fl |= S_IMMUTABLE; 5943 if (flags & EXT4_NOATIME_FL) 5944 new_fl |= S_NOATIME; 5945 if (flags & EXT4_DIRSYNC_FL) 5946 new_fl |= S_DIRSYNC; 5947 5948 /* Because of the way inode_set_flags() works we must preserve S_DAX 5949 * here if already set. */ 5950 new_fl |= (inode->i_flags & S_DAX); 5951 if (init && ext4_should_enable_dax(inode)) 5952 new_fl |= S_DAX; 5953 5954 if (flags & EXT4_ENCRYPT_FL) 5955 new_fl |= S_ENCRYPTED; 5956 if (flags & EXT4_CASEFOLD_FL) 5957 new_fl |= S_CASEFOLD; 5958 if (flags & EXT4_VERITY_FL) 5959 new_fl |= S_VERITY; 5960 inode_set_flags(inode, new_fl, 5961 S_SYNC|S_APPEND|S_IMMUTABLE|S_NOATIME|S_DIRSYNC|S_DAX| 5962 S_ENCRYPTED|S_CASEFOLD|S_VERITY); 5963 } 5964 5965 static blkcnt_t ext4_inode_blocks(struct ext4_inode *raw_inode, 5966 struct ext4_inode_info *ei) 5967 { 5968 blkcnt_t i_blocks ; 5969 struct inode *inode = &(ei->vfs_inode); 5970 struct super_block *sb = inode->i_sb; 5971 5972 if (ext4_has_feature_huge_file(sb)) { 5973 /* we are using combined 48 bit field */ 5974 i_blocks = ((u64)le16_to_cpu(raw_inode->i_blocks_high)) << 32 | 5975 le32_to_cpu(raw_inode->i_blocks_lo); 5976 if (ext4_test_inode_flag(inode, EXT4_INODE_HUGE_FILE)) { 5977 /* i_blocks represent file system block size */ 5978 return i_blocks << (inode->i_blkbits - 9); 5979 } else { 5980 return i_blocks; 5981 } 5982 } else { 5983 return le32_to_cpu(raw_inode->i_blocks_lo); 5984 } 5985 } 5986 5987 static inline int ext4_iget_extra_inode(struct inode *inode, 5988 struct ext4_inode *raw_inode, 5989 struct ext4_inode_info *ei) 5990 { 5991 __le32 *magic = (void *)raw_inode + 5992 EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize; 5993 5994 if (EXT4_INODE_HAS_XATTR_SPACE(inode) && 5995 *magic == cpu_to_le32(EXT4_XATTR_MAGIC)) { 5996 int err; 5997 5998 err = xattr_check_inode(inode, IHDR(inode, raw_inode), 5999 ITAIL(inode, raw_inode)); 6000 if (err) 6001 return err; 6002 6003 ext4_set_inode_state(inode, EXT4_STATE_XATTR); 6004 err = ext4_find_inline_data_nolock(inode); 6005 if (!err && ext4_has_inline_data(inode)) 6006 ext4_set_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA); 6007 return err; 6008 } else 6009 EXT4_I(inode)->i_inline_off = 0; 6010 return 0; 6011 } 6012 6013 int ext4_get_projid(struct inode *inode, kprojid_t *projid) 6014 { 6015 if (!ext4_has_feature_project(inode->i_sb)) 6016 return -EOPNOTSUPP; 6017 *projid = EXT4_I(inode)->i_projid; 6018 return 0; 6019 } 6020 6021 /* 6022 * ext4 has self-managed i_version for ea inodes, it stores the lower 32bit of 6023 * refcount in i_version, so use raw values if inode has EXT4_EA_INODE_FL flag 6024 * set. 6025 */ 6026 static inline void ext4_inode_set_iversion_queried(struct inode *inode, u64 val) 6027 { 6028 if (unlikely(EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL)) 6029 inode_set_iversion_raw(inode, val); 6030 else 6031 inode_set_iversion_queried(inode, val); 6032 } 6033 6034 static int check_igot_inode(struct inode *inode, ext4_iget_flags flags, 6035 const char *function, unsigned int line) 6036 { 6037 const char *err_str; 6038 6039 if (flags & EXT4_IGET_EA_INODE) { 6040 if (!(EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL)) { 6041 err_str = "missing EA_INODE flag"; 6042 goto error; 6043 } 6044 if (ext4_test_inode_state(inode, EXT4_STATE_XATTR) || 6045 EXT4_I(inode)->i_file_acl) { 6046 err_str = "ea_inode with extended attributes"; 6047 goto error; 6048 } 6049 } else { 6050 if ((EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL)) { 6051 /* 6052 * open_by_handle_at() could provide an old inode number 6053 * that has since been reused for an ea_inode; this does 6054 * not indicate filesystem corruption 6055 */ 6056 if (flags & EXT4_IGET_HANDLE) 6057 return -ESTALE; 6058 err_str = "unexpected EA_INODE flag"; 6059 goto error; 6060 } 6061 } 6062 if (is_bad_inode(inode) && !(flags & EXT4_IGET_BAD)) { 6063 err_str = "unexpected bad inode w/o EXT4_IGET_BAD"; 6064 goto error; 6065 } 6066 return 0; 6067 6068 error: 6069 ext4_error_inode(inode, function, line, 0, "%s", err_str); 6070 return -EFSCORRUPTED; 6071 } 6072 6073 /* 6074 * Determine whether an inode should use the iomap buffered I/O path. 6075 * EXT4_STATE_BUFFERED_IOMAP is generally set at inode initialization 6076 * time. Online switching of the buffered I/O path on an active inode is 6077 * NOT supported, with the exception of changing a per-inode journal 6078 * flag. 6079 * 6080 * For features like inline data, fsverity, and encryption that can be 6081 * dynamically enabled or disabled, we check the superblock-level 6082 * feature flags. If any of these is globally enabled, no inode is 6083 * allowed into the iomap buffered I/O path. This avoids the complexity 6084 * of dynamic toggling. 6085 * 6086 * For the global data journal mode (EXT4_MOUNT_JOURNAL_DATA), dynamic 6087 * change through remount is deferred. It will only become available 6088 * after the inode is re-initialized (i.e., after the last reference 6089 * drops and the inode is re-read from disk with the journal flag 6090 * cleared). 6091 * 6092 * For the per-inode data journal mode (EXT4_INODE_JOURNAL_DATA), 6093 * dynamic changes take effect immediately. This is safe because 6094 * address_space operations can be switched and all page cache can be 6095 * dropped under i_rwsem and filemap_invalidate_lock. 6096 * 6097 * For extent-to-indirect block migration (via EXT4_IOC_SETFLAGS 6098 * clearing EXT4_EXTENTS_FL), this operation is directly rejected for 6099 * inodes using the iomap path. 6100 * 6101 * When remounting to toggle the buffered_iomap mount option, the change 6102 * of I/O path is deferred as well, it will be available after the inode 6103 * is re-initialized. 6104 */ 6105 void ext4_enable_buffered_iomap(struct inode *inode) 6106 { 6107 struct super_block *sb = inode->i_sb; 6108 6109 if (!test_opt2(sb, BUFFERED_IOMAP)) 6110 return; 6111 if (!S_ISREG(inode->i_mode)) 6112 return; 6113 if (ext4_test_inode_flag(inode, EXT4_INODE_EA_INODE)) 6114 return; 6115 6116 /* Unsupported Features */ 6117 if (ext4_has_feature_inline_data(sb)) 6118 return; 6119 if (ext4_has_feature_verity(sb)) 6120 return; 6121 if (ext4_has_feature_encrypt(sb)) 6122 return; 6123 if (test_opt(sb, DATA_FLAGS) == EXT4_MOUNT_JOURNAL_DATA || 6124 ext4_test_inode_flag(inode, EXT4_INODE_JOURNAL_DATA)) 6125 return; 6126 if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) 6127 return; 6128 6129 ext4_set_inode_state(inode, EXT4_STATE_BUFFERED_IOMAP); 6130 6131 /* 6132 * Install the iomap end_io handler on the shared conversion 6133 * work. This is safe at inode initialization and during the 6134 * buffered I/O path changes where we flush all pending 6135 * writebacks and drop page cache under i_rwsem and 6136 * filemap_invalidate_lock. 6137 */ 6138 INIT_WORK(&EXT4_I(inode)->i_rsv_conversion_work, ext4_iomap_end_io); 6139 } 6140 6141 static void ext4_disable_buffered_iomap(struct inode *inode) 6142 { 6143 ext4_clear_inode_state(inode, EXT4_STATE_BUFFERED_IOMAP); 6144 6145 /* 6146 * Reinstall the buffer_head end_io handler on the shared 6147 * conversion work. This is safe during the buffered I/O path 6148 * changes where we flush all pending writebacks and drop page 6149 * cache under i_rwsem and filemap_invalidate_lock. 6150 */ 6151 INIT_WORK(&EXT4_I(inode)->i_rsv_conversion_work, ext4_end_io_rsv_work); 6152 } 6153 6154 void ext4_set_inode_mapping_order(struct inode *inode) 6155 { 6156 struct super_block *sb = inode->i_sb; 6157 u16 min_order, max_order; 6158 6159 max_order = EXT4_SB(sb)->s_max_folio_order; 6160 if (!max_order) 6161 return; 6162 6163 min_order = EXT4_SB(sb)->s_min_folio_order; 6164 if (!min_order && !S_ISREG(inode->i_mode)) 6165 return; 6166 6167 if (ext4_test_inode_flag(inode, EXT4_INODE_JOURNAL_DATA)) 6168 max_order = min_order; 6169 6170 mapping_set_folio_order_range(inode->i_mapping, min_order, max_order); 6171 } 6172 6173 static int ext4_iget_match(struct inode *inode, u64 ino, void *data) 6174 { 6175 if (inode->i_ino != ino) 6176 return 0; 6177 spin_lock(&inode->i_lock); 6178 if (inode_state_read(inode) & (I_FREEING | I_WILL_FREE | I_CREATING)) { 6179 spin_unlock(&inode->i_lock); 6180 return -1; 6181 } 6182 __iget(inode); 6183 spin_unlock(&inode->i_lock); 6184 return 1; 6185 } 6186 6187 struct inode *__ext4_iget(struct super_block *sb, unsigned long ino, 6188 ext4_iget_flags flags, const char *function, 6189 unsigned int line) 6190 { 6191 struct ext4_iloc iloc; 6192 struct ext4_inode *raw_inode; 6193 struct ext4_inode_info *ei; 6194 struct ext4_super_block *es = EXT4_SB(sb)->s_es; 6195 struct inode *inode; 6196 journal_t *journal = EXT4_SB(sb)->s_journal; 6197 long ret; 6198 loff_t size; 6199 int block; 6200 uid_t i_uid; 6201 gid_t i_gid; 6202 projid_t i_projid; 6203 6204 if ((!(flags & EXT4_IGET_SPECIAL) && is_special_ino(sb, ino)) || 6205 (ino < EXT4_ROOT_INO) || 6206 (ino > le32_to_cpu(es->s_inodes_count))) { 6207 if (flags & EXT4_IGET_HANDLE) 6208 return ERR_PTR(-ESTALE); 6209 __ext4_error(sb, function, line, false, EFSCORRUPTED, 0, 6210 "inode #%lu: comm %s: iget: illegal inode #", 6211 ino, current->comm); 6212 return ERR_PTR(-EFSCORRUPTED); 6213 } 6214 6215 if (flags & EXT4_IGET_NOWAIT) { 6216 inode = find_inode_nowait(sb, ino, ext4_iget_match, NULL); 6217 if (!inode) 6218 return ERR_PTR(-ENOENT); 6219 6220 if (inode_state_read_once(inode) & I_NEW) 6221 wait_on_new_inode(inode); 6222 6223 if (unlikely(inode_unhashed(inode))) { 6224 iput(inode); 6225 return ERR_PTR(-ENOENT); 6226 } 6227 } else { 6228 inode = iget_locked(sb, ino); 6229 if (!inode) 6230 return ERR_PTR(-ENOMEM); 6231 } 6232 6233 if (!(inode_state_read_once(inode) & I_NEW)) { 6234 ret = check_igot_inode(inode, flags, function, line); 6235 if (ret) { 6236 iput(inode); 6237 return ERR_PTR(ret); 6238 } 6239 return inode; 6240 } 6241 6242 ei = EXT4_I(inode); 6243 iloc.bh = NULL; 6244 6245 ret = __ext4_get_inode_loc_noinmem(inode, &iloc); 6246 if (ret < 0) 6247 goto bad_inode; 6248 raw_inode = ext4_raw_inode(&iloc); 6249 6250 if ((flags & EXT4_IGET_HANDLE) && 6251 (raw_inode->i_links_count == 0) && (raw_inode->i_mode == 0)) { 6252 ret = -ESTALE; 6253 goto bad_inode; 6254 } 6255 6256 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) { 6257 ei->i_extra_isize = le16_to_cpu(raw_inode->i_extra_isize); 6258 if (EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize > 6259 EXT4_INODE_SIZE(inode->i_sb) || 6260 (ei->i_extra_isize & 3)) { 6261 ext4_error_inode(inode, function, line, 0, 6262 "iget: bad extra_isize %u " 6263 "(inode size %u)", 6264 ei->i_extra_isize, 6265 EXT4_INODE_SIZE(inode->i_sb)); 6266 ret = -EFSCORRUPTED; 6267 goto bad_inode; 6268 } 6269 } else 6270 ei->i_extra_isize = 0; 6271 6272 /* Precompute checksum seed for inode metadata */ 6273 if (ext4_has_feature_metadata_csum(sb)) { 6274 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); 6275 __u32 csum; 6276 __le32 inum = cpu_to_le32(inode->i_ino); 6277 __le32 gen = raw_inode->i_generation; 6278 csum = ext4_chksum(sbi->s_csum_seed, (__u8 *)&inum, 6279 sizeof(inum)); 6280 ei->i_csum_seed = ext4_chksum(csum, (__u8 *)&gen, sizeof(gen)); 6281 } 6282 6283 if ((!ext4_inode_csum_verify(inode, raw_inode, ei) || 6284 ext4_simulate_fail(sb, EXT4_SIM_INODE_CRC)) && 6285 (!(EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY))) { 6286 ext4_error_inode_err(inode, function, line, 0, 6287 EFSBADCRC, "iget: checksum invalid"); 6288 ret = -EFSBADCRC; 6289 goto bad_inode; 6290 } 6291 6292 inode->i_mode = le16_to_cpu(raw_inode->i_mode); 6293 i_uid = (uid_t)le16_to_cpu(raw_inode->i_uid_low); 6294 i_gid = (gid_t)le16_to_cpu(raw_inode->i_gid_low); 6295 if (ext4_has_feature_project(sb) && 6296 EXT4_INODE_SIZE(sb) > EXT4_GOOD_OLD_INODE_SIZE && 6297 EXT4_FITS_IN_INODE(raw_inode, ei, i_projid)) 6298 i_projid = (projid_t)le32_to_cpu(raw_inode->i_projid); 6299 else 6300 i_projid = EXT4_DEF_PROJID; 6301 6302 if (!(test_opt(inode->i_sb, NO_UID32))) { 6303 i_uid |= le16_to_cpu(raw_inode->i_uid_high) << 16; 6304 i_gid |= le16_to_cpu(raw_inode->i_gid_high) << 16; 6305 } 6306 i_uid_write(inode, i_uid); 6307 i_gid_write(inode, i_gid); 6308 ei->i_projid = make_kprojid(&init_user_ns, i_projid); 6309 set_nlink(inode, le16_to_cpu(raw_inode->i_links_count)); 6310 6311 ei->i_inline_off = 0; 6312 ei->i_dir_start_lookup = 0; 6313 ei->i_dtime = le32_to_cpu(raw_inode->i_dtime); 6314 /* We now have enough fields to check if the inode was active or not. 6315 * This is needed because nfsd might try to access dead inodes 6316 * the test is that same one that e2fsck uses 6317 * NeilBrown 1999oct15 6318 */ 6319 if (inode->i_nlink == 0) { 6320 if ((inode->i_mode == 0 || flags & EXT4_IGET_SPECIAL || 6321 !(EXT4_SB(inode->i_sb)->s_mount_state & EXT4_ORPHAN_FS)) && 6322 ino != EXT4_BOOT_LOADER_INO) { 6323 /* this inode is deleted or unallocated */ 6324 if (flags & EXT4_IGET_SPECIAL) { 6325 ext4_error_inode(inode, function, line, 0, 6326 "iget: special inode unallocated"); 6327 ret = -EFSCORRUPTED; 6328 } else 6329 ret = -ESTALE; 6330 goto bad_inode; 6331 } 6332 /* The only unlinked inodes we let through here have 6333 * valid i_mode and are being read by the orphan 6334 * recovery code: that's fine, we're about to complete 6335 * the process of deleting those. 6336 * OR it is the EXT4_BOOT_LOADER_INO which is 6337 * not initialized on a new filesystem. */ 6338 } 6339 ei->i_flags = le32_to_cpu(raw_inode->i_flags); 6340 ext4_set_inode_flags(inode, true); 6341 /* Detect invalid flag combination - can't have both inline data and extents */ 6342 if (ext4_test_inode_flag(inode, EXT4_INODE_INLINE_DATA) && 6343 ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) { 6344 ext4_error_inode(inode, function, line, 0, 6345 "inode has both inline data and extents flags"); 6346 ret = -EFSCORRUPTED; 6347 goto bad_inode; 6348 } 6349 inode->i_blocks = ext4_inode_blocks(raw_inode, ei); 6350 ei->i_file_acl = le32_to_cpu(raw_inode->i_file_acl_lo); 6351 if (ext4_has_feature_64bit(sb)) 6352 ei->i_file_acl |= 6353 ((__u64)le16_to_cpu(raw_inode->i_file_acl_high)) << 32; 6354 inode->i_size = ext4_isize(sb, raw_inode); 6355 size = i_size_read(inode); 6356 if (size < 0 || size > ext4_get_maxbytes(inode)) { 6357 ext4_error_inode(inode, function, line, 0, 6358 "iget: bad i_size value: %lld", size); 6359 ret = -EFSCORRUPTED; 6360 goto bad_inode; 6361 } 6362 /* 6363 * If dir_index is not enabled but there's dir with INDEX flag set, 6364 * we'd normally treat htree data as empty space. But with metadata 6365 * checksumming that corrupts checksums so forbid that. 6366 */ 6367 if (!ext4_has_feature_dir_index(sb) && 6368 ext4_has_feature_metadata_csum(sb) && 6369 ext4_test_inode_flag(inode, EXT4_INODE_INDEX)) { 6370 ext4_error_inode(inode, function, line, 0, 6371 "iget: Dir with htree data on filesystem without dir_index feature."); 6372 ret = -EFSCORRUPTED; 6373 goto bad_inode; 6374 } 6375 ei->i_disksize = inode->i_size; 6376 #ifdef CONFIG_QUOTA 6377 ei->i_reserved_quota = 0; 6378 #endif 6379 inode->i_generation = le32_to_cpu(raw_inode->i_generation); 6380 ei->i_block_group = iloc.block_group; 6381 ei->i_last_alloc_group = ~0; 6382 /* 6383 * NOTE! The in-memory inode i_data array is in little-endian order 6384 * even on big-endian machines: we do NOT byteswap the block numbers! 6385 */ 6386 for (block = 0; block < EXT4_N_BLOCKS; block++) 6387 ei->i_data[block] = raw_inode->i_block[block]; 6388 INIT_LIST_HEAD(&ei->i_orphan); 6389 ext4_fc_init_inode(&ei->vfs_inode); 6390 6391 /* 6392 * Set transaction id's of transactions that have to be committed 6393 * to finish f[data]sync. We set them to currently running transaction 6394 * as we cannot be sure that the inode or some of its metadata isn't 6395 * part of the transaction - the inode could have been reclaimed and 6396 * now it is reread from disk. 6397 */ 6398 if (journal) { 6399 transaction_t *transaction; 6400 tid_t tid; 6401 6402 read_lock(&journal->j_state_lock); 6403 if (journal->j_running_transaction) 6404 transaction = journal->j_running_transaction; 6405 else 6406 transaction = journal->j_committing_transaction; 6407 if (transaction) 6408 tid = transaction->t_tid; 6409 else 6410 tid = journal->j_commit_sequence; 6411 read_unlock(&journal->j_state_lock); 6412 ei->i_sync_tid = tid; 6413 ei->i_datasync_tid = tid; 6414 } 6415 6416 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) { 6417 if (ei->i_extra_isize == 0) { 6418 /* The extra space is currently unused. Use it. */ 6419 BUILD_BUG_ON(sizeof(struct ext4_inode) & 3); 6420 ei->i_extra_isize = sizeof(struct ext4_inode) - 6421 EXT4_GOOD_OLD_INODE_SIZE; 6422 } else { 6423 ret = ext4_iget_extra_inode(inode, raw_inode, ei); 6424 if (ret) 6425 goto bad_inode; 6426 } 6427 } 6428 6429 EXT4_INODE_GET_CTIME(inode, raw_inode); 6430 EXT4_INODE_GET_ATIME(inode, raw_inode); 6431 EXT4_INODE_GET_MTIME(inode, raw_inode); 6432 EXT4_EINODE_GET_XTIME(i_crtime, ei, raw_inode); 6433 6434 if (likely(!test_opt2(inode->i_sb, HURD_COMPAT))) { 6435 u64 ivers = le32_to_cpu(raw_inode->i_disk_version); 6436 6437 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) { 6438 if (EXT4_FITS_IN_INODE(raw_inode, ei, i_version_hi)) 6439 ivers |= 6440 (__u64)(le32_to_cpu(raw_inode->i_version_hi)) << 32; 6441 } 6442 ext4_inode_set_iversion_queried(inode, ivers); 6443 } 6444 6445 ret = 0; 6446 if (ei->i_file_acl && 6447 !ext4_inode_block_valid(inode, ei->i_file_acl, 1)) { 6448 ext4_error_inode(inode, function, line, 0, 6449 "iget: bad extended attribute block %llu", 6450 ei->i_file_acl); 6451 ret = -EFSCORRUPTED; 6452 goto bad_inode; 6453 } else if (!ext4_has_inline_data(inode)) { 6454 /* validate the block references in the inode */ 6455 if (!(EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY) && 6456 (S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode) || 6457 (S_ISLNK(inode->i_mode) && 6458 !ext4_inode_is_fast_symlink(inode)))) { 6459 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) 6460 ret = ext4_ext_check_inode(inode); 6461 else 6462 ret = ext4_ind_check_inode(inode); 6463 } 6464 } 6465 if (ret) 6466 goto bad_inode; 6467 6468 ext4_enable_buffered_iomap(inode); 6469 6470 if (S_ISREG(inode->i_mode)) { 6471 inode->i_op = &ext4_file_inode_operations; 6472 inode->i_fop = &ext4_file_operations; 6473 ext4_set_aops(inode); 6474 } else if (S_ISDIR(inode->i_mode)) { 6475 inode->i_op = &ext4_dir_inode_operations; 6476 inode->i_fop = &ext4_dir_operations; 6477 } else if (S_ISLNK(inode->i_mode)) { 6478 /* VFS does not allow setting these so must be corruption */ 6479 if (IS_APPEND(inode) || IS_IMMUTABLE(inode)) { 6480 ext4_error_inode(inode, function, line, 0, 6481 "iget: immutable or append flags " 6482 "not allowed on symlinks"); 6483 ret = -EFSCORRUPTED; 6484 goto bad_inode; 6485 } 6486 if (IS_ENCRYPTED(inode)) { 6487 inode->i_op = &ext4_encrypted_symlink_inode_operations; 6488 } else if (ext4_inode_is_fast_symlink(inode)) { 6489 inode->i_op = &ext4_fast_symlink_inode_operations; 6490 6491 /* 6492 * Orphan cleanup can see inodes with i_size == 0 6493 * and i_data uninitialized. Skip size checks in 6494 * that case. This is safe because the first thing 6495 * ext4_evict_inode() does for fast symlinks is 6496 * clearing of i_data and i_size. 6497 */ 6498 if ((EXT4_SB(sb)->s_mount_state & EXT4_ORPHAN_FS)) { 6499 if (inode->i_nlink != 0) { 6500 ext4_error_inode(inode, function, line, 0, 6501 "invalid orphan symlink nlink %d", 6502 inode->i_nlink); 6503 ret = -EFSCORRUPTED; 6504 goto bad_inode; 6505 } 6506 } else { 6507 if (inode->i_size == 0 || 6508 inode->i_size >= sizeof(ei->i_data) || 6509 strnlen((char *)ei->i_data, inode->i_size + 1) != 6510 inode->i_size) { 6511 ext4_error_inode(inode, function, line, 0, 6512 "invalid fast symlink length %llu", 6513 (unsigned long long)inode->i_size); 6514 ret = -EFSCORRUPTED; 6515 goto bad_inode; 6516 } 6517 inode_set_cached_link(inode, (char *)ei->i_data, 6518 inode->i_size); 6519 } 6520 } else { 6521 inode->i_op = &ext4_symlink_inode_operations; 6522 } 6523 } else if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode) || 6524 S_ISFIFO(inode->i_mode) || S_ISSOCK(inode->i_mode)) { 6525 inode->i_op = &ext4_special_inode_operations; 6526 if (raw_inode->i_block[0]) 6527 init_special_inode(inode, inode->i_mode, 6528 old_decode_dev(le32_to_cpu(raw_inode->i_block[0]))); 6529 else 6530 init_special_inode(inode, inode->i_mode, 6531 new_decode_dev(le32_to_cpu(raw_inode->i_block[1]))); 6532 } else if (ino == EXT4_BOOT_LOADER_INO) { 6533 make_bad_inode(inode); 6534 } else { 6535 ret = -EFSCORRUPTED; 6536 ext4_error_inode(inode, function, line, 0, 6537 "iget: bogus i_mode (%o)", inode->i_mode); 6538 goto bad_inode; 6539 } 6540 if (IS_CASEFOLDED(inode) && !ext4_has_feature_casefold(inode->i_sb)) { 6541 ext4_error_inode(inode, function, line, 0, 6542 "casefold flag without casefold feature"); 6543 ret = -EFSCORRUPTED; 6544 goto bad_inode; 6545 } 6546 if (IS_ENCRYPTED(inode) && sb->s_blocksize > PAGE_SIZE) { 6547 ext4_error_inode(inode, function, line, 0, 6548 "encrypted inode with block size larger than page size"); 6549 ret = -EFSCORRUPTED; 6550 goto bad_inode; 6551 } 6552 6553 ext4_set_inode_mapping_order(inode); 6554 6555 ret = check_igot_inode(inode, flags, function, line); 6556 /* 6557 * -ESTALE here means there is nothing inherently wrong with the inode, 6558 * it's just not an inode we can return for an fhandle lookup. 6559 */ 6560 if (ret == -ESTALE) { 6561 brelse(iloc.bh); 6562 unlock_new_inode(inode); 6563 iput(inode); 6564 return ERR_PTR(-ESTALE); 6565 } 6566 if (ret) 6567 goto bad_inode; 6568 brelse(iloc.bh); 6569 /* Initialize the "no ACL's" state for the simple cases */ 6570 if (!ext4_test_inode_state(inode, EXT4_STATE_XATTR) && !ei->i_file_acl) 6571 cache_no_acl(inode); 6572 unlock_new_inode(inode); 6573 return inode; 6574 6575 bad_inode: 6576 brelse(iloc.bh); 6577 iget_failed(inode); 6578 return ERR_PTR(ret); 6579 } 6580 6581 static void __ext4_update_other_inode_time(struct super_block *sb, 6582 unsigned long orig_ino, 6583 unsigned long ino, 6584 struct ext4_inode *raw_inode) 6585 { 6586 struct inode *inode; 6587 6588 inode = find_inode_by_ino_rcu(sb, ino); 6589 if (!inode) 6590 return; 6591 6592 if (!inode_is_dirtytime_only(inode)) 6593 return; 6594 6595 spin_lock(&inode->i_lock); 6596 if (inode_is_dirtytime_only(inode)) { 6597 struct ext4_inode_info *ei = EXT4_I(inode); 6598 6599 inode_state_clear(inode, I_DIRTY_TIME); 6600 spin_unlock(&inode->i_lock); 6601 6602 spin_lock(&ei->i_raw_lock); 6603 EXT4_INODE_SET_CTIME(inode, raw_inode); 6604 EXT4_INODE_SET_MTIME(inode, raw_inode); 6605 EXT4_INODE_SET_ATIME(inode, raw_inode); 6606 ext4_inode_csum_set(inode, raw_inode, ei); 6607 spin_unlock(&ei->i_raw_lock); 6608 trace_ext4_other_inode_update_time(inode, orig_ino); 6609 return; 6610 } 6611 spin_unlock(&inode->i_lock); 6612 } 6613 6614 /* 6615 * Opportunistically update the other time fields for other inodes in 6616 * the same inode table block. 6617 */ 6618 static void ext4_update_other_inodes_time(struct super_block *sb, 6619 unsigned long orig_ino, char *buf) 6620 { 6621 unsigned long ino; 6622 int i, inodes_per_block = EXT4_SB(sb)->s_inodes_per_block; 6623 int inode_size = EXT4_INODE_SIZE(sb); 6624 6625 /* 6626 * Calculate the first inode in the inode table block. Inode 6627 * numbers are one-based. That is, the first inode in a block 6628 * (assuming 4k blocks and 256 byte inodes) is (n*16 + 1). 6629 */ 6630 ino = ((orig_ino - 1) & ~(inodes_per_block - 1)) + 1; 6631 rcu_read_lock(); 6632 for (i = 0; i < inodes_per_block; i++, ino++, buf += inode_size) { 6633 if (ino == orig_ino) 6634 continue; 6635 __ext4_update_other_inode_time(sb, orig_ino, ino, 6636 (struct ext4_inode *)buf); 6637 } 6638 rcu_read_unlock(); 6639 } 6640 6641 /* 6642 * Post the struct inode info into an on-disk inode location in the 6643 * buffer-cache. This gobbles the caller's reference to the 6644 * buffer_head in the inode location struct. 6645 * 6646 * The caller must have write access to iloc->bh. 6647 */ 6648 static int ext4_do_update_inode(handle_t *handle, 6649 struct inode *inode, 6650 struct ext4_iloc *iloc) 6651 { 6652 struct ext4_inode *raw_inode = ext4_raw_inode(iloc); 6653 struct ext4_inode_info *ei = EXT4_I(inode); 6654 struct buffer_head *bh = iloc->bh; 6655 struct super_block *sb = inode->i_sb; 6656 int err; 6657 int need_datasync = 0, set_large_file = 0; 6658 6659 spin_lock(&ei->i_raw_lock); 6660 6661 /* 6662 * For fields not tracked in the in-memory inode, initialise them 6663 * to zero for new inodes. 6664 */ 6665 if (ext4_test_inode_state(inode, EXT4_STATE_NEW)) 6666 memset(raw_inode, 0, EXT4_SB(inode->i_sb)->s_inode_size); 6667 6668 if (READ_ONCE(ei->i_disksize) != ext4_isize(inode->i_sb, raw_inode)) 6669 need_datasync = 1; 6670 if (ei->i_disksize > 0x7fffffffULL) { 6671 if (!ext4_has_feature_large_file(sb) || 6672 EXT4_SB(sb)->s_es->s_rev_level == cpu_to_le32(EXT4_GOOD_OLD_REV)) 6673 set_large_file = 1; 6674 } 6675 6676 err = ext4_fill_raw_inode(inode, raw_inode); 6677 spin_unlock(&ei->i_raw_lock); 6678 if (err) { 6679 EXT4_ERROR_INODE(inode, "corrupted inode contents"); 6680 goto out_brelse; 6681 } 6682 6683 if (inode->i_sb->s_flags & SB_LAZYTIME) 6684 ext4_update_other_inodes_time(inode->i_sb, inode->i_ino, 6685 bh->b_data); 6686 6687 BUFFER_TRACE(bh, "call ext4_handle_dirty_metadata"); 6688 err = ext4_handle_dirty_metadata(handle, NULL, bh); 6689 if (err) 6690 goto out_error; 6691 ext4_clear_inode_state(inode, EXT4_STATE_NEW); 6692 if (set_large_file) { 6693 BUFFER_TRACE(EXT4_SB(sb)->s_sbh, "get write access"); 6694 err = ext4_journal_get_write_access(handle, sb, 6695 EXT4_SB(sb)->s_sbh, 6696 EXT4_JTR_NONE); 6697 if (err) 6698 goto out_error; 6699 lock_buffer(EXT4_SB(sb)->s_sbh); 6700 ext4_set_feature_large_file(sb); 6701 ext4_superblock_csum_set(sb); 6702 unlock_buffer(EXT4_SB(sb)->s_sbh); 6703 ext4_handle_sync(handle); 6704 err = ext4_handle_dirty_metadata(handle, NULL, 6705 EXT4_SB(sb)->s_sbh); 6706 } 6707 ext4_update_inode_fsync_trans(handle, inode, need_datasync); 6708 out_error: 6709 ext4_std_error(inode->i_sb, err); 6710 out_brelse: 6711 brelse(bh); 6712 return err; 6713 } 6714 6715 /* 6716 * ext4_write_inode() 6717 * 6718 * We are called from a few places: 6719 * 6720 * - Within generic_file_aio_write() -> generic_write_sync() for O_SYNC files. 6721 * Here, there will be no transaction running. We wait for any running 6722 * transaction to commit. 6723 * 6724 * - Within flush work (sys_sync(), kupdate and such). 6725 * We wait on commit, if told to. 6726 * 6727 * - Within iput_final() -> write_inode_now() 6728 * We wait on commit, if told to. 6729 * 6730 * In all cases it is actually safe for us to return without doing anything, 6731 * because the inode has been copied into a raw inode buffer in 6732 * ext4_mark_inode_dirty(). This is a correctness thing for WB_SYNC_ALL 6733 * writeback. 6734 * 6735 * For nojournal mode all the work is done in ext4_sync_inode_metadata() 6736 * because inode content is already copied into raw inode buffer and inode 6737 * is marked with I_METADATA_WRITEBACK. 6738 * 6739 * Note that we are absolutely dependent upon all inode dirtiers doing the 6740 * right thing: they *must* call mark_inode_dirty() after dirtying info in 6741 * which we are interested. 6742 * 6743 * It would be a bug for them to not do this. The code: 6744 * 6745 * mark_inode_dirty(inode) 6746 * stuff(); 6747 * inode->i_size = expr; 6748 * 6749 * is in error because write_inode() could occur while `stuff()' is running, 6750 * and the new i_size will be lost. Plus the inode will no longer be on the 6751 * superblock's dirty inode list. 6752 */ 6753 int ext4_write_inode(struct inode *inode, struct writeback_control *wbc) 6754 { 6755 int err; 6756 6757 if (WARN_ON_ONCE(current->flags & PF_MEMALLOC)) 6758 return 0; 6759 6760 err = ext4_emergency_state(inode->i_sb); 6761 if (unlikely(err)) 6762 return err; 6763 6764 if (!EXT4_SB(inode->i_sb)->s_journal) 6765 return 0; 6766 6767 if (ext4_journal_current_handle()) { 6768 ext4_debug("called recursively, non-PF_MEMALLOC!\n"); 6769 dump_stack(); 6770 return -EIO; 6771 } 6772 6773 /* 6774 * No need to force transaction in WB_SYNC_NONE mode. Also 6775 * ext4_sync_fs() will force the commit after everything is 6776 * written. 6777 */ 6778 if (wbc->sync_mode != WB_SYNC_ALL || wbc->for_sync) 6779 return 0; 6780 6781 return ext4_fc_commit(EXT4_SB(inode->i_sb)->s_journal, 6782 EXT4_I(inode)->i_sync_tid); 6783 } 6784 6785 int ext4_sync_inode_metadata(struct inode *inode, struct writeback_control *wbc) 6786 { 6787 struct ext4_iloc iloc; 6788 struct mapping_metadata_bhs *mmb; 6789 int err; 6790 6791 /* We should only get here in nojournal mode */ 6792 if (WARN_ON_ONCE(EXT4_SB(inode->i_sb)->s_journal)) 6793 return -EFSCORRUPTED; 6794 6795 mmb = READ_ONCE(EXT4_I(inode)->i_metadata_bhs); 6796 if (mmb) { 6797 err = mmb_sync(mmb); 6798 if (err) 6799 return err; 6800 } 6801 /* A block that isn't cached is neither dirty nor under writeback. */ 6802 err = ext4_get_inode_loc_noio(inode, &iloc); 6803 if (err == -EAGAIN) 6804 return 0; 6805 if (err) 6806 return err; 6807 sync_dirty_buffer(iloc.bh); 6808 if (buffer_write_io_error(iloc.bh)) { 6809 ext4_error_inode_block(inode, iloc.bh->b_blocknr, EIO, 6810 "IO error syncing inode"); 6811 err = -EIO; 6812 } 6813 brelse(iloc.bh); 6814 return err; 6815 } 6816 6817 /* 6818 * In data=journal mode ext4_journalled_invalidate_folio() may fail to invalidate 6819 * buffers that are attached to a folio straddling i_size and are undergoing 6820 * commit. In that case we have to wait for commit to finish and try again. 6821 */ 6822 static void ext4_wait_for_tail_page_commit(struct inode *inode) 6823 { 6824 unsigned offset; 6825 journal_t *journal = EXT4_SB(inode->i_sb)->s_journal; 6826 tid_t commit_tid; 6827 int ret; 6828 bool has_transaction; 6829 6830 offset = inode->i_size & (PAGE_SIZE - 1); 6831 /* 6832 * If the folio is fully truncated, we don't need to wait for any commit 6833 * (and we even should not as __ext4_journalled_invalidate_folio() may 6834 * strip all buffers from the folio but keep the folio dirty which can then 6835 * confuse e.g. concurrent ext4_writepages() seeing dirty folio without 6836 * buffers). Also we don't need to wait for any commit if all buffers in 6837 * the folio remain valid. This is most beneficial for the common case of 6838 * blocksize == PAGESIZE. 6839 */ 6840 if (!offset || offset > (PAGE_SIZE - i_blocksize(inode))) 6841 return; 6842 while (1) { 6843 struct folio *folio = filemap_lock_folio(inode->i_mapping, 6844 inode->i_size >> PAGE_SHIFT); 6845 if (IS_ERR(folio)) 6846 return; 6847 ret = __ext4_journalled_invalidate_folio(folio, offset, 6848 folio_size(folio) - offset); 6849 folio_unlock(folio); 6850 folio_put(folio); 6851 if (ret != -EBUSY) 6852 return; 6853 has_transaction = false; 6854 read_lock(&journal->j_state_lock); 6855 if (journal->j_committing_transaction) { 6856 commit_tid = journal->j_committing_transaction->t_tid; 6857 has_transaction = true; 6858 } 6859 read_unlock(&journal->j_state_lock); 6860 if (has_transaction) 6861 jbd2_log_wait_commit(journal, commit_tid); 6862 } 6863 } 6864 6865 /* 6866 * Set i_size and i_disksize to 'newsize'. 6867 * 6868 * Both i_rwsem and i_data_sem are required here to avoid races between 6869 * generic append writeback (or zeroed pending I/O writeback) and 6870 * concurrent operations (e.g., fallocate, truncate) that also modify 6871 * i_size and i_disksize. This also ensures that the writeback ioend worker 6872 * observes the latest i_size under the same lock protection. 6873 */ 6874 static inline void ext4_set_inode_size(struct inode *inode, loff_t newsize) 6875 { 6876 WARN_ON_ONCE(S_ISREG(inode->i_mode) && !inode_is_locked(inode)); 6877 6878 down_write(&EXT4_I(inode)->i_data_sem); 6879 i_size_write(inode, newsize); 6880 __ext4_set_i_disksize(inode, newsize); 6881 up_write(&EXT4_I(inode)->i_data_sem); 6882 } 6883 6884 static int ext4_truncate_up(struct inode *inode, loff_t oldsize, loff_t newsize) 6885 { 6886 ext4_lblk_t old_lblk, new_lblk; 6887 handle_t *handle; 6888 int ret; 6889 6890 if (!IS_ALIGNED(oldsize | newsize, i_blocksize(inode))) { 6891 ret = ext4_inode_attach_jinode(inode); 6892 if (ret) 6893 return ret; 6894 } 6895 6896 inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); 6897 if (!IS_ALIGNED(oldsize, i_blocksize(inode))) { 6898 ret = ext4_block_zero_eof(inode, oldsize, LLONG_MAX); 6899 if (ret) 6900 return ret; 6901 } 6902 6903 handle = ext4_journal_start(inode, EXT4_HT_INODE, 3); 6904 if (IS_ERR(handle)) 6905 return PTR_ERR(handle); 6906 6907 old_lblk = oldsize > 0 ? (oldsize - 1) >> inode->i_blkbits : 0; 6908 new_lblk = newsize > 0 ? (newsize - 1) >> inode->i_blkbits : 0; 6909 ext4_fc_track_range(handle, inode, old_lblk, new_lblk); 6910 6911 ext4_set_inode_size(inode, newsize); 6912 6913 ret = ext4_mark_inode_dirty(handle, inode); 6914 ext4_journal_stop(handle); 6915 if (ret) 6916 return ret; 6917 /* 6918 * isize extend must be called outside an active handle due to 6919 * the lock ordering of transaction start and folio lock in the 6920 * iomap buffered I/O path (folio lock -> transaction start). 6921 */ 6922 pagecache_isize_extended(inode, oldsize, newsize); 6923 return 0; 6924 } 6925 6926 static int ext4_truncate_down(struct inode *inode, loff_t oldsize, 6927 loff_t newsize, int *orphan) 6928 { 6929 ext4_lblk_t start_lblk; 6930 handle_t *handle; 6931 int ret; 6932 6933 /* Do not change i_size. */ 6934 if (newsize == oldsize) 6935 goto truncate; 6936 6937 /* Shrink. */ 6938 handle = ext4_journal_start(inode, EXT4_HT_INODE, 3); 6939 if (IS_ERR(handle)) 6940 return PTR_ERR(handle); 6941 6942 if (ext4_handle_valid(handle)) { 6943 ret = ext4_orphan_add(handle, inode); 6944 *orphan = 1; 6945 if (ret) { 6946 ext4_journal_stop(handle); 6947 return ret; 6948 } 6949 } 6950 6951 start_lblk = newsize > 0 ? (newsize - 1) >> inode->i_blkbits : 0; 6952 ext4_fc_track_range(handle, inode, start_lblk, EXT_MAX_BLOCKS - 1); 6953 6954 down_write(&EXT4_I(inode)->i_data_sem); 6955 /* 6956 * Truncate the zeroed EOF block invalidates the pending disksize 6957 * update, so clear the disksize-grow-pending state. 6958 */ 6959 if (ext4_test_inode_state(inode, EXT4_STATE_DISKSIZE_GROW_PENDING) && 6960 (newsize <= EXT4_I(inode)->i_disksize)) 6961 ext4_iomap_clear_disksize_pending(inode); 6962 6963 i_size_write(inode, newsize); 6964 __ext4_set_i_disksize(inode, newsize); 6965 up_write(&EXT4_I(inode)->i_data_sem); 6966 6967 ret = ext4_mark_inode_dirty(handle, inode); 6968 ext4_journal_stop(handle); 6969 if (ret) 6970 return ret; 6971 6972 if (ext4_should_journal_data(inode)) 6973 ext4_wait_for_tail_page_commit(inode); 6974 truncate: 6975 /* 6976 * Truncate pagecache after we've waited for commit in data=journal 6977 * mode to make pages freeable. Call ext4_truncate() even if 6978 * i_size didn't change to truncate possible preallocated blocks. 6979 */ 6980 truncate_pagecache(inode, newsize); 6981 return ext4_truncate(inode); 6982 } 6983 6984 /* 6985 * ext4_setattr() 6986 * 6987 * Called from notify_change. 6988 * 6989 * We want to trap VFS attempts to truncate the file as soon as 6990 * possible. In particular, we want to make sure that when the VFS 6991 * shrinks i_size, we put the inode on the orphan list and modify 6992 * i_disksize immediately, so that during the subsequent flushing of 6993 * dirty pages and freeing of disk blocks, we can guarantee that any 6994 * commit will leave the blocks being flushed in an unused state on 6995 * disk. (On recovery, the inode will get truncated and the blocks will 6996 * be freed, so we have a strong guarantee that no future commit will 6997 * leave these blocks visible to the user.) 6998 * 6999 * Another thing we have to assure is that if we are in ordered mode 7000 * and inode is still attached to the committing transaction, we must 7001 * we start writeout of all the dirty pages which are being truncated. 7002 * This way we are sure that all the data written in the previous 7003 * transaction are already on disk (truncate waits for pages under 7004 * writeback). 7005 * 7006 * Called with inode->i_rwsem down. 7007 */ 7008 int ext4_setattr(const struct mnt_idmap *idmap, struct dentry *dentry, 7009 struct iattr *attr) 7010 { 7011 struct inode *inode = d_inode(dentry); 7012 int error, rc = 0; 7013 int orphan = 0; 7014 const unsigned int ia_valid = attr->ia_valid; 7015 bool inc_ivers = true; 7016 7017 error = ext4_emergency_state(inode->i_sb); 7018 if (unlikely(error)) 7019 return error; 7020 7021 if (unlikely(IS_IMMUTABLE(inode))) 7022 return -EPERM; 7023 7024 if (unlikely(IS_APPEND(inode) && 7025 (ia_valid & (ATTR_MODE | ATTR_UID | 7026 ATTR_GID | ATTR_TIMES_SET)))) 7027 return -EPERM; 7028 7029 error = setattr_prepare(idmap, dentry, attr); 7030 if (error) 7031 return error; 7032 7033 error = fscrypt_prepare_setattr(dentry, attr); 7034 if (error) 7035 return error; 7036 7037 if (is_quota_modification(idmap, inode, attr)) { 7038 error = dquot_initialize(inode); 7039 if (error) 7040 return error; 7041 } 7042 7043 if (i_uid_needs_update(idmap, attr, inode) || 7044 i_gid_needs_update(idmap, attr, inode)) { 7045 handle_t *handle; 7046 7047 /* (user+group)*(old+new) structure, inode write (sb, 7048 * inode block, ? - but truncate inode update has it) */ 7049 handle = ext4_journal_start(inode, EXT4_HT_QUOTA, 7050 (EXT4_MAXQUOTAS_INIT_BLOCKS(inode->i_sb) + 7051 EXT4_MAXQUOTAS_DEL_BLOCKS(inode->i_sb)) + 3); 7052 if (IS_ERR(handle)) { 7053 error = PTR_ERR(handle); 7054 goto err_out; 7055 } 7056 7057 /* dquot_transfer() calls back ext4_get_inode_usage() which 7058 * counts xattr inode references. 7059 */ 7060 down_read(&EXT4_I(inode)->xattr_sem); 7061 error = dquot_transfer(idmap, inode, attr); 7062 up_read(&EXT4_I(inode)->xattr_sem); 7063 7064 if (error) { 7065 ext4_journal_stop(handle); 7066 return error; 7067 } 7068 /* Update corresponding info in inode so that everything is in 7069 * one transaction */ 7070 i_uid_update(idmap, attr, inode); 7071 i_gid_update(idmap, attr, inode); 7072 error = ext4_mark_inode_dirty(handle, inode); 7073 ext4_journal_stop(handle); 7074 if (unlikely(error)) { 7075 return error; 7076 } 7077 } 7078 7079 if (attr->ia_valid & ATTR_SIZE) { 7080 loff_t oldsize = inode->i_size; 7081 int shrink = (attr->ia_size < inode->i_size); 7082 7083 if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) { 7084 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); 7085 7086 if (attr->ia_size > sbi->s_bitmap_maxbytes) { 7087 return -EFBIG; 7088 } 7089 } 7090 if (!S_ISREG(inode->i_mode)) { 7091 return -EINVAL; 7092 } 7093 7094 if (attr->ia_size == inode->i_size) 7095 inc_ivers = false; 7096 7097 /* 7098 * If file has inline data but new size exceeds inline capacity, 7099 * convert to extent-based storage first to prevent inconsistent 7100 * state (inline flag set but size exceeds inline capacity). 7101 */ 7102 if (ext4_has_inline_data(inode) && 7103 attr->ia_size > EXT4_I(inode)->i_inline_size) { 7104 error = ext4_convert_inline_data(inode); 7105 if (error) 7106 goto err_out; 7107 } 7108 7109 if (shrink) { 7110 if (ext4_should_order_data(inode)) { 7111 error = ext4_begin_ordered_truncate(inode, 7112 attr->ia_size); 7113 if (error) 7114 goto err_out; 7115 } 7116 /* 7117 * Blocks are going to be removed from the inode. Wait 7118 * for dio in flight. 7119 */ 7120 inode_dio_wait(inode); 7121 } 7122 7123 filemap_invalidate_lock(inode->i_mapping); 7124 7125 rc = ext4_break_layouts(inode); 7126 if (rc) { 7127 filemap_invalidate_unlock(inode->i_mapping); 7128 goto err_out; 7129 } 7130 7131 if (attr->ia_size > oldsize) 7132 error = ext4_truncate_up(inode, oldsize, attr->ia_size); 7133 else { 7134 /* Shrink or do not change i_size. */ 7135 error = ext4_truncate_down(inode, oldsize, 7136 attr->ia_size, &orphan); 7137 } 7138 7139 filemap_invalidate_unlock(inode->i_mapping); 7140 } 7141 7142 if (!error) { 7143 if (inc_ivers) 7144 inode_inc_iversion(inode); 7145 setattr_copy(idmap, inode, attr); 7146 mark_inode_dirty(inode); 7147 } 7148 7149 /* 7150 * If the call to ext4_truncate failed to get a transaction handle at 7151 * all, we need to clean up the in-core orphan list manually. 7152 */ 7153 if (orphan && inode->i_nlink) 7154 ext4_orphan_del(NULL, inode); 7155 7156 if (!error && (ia_valid & ATTR_MODE)) 7157 rc = posix_acl_chmod(idmap, dentry, inode->i_mode); 7158 7159 err_out: 7160 if (error) 7161 ext4_std_error(inode->i_sb, error); 7162 if (!error) 7163 error = rc; 7164 return error; 7165 } 7166 7167 u32 ext4_dio_alignment(struct inode *inode) 7168 { 7169 if (fsverity_active(inode)) 7170 return 0; 7171 if (ext4_should_journal_data(inode)) 7172 return 0; 7173 if (ext4_has_inline_data(inode)) 7174 return 0; 7175 if (IS_ENCRYPTED(inode)) 7176 return i_blocksize(inode); 7177 return 1; /* use the iomap defaults */ 7178 } 7179 7180 int ext4_getattr(const struct mnt_idmap *idmap, const struct path *path, 7181 struct kstat *stat, u32 request_mask, unsigned int query_flags) 7182 { 7183 struct inode *inode = d_inode(path->dentry); 7184 struct ext4_inode *raw_inode; 7185 struct ext4_inode_info *ei = EXT4_I(inode); 7186 unsigned int flags; 7187 7188 if ((request_mask & STATX_BTIME) && 7189 EXT4_FITS_IN_INODE(raw_inode, ei, i_crtime)) { 7190 stat->result_mask |= STATX_BTIME; 7191 stat->btime.tv_sec = ei->i_crtime.tv_sec; 7192 stat->btime.tv_nsec = ei->i_crtime.tv_nsec; 7193 } 7194 7195 /* Return the DIO alignment restrictions if requested. */ 7196 if ((request_mask & STATX_DIOALIGN) && S_ISREG(inode->i_mode)) { 7197 u32 dio_align = ext4_dio_alignment(inode); 7198 7199 stat->result_mask |= STATX_DIOALIGN; 7200 if (dio_align == 1) { 7201 struct block_device *bdev = inode->i_sb->s_bdev; 7202 7203 /* iomap defaults */ 7204 stat->dio_mem_align = bdev_dma_alignment(bdev) + 1; 7205 stat->dio_offset_align = bdev_logical_block_size(bdev); 7206 } else { 7207 stat->dio_mem_align = dio_align; 7208 stat->dio_offset_align = dio_align; 7209 } 7210 } 7211 7212 if ((request_mask & STATX_WRITE_ATOMIC) && S_ISREG(inode->i_mode)) { 7213 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); 7214 unsigned int awu_min = 0, awu_max = 0; 7215 7216 if (ext4_inode_can_atomic_write(inode)) { 7217 awu_min = sbi->s_awu_min; 7218 awu_max = sbi->s_awu_max; 7219 } 7220 7221 generic_fill_statx_atomic_writes(stat, awu_min, awu_max, 0); 7222 } 7223 7224 flags = ei->i_flags & EXT4_FL_USER_VISIBLE; 7225 if (flags & EXT4_APPEND_FL) 7226 stat->attributes |= STATX_ATTR_APPEND; 7227 if (flags & EXT4_COMPR_FL) 7228 stat->attributes |= STATX_ATTR_COMPRESSED; 7229 if (flags & EXT4_ENCRYPT_FL) 7230 stat->attributes |= STATX_ATTR_ENCRYPTED; 7231 if (flags & EXT4_IMMUTABLE_FL) 7232 stat->attributes |= STATX_ATTR_IMMUTABLE; 7233 if (flags & EXT4_NODUMP_FL) 7234 stat->attributes |= STATX_ATTR_NODUMP; 7235 7236 stat->attributes_mask |= (STATX_ATTR_APPEND | 7237 STATX_ATTR_COMPRESSED | 7238 STATX_ATTR_ENCRYPTED | 7239 STATX_ATTR_IMMUTABLE | 7240 STATX_ATTR_NODUMP); 7241 7242 generic_fillattr(idmap, request_mask, inode, stat); 7243 return 0; 7244 } 7245 7246 int ext4_file_getattr(const struct mnt_idmap *idmap, 7247 const struct path *path, struct kstat *stat, 7248 u32 request_mask, unsigned int query_flags) 7249 { 7250 struct inode *inode = d_inode(path->dentry); 7251 u64 delalloc_blocks; 7252 7253 ext4_getattr(idmap, path, stat, request_mask, query_flags); 7254 7255 /* 7256 * If there is inline data in the inode, the inode will normally not 7257 * have data blocks allocated (it may have an external xattr block). 7258 * Report at least one sector for such files, so tools like tar, rsync, 7259 * others don't incorrectly think the file is completely sparse. 7260 */ 7261 if (unlikely(ext4_has_inline_data(inode))) 7262 stat->blocks += (stat->size + 511) >> 9; 7263 7264 /* 7265 * We can't update i_blocks if the block allocation is delayed 7266 * otherwise in the case of system crash before the real block 7267 * allocation is done, we will have i_blocks inconsistent with 7268 * on-disk file blocks. 7269 * We always keep i_blocks updated together with real 7270 * allocation. But to not confuse with user, stat 7271 * will return the blocks that include the delayed allocation 7272 * blocks for this file. 7273 */ 7274 delalloc_blocks = EXT4_C2B(EXT4_SB(inode->i_sb), 7275 EXT4_I(inode)->i_reserved_data_blocks); 7276 stat->blocks += delalloc_blocks << (inode->i_sb->s_blocksize_bits - 9); 7277 return 0; 7278 } 7279 7280 static int ext4_index_trans_blocks(struct inode *inode, int lblocks, 7281 int pextents) 7282 { 7283 if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) 7284 return ext4_ind_trans_blocks(inode, lblocks); 7285 return ext4_ext_index_trans_blocks(inode, pextents); 7286 } 7287 7288 /* 7289 * Calculate number of credits needed in a transaction to: 7290 * * Allocate data blocks from @alloc_extents different groups - note that 7291 * with flexbg a single physical extent can span multiple groups but 7292 * single mballoc request only returns extent within one group. 7293 * * Allocate metatadata (extent tree blocks, indirect blocks) to store 7294 * pointers to @pextents data extents having @lblocks in total. 7295 * * Modify extent tree / indirect block tree, inode, superblock, quota 7296 * tracking, xattr blocks 7297 */ 7298 int ext4_meta_trans_blocks(struct inode *inode, int lblocks, int pextents, 7299 int alloc_extents) 7300 { 7301 ext4_group_t groups, ngroups = ext4_get_groups_count(inode->i_sb); 7302 int gdpblocks; 7303 int idxblocks; 7304 int ret; 7305 7306 /* 7307 * How many index and leaf blocks need to touch to map @lblocks 7308 * logical blocks to @pextents physical extents? 7309 */ 7310 idxblocks = ext4_index_trans_blocks(inode, lblocks, pextents); 7311 7312 /* 7313 * Now let's see how many group bitmaps and group descriptors need 7314 * to account 7315 */ 7316 groups = idxblocks + alloc_extents; 7317 gdpblocks = groups; 7318 if (groups > ngroups) 7319 groups = ngroups; 7320 if (groups > EXT4_SB(inode->i_sb)->s_gdb_count) 7321 gdpblocks = EXT4_SB(inode->i_sb)->s_gdb_count; 7322 7323 /* bitmaps and block group descriptor blocks */ 7324 ret = idxblocks + groups + gdpblocks; 7325 7326 /* Blocks for super block, inode, quota and xattr blocks */ 7327 ret += EXT4_META_TRANS_BLOCKS(inode->i_sb); 7328 7329 return ret; 7330 } 7331 7332 /* 7333 * Calculate the journal credits for modifying the number of blocks 7334 * in a single extent within one transaction. 'nrblocks' is used only 7335 * for non-extent inodes. For extent type inodes, 'nrblocks' can be 7336 * zero if the exact number of blocks is unknown. 7337 */ 7338 int ext4_chunk_trans_extent(struct inode *inode, int nrblocks) 7339 { 7340 int ret; 7341 7342 ret = ext4_meta_trans_blocks(inode, nrblocks, 1, 1); 7343 /* Account for data blocks for journalled mode */ 7344 if (ext4_should_journal_data(inode)) 7345 ret += nrblocks; 7346 return ret; 7347 } 7348 7349 /* 7350 * Calculate the journal credits for a chunk of data modification. 7351 * 7352 * This is called from DIO, fallocate or whoever calling 7353 * ext4_map_blocks() to map/allocate a chunk of contiguous disk blocks. 7354 * 7355 * journal buffers for data blocks are not included here, as DIO 7356 * and fallocate do no need to journal data buffers. 7357 */ 7358 int ext4_chunk_trans_blocks(struct inode *inode, int nrblocks) 7359 { 7360 return ext4_meta_trans_blocks(inode, nrblocks, 1, 1); 7361 } 7362 7363 /* 7364 * The caller must have previously called ext4_reserve_inode_write(). 7365 * Give this, we know that the caller already has write access to iloc->bh. 7366 */ 7367 int ext4_mark_iloc_dirty(handle_t *handle, 7368 struct inode *inode, struct ext4_iloc *iloc) 7369 { 7370 struct super_block *sb = inode->i_sb; 7371 int err = 0; 7372 7373 err = ext4_emergency_state(sb); 7374 if (unlikely(err)) { 7375 put_bh(iloc->bh); 7376 return err; 7377 } 7378 ext4_fc_track_inode(handle, inode); 7379 7380 /* the do_update_inode consumes one bh->b_count */ 7381 get_bh(iloc->bh); 7382 7383 /* ext4_do_update_inode() does jbd2_journal_dirty_metadata */ 7384 err = ext4_do_update_inode(handle, inode, iloc); 7385 put_bh(iloc->bh); 7386 /* 7387 * Mark that there's metadata writeout pending for the inode so that it 7388 * gets properly flushed on fsync(2) and similar. We don't bother for 7389 * fastcommit replay as that flushes the whole bdev afterwards anyway. 7390 * It is faster this way and we avoid entering fs writeback paths which 7391 * aren't fully initialized yet. 7392 */ 7393 if (!ext4_handle_valid(handle) && 7394 !(EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY)) { 7395 /* 7396 * Inode didn't need to go through dirtying, make sure it is 7397 * attached to wb so that writeback can handle it. 7398 */ 7399 spin_lock(&inode->i_lock); 7400 inode_attach_wb(inode, NULL); 7401 spin_unlock(&inode->i_lock); 7402 set_inode_metadata_writeback(inode); 7403 } 7404 return err; 7405 } 7406 7407 /* 7408 * On success, We end up with an outstanding reference count against 7409 * iloc->bh. This _must_ be cleaned up later. 7410 */ 7411 7412 int 7413 ext4_reserve_inode_write(handle_t *handle, struct inode *inode, 7414 struct ext4_iloc *iloc) 7415 { 7416 int err; 7417 7418 err = ext4_emergency_state(inode->i_sb); 7419 if (unlikely(err)) 7420 return err; 7421 7422 err = ext4_get_inode_loc(inode, iloc); 7423 if (!err) { 7424 BUFFER_TRACE(iloc->bh, "get_write_access"); 7425 err = ext4_journal_get_write_access(handle, inode->i_sb, 7426 iloc->bh, EXT4_JTR_NONE); 7427 if (err) { 7428 brelse(iloc->bh); 7429 iloc->bh = NULL; 7430 } 7431 ext4_fc_track_inode(handle, inode); 7432 } 7433 ext4_std_error(inode->i_sb, err); 7434 return err; 7435 } 7436 7437 static int __ext4_expand_extra_isize(struct inode *inode, 7438 unsigned int new_extra_isize, 7439 struct ext4_iloc *iloc, 7440 handle_t *handle, int *no_expand) 7441 { 7442 struct ext4_inode *raw_inode; 7443 struct ext4_xattr_ibody_header *header; 7444 unsigned int inode_size = EXT4_INODE_SIZE(inode->i_sb); 7445 struct ext4_inode_info *ei = EXT4_I(inode); 7446 int error; 7447 7448 /* this was checked at iget time, but double check for good measure */ 7449 if ((EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize > inode_size) || 7450 (ei->i_extra_isize & 3)) { 7451 EXT4_ERROR_INODE(inode, "bad extra_isize %u (inode size %u)", 7452 ei->i_extra_isize, 7453 EXT4_INODE_SIZE(inode->i_sb)); 7454 return -EFSCORRUPTED; 7455 } 7456 if ((new_extra_isize < ei->i_extra_isize) || 7457 (new_extra_isize < 4) || 7458 (new_extra_isize > inode_size - EXT4_GOOD_OLD_INODE_SIZE)) 7459 return -EINVAL; /* Should never happen */ 7460 7461 raw_inode = ext4_raw_inode(iloc); 7462 7463 header = IHDR(inode, raw_inode); 7464 7465 /* No extended attributes present */ 7466 if (!ext4_test_inode_state(inode, EXT4_STATE_XATTR) || 7467 header->h_magic != cpu_to_le32(EXT4_XATTR_MAGIC)) { 7468 memset((void *)raw_inode + EXT4_GOOD_OLD_INODE_SIZE + 7469 EXT4_I(inode)->i_extra_isize, 0, 7470 new_extra_isize - EXT4_I(inode)->i_extra_isize); 7471 EXT4_I(inode)->i_extra_isize = new_extra_isize; 7472 return 0; 7473 } 7474 7475 /* 7476 * We may need to allocate external xattr block so we need quotas 7477 * initialized. Here we can be called with various locks held so we 7478 * cannot affort to initialize quotas ourselves. So just bail. 7479 */ 7480 if (dquot_initialize_needed(inode)) 7481 return -EAGAIN; 7482 7483 /* try to expand with EAs present */ 7484 error = ext4_expand_extra_isize_ea(inode, new_extra_isize, 7485 raw_inode, handle); 7486 if (error) { 7487 /* 7488 * Inode size expansion failed; don't try again 7489 */ 7490 *no_expand = 1; 7491 } 7492 7493 return error; 7494 } 7495 7496 /* 7497 * Expand an inode by new_extra_isize bytes. 7498 * Returns 0 on success or negative error number on failure. 7499 */ 7500 static int ext4_try_to_expand_extra_isize(struct inode *inode, 7501 unsigned int new_extra_isize, 7502 struct ext4_iloc iloc, 7503 handle_t *handle) 7504 { 7505 int no_expand; 7506 int error; 7507 7508 if (ext4_test_inode_state(inode, EXT4_STATE_NO_EXPAND)) 7509 return -EOVERFLOW; 7510 7511 /* 7512 * Skip expansion during mount (!SB_ACTIVE). Expanding extra isize 7513 * may move xattrs to external blocks and release ea_inodes via iput. 7514 * When !SB_ACTIVE, iput triggers write_inode_now() which acquires 7515 * s_writepages_rwsem, causing a deadlock with the caller's active 7516 * jbd2 handle (lock order: s_writepages_rwsem -> jbd2_handle). 7517 * 7518 * Skip it while unmounting as well. shrink_dcache_for_umount() 7519 * clears s_root before generic_shutdown_super() clears SB_ACTIVE, and 7520 * the last iput() of a lazytime inode in that window redirties it and 7521 * lands here. Moving xattrs out to a block then needs a new EA inode, 7522 * which ext4_xattr_inode_create() refuses without s_root. 7523 */ 7524 if (unlikely(!(inode->i_sb->s_flags & SB_ACTIVE) || !inode->i_sb->s_root)) 7525 return -EBUSY; 7526 7527 /* 7528 * In nojournal mode, we can immediately attempt to expand 7529 * the inode. When journaled, we first need to obtain extra 7530 * buffer credits since we may write into the EA block 7531 * with this same handle. If journal_extend fails, then it will 7532 * only result in a minor loss of functionality for that inode. 7533 * If this is felt to be critical, then e2fsck should be run to 7534 * force a large enough s_min_extra_isize. 7535 */ 7536 if (ext4_journal_extend(handle, 7537 EXT4_DATA_TRANS_BLOCKS(inode->i_sb), 0) != 0) 7538 return -ENOSPC; 7539 7540 if (ext4_write_trylock_xattr(inode, &no_expand) == 0) 7541 return -EBUSY; 7542 7543 error = __ext4_expand_extra_isize(inode, new_extra_isize, &iloc, 7544 handle, &no_expand); 7545 ext4_write_unlock_xattr(inode, &no_expand); 7546 7547 return error; 7548 } 7549 7550 int ext4_expand_extra_isize(struct inode *inode, 7551 unsigned int new_extra_isize, 7552 struct ext4_iloc *iloc) 7553 { 7554 handle_t *handle; 7555 int no_expand; 7556 int error, rc; 7557 7558 if (ext4_test_inode_state(inode, EXT4_STATE_NO_EXPAND)) { 7559 brelse(iloc->bh); 7560 return -EOVERFLOW; 7561 } 7562 7563 handle = ext4_journal_start(inode, EXT4_HT_INODE, 7564 EXT4_DATA_TRANS_BLOCKS(inode->i_sb)); 7565 if (IS_ERR(handle)) { 7566 error = PTR_ERR(handle); 7567 brelse(iloc->bh); 7568 return error; 7569 } 7570 7571 ext4_write_lock_xattr(inode, &no_expand); 7572 7573 BUFFER_TRACE(iloc->bh, "get_write_access"); 7574 error = ext4_journal_get_write_access(handle, inode->i_sb, iloc->bh, 7575 EXT4_JTR_NONE); 7576 if (error) { 7577 brelse(iloc->bh); 7578 goto out_unlock; 7579 } 7580 7581 error = __ext4_expand_extra_isize(inode, new_extra_isize, iloc, 7582 handle, &no_expand); 7583 7584 rc = ext4_mark_iloc_dirty(handle, inode, iloc); 7585 if (!error) 7586 error = rc; 7587 7588 out_unlock: 7589 ext4_write_unlock_xattr(inode, &no_expand); 7590 ext4_journal_stop(handle); 7591 return error; 7592 } 7593 7594 /* 7595 * What we do here is to mark the in-core inode as clean with respect to inode 7596 * dirtiness (it may still be data-dirty). 7597 * This means that the in-core inode may be reaped by prune_icache 7598 * without having to perform any I/O. This is a very good thing, 7599 * because *any* task may call prune_icache - even ones which 7600 * have a transaction open against a different journal. 7601 * 7602 * Is this cheating? Not really. Sure, we haven't written the 7603 * inode out, but prune_icache isn't a user-visible syncing function. 7604 * Whenever the user wants stuff synced (sys_sync, sys_msync, sys_fsync) 7605 * we start and wait on commits. 7606 */ 7607 int __ext4_mark_inode_dirty(handle_t *handle, struct inode *inode, 7608 const char *func, unsigned int line) 7609 { 7610 struct ext4_iloc iloc; 7611 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); 7612 int err; 7613 7614 might_sleep(); 7615 trace_ext4_mark_inode_dirty(inode, _RET_IP_); 7616 err = ext4_reserve_inode_write(handle, inode, &iloc); 7617 if (err) 7618 goto out; 7619 7620 if (EXT4_I(inode)->i_extra_isize < sbi->s_want_extra_isize) 7621 ext4_try_to_expand_extra_isize(inode, sbi->s_want_extra_isize, 7622 iloc, handle); 7623 7624 err = ext4_mark_iloc_dirty(handle, inode, &iloc); 7625 out: 7626 if (unlikely(err)) 7627 ext4_error_inode_err(inode, func, line, 0, err, 7628 "mark_inode_dirty error"); 7629 return err; 7630 } 7631 7632 /* 7633 * ext4_dirty_inode() is called from __mark_inode_dirty() 7634 * 7635 * We're really interested in the case where a file is being extended. 7636 * i_size has been changed by generic_commit_write() and we thus need 7637 * to include the updated inode in the current transaction. 7638 * 7639 * Also, dquot_alloc_block() will always dirty the inode when blocks 7640 * are allocated to the file. 7641 * 7642 * If the inode is marked synchronous, we don't honour that here - doing 7643 * so would cause a commit on atime updates, which we don't bother doing. 7644 * We handle synchronous inodes at the highest possible level. 7645 */ 7646 void ext4_dirty_inode(struct inode *inode, int flags) 7647 { 7648 handle_t *handle; 7649 7650 handle = ext4_journal_start(inode, EXT4_HT_INODE, 2); 7651 if (IS_ERR(handle)) 7652 return; 7653 ext4_mark_inode_dirty(handle, inode); 7654 ext4_journal_stop(handle); 7655 } 7656 7657 int ext4_change_inode_journal_flag(struct inode *inode, int val) 7658 { 7659 journal_t *journal; 7660 handle_t *handle; 7661 int err; 7662 int alloc_ctx; 7663 7664 /* 7665 * We have to be very careful here: changing a data block's 7666 * journaling status dynamically is dangerous. If we write a 7667 * data block to the journal, change the status and then delete 7668 * that block, we risk forgetting to revoke the old log record 7669 * from the journal and so a subsequent replay can corrupt data. 7670 * So, first we make sure that the journal is empty and that 7671 * nobody is changing anything. 7672 */ 7673 7674 journal = EXT4_JOURNAL(inode); 7675 if (!journal) 7676 return 0; 7677 if (is_journal_aborted(journal)) 7678 return -EROFS; 7679 7680 /* Wait for all existing dio workers */ 7681 inode_dio_wait(inode); 7682 7683 /* 7684 * Before flushing the journal and switching inode's aops, we have 7685 * to flush all dirty data the inode has. There can be outstanding 7686 * delayed allocations, there can be unwritten extents created by 7687 * fallocate or buffered writes in dioread_nolock mode covered by 7688 * dirty data which can be converted only after flushing the dirty 7689 * data (and journalled aops don't know how to handle these cases). 7690 */ 7691 filemap_invalidate_lock(inode->i_mapping); 7692 err = filemap_write_and_wait(inode->i_mapping); 7693 if (err < 0) { 7694 filemap_invalidate_unlock(inode->i_mapping); 7695 return err; 7696 } 7697 /* Before switch the inode journalling mode evict all the page cache. */ 7698 truncate_pagecache(inode, 0); 7699 7700 alloc_ctx = ext4_writepages_down_write(inode->i_sb); 7701 jbd2_journal_lock_updates(journal); 7702 7703 /* 7704 * OK, there are no updates running now, and all cached data is 7705 * synced to disk. We are now in a completely consistent state 7706 * which doesn't have anything in the journal, and we know that 7707 * no filesystem updates are running, so it is safe to modify 7708 * the inode's in-core data-journaling state flag now. 7709 */ 7710 7711 if (val) { 7712 ext4_set_inode_flag(inode, EXT4_INODE_JOURNAL_DATA); 7713 ext4_disable_buffered_iomap(inode); 7714 } else { 7715 err = jbd2_journal_flush(journal, 0); 7716 if (err < 0) { 7717 jbd2_journal_unlock_updates(journal); 7718 ext4_writepages_up_write(inode->i_sb, alloc_ctx); 7719 filemap_invalidate_unlock(inode->i_mapping); 7720 return err; 7721 } 7722 ext4_clear_inode_flag(inode, EXT4_INODE_JOURNAL_DATA); 7723 ext4_enable_buffered_iomap(inode); 7724 } 7725 ext4_set_aops(inode); 7726 ext4_set_inode_mapping_order(inode); 7727 7728 jbd2_journal_unlock_updates(journal); 7729 ext4_writepages_up_write(inode->i_sb, alloc_ctx); 7730 filemap_invalidate_unlock(inode->i_mapping); 7731 7732 /* Finally we can mark the inode as dirty. */ 7733 7734 handle = ext4_journal_start(inode, EXT4_HT_INODE, 1); 7735 if (IS_ERR(handle)) 7736 return PTR_ERR(handle); 7737 7738 ext4_fc_mark_ineligible(inode->i_sb, 7739 EXT4_FC_REASON_JOURNAL_FLAG_CHANGE, handle); 7740 err = ext4_mark_inode_dirty(handle, inode); 7741 ext4_handle_sync(handle); 7742 ext4_journal_stop(handle); 7743 ext4_std_error(inode->i_sb, err); 7744 7745 return err; 7746 } 7747 7748 static int ext4_bh_unmapped(handle_t *handle, struct inode *inode, 7749 struct buffer_head *bh) 7750 { 7751 return !buffer_mapped(bh); 7752 } 7753 7754 static int ext4_block_page_mkwrite(struct inode *inode, struct folio *folio, 7755 get_block_t get_block) 7756 { 7757 handle_t *handle; 7758 loff_t size; 7759 unsigned long len; 7760 int credits; 7761 int ret; 7762 7763 credits = ext4_chunk_trans_extent(inode, 7764 ext4_journal_blocks_per_folio(inode)); 7765 handle = ext4_journal_start(inode, EXT4_HT_WRITE_PAGE, credits); 7766 if (IS_ERR(handle)) 7767 return PTR_ERR(handle); 7768 7769 folio_lock(folio); 7770 size = i_size_read(inode); 7771 /* Page got truncated from under us? */ 7772 if (folio->mapping != inode->i_mapping || folio_pos(folio) > size) { 7773 ret = -EFAULT; 7774 goto out_error; 7775 } 7776 7777 len = folio_size(folio); 7778 if (folio_pos(folio) + len > size) 7779 len = size - folio_pos(folio); 7780 7781 ret = ext4_block_write_begin(handle, folio, 0, len, get_block); 7782 if (ret) 7783 goto out_error; 7784 7785 if (!ext4_should_journal_data(inode)) { 7786 block_commit_write(folio, 0, len); 7787 folio_mark_dirty(folio); 7788 } else { 7789 ret = ext4_journal_folio_buffers(handle, folio, len); 7790 if (ret) 7791 goto out_error; 7792 } 7793 ext4_journal_stop(handle); 7794 folio_wait_stable(folio); 7795 return ret; 7796 7797 out_error: 7798 folio_unlock(folio); 7799 ext4_journal_stop(handle); 7800 return ret; 7801 } 7802 7803 static vm_fault_t ext4_iomap_page_mkwrite(struct vm_fault *vmf) 7804 { 7805 struct inode *inode = file_inode(vmf->vma->vm_file); 7806 iomap_iter_next_fn next; 7807 7808 /* 7809 * ext4_nonda_switch() could writeback this folio, so have to 7810 * call it before lock folio. 7811 */ 7812 if (test_opt(inode->i_sb, DELALLOC) && !ext4_nonda_switch(inode->i_sb)) 7813 next = ext4_iomap_buffered_da_write_next; 7814 else 7815 next = ext4_iomap_buffered_write_next; 7816 7817 return iomap_page_mkwrite(vmf, next, NULL); 7818 } 7819 7820 vm_fault_t ext4_page_mkwrite(struct vm_fault *vmf) 7821 { 7822 struct vm_area_struct *vma = vmf->vma; 7823 struct folio *folio = page_folio(vmf->page); 7824 loff_t size; 7825 unsigned long len; 7826 int err; 7827 vm_fault_t ret; 7828 struct file *file = vma->vm_file; 7829 struct inode *inode = file_inode(file); 7830 struct address_space *mapping = inode->i_mapping; 7831 get_block_t *get_block = ext4_get_block; 7832 int retries = 0; 7833 7834 if (unlikely(IS_IMMUTABLE(inode))) 7835 return VM_FAULT_SIGBUS; 7836 7837 sb_start_pagefault(inode->i_sb); 7838 file_update_time(vma->vm_file); 7839 7840 filemap_invalidate_lock_shared(mapping); 7841 7842 err = ext4_convert_inline_data(inode); 7843 if (err) 7844 goto out_ret; 7845 7846 if (ext4_inode_buffered_iomap(inode)) { 7847 ret = ext4_iomap_page_mkwrite(vmf); 7848 goto out; 7849 } 7850 7851 /* 7852 * On data journalling we skip straight to the transaction handle: 7853 * there's no delalloc; page truncated will be checked later; the 7854 * early return w/ all buffers mapped (calculates size/len) can't 7855 * be used; and there's no dioread_nolock, so only ext4_get_block. 7856 */ 7857 if (ext4_should_journal_data(inode)) 7858 goto retry_alloc; 7859 7860 /* Delalloc case is easy... */ 7861 if (test_opt(inode->i_sb, DELALLOC) && 7862 !ext4_nonda_switch(inode->i_sb)) { 7863 do { 7864 err = block_page_mkwrite(vma, vmf, 7865 ext4_da_get_block_prep); 7866 } while (err == -ENOSPC && 7867 ext4_should_retry_alloc(inode->i_sb, &retries)); 7868 goto out_ret; 7869 } 7870 7871 folio_lock(folio); 7872 size = i_size_read(inode); 7873 /* Page got truncated from under us? */ 7874 if (folio->mapping != mapping || folio_pos(folio) > size) { 7875 folio_unlock(folio); 7876 ret = VM_FAULT_NOPAGE; 7877 goto out; 7878 } 7879 7880 len = folio_size(folio); 7881 if (folio_pos(folio) + len > size) 7882 len = size - folio_pos(folio); 7883 /* 7884 * Return if we have all the buffers mapped. This avoids the need to do 7885 * journal_start/journal_stop which can block and take a long time 7886 * 7887 * This cannot be done for data journalling, as we have to add the 7888 * inode to the transaction's list to writeprotect pages on commit. 7889 */ 7890 if (folio_buffers(folio)) { 7891 if (!ext4_walk_page_buffers(NULL, inode, folio_buffers(folio), 7892 0, len, NULL, 7893 ext4_bh_unmapped)) { 7894 /* Wait so that we don't change page under IO */ 7895 folio_wait_stable(folio); 7896 ret = VM_FAULT_LOCKED; 7897 goto out; 7898 } 7899 } 7900 folio_unlock(folio); 7901 /* OK, we need to fill the hole... */ 7902 if (ext4_should_dioread_nolock(inode)) 7903 get_block = ext4_get_block_unwritten; 7904 retry_alloc: 7905 /* Start journal and allocate blocks */ 7906 err = ext4_block_page_mkwrite(inode, folio, get_block); 7907 if (err == -EAGAIN || 7908 (err == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries))) 7909 goto retry_alloc; 7910 out_ret: 7911 ret = vmf_fs_error(err); 7912 out: 7913 filemap_invalidate_unlock_shared(mapping); 7914 sb_end_pagefault(inode->i_sb); 7915 return ret; 7916 } 7917