xref: /linux/fs/ext4/inode.c (revision df415fc340d656f8e65986393c464a39ce61581d)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  *  linux/fs/ext4/inode.c
4  *
5  * Copyright (C) 1992, 1993, 1994, 1995
6  * Remy Card (card@masi.ibp.fr)
7  * Laboratoire MASI - Institut Blaise Pascal
8  * Universite Pierre et Marie Curie (Paris VI)
9  *
10  *  from
11  *
12  *  linux/fs/minix/inode.c
13  *
14  *  Copyright (C) 1991, 1992  Linus Torvalds
15  *
16  *  64-bit file support on 64-bit platforms by Jakub Jelinek
17  *	(jj@sunsite.ms.mff.cuni.cz)
18  *
19  *  Assorted race fixes, rewrite of ext4_get_block() by Al Viro, 2000
20  */
21 
22 #include <linux/fs.h>
23 #include <linux/mount.h>
24 #include <linux/time.h>
25 #include <linux/highuid.h>
26 #include <linux/pagemap.h>
27 #include <linux/dax.h>
28 #include <linux/quotaops.h>
29 #include <linux/string.h>
30 #include <linux/buffer_head.h>
31 #include <linux/writeback.h>
32 #include <linux/folio_batch.h>
33 #include <linux/mpage.h>
34 #include <linux/rmap.h>
35 #include <linux/namei.h>
36 #include <linux/uio.h>
37 #include <linux/bio.h>
38 #include <linux/workqueue.h>
39 #include <linux/kernel.h>
40 #include <linux/printk.h>
41 #include <linux/slab.h>
42 #include <linux/bitops.h>
43 #include <linux/iomap.h>
44 #include <linux/iversion.h>
45 
46 #include "ext4_jbd2.h"
47 #include "ext4_extents.h"
48 #include "xattr.h"
49 #include "acl.h"
50 #include "truncate.h"
51 
52 #include <kunit/static_stub.h>
53 
54 #include <trace/events/ext4.h>
55 
56 static void ext4_journalled_zero_new_buffers(handle_t *handle,
57 					    struct inode *inode,
58 					    struct folio *folio,
59 					    unsigned from, unsigned to);
60 
61 static __u32 ext4_inode_csum(struct inode *inode, struct ext4_inode *raw,
62 			      struct ext4_inode_info *ei)
63 {
64 	__u32 csum;
65 	__u16 dummy_csum = 0;
66 	int offset = offsetof(struct ext4_inode, i_checksum_lo);
67 	unsigned int csum_size = sizeof(dummy_csum);
68 
69 	csum = ext4_chksum(ei->i_csum_seed, (__u8 *)raw, offset);
70 	csum = ext4_chksum(csum, (__u8 *)&dummy_csum, csum_size);
71 	offset += csum_size;
72 	csum = ext4_chksum(csum, (__u8 *)raw + offset,
73 			   EXT4_GOOD_OLD_INODE_SIZE - offset);
74 
75 	if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) {
76 		offset = offsetof(struct ext4_inode, i_checksum_hi);
77 		csum = ext4_chksum(csum, (__u8 *)raw + EXT4_GOOD_OLD_INODE_SIZE,
78 				   offset - EXT4_GOOD_OLD_INODE_SIZE);
79 		if (EXT4_FITS_IN_INODE(raw, ei, i_checksum_hi)) {
80 			csum = ext4_chksum(csum, (__u8 *)&dummy_csum,
81 					   csum_size);
82 			offset += csum_size;
83 		}
84 		csum = ext4_chksum(csum, (__u8 *)raw + offset,
85 				   EXT4_INODE_SIZE(inode->i_sb) - offset);
86 	}
87 
88 	return csum;
89 }
90 
91 static int ext4_inode_csum_verify(struct inode *inode, struct ext4_inode *raw,
92 				  struct ext4_inode_info *ei)
93 {
94 	__u32 provided, calculated;
95 
96 	if (EXT4_SB(inode->i_sb)->s_es->s_creator_os !=
97 	    cpu_to_le32(EXT4_OS_LINUX) ||
98 	    !ext4_has_feature_metadata_csum(inode->i_sb))
99 		return 1;
100 
101 	provided = le16_to_cpu(raw->i_checksum_lo);
102 	calculated = ext4_inode_csum(inode, raw, ei);
103 	if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE &&
104 	    EXT4_FITS_IN_INODE(raw, ei, i_checksum_hi))
105 		provided |= ((__u32)le16_to_cpu(raw->i_checksum_hi)) << 16;
106 	else
107 		calculated &= 0xFFFF;
108 
109 	return provided == calculated;
110 }
111 
112 void ext4_inode_csum_set(struct inode *inode, struct ext4_inode *raw,
113 			 struct ext4_inode_info *ei)
114 {
115 	__u32 csum;
116 
117 	if (EXT4_SB(inode->i_sb)->s_es->s_creator_os !=
118 	    cpu_to_le32(EXT4_OS_LINUX) ||
119 	    !ext4_has_feature_metadata_csum(inode->i_sb))
120 		return;
121 
122 	csum = ext4_inode_csum(inode, raw, ei);
123 	raw->i_checksum_lo = cpu_to_le16(csum & 0xFFFF);
124 	if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE &&
125 	    EXT4_FITS_IN_INODE(raw, ei, i_checksum_hi))
126 		raw->i_checksum_hi = cpu_to_le16(csum >> 16);
127 }
128 
129 /*
130  * Clear the disksize-grow-pending state and wake up all waiters.
131  * Called when the pending zeroed EOF block which straddles i_disksize
132  * has completed writeback or its folio is discarded.
133  */
134 void ext4_iomap_clear_disksize_pending(struct inode *inode)
135 {
136 	if (!test_and_clear_wake_up_bit(
137 		ext4_inode_state_wait_bit(EXT4_STATE_DISKSIZE_GROW_PENDING),
138 		ext4_inode_state_wait_word(inode)))
139 		return;
140 
141 	trace_ext4_iomap_clear_disksize_pending(inode);
142 }
143 
144 /*
145  * Wait for the disksize-grow-pending zeroed EOF block which straddles
146  * i_disksize to be written back or cleared.
147  */
148 void ext4_iomap_wait_disksize_pending(struct inode *inode)
149 {
150 	/* Only emit the trace when the bit is actually set */
151 	if (ext4_test_inode_state(inode, EXT4_STATE_DISKSIZE_GROW_PENDING))
152 		trace_ext4_iomap_wait_disksize_pending(inode);
153 
154 	wait_on_bit(ext4_inode_state_wait_word(inode),
155 		    ext4_inode_state_wait_bit(EXT4_STATE_DISKSIZE_GROW_PENDING),
156 		    TASK_UNINTERRUPTIBLE);
157 }
158 
159 /*
160  * Get the range of the disksize-grow-pending zeroed EOF block range
161  * which straddles i_disksize if the EXT4_STATE_DISKSIZE_GROW_PENDING
162  * bit is set.
163  *
164  * Return the pending range, or zero if the BIT has already been cleared.
165  */
166 unsigned int ext4_iomap_get_disksize_pending_range(struct inode *inode,
167 						   loff_t *start)
168 {
169 	unsigned int blocksize = i_blocksize(inode);
170 	loff_t disksize;
171 
172 	if (!ext4_test_inode_state(inode, EXT4_STATE_DISKSIZE_GROW_PENDING))
173 		return 0;
174 
175 	/*
176 	 * The pending bit should be set only when i_disksize is not
177 	 * block-size aligned. While set, i_disksize must not be advanced,
178 	 * and the bit must be cleared when i_disksize is shrunk.
179 	 */
180 	down_read(&EXT4_I(inode)->i_data_sem);
181 	disksize = READ_ONCE(EXT4_I(inode)->i_disksize);
182 	if (!ext4_test_inode_state(inode, EXT4_STATE_DISKSIZE_GROW_PENDING) ||
183 	    WARN_ON_ONCE(IS_ALIGNED(disksize, blocksize))) {
184 		up_read(&EXT4_I(inode)->i_data_sem);
185 		return 0;
186 	}
187 
188 	up_read(&EXT4_I(inode)->i_data_sem);
189 	*start = disksize;
190 	return blocksize - (disksize & (blocksize - 1));
191 }
192 
193 static inline int ext4_begin_ordered_truncate(struct inode *inode,
194 					      loff_t new_size)
195 {
196 	struct jbd2_inode *jinode = READ_ONCE(EXT4_I(inode)->jinode);
197 
198 	trace_ext4_begin_ordered_truncate(inode, new_size);
199 	/*
200 	 * If jinode is zero, then we never opened the file for
201 	 * writing, so there's no need to call
202 	 * jbd2_journal_begin_ordered_truncate() since there's no
203 	 * outstanding writes we need to flush.
204 	 */
205 	if (!jinode)
206 		return 0;
207 	return jbd2_journal_begin_ordered_truncate(EXT4_JOURNAL(inode),
208 						   jinode,
209 						   new_size);
210 }
211 
212 /*
213  * Test whether an inode is a fast symlink.
214  * A fast symlink has its symlink data stored in ext4_inode_info->i_data.
215  */
216 int ext4_inode_is_fast_symlink(struct inode *inode)
217 {
218 	if (!ext4_has_feature_ea_inode(inode->i_sb)) {
219 		int ea_blocks = EXT4_I(inode)->i_file_acl ?
220 				EXT4_CLUSTER_SIZE(inode->i_sb) >> 9 : 0;
221 
222 		if (ext4_has_inline_data(inode))
223 			return 0;
224 
225 		return (S_ISLNK(inode->i_mode) && inode->i_blocks - ea_blocks == 0);
226 	}
227 	return S_ISLNK(inode->i_mode) && inode->i_size &&
228 	       (inode->i_size < EXT4_N_BLOCKS * 4);
229 }
230 
231 /*
232  * Called at the last iput() if i_nlink is zero.
233  */
234 void ext4_evict_inode(struct inode *inode)
235 {
236 	handle_t *handle;
237 	int err;
238 	/*
239 	 * Credits for final inode cleanup and freeing:
240 	 * sb + inode (ext4_orphan_del()), block bitmap, group descriptor
241 	 * (xattr block freeing), bitmap, group descriptor (inode freeing)
242 	 */
243 	int extra_credits = 6;
244 	bool freeze_protected = false;
245 
246 	trace_ext4_evict_inode(inode);
247 
248 	dax_break_layout_final(inode);
249 
250 	if (EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL)
251 		ext4_evict_ea_inode(inode);
252 	if (inode->i_nlink) {
253 		/*
254 		 * If there's dirty page will lead to data loss, user
255 		 * could see stale data.
256 		 */
257 		if (unlikely(!ext4_emergency_state(inode->i_sb) &&
258 		    mapping_tagged(&inode->i_data, PAGECACHE_TAG_DIRTY)))
259 			ext4_warning_inode(inode, "data will be lost");
260 
261 		truncate_inode_pages_final(&inode->i_data);
262 		goto no_delete;
263 	}
264 
265 	if (is_bad_inode(inode))
266 		goto no_delete;
267 	dquot_initialize(inode);
268 
269 	if (ext4_should_order_data(inode))
270 		ext4_begin_ordered_truncate(inode, 0);
271 	if (ext4_inode_buffered_iomap(inode))
272 		ext4_iomap_clear_disksize_pending(inode);
273 	truncate_inode_pages_final(&inode->i_data);
274 
275 	/*
276 	 * For inodes with journalled data, transaction commit could have
277 	 * dirtied the inode. And for inodes with dioread_nolock, unwritten
278 	 * extents converting worker could merge extents and also have dirtied
279 	 * the inode. Flush worker is ignoring it because of I_FREEING flag but
280 	 * we still need to remove the inode from the writeback lists.
281 	 */
282 	inode_io_list_del(inode);
283 
284 	/*
285 	 * Protect us against freezing - iput() caller didn't have to have any
286 	 * protection against it. When we are in a running transaction though,
287 	 * we are already protected against freezing and we cannot grab further
288 	 * protection due to lock ordering constraints.
289 	 */
290 	if (!ext4_journal_current_handle()) {
291 		sb_start_intwrite(inode->i_sb);
292 		freeze_protected = true;
293 	}
294 
295 	if (!IS_NOQUOTA(inode))
296 		extra_credits += EXT4_MAXQUOTAS_DEL_BLOCKS(inode->i_sb);
297 
298 	/*
299 	 * Block bitmap, group descriptor, and inode are accounted in both
300 	 * ext4_blocks_for_truncate() and extra_credits. So subtract 3.
301 	 */
302 	handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE,
303 			 ext4_blocks_for_truncate(inode) + extra_credits - 3);
304 	if (IS_ERR(handle)) {
305 		ext4_std_error(inode->i_sb, PTR_ERR(handle));
306 		/*
307 		 * If we're going to skip the normal cleanup, we still need to
308 		 * make sure that the in-core orphan linked list is properly
309 		 * cleaned up.
310 		 */
311 		ext4_orphan_del(NULL, inode);
312 		if (freeze_protected)
313 			sb_end_intwrite(inode->i_sb);
314 		goto no_delete;
315 	}
316 
317 	if (IS_SYNC(inode))
318 		ext4_handle_sync(handle);
319 
320 	/*
321 	 * Set inode->i_size to 0 before calling ext4_truncate(). We need
322 	 * special handling of symlinks here because i_size is used to
323 	 * determine whether ext4_inode_info->i_data contains symlink data or
324 	 * block mappings. Setting i_size to 0 will remove its fast symlink
325 	 * status. Erase i_data so that it becomes a valid empty block map.
326 	 */
327 	if (ext4_inode_is_fast_symlink(inode))
328 		memset(EXT4_I(inode)->i_data, 0, sizeof(EXT4_I(inode)->i_data));
329 	inode->i_size = 0;
330 	ext4_set_inode_state(inode, EXT4_STATE_NO_EXPAND);
331 	err = ext4_mark_inode_dirty(handle, inode);
332 	if (err) {
333 		ext4_warning(inode->i_sb,
334 			     "couldn't mark inode dirty (err %d)", err);
335 		goto stop_handle;
336 	}
337 	if (inode->i_blocks) {
338 		err = ext4_truncate(inode);
339 		if (err) {
340 			ext4_error_err(inode->i_sb, -err,
341 				       "couldn't truncate inode %llu (err %d)",
342 				       inode->i_ino, err);
343 			goto stop_handle;
344 		}
345 	}
346 
347 	/* Remove xattr references. */
348 	err = ext4_xattr_delete_inode(handle, inode, extra_credits);
349 	if (err) {
350 		ext4_warning(inode->i_sb, "xattr delete (err %d)", err);
351 stop_handle:
352 		ext4_journal_stop(handle);
353 		ext4_orphan_del(NULL, inode);
354 		if (freeze_protected)
355 			sb_end_intwrite(inode->i_sb);
356 		goto no_delete;
357 	}
358 
359 	/*
360 	 * Kill off the orphan record which ext4_truncate created.
361 	 * AKPM: I think this can be inside the above `if'.
362 	 * Note that ext4_orphan_del() has to be able to cope with the
363 	 * deletion of a non-existent orphan - this is because we don't
364 	 * know if ext4_truncate() actually created an orphan record.
365 	 * (Well, we could do this if we need to, but heck - it works)
366 	 */
367 	ext4_orphan_del(handle, inode);
368 	EXT4_I(inode)->i_dtime	= (__u32)ktime_get_real_seconds();
369 
370 	/*
371 	 * One subtle ordering requirement: if anything has gone wrong
372 	 * (transaction abort, IO errors, whatever), then we can still
373 	 * do these next steps (the fs will already have been marked as
374 	 * having errors), but we can't free the inode if the mark_dirty
375 	 * fails.
376 	 */
377 	if (ext4_mark_inode_dirty(handle, inode))
378 		/* If that failed, just do the required in-core inode clear. */
379 		ext4_clear_inode(inode);
380 	else
381 		ext4_free_inode(handle, inode);
382 	ext4_journal_stop(handle);
383 	if (freeze_protected)
384 		sb_end_intwrite(inode->i_sb);
385 	return;
386 no_delete:
387 	/*
388 	 * Check out some where else accidentally dirty the evicting inode,
389 	 * which may probably cause inode use-after-free issues later.
390 	 */
391 	WARN_ON_ONCE(!list_empty_careful(&inode->i_io_list));
392 
393 	if (!list_empty(&EXT4_I(inode)->i_fc_list))
394 		ext4_fc_mark_ineligible(inode->i_sb, EXT4_FC_REASON_NOMEM, NULL);
395 	ext4_clear_inode(inode);	/* We must guarantee clearing of inode... */
396 }
397 
398 #ifdef CONFIG_QUOTA
399 qsize_t *ext4_get_reserved_space(struct inode *inode)
400 {
401 	return &EXT4_I(inode)->i_reserved_quota;
402 }
403 #endif
404 
405 /*
406  * Called with i_data_sem down, which is important since we can call
407  * ext4_discard_preallocations() from here.
408  */
409 void ext4_da_update_reserve_space(struct inode *inode,
410 					int used, int quota_claim)
411 {
412 	struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
413 	struct ext4_inode_info *ei = EXT4_I(inode);
414 
415 	spin_lock(&ei->i_block_reservation_lock);
416 	trace_ext4_da_update_reserve_space(inode, used, quota_claim);
417 	if (unlikely(used > ei->i_reserved_data_blocks)) {
418 		ext4_warning(inode->i_sb, "%s: ino %llu, used %d "
419 			 "with only %d reserved data blocks",
420 			 __func__, inode->i_ino, used,
421 			 ei->i_reserved_data_blocks);
422 		WARN_ON(1);
423 		used = ei->i_reserved_data_blocks;
424 	}
425 
426 	/* Update per-inode reservations */
427 	ei->i_reserved_data_blocks -= used;
428 	percpu_counter_sub(&sbi->s_dirtyclusters_counter, used);
429 
430 	spin_unlock(&ei->i_block_reservation_lock);
431 
432 	/* Update quota subsystem for data blocks */
433 	if (quota_claim)
434 		dquot_claim_block(inode, EXT4_C2B(sbi, used));
435 	else {
436 		/*
437 		 * We did fallocate with an offset that is already delayed
438 		 * allocated. So on delayed allocated writeback we should
439 		 * not re-claim the quota for fallocated blocks.
440 		 */
441 		dquot_release_reservation_block(inode, EXT4_C2B(sbi, used));
442 	}
443 
444 	/*
445 	 * If we have done all the pending block allocations and if
446 	 * there aren't any writers on the inode, we can discard the
447 	 * inode's preallocations.
448 	 */
449 	if ((ei->i_reserved_data_blocks == 0) &&
450 	    !inode_is_open_for_write(inode))
451 		ext4_discard_preallocations(inode);
452 }
453 
454 static int __check_block_validity(struct inode *inode, const char *func,
455 				unsigned int line,
456 				struct ext4_map_blocks *map)
457 {
458 	journal_t *journal = EXT4_SB(inode->i_sb)->s_journal;
459 
460 	if (journal && inode == journal->j_inode)
461 		return 0;
462 
463 	if (!ext4_inode_block_valid(inode, map->m_pblk, map->m_len)) {
464 		ext4_error_inode(inode, func, line, map->m_pblk,
465 				 "lblock %lu mapped to illegal pblock %llu "
466 				 "(length %d)", (unsigned long) map->m_lblk,
467 				 map->m_pblk, map->m_len);
468 		return -EFSCORRUPTED;
469 	}
470 	return 0;
471 }
472 
473 int ext4_issue_zeroout(struct inode *inode, ext4_lblk_t lblk, ext4_fsblk_t pblk,
474 		       ext4_lblk_t len)
475 {
476 	int ret;
477 
478 	KUNIT_STATIC_STUB_REDIRECT(ext4_issue_zeroout, inode, lblk, pblk, len);
479 
480 	if (IS_ENCRYPTED(inode) && S_ISREG(inode->i_mode))
481 		return fscrypt_zeroout_range(inode,
482 				(loff_t)lblk << inode->i_blkbits,
483 				pblk << (inode->i_blkbits - SECTOR_SHIFT),
484 				(u64)len << inode->i_blkbits);
485 
486 	ret = sb_issue_zeroout(inode->i_sb, pblk, len, GFP_NOFS);
487 	if (ret > 0)
488 		ret = 0;
489 
490 	return ret;
491 }
492 
493 /*
494  * For generic regular files, when updating the extent tree, Ext4 should
495  * hold the i_rwsem and invalidate_lock exclusively. This ensures
496  * exclusion against concurrent page faults, as well as reads and writes.
497  */
498 #ifdef CONFIG_EXT4_DEBUG
499 void ext4_check_map_extents_env(struct inode *inode)
500 {
501 	if (EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY)
502 		return;
503 
504 	if (!S_ISREG(inode->i_mode) ||
505 	    IS_NOQUOTA(inode) || IS_VERITY(inode) ||
506 	    is_special_ino(inode->i_sb, inode->i_ino) ||
507 	    (inode_state_read_once(inode) & (I_FREEING | I_WILL_FREE | I_NEW)) ||
508 	    ext4_test_inode_flag(inode, EXT4_INODE_EA_INODE) ||
509 	    ext4_verity_in_progress(inode))
510 		return;
511 
512 	WARN_ON_ONCE(!inode_is_locked(inode) &&
513 		     !rwsem_is_locked(&inode->i_mapping->invalidate_lock));
514 }
515 #else
516 void ext4_check_map_extents_env(struct inode *inode) {}
517 #endif
518 
519 #define check_block_validity(inode, map)	\
520 	__check_block_validity((inode), __func__, __LINE__, (map))
521 
522 #ifdef ES_AGGRESSIVE_TEST
523 static void ext4_map_blocks_es_recheck(handle_t *handle,
524 				       struct inode *inode,
525 				       struct ext4_map_blocks *es_map,
526 				       struct ext4_map_blocks *map,
527 				       int flags)
528 {
529 	int retval;
530 
531 	map->m_flags = 0;
532 	/*
533 	 * There is a race window that the result is not the same.
534 	 * e.g. xfstests #223 when dioread_nolock enables.  The reason
535 	 * is that we lookup a block mapping in extent status tree with
536 	 * out taking i_data_sem.  So at the time the unwritten extent
537 	 * could be converted.
538 	 */
539 	down_read(&EXT4_I(inode)->i_data_sem);
540 	if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) {
541 		retval = ext4_ext_map_blocks(handle, inode, map, 0);
542 	} else {
543 		retval = ext4_ind_map_blocks(handle, inode, map, 0);
544 	}
545 	up_read((&EXT4_I(inode)->i_data_sem));
546 
547 	/*
548 	 * We don't check m_len because extent will be collpased in status
549 	 * tree.  So the m_len might not equal.
550 	 */
551 	if (es_map->m_lblk != map->m_lblk ||
552 	    es_map->m_flags != map->m_flags ||
553 	    es_map->m_pblk != map->m_pblk) {
554 		printk("ES cache assertion failed for inode: %llu "
555 		       "es_cached ex [%d/%d/%llu/%x] != "
556 		       "found ex [%d/%d/%llu/%x] retval %d flags %x\n",
557 		       inode->i_ino, es_map->m_lblk, es_map->m_len,
558 		       es_map->m_pblk, es_map->m_flags, map->m_lblk,
559 		       map->m_len, map->m_pblk, map->m_flags,
560 		       retval, flags);
561 	}
562 }
563 #endif /* ES_AGGRESSIVE_TEST */
564 
565 static int ext4_map_query_blocks_next_in_leaf(handle_t *handle,
566 			struct inode *inode, struct ext4_map_blocks *map,
567 			unsigned int orig_mlen)
568 {
569 	struct ext4_map_blocks map2;
570 	unsigned int status, status2;
571 	int retval;
572 
573 	status = map->m_flags & EXT4_MAP_UNWRITTEN ?
574 		EXTENT_STATUS_UNWRITTEN : EXTENT_STATUS_WRITTEN;
575 
576 	WARN_ON_ONCE(!(map->m_flags & EXT4_MAP_QUERY_LAST_IN_LEAF));
577 	WARN_ON_ONCE(orig_mlen <= map->m_len);
578 
579 	/* Prepare map2 for lookup in next leaf block */
580 	map2.m_lblk = map->m_lblk + map->m_len;
581 	map2.m_len = orig_mlen - map->m_len;
582 	map2.m_flags = 0;
583 	retval = ext4_ext_map_blocks(handle, inode, &map2, 0);
584 
585 	if (retval <= 0) {
586 		ext4_es_cache_extent(inode, map->m_lblk, map->m_len,
587 				     map->m_pblk, status);
588 		return map->m_len;
589 	}
590 
591 	if (unlikely(retval != map2.m_len)) {
592 		ext4_warning(inode->i_sb,
593 			     "ES len assertion failed for inode "
594 			     "%llu: retval %d != map->m_len %d",
595 			     inode->i_ino, retval, map2.m_len);
596 		WARN_ON(1);
597 	}
598 
599 	status2 = map2.m_flags & EXT4_MAP_UNWRITTEN ?
600 		EXTENT_STATUS_UNWRITTEN : EXTENT_STATUS_WRITTEN;
601 
602 	/*
603 	 * If map2 is contiguous with map, then let's insert it as a single
604 	 * extent in es cache and return the combined length of both the maps.
605 	 */
606 	if (map->m_pblk + map->m_len == map2.m_pblk &&
607 			status == status2) {
608 		ext4_es_cache_extent(inode, map->m_lblk,
609 				     map->m_len + map2.m_len, map->m_pblk,
610 				     status);
611 		map->m_len += map2.m_len;
612 	} else {
613 		ext4_es_cache_extent(inode, map->m_lblk, map->m_len,
614 				     map->m_pblk, status);
615 	}
616 
617 	return map->m_len;
618 }
619 
620 int ext4_map_query_blocks(handle_t *handle, struct inode *inode,
621 			  struct ext4_map_blocks *map, int flags)
622 {
623 	unsigned int status;
624 	int retval;
625 	unsigned int orig_mlen = map->m_len;
626 
627 	flags &= EXT4_EX_QUERY_FILTER;
628 	if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
629 		retval = ext4_ext_map_blocks(handle, inode, map, flags);
630 	else
631 		retval = ext4_ind_map_blocks(handle, inode, map, flags);
632 	if (retval < 0)
633 		return retval;
634 
635 	/* A hole? */
636 	if (retval == 0)
637 		goto out;
638 
639 	if (unlikely(retval != map->m_len)) {
640 		ext4_warning(inode->i_sb,
641 			     "ES len assertion failed for inode "
642 			     "%llu: retval %d != map->m_len %d",
643 			     inode->i_ino, retval, map->m_len);
644 		WARN_ON(1);
645 	}
646 
647 	/*
648 	 * No need to query next in leaf:
649 	 * - if returned extent is not last in leaf or
650 	 * - if the last in leaf is the full requested range
651 	 */
652 	if (!(map->m_flags & EXT4_MAP_QUERY_LAST_IN_LEAF) ||
653 			map->m_len == orig_mlen) {
654 		status = map->m_flags & EXT4_MAP_UNWRITTEN ?
655 				EXTENT_STATUS_UNWRITTEN : EXTENT_STATUS_WRITTEN;
656 		ext4_es_cache_extent(inode, map->m_lblk, map->m_len,
657 				     map->m_pblk, status);
658 	} else {
659 		retval = ext4_map_query_blocks_next_in_leaf(handle, inode, map,
660 							    orig_mlen);
661 	}
662 out:
663 	map->m_seq = READ_ONCE(EXT4_I(inode)->i_es_seq);
664 	return retval;
665 }
666 
667 int ext4_map_create_blocks(handle_t *handle, struct inode *inode,
668 			   struct ext4_map_blocks *map, int flags)
669 {
670 	unsigned int status;
671 	int err, retval = 0;
672 
673 	/*
674 	 * We pass in the magic EXT4_GET_BLOCKS_DELALLOC_RESERVE
675 	 * indicates that the blocks and quotas has already been
676 	 * checked when the data was copied into the page cache.
677 	 */
678 	if (map->m_flags & EXT4_MAP_DELAYED)
679 		flags |= EXT4_GET_BLOCKS_DELALLOC_RESERVE;
680 
681 	/*
682 	 * Here we clear m_flags because after allocating an new extent,
683 	 * it will be set again.
684 	 */
685 	map->m_flags &= ~EXT4_MAP_FLAGS;
686 
687 	/*
688 	 * We need to check for EXT4 here because migrate could have
689 	 * changed the inode type in between.
690 	 */
691 	if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) {
692 		retval = ext4_ext_map_blocks(handle, inode, map, flags);
693 	} else {
694 		retval = ext4_ind_map_blocks(handle, inode, map, flags);
695 
696 		/*
697 		 * We allocated new blocks which will result in i_data's
698 		 * format changing. Force the migrate to fail by clearing
699 		 * migrate flags.
700 		 */
701 		if (retval > 0 && map->m_flags & EXT4_MAP_NEW)
702 			ext4_clear_inode_state(inode, EXT4_STATE_EXT_MIGRATE);
703 	}
704 	if (retval <= 0)
705 		return retval;
706 
707 	if (unlikely(retval != map->m_len)) {
708 		ext4_warning(inode->i_sb,
709 			     "ES len assertion failed for inode %llu: "
710 			     "retval %d != map->m_len %d",
711 			     inode->i_ino, retval, map->m_len);
712 		WARN_ON(1);
713 	}
714 
715 	/*
716 	 * We have to zeroout blocks before inserting them into extent
717 	 * status tree. Otherwise someone could look them up there and
718 	 * use them before they are really zeroed. We also have to
719 	 * unmap metadata before zeroing as otherwise writeback can
720 	 * overwrite zeros with stale data from block device.
721 	 */
722 	if (flags & EXT4_GET_BLOCKS_ZERO &&
723 	    map->m_flags & EXT4_MAP_MAPPED && map->m_flags & EXT4_MAP_NEW) {
724 		err = ext4_issue_zeroout(inode, map->m_lblk, map->m_pblk,
725 					 map->m_len);
726 		if (err)
727 			return err;
728 	}
729 
730 	status = map->m_flags & EXT4_MAP_UNWRITTEN ?
731 			EXTENT_STATUS_UNWRITTEN : EXTENT_STATUS_WRITTEN;
732 	ext4_es_insert_extent(inode, map->m_lblk, map->m_len, map->m_pblk,
733 			      status, flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE);
734 	map->m_seq = READ_ONCE(EXT4_I(inode)->i_es_seq);
735 
736 	return retval;
737 }
738 
739 /*
740  * The ext4_map_blocks() function tries to look up the requested blocks,
741  * and returns if the blocks are already mapped.
742  *
743  * Otherwise it takes the write lock of the i_data_sem and allocate blocks
744  * and store the allocated blocks in the result buffer head and mark it
745  * mapped.
746  *
747  * If file type is extents based, it will call ext4_ext_map_blocks(),
748  * Otherwise, call with ext4_ind_map_blocks() to handle indirect mapping
749  * based files
750  *
751  * ext4_map_blocks() may also be called with a NULL @handle. In that case
752  * it starts its own transaction handle only when block allocation is
753  * actually needed, i.e. after the lookup confirms the blocks are not yet
754  * mapped, and stops it before returning.
755  *
756  * On success, it returns the number of blocks being mapped or allocated.
757  * If flags doesn't contain EXT4_GET_BLOCKS_CREATE the blocks are
758  * pre-allocated and unwritten, the resulting @map is marked as unwritten.
759  * If the flags contain EXT4_GET_BLOCKS_CREATE, it will mark @map as mapped.
760  *
761  * It returns 0 if plain look up failed (blocks have not been allocated), in
762  * that case, @map is returned as unmapped but we still do fill map->m_len to
763  * indicate the length of a hole starting at map->m_lblk.
764  *
765  * It returns the error in case of allocation failure.
766  */
767 int ext4_map_blocks(handle_t *handle, struct inode *inode,
768 		    struct ext4_map_blocks *map, int flags)
769 {
770 	struct extent_status es;
771 	int retval;
772 	int ret = 0;
773 	bool internal_handle = false;
774 	unsigned int orig_mlen;
775 #ifdef ES_AGGRESSIVE_TEST
776 	struct ext4_map_blocks orig_map;
777 
778 	memcpy(&orig_map, map, sizeof(*map));
779 #endif
780 
781 	map->m_flags = 0;
782 	ext_debug(inode, "flag 0x%x, max_blocks %u, logical block %lu\n",
783 		  flags, map->m_len, (unsigned long) map->m_lblk);
784 
785 	/*
786 	 * ext4_map_blocks returns an int, and m_len is an unsigned int
787 	 */
788 	if (unlikely(map->m_len > INT_MAX))
789 		map->m_len = INT_MAX;
790 	orig_mlen = map->m_len;
791 
792 	/* We can handle the block number less than EXT_MAX_BLOCKS */
793 	if (unlikely(map->m_lblk >= EXT_MAX_BLOCKS))
794 		return -EFSCORRUPTED;
795 
796 	/*
797 	 * Callers from the context of data submission are the only exceptions
798 	 * for regular files that do not hold the i_rwsem or invalidate_lock.
799 	 * However, caching unrelated ranges is not permitted.
800 	 */
801 	if (flags & EXT4_GET_BLOCKS_IO_SUBMIT)
802 		WARN_ON_ONCE(!(flags & EXT4_EX_NOCACHE));
803 	else
804 		ext4_check_map_extents_env(inode);
805 
806 create_retry:
807 	/* Lookup extent status tree firstly */
808 	if (ext4_es_lookup_extent(inode, map->m_lblk, NULL, &es, &map->m_seq)) {
809 		if (ext4_es_is_written(&es) || ext4_es_is_unwritten(&es)) {
810 			map->m_pblk = ext4_es_pblock(&es) +
811 					map->m_lblk - es.es_lblk;
812 			map->m_flags |= ext4_es_is_written(&es) ?
813 					EXT4_MAP_MAPPED : EXT4_MAP_UNWRITTEN;
814 			retval = es.es_len - (map->m_lblk - es.es_lblk);
815 			if (retval > map->m_len)
816 				retval = map->m_len;
817 			map->m_len = retval;
818 		} else if (ext4_es_is_delayed(&es) || ext4_es_is_hole(&es)) {
819 			map->m_pblk = 0;
820 			map->m_flags |= ext4_es_is_delayed(&es) ?
821 					EXT4_MAP_DELAYED : 0;
822 			retval = es.es_len - (map->m_lblk - es.es_lblk);
823 			if (retval > map->m_len)
824 				retval = map->m_len;
825 			map->m_len = retval;
826 			retval = 0;
827 		} else {
828 			BUG();
829 		}
830 
831 		if (flags & EXT4_GET_BLOCKS_CACHED_NOWAIT)
832 			return retval;
833 #ifdef ES_AGGRESSIVE_TEST
834 		ext4_map_blocks_es_recheck(handle, inode, map,
835 					   &orig_map, flags);
836 #endif
837 		if (!(flags & EXT4_GET_BLOCKS_QUERY_LAST_IN_LEAF) ||
838 				orig_mlen == map->m_len)
839 			goto found;
840 
841 		map->m_len = orig_mlen;
842 	}
843 	/*
844 	 * In the query cache no-wait mode, nothing we can do more if we
845 	 * cannot find extent in the cache.
846 	 */
847 	if (flags & EXT4_GET_BLOCKS_CACHED_NOWAIT)
848 		return 0;
849 
850 	/*
851 	 * Try to see if we can get the block without requesting a new
852 	 * file system block.
853 	 */
854 	down_read(&EXT4_I(inode)->i_data_sem);
855 	retval = ext4_map_query_blocks(handle, inode, map, flags);
856 	up_read((&EXT4_I(inode)->i_data_sem));
857 	if (retval < 0)
858 		goto out;
859 
860 found:
861 	if (retval > 0 && map->m_flags & EXT4_MAP_MAPPED) {
862 		ret = check_block_validity(inode, map);
863 		if (ret != 0)
864 			goto out;
865 	}
866 
867 	/* If it is only a block(s) look up */
868 	if ((flags & EXT4_GET_BLOCKS_CREATE) == 0)
869 		return retval;
870 
871 	/*
872 	 * Returns if the blocks have already allocated
873 	 *
874 	 * Note that if blocks have been preallocated
875 	 * ext4_ext_map_blocks() returns with buffer head unmapped
876 	 */
877 	if (retval > 0) {
878 		/*
879 		 * If we need to convert written extent to unwritten or
880 		 * convert unwritten extent to written, continue and do
881 		 * the actual work in ext4_ext_map_blocks().
882 		 */
883 		if (map->m_flags & EXT4_MAP_MAPPED &&
884 		    !(flags & EXT4_GET_BLOCKS_CONVERT_UNWRITTEN))
885 			goto out;
886 		if (map->m_flags & EXT4_MAP_UNWRITTEN &&
887 		    (flags & EXT4_GET_BLOCKS_UNWRIT_EXT) &&
888 		    !(flags & EXT4_GET_BLOCKS_CONVERT)) {
889 			/* Contains EXT4_GET_BLOCKS_CREATE - mark mapped. */
890 			ret = check_block_validity(inode, map);
891 			if (!ret)
892 				map->m_flags |= EXT4_MAP_MAPPED;
893 			goto out;
894 		}
895 	} else if (retval == 0) {
896 		/*
897 		 * Do not allocate blocks for holes in the context of
898 		 * data submission path.
899 		 */
900 		if (!map->m_flags && (flags & EXT4_GET_BLOCKS_IO_SUBMIT))
901 			goto out;
902 	}
903 
904 	if (!handle) {
905 		handle = ext4_journal_start(inode, EXT4_HT_MAP_BLOCKS,
906 				ext4_chunk_trans_blocks(inode, orig_mlen));
907 		if (IS_ERR(handle)) {
908 			ret = PTR_ERR(handle);
909 			goto out;
910 		}
911 		internal_handle = true;
912 	}
913 
914 	ext4_fc_track_inode(handle, inode);
915 	/*
916 	 * New blocks allocate and/or writing to unwritten extent
917 	 * will possibly result in updating i_data, so we take
918 	 * the write lock of i_data_sem, and call get_block()
919 	 * with create == 1 flag.
920 	 */
921 	down_write(&EXT4_I(inode)->i_data_sem);
922 
923 	/*
924 	 * Check the validity of the mapping found via the extent status
925 	 * tree or the disk query. A racing truncate may have changed the
926 	 * extent, since writeback may not hold i_rwsem or the folio locks
927 	 * covering the full extent(e.g., the iomap writeback path may
928 	 * allocate an extent that extends beyond the length of the locked
929 	 * folio in advance).
930 	 */
931 	if (map->m_seq != READ_ONCE(EXT4_I(inode)->i_es_seq)) {
932 		up_write(&EXT4_I(inode)->i_data_sem);
933 		map->m_flags = 0;
934 		map->m_len = orig_mlen;
935 		goto create_retry;
936 	}
937 	retval = ext4_map_create_blocks(handle, inode, map, flags);
938 	up_write((&EXT4_I(inode)->i_data_sem));
939 
940 	if (retval < 0)
941 		ext_debug(inode, "failed with err %d\n", retval);
942 	if (retval <= 0)
943 		goto out;
944 
945 	if (map->m_flags & EXT4_MAP_MAPPED) {
946 		ret = check_block_validity(inode, map);
947 		if (ret != 0)
948 			goto out;
949 
950 		/*
951 		 * Inodes with freshly allocated blocks where contents will be
952 		 * visible after transaction commit must be on transaction's
953 		 * ordered data list.
954 		 */
955 		if (map->m_flags & EXT4_MAP_NEW &&
956 		    !(map->m_flags & EXT4_MAP_UNWRITTEN) &&
957 		    !(flags & EXT4_GET_BLOCKS_ZERO) &&
958 		    !ext4_is_quota_file(inode) &&
959 		    ext4_should_order_data(inode)) {
960 			loff_t start_byte = EXT4_LBLK_TO_B(inode, map->m_lblk);
961 			loff_t length = EXT4_LBLK_TO_B(inode, map->m_len);
962 
963 			if (flags & EXT4_GET_BLOCKS_IO_SUBMIT)
964 				ret = ext4_jbd2_inode_add_wait(handle, inode,
965 						start_byte, length);
966 			else
967 				ret = ext4_jbd2_inode_add_write(handle, inode,
968 						start_byte, length);
969 			if (ret)
970 				goto out;
971 		}
972 	}
973 	ext4_fc_track_range(handle, inode, map->m_lblk, map->m_lblk +
974 			    map->m_len - 1);
975 out:
976 	if (internal_handle)
977 		ext4_journal_stop(handle);
978 
979 	return ret ? ret : retval;
980 }
981 
982 /*
983  * Update EXT4_MAP_FLAGS in bh->b_state. For buffer heads attached to pages
984  * we have to be careful as someone else may be manipulating b_state as well.
985  */
986 static void ext4_update_bh_state(struct buffer_head *bh, unsigned long flags)
987 {
988 	unsigned long old_state;
989 	unsigned long new_state;
990 
991 	flags &= EXT4_MAP_FLAGS;
992 
993 	/* Dummy buffer_head? Set non-atomically. */
994 	if (!bh->b_folio) {
995 		bh->b_state = (bh->b_state & ~EXT4_MAP_FLAGS) | flags;
996 		return;
997 	}
998 	/*
999 	 * Someone else may be modifying b_state. Be careful! This is ugly but
1000 	 * once we get rid of using bh as a container for mapping information
1001 	 * to pass to / from get_block functions, this can go away.
1002 	 */
1003 	old_state = READ_ONCE(bh->b_state);
1004 	do {
1005 		new_state = (old_state & ~EXT4_MAP_FLAGS) | flags;
1006 	} while (unlikely(!try_cmpxchg(&bh->b_state, &old_state, new_state)));
1007 }
1008 
1009 /*
1010  * Make sure that the current journal transaction has enough credits to map
1011  * one extent. Return -EAGAIN if it cannot extend the current running
1012  * transaction.
1013  */
1014 static inline int ext4_journal_ensure_extent_credits(handle_t *handle,
1015 						     struct inode *inode)
1016 {
1017 	int credits;
1018 	int ret;
1019 
1020 	/* Called from ext4_da_write_begin() which has no handle started? */
1021 	if (!handle)
1022 		return 0;
1023 
1024 	credits = ext4_chunk_trans_blocks(inode, 1);
1025 	ret = __ext4_journal_ensure_credits(handle, credits, credits, 0);
1026 	return ret <= 0 ? ret : -EAGAIN;
1027 }
1028 
1029 static int _ext4_get_block(struct inode *inode, sector_t iblock,
1030 			   struct buffer_head *bh, int flags)
1031 {
1032 	struct ext4_map_blocks map;
1033 	int ret = 0;
1034 
1035 	if (ext4_has_inline_data(inode))
1036 		return -ERANGE;
1037 	/* inode using the iomap buffered I/O path should not go here. */
1038 	if (WARN_ON_ONCE(ext4_inode_buffered_iomap(inode)))
1039 		return -EINVAL;
1040 
1041 	map.m_lblk = iblock;
1042 	map.m_len = bh->b_size >> inode->i_blkbits;
1043 
1044 	ret = ext4_map_blocks(ext4_journal_current_handle(), inode, &map,
1045 			      flags);
1046 	if (ret > 0) {
1047 		map_bh(bh, inode->i_sb, map.m_pblk);
1048 		ext4_update_bh_state(bh, map.m_flags);
1049 		bh->b_size = inode->i_sb->s_blocksize * map.m_len;
1050 		ret = 0;
1051 	} else if (ret == 0) {
1052 		/* hole case, need to fill in bh->b_size */
1053 		bh->b_size = inode->i_sb->s_blocksize * map.m_len;
1054 	}
1055 	return ret;
1056 }
1057 
1058 int ext4_get_block(struct inode *inode, sector_t iblock,
1059 		   struct buffer_head *bh, int create)
1060 {
1061 	return _ext4_get_block(inode, iblock, bh,
1062 			       create ? EXT4_GET_BLOCKS_CREATE : 0);
1063 }
1064 
1065 /*
1066  * Get block function used when preparing for buffered write if we require
1067  * creating an unwritten extent if blocks haven't been allocated.  The extent
1068  * will be converted to written after the IO is complete.
1069  */
1070 int ext4_get_block_unwritten(struct inode *inode, sector_t iblock,
1071 			     struct buffer_head *bh_result, int create)
1072 {
1073 	int ret = 0;
1074 
1075 	ext4_debug("ext4_get_block_unwritten: inode %llu, create flag %d\n",
1076 		   inode->i_ino, create);
1077 	ret = _ext4_get_block(inode, iblock, bh_result,
1078 			       EXT4_GET_BLOCKS_CREATE_UNWRIT_EXT);
1079 
1080 	/*
1081 	 * If the buffer is marked unwritten, mark it as new to make sure it is
1082 	 * zeroed out correctly in case of partial writes. Otherwise, there is
1083 	 * a chance of stale data getting exposed.
1084 	 */
1085 	if (ret == 0 && buffer_unwritten(bh_result))
1086 		set_buffer_new(bh_result);
1087 
1088 	return ret;
1089 }
1090 
1091 /* Maximum number of blocks we map for direct IO at once. */
1092 #define DIO_MAX_BLOCKS 4096
1093 
1094 /*
1095  * `handle' can be NULL if create is zero
1096  */
1097 struct buffer_head *ext4_getblk(handle_t *handle, struct inode *inode,
1098 				ext4_lblk_t block, int map_flags)
1099 {
1100 	struct ext4_map_blocks map;
1101 	struct buffer_head *bh;
1102 	int create = map_flags & EXT4_GET_BLOCKS_CREATE;
1103 	bool nowait = map_flags & EXT4_GET_BLOCKS_CACHED_NOWAIT;
1104 	int err;
1105 
1106 	ASSERT((EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY)
1107 		    || handle != NULL || create == 0);
1108 	ASSERT(create == 0 || !nowait);
1109 
1110 	map.m_lblk = block;
1111 	map.m_len = 1;
1112 	err = ext4_map_blocks(handle, inode, &map, map_flags);
1113 
1114 	if (err == 0)
1115 		return create ? ERR_PTR(-ENOSPC) : NULL;
1116 	if (err < 0)
1117 		return ERR_PTR(err);
1118 
1119 	if (nowait)
1120 		return sb_find_get_block(inode->i_sb, map.m_pblk);
1121 
1122 	/*
1123 	 * Since bh could introduce extra ref count such as referred by
1124 	 * journal_head etc. Try to avoid using __GFP_MOVABLE here
1125 	 * as it may fail the migration when journal_head remains.
1126 	 */
1127 	bh = getblk_unmovable(inode->i_sb->s_bdev, map.m_pblk,
1128 				inode->i_sb->s_blocksize);
1129 
1130 	if (unlikely(!bh))
1131 		return ERR_PTR(-ENOMEM);
1132 	if (map.m_flags & EXT4_MAP_NEW) {
1133 		ASSERT(create != 0);
1134 		ASSERT((EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY)
1135 			    || (handle != NULL));
1136 
1137 		/*
1138 		 * Now that we do not always journal data, we should
1139 		 * keep in mind whether this should always journal the
1140 		 * new buffer as metadata.  For now, regular file
1141 		 * writes use ext4_get_block instead, so it's not a
1142 		 * problem.
1143 		 */
1144 		lock_buffer(bh);
1145 		BUFFER_TRACE(bh, "call get_create_access");
1146 		err = ext4_journal_get_create_access(handle, inode->i_sb, bh,
1147 						     EXT4_JTR_NONE);
1148 		if (unlikely(err)) {
1149 			unlock_buffer(bh);
1150 			goto errout;
1151 		}
1152 		if (!buffer_uptodate(bh)) {
1153 			memset(bh->b_data, 0, inode->i_sb->s_blocksize);
1154 			set_buffer_uptodate(bh);
1155 		}
1156 		unlock_buffer(bh);
1157 		BUFFER_TRACE(bh, "call ext4_handle_dirty_metadata");
1158 		err = ext4_handle_dirty_metadata(handle, inode, bh);
1159 		if (unlikely(err))
1160 			goto errout;
1161 	} else
1162 		BUFFER_TRACE(bh, "not a new buffer");
1163 	return bh;
1164 errout:
1165 	brelse(bh);
1166 	return ERR_PTR(err);
1167 }
1168 
1169 struct buffer_head *ext4_bread(handle_t *handle, struct inode *inode,
1170 			       ext4_lblk_t block, int map_flags)
1171 {
1172 	struct buffer_head *bh;
1173 	int ret;
1174 
1175 	bh = ext4_getblk(handle, inode, block, map_flags);
1176 	if (IS_ERR(bh))
1177 		return bh;
1178 	if (!bh || ext4_buffer_uptodate(bh))
1179 		return bh;
1180 
1181 	ret = ext4_read_bh_lock(bh, REQ_META | REQ_PRIO, true);
1182 	if (ret) {
1183 		put_bh(bh);
1184 		return ERR_PTR(ret);
1185 	}
1186 	return bh;
1187 }
1188 
1189 /* Read a contiguous batch of blocks. */
1190 int ext4_bread_batch(struct inode *inode, ext4_lblk_t block, int bh_count,
1191 		     bool wait, struct buffer_head **bhs)
1192 {
1193 	int i, err;
1194 
1195 	for (i = 0; i < bh_count; i++) {
1196 		bhs[i] = ext4_getblk(NULL, inode, block + i, 0 /* map_flags */);
1197 		if (IS_ERR(bhs[i])) {
1198 			err = PTR_ERR(bhs[i]);
1199 			bh_count = i;
1200 			goto out_brelse;
1201 		}
1202 	}
1203 
1204 	for (i = 0; i < bh_count; i++)
1205 		/* Note that NULL bhs[i] is valid because of holes. */
1206 		if (bhs[i] && !ext4_buffer_uptodate(bhs[i]))
1207 			ext4_read_bh_lock(bhs[i], REQ_META | REQ_PRIO, false);
1208 
1209 	if (!wait)
1210 		return 0;
1211 
1212 	for (i = 0; i < bh_count; i++)
1213 		if (bhs[i])
1214 			wait_on_buffer(bhs[i]);
1215 
1216 	for (i = 0; i < bh_count; i++) {
1217 		if (bhs[i] && !buffer_uptodate(bhs[i])) {
1218 			err = -EIO;
1219 			goto out_brelse;
1220 		}
1221 	}
1222 	return 0;
1223 
1224 out_brelse:
1225 	for (i = 0; i < bh_count; i++) {
1226 		brelse(bhs[i]);
1227 		bhs[i] = NULL;
1228 	}
1229 	return err;
1230 }
1231 
1232 int ext4_walk_page_buffers(handle_t *handle, struct inode *inode,
1233 			   struct buffer_head *head,
1234 			   unsigned from,
1235 			   unsigned to,
1236 			   int *partial,
1237 			   int (*fn)(handle_t *handle, struct inode *inode,
1238 				     struct buffer_head *bh))
1239 {
1240 	struct buffer_head *bh;
1241 	unsigned block_start, block_end;
1242 	unsigned blocksize = head->b_size;
1243 	int err, ret = 0;
1244 	struct buffer_head *next;
1245 
1246 	for (bh = head, block_start = 0;
1247 	     ret == 0 && (bh != head || !block_start);
1248 	     block_start = block_end, bh = next) {
1249 		next = bh->b_this_page;
1250 		block_end = block_start + blocksize;
1251 		if (block_end <= from || block_start >= to) {
1252 			if (partial && !buffer_uptodate(bh))
1253 				*partial = 1;
1254 			continue;
1255 		}
1256 		err = (*fn)(handle, inode, bh);
1257 		if (!ret)
1258 			ret = err;
1259 	}
1260 	return ret;
1261 }
1262 
1263 /*
1264  * Helper for handling dirtying of journalled data. We also mark the folio as
1265  * dirty so that writeback code knows about this page (and inode) contains
1266  * dirty data. ext4_writepages() then commits appropriate transaction to
1267  * make data stable.
1268  */
1269 static int ext4_dirty_journalled_data(handle_t *handle, struct buffer_head *bh)
1270 {
1271 	struct folio *folio = bh->b_folio;
1272 	struct inode *inode = folio->mapping->host;
1273 
1274 	/* only regular files have a_ops */
1275 	if (S_ISREG(inode->i_mode))
1276 		folio_mark_dirty(folio);
1277 	return ext4_handle_dirty_metadata(handle, NULL, bh);
1278 }
1279 
1280 int do_journal_get_write_access(handle_t *handle, struct inode *inode,
1281 				struct buffer_head *bh)
1282 {
1283 	if (!buffer_mapped(bh) || buffer_freed(bh))
1284 		return 0;
1285 	BUFFER_TRACE(bh, "get write access");
1286 	return ext4_journal_get_write_access(handle, inode->i_sb, bh,
1287 					    EXT4_JTR_NONE);
1288 }
1289 
1290 int ext4_block_write_begin(handle_t *handle, struct folio *folio,
1291 			   loff_t pos, unsigned len,
1292 			   get_block_t *get_block)
1293 {
1294 	unsigned int from = offset_in_folio(folio, pos);
1295 	unsigned to = from + len;
1296 	struct inode *inode = folio->mapping->host;
1297 	unsigned block_start, block_end;
1298 	sector_t block;
1299 	int err = 0;
1300 	unsigned int blocksize = i_blocksize(inode);
1301 	struct buffer_head *bh, *head, *wait[2];
1302 	int nr_wait = 0;
1303 	int i;
1304 	bool should_journal_data = ext4_should_journal_data(inode);
1305 	bool folio_uptodate = folio_test_uptodate(folio);
1306 
1307 	BUG_ON(!folio_test_locked(folio));
1308 	BUG_ON(to > folio_size(folio));
1309 	BUG_ON(from > to);
1310 	WARN_ON_ONCE(blocksize > folio_size(folio));
1311 
1312 	head = folio_buffers(folio);
1313 	if (!head)
1314 		head = create_empty_buffers(folio, blocksize, 0);
1315 	block = EXT4_PG_TO_LBLK(inode, folio->index);
1316 
1317 	for (bh = head, block_start = 0;
1318 	     block_start < to || (!folio_uptodate && bh != head);
1319 	    block++, block_start = block_end, bh = bh->b_this_page) {
1320 		block_end = block_start + blocksize;
1321 		if (block_end <= from || block_start >= to) {
1322 			if (folio_uptodate)
1323 				set_buffer_uptodate(bh);
1324 			continue;
1325 		}
1326 		if (WARN_ON_ONCE(buffer_new(bh)))
1327 			clear_buffer_new(bh);
1328 		if (!buffer_mapped(bh)) {
1329 			WARN_ON(bh->b_size != blocksize);
1330 			err = ext4_journal_ensure_extent_credits(handle, inode);
1331 			if (!err)
1332 				err = get_block(inode, block, bh, 1);
1333 			if (err)
1334 				break;
1335 			if (buffer_new(bh)) {
1336 				/*
1337 				 * We may be zeroing partial buffers or all new
1338 				 * buffers in case of failure. Prepare JBD2 for
1339 				 * that.
1340 				 */
1341 				if (should_journal_data)
1342 					do_journal_get_write_access(handle,
1343 								    inode, bh);
1344 				if (folio_uptodate) {
1345 					/*
1346 					 * Unlike __block_write_begin() we leave
1347 					 * dirtying of new uptodate buffers to
1348 					 * ->write_end() time or
1349 					 * folio_zero_new_buffers().
1350 					 */
1351 					set_buffer_uptodate(bh);
1352 					continue;
1353 				}
1354 				if (block_end > to || block_start < from)
1355 					folio_zero_segments(folio, to,
1356 							    block_end,
1357 							    block_start, from);
1358 				continue;
1359 			}
1360 		}
1361 		if (folio_uptodate) {
1362 			set_buffer_uptodate(bh);
1363 			continue;
1364 		}
1365 		if (!buffer_uptodate(bh) && !buffer_delay(bh) &&
1366 		    !buffer_unwritten(bh) &&
1367 		    (block_start < from || block_end > to)) {
1368 			ext4_read_bh_lock(bh, 0, false);
1369 			wait[nr_wait++] = bh;
1370 		}
1371 	}
1372 	/*
1373 	 * If we issued read requests, let them complete.
1374 	 */
1375 	for (i = 0; i < nr_wait; i++) {
1376 		wait_on_buffer(wait[i]);
1377 		if (!buffer_uptodate(wait[i]))
1378 			err = -EIO;
1379 	}
1380 	if (unlikely(err)) {
1381 		if (should_journal_data)
1382 			ext4_journalled_zero_new_buffers(handle, inode, folio,
1383 							 from, to);
1384 		else
1385 			folio_zero_new_buffers(folio, from, to);
1386 	}
1387 
1388 	return err;
1389 }
1390 
1391 /*
1392  * To preserve ordering, it is essential that the hole instantiation and
1393  * the data write be encapsulated in a single transaction.  We cannot
1394  * close off a transaction and start a new one between the ext4_get_block()
1395  * and the ext4_write_end().  So doing the jbd2_journal_start at the start of
1396  * ext4_write_begin() is the right place.
1397  */
1398 static int ext4_write_begin(const struct kiocb *iocb,
1399 			    struct address_space *mapping,
1400 			    loff_t pos, unsigned len,
1401 			    struct folio **foliop, void **fsdata)
1402 {
1403 	struct inode *inode = mapping->host;
1404 	int ret, needed_blocks;
1405 	handle_t *handle;
1406 	int retries = 0;
1407 	struct folio *folio;
1408 	pgoff_t index;
1409 	unsigned from, to;
1410 
1411 	ret = ext4_emergency_state(inode->i_sb);
1412 	if (unlikely(ret))
1413 		return ret;
1414 
1415 	*fsdata = (void *)((unsigned long)*fsdata & ~EXT4_WRITE_DATA_INLINE);
1416 
1417 	trace_ext4_write_begin(inode, pos, len);
1418 	/*
1419 	 * Reserve one block more for addition to orphan list in case
1420 	 * we allocate blocks but write fails for some reason
1421 	 */
1422 	needed_blocks = ext4_chunk_trans_extent(inode,
1423 			ext4_journal_blocks_per_folio(inode)) + 1;
1424 	index = pos >> PAGE_SHIFT;
1425 
1426 	if (ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) {
1427 		ret = ext4_try_to_write_inline_data(mapping, inode, pos, len,
1428 						    foliop);
1429 		if (ret < 0)
1430 			return ret;
1431 		if (ret == 1) {
1432 			*fsdata = (void *)((unsigned long)*fsdata | EXT4_WRITE_DATA_INLINE);
1433 			return 0;
1434 		}
1435 	}
1436 
1437 	/*
1438 	 * write_begin_get_folio() can take a long time if the
1439 	 * system is thrashing due to memory pressure, or if the folio
1440 	 * is being written back.  So grab it first before we start
1441 	 * the transaction handle.  This also allows us to allocate
1442 	 * the folio (if needed) without using GFP_NOFS.
1443 	 */
1444 retry_grab:
1445 	folio = write_begin_get_folio(iocb, mapping, index, len);
1446 	if (IS_ERR(folio))
1447 		return PTR_ERR(folio);
1448 
1449 	if (len > folio_next_pos(folio) - pos)
1450 		len = folio_next_pos(folio) - pos;
1451 
1452 	from = offset_in_folio(folio, pos);
1453 	to = from + len;
1454 
1455 	/*
1456 	 * The same as page allocation, we prealloc buffer heads before
1457 	 * starting the handle.
1458 	 */
1459 	if (!folio_buffers(folio))
1460 		create_empty_buffers(folio, inode->i_sb->s_blocksize, 0);
1461 
1462 	folio_unlock(folio);
1463 
1464 retry_journal:
1465 	handle = ext4_journal_start(inode, EXT4_HT_WRITE_PAGE, needed_blocks);
1466 	if (IS_ERR(handle)) {
1467 		folio_put(folio);
1468 		return PTR_ERR(handle);
1469 	}
1470 
1471 	folio_lock(folio);
1472 	if (folio->mapping != mapping) {
1473 		/* The folio got truncated from under us */
1474 		folio_unlock(folio);
1475 		folio_put(folio);
1476 		ext4_journal_stop(handle);
1477 		goto retry_grab;
1478 	}
1479 	/* In case writeback began while the folio was unlocked */
1480 	folio_wait_stable(folio);
1481 
1482 	if (ext4_should_dioread_nolock(inode))
1483 		ret = ext4_block_write_begin(handle, folio, pos, len,
1484 					     ext4_get_block_unwritten);
1485 	else
1486 		ret = ext4_block_write_begin(handle, folio, pos, len,
1487 					     ext4_get_block);
1488 	if (!ret && ext4_should_journal_data(inode)) {
1489 		ret = ext4_walk_page_buffers(handle, inode,
1490 					     folio_buffers(folio), from, to,
1491 					     NULL, do_journal_get_write_access);
1492 	}
1493 
1494 	if (ret) {
1495 		bool extended = (pos + len > inode->i_size) &&
1496 				!ext4_verity_in_progress(inode);
1497 
1498 		folio_unlock(folio);
1499 		/*
1500 		 * ext4_block_write_begin may have instantiated a few blocks
1501 		 * outside i_size.  Trim these off again. Don't need
1502 		 * i_size_read because we hold i_rwsem.
1503 		 *
1504 		 * Add inode to orphan list in case we crash before
1505 		 * truncate finishes
1506 		 */
1507 		if (extended && ext4_can_truncate(inode))
1508 			ext4_orphan_add(handle, inode);
1509 
1510 		ext4_journal_stop(handle);
1511 		if (extended) {
1512 			ext4_truncate_failed_write(inode);
1513 			/*
1514 			 * If truncate failed early the inode might
1515 			 * still be on the orphan list; we need to
1516 			 * make sure the inode is removed from the
1517 			 * orphan list in that case.
1518 			 */
1519 			if (inode->i_nlink)
1520 				ext4_orphan_del(NULL, inode);
1521 		}
1522 
1523 		if (ret == -EAGAIN ||
1524 		    (ret == -ENOSPC &&
1525 		     ext4_should_retry_alloc(inode->i_sb, &retries)))
1526 			goto retry_journal;
1527 		folio_put(folio);
1528 		return ret;
1529 	}
1530 	*foliop = folio;
1531 	return ret;
1532 }
1533 
1534 /* For write_end() in data=journal mode */
1535 static int write_end_fn(handle_t *handle, struct inode *inode,
1536 			struct buffer_head *bh)
1537 {
1538 	int ret;
1539 	if (!buffer_mapped(bh) || buffer_freed(bh))
1540 		return 0;
1541 	set_buffer_uptodate(bh);
1542 	ret = ext4_dirty_journalled_data(handle, bh);
1543 	clear_buffer_meta(bh);
1544 	clear_buffer_prio(bh);
1545 	clear_buffer_new(bh);
1546 	return ret;
1547 }
1548 
1549 /*
1550  * We need to pick up the new inode size which generic_commit_write gave us
1551  * `iocb` can be NULL - eg, when called from page_symlink().
1552  */
1553 static int ext4_write_end(const struct kiocb *iocb,
1554 			  struct address_space *mapping,
1555 			  loff_t pos, unsigned len, unsigned copied,
1556 			  struct folio *folio, void *fsdata)
1557 {
1558 	handle_t *handle = ext4_journal_current_handle();
1559 	struct inode *inode = mapping->host;
1560 	loff_t old_size = inode->i_size;
1561 	int ret = 0, ret2;
1562 	int i_size_changed = 0;
1563 	bool verity = ext4_verity_in_progress(inode);
1564 
1565 	trace_ext4_write_end(inode, pos, len, copied);
1566 
1567 	if ((unsigned long)fsdata & EXT4_WRITE_DATA_INLINE)
1568 		return ext4_write_inline_data_end(inode, pos, len, copied,
1569 						  folio);
1570 
1571 	copied = block_write_end(pos, len, copied, folio);
1572 	/*
1573 	 * it's important to update i_size while still holding folio lock:
1574 	 * page writeout could otherwise come in and zero beyond i_size.
1575 	 *
1576 	 * If FS_IOC_ENABLE_VERITY is running on this inode, then Merkle tree
1577 	 * blocks are being written past EOF, so skip the i_size update.
1578 	 */
1579 	if (!verity)
1580 		i_size_changed = ext4_update_inode_size(inode, pos + copied);
1581 	folio_unlock(folio);
1582 	folio_put(folio);
1583 
1584 	if (old_size < pos && !verity)
1585 		pagecache_isize_extended(inode, old_size, pos);
1586 
1587 	/*
1588 	 * Don't mark the inode dirty under folio lock. First, it unnecessarily
1589 	 * makes the holding time of folio lock longer. Second, it forces lock
1590 	 * ordering of folio lock and transaction start for journaling
1591 	 * filesystems.
1592 	 */
1593 	if (i_size_changed)
1594 		ret = ext4_mark_inode_dirty(handle, inode);
1595 
1596 	if (pos + len > inode->i_size && !verity && ext4_can_truncate(inode))
1597 		/* if we have allocated more blocks and copied
1598 		 * less. We will have blocks allocated outside
1599 		 * inode->i_size. So truncate them
1600 		 */
1601 		ext4_orphan_add(handle, inode);
1602 
1603 	ret2 = ext4_journal_stop(handle);
1604 	if (!ret)
1605 		ret = ret2;
1606 
1607 	if (pos + len > inode->i_size && !verity) {
1608 		ext4_truncate_failed_write(inode);
1609 		/*
1610 		 * If truncate failed early the inode might still be
1611 		 * on the orphan list; we need to make sure the inode
1612 		 * is removed from the orphan list in that case.
1613 		 */
1614 		if (inode->i_nlink)
1615 			ext4_orphan_del(NULL, inode);
1616 	}
1617 
1618 	return ret ? ret : copied;
1619 }
1620 
1621 /*
1622  * This is a private version of folio_zero_new_buffers() which doesn't
1623  * set the buffer to be dirty, since in data=journalled mode we need
1624  * to call ext4_dirty_journalled_data() instead.
1625  */
1626 static void ext4_journalled_zero_new_buffers(handle_t *handle,
1627 					    struct inode *inode,
1628 					    struct folio *folio,
1629 					    unsigned from, unsigned to)
1630 {
1631 	unsigned int block_start = 0, block_end;
1632 	struct buffer_head *head, *bh;
1633 
1634 	bh = head = folio_buffers(folio);
1635 	do {
1636 		block_end = block_start + bh->b_size;
1637 		if (buffer_new(bh)) {
1638 			if (block_end > from && block_start < to) {
1639 				if (!folio_test_uptodate(folio)) {
1640 					unsigned start, size;
1641 
1642 					start = max(from, block_start);
1643 					size = min(to, block_end) - start;
1644 
1645 					folio_zero_range(folio, start, size);
1646 				}
1647 				clear_buffer_new(bh);
1648 				write_end_fn(handle, inode, bh);
1649 			}
1650 		}
1651 		block_start = block_end;
1652 		bh = bh->b_this_page;
1653 	} while (bh != head);
1654 }
1655 
1656 static int ext4_journalled_write_end(const struct kiocb *iocb,
1657 				     struct address_space *mapping,
1658 				     loff_t pos, unsigned len, unsigned copied,
1659 				     struct folio *folio, void *fsdata)
1660 {
1661 	handle_t *handle = ext4_journal_current_handle();
1662 	struct inode *inode = mapping->host;
1663 	loff_t old_size = inode->i_size;
1664 	int ret = 0, ret2;
1665 	int partial = 0;
1666 	unsigned from, to;
1667 	int size_changed = 0;
1668 	bool verity = ext4_verity_in_progress(inode);
1669 
1670 	trace_ext4_journalled_write_end(inode, pos, len, copied);
1671 	from = pos & (PAGE_SIZE - 1);
1672 	to = from + len;
1673 
1674 	BUG_ON(!ext4_handle_valid(handle));
1675 
1676 	if ((unsigned long)fsdata & EXT4_WRITE_DATA_INLINE)
1677 		return ext4_write_inline_data_end(inode, pos, len, copied,
1678 						  folio);
1679 
1680 	if (unlikely(copied < len) && !folio_test_uptodate(folio)) {
1681 		copied = 0;
1682 		ext4_journalled_zero_new_buffers(handle, inode, folio,
1683 						 from, to);
1684 	} else {
1685 		if (unlikely(copied < len))
1686 			ext4_journalled_zero_new_buffers(handle, inode, folio,
1687 							 from + copied, to);
1688 		ret = ext4_walk_page_buffers(handle, inode,
1689 					     folio_buffers(folio),
1690 					     from, from + copied, &partial,
1691 					     write_end_fn);
1692 		if (!partial)
1693 			folio_mark_uptodate(folio);
1694 	}
1695 	if (!verity)
1696 		size_changed = ext4_update_inode_size(inode, pos + copied);
1697 	EXT4_I(inode)->i_datasync_tid = handle->h_transaction->t_tid;
1698 	folio_unlock(folio);
1699 	folio_put(folio);
1700 
1701 	if (old_size < pos && !verity)
1702 		pagecache_isize_extended(inode, old_size, pos);
1703 
1704 	if (size_changed) {
1705 		ret2 = ext4_mark_inode_dirty(handle, inode);
1706 		if (!ret)
1707 			ret = ret2;
1708 	}
1709 
1710 	if (pos + len > inode->i_size && !verity && ext4_can_truncate(inode))
1711 		/* if we have allocated more blocks and copied
1712 		 * less. We will have blocks allocated outside
1713 		 * inode->i_size. So truncate them
1714 		 */
1715 		ext4_orphan_add(handle, inode);
1716 
1717 	ret2 = ext4_journal_stop(handle);
1718 	if (!ret)
1719 		ret = ret2;
1720 	if (pos + len > inode->i_size && !verity) {
1721 		ext4_truncate_failed_write(inode);
1722 		/*
1723 		 * If truncate failed early the inode might still be
1724 		 * on the orphan list; we need to make sure the inode
1725 		 * is removed from the orphan list in that case.
1726 		 */
1727 		if (inode->i_nlink)
1728 			ext4_orphan_del(NULL, inode);
1729 	}
1730 
1731 	return ret ? ret : copied;
1732 }
1733 
1734 /*
1735  * Reserve space for 'nr_resv' clusters
1736  */
1737 static int ext4_da_reserve_space(struct inode *inode, int nr_resv)
1738 {
1739 	struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
1740 	struct ext4_inode_info *ei = EXT4_I(inode);
1741 	int ret;
1742 
1743 	/*
1744 	 * We will charge metadata quota at writeout time; this saves
1745 	 * us from metadata over-estimation, though we may go over by
1746 	 * a small amount in the end.  Here we just reserve for data.
1747 	 */
1748 	ret = dquot_reserve_block(inode, EXT4_C2B(sbi, nr_resv));
1749 	if (ret)
1750 		return ret;
1751 
1752 	spin_lock(&ei->i_block_reservation_lock);
1753 	if (ext4_claim_free_clusters(sbi, nr_resv, 0)) {
1754 		spin_unlock(&ei->i_block_reservation_lock);
1755 		dquot_release_reservation_block(inode, EXT4_C2B(sbi, nr_resv));
1756 		return -ENOSPC;
1757 	}
1758 	ei->i_reserved_data_blocks += nr_resv;
1759 	trace_ext4_da_reserve_space(inode, nr_resv);
1760 	spin_unlock(&ei->i_block_reservation_lock);
1761 
1762 	return 0;       /* success */
1763 }
1764 
1765 void ext4_da_release_space(struct inode *inode, int to_free)
1766 {
1767 	struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
1768 	struct ext4_inode_info *ei = EXT4_I(inode);
1769 
1770 	if (!to_free)
1771 		return;		/* Nothing to release, exit */
1772 
1773 	spin_lock(&EXT4_I(inode)->i_block_reservation_lock);
1774 
1775 	trace_ext4_da_release_space(inode, to_free);
1776 	if (unlikely(to_free > ei->i_reserved_data_blocks)) {
1777 		/*
1778 		 * if there aren't enough reserved blocks, then the
1779 		 * counter is messed up somewhere.  Since this
1780 		 * function is called from invalidate page, it's
1781 		 * harmless to return without any action.
1782 		 */
1783 		ext4_warning(inode->i_sb, "ext4_da_release_space: "
1784 			 "ino %llu, to_free %d with only %d reserved "
1785 			 "data blocks", inode->i_ino, to_free,
1786 			 ei->i_reserved_data_blocks);
1787 		WARN_ON(1);
1788 		to_free = ei->i_reserved_data_blocks;
1789 	}
1790 	ei->i_reserved_data_blocks -= to_free;
1791 
1792 	/* update fs dirty data blocks counter */
1793 	percpu_counter_sub(&sbi->s_dirtyclusters_counter, to_free);
1794 
1795 	spin_unlock(&EXT4_I(inode)->i_block_reservation_lock);
1796 
1797 	dquot_release_reservation_block(inode, EXT4_C2B(sbi, to_free));
1798 }
1799 
1800 /*
1801  * Delayed allocation stuff
1802  */
1803 
1804 struct mpage_da_data {
1805 	/* These are input fields for ext4_do_writepages() */
1806 	struct inode *inode;
1807 	struct writeback_control *wbc;
1808 	unsigned int can_map:1;	/* Can writepages call map blocks? */
1809 
1810 	/* These are internal state of ext4_do_writepages() */
1811 	loff_t start_pos;	/* The start pos to write */
1812 	loff_t next_pos;	/* Current pos to examine */
1813 	loff_t end_pos;		/* Last pos to examine */
1814 
1815 	/*
1816 	 * Extent to map - this can be after start_pos because that can be
1817 	 * fully mapped. We somewhat abuse m_flags to store whether the extent
1818 	 * is delalloc or unwritten.
1819 	 */
1820 	struct ext4_map_blocks map;
1821 	struct ext4_io_submit io_submit;	/* IO submission data */
1822 	unsigned int do_map:1;
1823 	unsigned int scanned_until_end:1;
1824 	unsigned int journalled_more_data:1;
1825 };
1826 
1827 static void mpage_release_unused_pages(struct mpage_da_data *mpd,
1828 				       bool invalidate)
1829 {
1830 	unsigned nr, i;
1831 	pgoff_t index, end;
1832 	struct folio_batch fbatch;
1833 	struct inode *inode = mpd->inode;
1834 	struct address_space *mapping = inode->i_mapping;
1835 
1836 	/* This is necessary when next_pos == 0. */
1837 	if (mpd->start_pos >= mpd->next_pos)
1838 		return;
1839 
1840 	mpd->scanned_until_end = 0;
1841 	if (invalidate) {
1842 		ext4_lblk_t start, last;
1843 		start = EXT4_B_TO_LBLK(inode, mpd->start_pos);
1844 		last = mpd->next_pos >> inode->i_blkbits;
1845 
1846 		/*
1847 		 * avoid racing with extent status tree scans made by
1848 		 * ext4_insert_delayed_block()
1849 		 */
1850 		down_write(&EXT4_I(inode)->i_data_sem);
1851 		ext4_es_remove_extent(inode, start, last - start);
1852 		up_write(&EXT4_I(inode)->i_data_sem);
1853 	}
1854 
1855 	folio_batch_init(&fbatch);
1856 	index = mpd->start_pos >> PAGE_SHIFT;
1857 	end = mpd->next_pos >> PAGE_SHIFT;
1858 	while (index < end) {
1859 		nr = filemap_get_folios(mapping, &index, end - 1, &fbatch);
1860 		if (nr == 0)
1861 			break;
1862 		for (i = 0; i < nr; i++) {
1863 			struct folio *folio = fbatch.folios[i];
1864 
1865 			if (folio_pos(folio) < mpd->start_pos)
1866 				continue;
1867 			if (folio_next_index(folio) > end)
1868 				continue;
1869 			BUG_ON(!folio_test_locked(folio));
1870 			BUG_ON(folio_test_writeback(folio));
1871 			if (invalidate) {
1872 				if (folio_mapped(folio)) {
1873 					folio_clear_dirty_for_io(folio);
1874 					/*
1875 					 * Unmap folio from page
1876 					 * tables to prevent
1877 					 * subsequent accesses through
1878 					 * stale PTEs. This ensures
1879 					 * future accesses trigger new
1880 					 * page faults rather than
1881 					 * reusing the invalidated
1882 					 * folio.
1883 					 */
1884 					unmap_mapping_pages(folio->mapping,
1885 						folio->index,
1886 						folio_nr_pages(folio), false);
1887 				}
1888 				block_invalidate_folio(folio, 0,
1889 						folio_size(folio));
1890 				folio_clear_uptodate(folio);
1891 			}
1892 			folio_unlock(folio);
1893 		}
1894 		folio_batch_release(&fbatch);
1895 	}
1896 }
1897 
1898 static void ext4_print_free_blocks(struct inode *inode)
1899 {
1900 	struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
1901 	struct super_block *sb = inode->i_sb;
1902 	struct ext4_inode_info *ei = EXT4_I(inode);
1903 
1904 	ext4_msg(sb, KERN_CRIT, "Total free blocks count %lld",
1905 	       EXT4_C2B(EXT4_SB(inode->i_sb),
1906 			ext4_count_free_clusters(sb)));
1907 	ext4_msg(sb, KERN_CRIT, "Free/Dirty block details");
1908 	ext4_msg(sb, KERN_CRIT, "free_blocks=%lld",
1909 	       (long long) EXT4_C2B(EXT4_SB(sb),
1910 		percpu_counter_sum(&sbi->s_freeclusters_counter)));
1911 	ext4_msg(sb, KERN_CRIT, "dirty_blocks=%lld",
1912 	       (long long) EXT4_C2B(EXT4_SB(sb),
1913 		percpu_counter_sum(&sbi->s_dirtyclusters_counter)));
1914 	ext4_msg(sb, KERN_CRIT, "Block reservation details");
1915 	ext4_msg(sb, KERN_CRIT, "i_reserved_data_blocks=%u",
1916 		 ei->i_reserved_data_blocks);
1917 	return;
1918 }
1919 
1920 /*
1921  * Check whether the cluster containing lblk has been allocated or has
1922  * delalloc reservation.
1923  *
1924  * Returns 0 if the cluster doesn't have either, 1 if it has delalloc
1925  * reservation, 2 if it's already been allocated, negative error code on
1926  * failure.
1927  */
1928 static int ext4_clu_alloc_state(struct inode *inode, ext4_lblk_t lblk)
1929 {
1930 	struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
1931 	int ret;
1932 
1933 	/* Has delalloc reservation? */
1934 	if (ext4_es_scan_clu(inode, &ext4_es_is_delayed, lblk))
1935 		return 1;
1936 
1937 	/* Already been allocated? */
1938 	if (ext4_es_scan_clu(inode, &ext4_es_is_mapped, lblk))
1939 		return 2;
1940 	ret = ext4_clu_mapped(inode, EXT4_B2C(sbi, lblk));
1941 	if (ret < 0)
1942 		return ret;
1943 	if (ret > 0)
1944 		return 2;
1945 
1946 	return 0;
1947 }
1948 
1949 /*
1950  * ext4_insert_delayed_blocks - adds a multiple delayed blocks to the extents
1951  *                              status tree, incrementing the reserved
1952  *                              cluster/block count or making pending
1953  *                              reservations where needed
1954  *
1955  * @inode - file containing the newly added block
1956  * @lblk - start logical block to be added
1957  * @len - length of blocks to be added
1958  *
1959  * Returns 0 on success, negative error code on failure.
1960  */
1961 static int ext4_insert_delayed_blocks(struct inode *inode, ext4_lblk_t lblk,
1962 				      ext4_lblk_t len)
1963 {
1964 	struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
1965 	int ret;
1966 	bool lclu_allocated = false;
1967 	bool end_allocated = false;
1968 	ext4_lblk_t resv_clu;
1969 	ext4_lblk_t end = lblk + len - 1;
1970 
1971 	/*
1972 	 * If the cluster containing lblk or end is shared with a delayed,
1973 	 * written, or unwritten extent in a bigalloc file system, it's
1974 	 * already been accounted for and does not need to be reserved.
1975 	 * A pending reservation must be made for the cluster if it's
1976 	 * shared with a written or unwritten extent and doesn't already
1977 	 * have one.  Written and unwritten extents can be purged from the
1978 	 * extents status tree if the system is under memory pressure, so
1979 	 * it's necessary to examine the extent tree if a search of the
1980 	 * extents status tree doesn't get a match.
1981 	 */
1982 	if (sbi->s_cluster_ratio == 1) {
1983 		ret = ext4_da_reserve_space(inode, len);
1984 		if (ret != 0)   /* ENOSPC */
1985 			return ret;
1986 	} else {   /* bigalloc */
1987 		resv_clu = EXT4_B2C(sbi, end) - EXT4_B2C(sbi, lblk) + 1;
1988 
1989 		ret = ext4_clu_alloc_state(inode, lblk);
1990 		if (ret < 0)
1991 			return ret;
1992 		if (ret > 0) {
1993 			resv_clu--;
1994 			lclu_allocated = (ret == 2);
1995 		}
1996 
1997 		if (EXT4_B2C(sbi, lblk) != EXT4_B2C(sbi, end)) {
1998 			ret = ext4_clu_alloc_state(inode, end);
1999 			if (ret < 0)
2000 				return ret;
2001 			if (ret > 0) {
2002 				resv_clu--;
2003 				end_allocated = (ret == 2);
2004 			}
2005 		}
2006 
2007 		if (resv_clu) {
2008 			ret = ext4_da_reserve_space(inode, resv_clu);
2009 			if (ret != 0)   /* ENOSPC */
2010 				return ret;
2011 		}
2012 	}
2013 
2014 	ext4_es_insert_delayed_extent(inode, lblk, len, lclu_allocated,
2015 				      end_allocated);
2016 	return 0;
2017 }
2018 
2019 /*
2020  * Looks up the requested blocks and sets the delalloc extent map.
2021  * First try to look up for the extent entry that contains the requested
2022  * blocks in the extent status tree without i_data_sem, then try to look
2023  * up for the ondisk extent mapping with i_data_sem in read mode,
2024  * finally hold i_data_sem in write mode, looks up again and add a
2025  * delalloc extent entry if it still couldn't find any extent. Pass out
2026  * the mapped extent through @map and return 0 on success.
2027  */
2028 static int ext4_da_map_blocks(struct inode *inode, struct ext4_map_blocks *map)
2029 {
2030 	struct extent_status es;
2031 	int retval;
2032 #ifdef ES_AGGRESSIVE_TEST
2033 	struct ext4_map_blocks orig_map;
2034 
2035 	memcpy(&orig_map, map, sizeof(*map));
2036 #endif
2037 
2038 	map->m_flags = 0;
2039 	ext_debug(inode, "max_blocks %u, logical block %lu\n", map->m_len,
2040 		  (unsigned long) map->m_lblk);
2041 
2042 	ext4_check_map_extents_env(inode);
2043 
2044 	/* Lookup extent status tree firstly */
2045 	if (ext4_es_lookup_extent(inode, map->m_lblk, NULL, &es, &map->m_seq)) {
2046 		map->m_len = min_t(unsigned int, map->m_len,
2047 				   es.es_len - (map->m_lblk - es.es_lblk));
2048 
2049 		if (ext4_es_is_hole(&es))
2050 			goto add_delayed;
2051 
2052 found:
2053 		/*
2054 		 * Delayed extent could be allocated by fallocate.
2055 		 * So we need to check it.
2056 		 */
2057 		if (ext4_es_is_delayed(&es)) {
2058 			map->m_flags |= EXT4_MAP_DELAYED;
2059 			return 0;
2060 		}
2061 
2062 		map->m_pblk = ext4_es_pblock(&es) + map->m_lblk - es.es_lblk;
2063 		if (ext4_es_is_written(&es))
2064 			map->m_flags |= EXT4_MAP_MAPPED;
2065 		else if (ext4_es_is_unwritten(&es))
2066 			map->m_flags |= EXT4_MAP_UNWRITTEN;
2067 		else
2068 			BUG();
2069 
2070 #ifdef ES_AGGRESSIVE_TEST
2071 		ext4_map_blocks_es_recheck(NULL, inode, map, &orig_map, 0);
2072 #endif
2073 		return 0;
2074 	}
2075 
2076 	/*
2077 	 * Try to see if we can get the block without requesting a new
2078 	 * file system block.
2079 	 */
2080 	down_read(&EXT4_I(inode)->i_data_sem);
2081 	if (ext4_has_inline_data(inode))
2082 		retval = 0;
2083 	else
2084 		retval = ext4_map_query_blocks(NULL, inode, map, 0);
2085 	up_read(&EXT4_I(inode)->i_data_sem);
2086 	if (retval)
2087 		return retval < 0 ? retval : 0;
2088 
2089 add_delayed:
2090 	down_write(&EXT4_I(inode)->i_data_sem);
2091 	/*
2092 	 * Page fault path (ext4_page_mkwrite does not take i_rwsem)
2093 	 * and fallocate path (no folio lock) can race. Make sure we
2094 	 * lookup the extent status tree here again while i_data_sem
2095 	 * is held in write mode, before inserting a new da entry in
2096 	 * the extent status tree.
2097 	 */
2098 	if (ext4_es_lookup_extent(inode, map->m_lblk, NULL, &es, &map->m_seq)) {
2099 		map->m_len = min_t(unsigned int, map->m_len,
2100 				   es.es_len - (map->m_lblk - es.es_lblk));
2101 
2102 		if (!ext4_es_is_hole(&es)) {
2103 			up_write(&EXT4_I(inode)->i_data_sem);
2104 			goto found;
2105 		}
2106 	} else if (!ext4_has_inline_data(inode)) {
2107 		retval = ext4_map_query_blocks(NULL, inode, map, 0);
2108 		if (retval) {
2109 			up_write(&EXT4_I(inode)->i_data_sem);
2110 			return retval < 0 ? retval : 0;
2111 		}
2112 	}
2113 
2114 	map->m_flags |= EXT4_MAP_DELAYED | EXT4_MAP_NEW;
2115 	retval = ext4_insert_delayed_blocks(inode, map->m_lblk, map->m_len);
2116 	if (!retval)
2117 		map->m_seq = READ_ONCE(EXT4_I(inode)->i_es_seq);
2118 	up_write(&EXT4_I(inode)->i_data_sem);
2119 
2120 	return retval;
2121 }
2122 
2123 /*
2124  * This is a special get_block_t callback which is used by
2125  * ext4_da_write_begin().  It will either return mapped block or
2126  * reserve space for a single block.
2127  *
2128  * For delayed buffer_head we have BH_Mapped, BH_New, BH_Delay set.
2129  * We also have b_blocknr = -1 and b_bdev initialized properly
2130  *
2131  * For unwritten buffer_head we have BH_Mapped, BH_New, BH_Unwritten set.
2132  * We also have b_blocknr = physicalblock mapping unwritten extent and b_bdev
2133  * initialized properly.
2134  */
2135 int ext4_da_get_block_prep(struct inode *inode, sector_t iblock,
2136 			   struct buffer_head *bh, int create)
2137 {
2138 	struct ext4_map_blocks map;
2139 	sector_t invalid_block = ~((sector_t) 0xffff);
2140 	int ret = 0;
2141 
2142 	BUG_ON(create == 0);
2143 	BUG_ON(bh->b_size != inode->i_sb->s_blocksize);
2144 
2145 	if (invalid_block < ext4_blocks_count(EXT4_SB(inode->i_sb)->s_es))
2146 		invalid_block = ~0;
2147 
2148 	map.m_lblk = iblock;
2149 	map.m_len = 1;
2150 
2151 	/*
2152 	 * first, we need to know whether the block is allocated already
2153 	 * preallocated blocks are unmapped but should treated
2154 	 * the same as allocated blocks.
2155 	 */
2156 	ret = ext4_da_map_blocks(inode, &map);
2157 	if (ret < 0)
2158 		return ret;
2159 
2160 	if (map.m_flags & EXT4_MAP_DELAYED) {
2161 		map_bh(bh, inode->i_sb, invalid_block);
2162 		set_buffer_new(bh);
2163 		set_buffer_delay(bh);
2164 		return 0;
2165 	}
2166 
2167 	map_bh(bh, inode->i_sb, map.m_pblk);
2168 	ext4_update_bh_state(bh, map.m_flags);
2169 
2170 	if (buffer_unwritten(bh)) {
2171 		/* A delayed write to unwritten bh should be marked
2172 		 * new and mapped.  Mapped ensures that we don't do
2173 		 * get_block multiple times when we write to the same
2174 		 * offset and new ensures that we do proper zero out
2175 		 * for partial write.
2176 		 */
2177 		set_buffer_new(bh);
2178 		set_buffer_mapped(bh);
2179 	}
2180 	return 0;
2181 }
2182 
2183 static void mpage_folio_done(struct mpage_da_data *mpd, struct folio *folio)
2184 {
2185 	mpd->start_pos += folio_size(folio);
2186 	mpd->wbc->nr_to_write -= folio_nr_pages(folio);
2187 	folio_unlock(folio);
2188 }
2189 
2190 static void mpage_submit_folio(struct mpage_da_data *mpd, struct folio *folio)
2191 {
2192 	size_t len;
2193 	loff_t size;
2194 
2195 	WARN_ON_ONCE(folio_pos(folio) != mpd->start_pos);
2196 	folio_clear_dirty_for_io(folio);
2197 	/*
2198 	 * We have to be very careful here!  Nothing protects writeback path
2199 	 * against i_size changes and the page can be writeably mapped into
2200 	 * page tables. So an application can be growing i_size and writing
2201 	 * data through mmap while writeback runs. folio_clear_dirty_for_io()
2202 	 * write-protects our page in page tables and the page cannot get
2203 	 * written to again until we release folio lock. So only after
2204 	 * folio_clear_dirty_for_io() we are safe to sample i_size for
2205 	 * ext4_bio_write_folio() to zero-out tail of the written page. We rely
2206 	 * on the barrier provided by folio_test_clear_dirty() in
2207 	 * folio_clear_dirty_for_io() to make sure i_size is really sampled only
2208 	 * after page tables are updated.
2209 	 */
2210 	size = i_size_read(mpd->inode);
2211 	len = folio_size(folio);
2212 	if (folio_pos(folio) + len > size &&
2213 	    !ext4_verity_in_progress(mpd->inode))
2214 		len = size & (len - 1);
2215 	ext4_bio_write_folio(&mpd->io_submit, folio, len);
2216 }
2217 
2218 #define BH_FLAGS (BIT(BH_Unwritten) | BIT(BH_Delay))
2219 
2220 /*
2221  * mballoc gives us at most this number of blocks...
2222  * XXX: That seems to be only a limitation of ext4_mb_normalize_request().
2223  * The rest of mballoc seems to handle chunks up to full group size.
2224  */
2225 #define MAX_WRITEPAGES_EXTENT_LEN 2048
2226 
2227 /*
2228  * mpage_add_bh_to_extent - try to add bh to extent of blocks to map
2229  *
2230  * @mpd - extent of blocks
2231  * @lblk - logical number of the block in the file
2232  * @bh - buffer head we want to add to the extent
2233  *
2234  * The function is used to collect contig. blocks in the same state. If the
2235  * buffer doesn't require mapping for writeback and we haven't started the
2236  * extent of buffers to map yet, the function returns 'true' immediately - the
2237  * caller can write the buffer right away. Otherwise the function returns true
2238  * if the block has been added to the extent, false if the block couldn't be
2239  * added.
2240  */
2241 static bool mpage_add_bh_to_extent(struct mpage_da_data *mpd, ext4_lblk_t lblk,
2242 				   struct buffer_head *bh)
2243 {
2244 	struct ext4_map_blocks *map = &mpd->map;
2245 
2246 	/* Buffer that doesn't need mapping for writeback? */
2247 	if (!buffer_dirty(bh) || !buffer_mapped(bh) ||
2248 	    (!buffer_delay(bh) && !buffer_unwritten(bh))) {
2249 		/* So far no extent to map => we write the buffer right away */
2250 		if (map->m_len == 0)
2251 			return true;
2252 		return false;
2253 	}
2254 
2255 	/* First block in the extent? */
2256 	if (map->m_len == 0) {
2257 		/* We cannot map unless handle is started... */
2258 		if (!mpd->do_map)
2259 			return false;
2260 		map->m_lblk = lblk;
2261 		map->m_len = 1;
2262 		map->m_flags = bh->b_state & BH_FLAGS;
2263 		return true;
2264 	}
2265 
2266 	/* Don't go larger than mballoc is willing to allocate */
2267 	if (map->m_len >= MAX_WRITEPAGES_EXTENT_LEN)
2268 		return false;
2269 
2270 	/* Can we merge the block to our big extent? */
2271 	if (lblk == map->m_lblk + map->m_len &&
2272 	    (bh->b_state & BH_FLAGS) == map->m_flags) {
2273 		map->m_len++;
2274 		return true;
2275 	}
2276 	return false;
2277 }
2278 
2279 /*
2280  * mpage_process_page_bufs - submit page buffers for IO or add them to extent
2281  *
2282  * @mpd - extent of blocks for mapping
2283  * @head - the first buffer in the page
2284  * @bh - buffer we should start processing from
2285  * @lblk - logical number of the block in the file corresponding to @bh
2286  *
2287  * Walk through page buffers from @bh upto @head (exclusive) and either submit
2288  * the page for IO if all buffers in this page were mapped and there's no
2289  * accumulated extent of buffers to map or add buffers in the page to the
2290  * extent of buffers to map. The function returns 1 if the caller can continue
2291  * by processing the next page, 0 if it should stop adding buffers to the
2292  * extent to map because we cannot extend it anymore.
2293  */
2294 static int mpage_process_page_bufs(struct mpage_da_data *mpd,
2295 				   struct buffer_head *head,
2296 				   struct buffer_head *bh,
2297 				   ext4_lblk_t lblk)
2298 {
2299 	struct inode *inode = mpd->inode;
2300 	ext4_lblk_t blocks = (i_size_read(inode) + i_blocksize(inode) - 1)
2301 							>> inode->i_blkbits;
2302 
2303 	if (ext4_verity_in_progress(inode))
2304 		blocks = EXT_MAX_BLOCKS;
2305 
2306 	do {
2307 		BUG_ON(buffer_locked(bh));
2308 
2309 		if (lblk >= blocks || !mpage_add_bh_to_extent(mpd, lblk, bh)) {
2310 			/* Found extent to map? */
2311 			if (mpd->map.m_len)
2312 				return 0;
2313 			/* Buffer needs mapping and handle is not started? */
2314 			if (!mpd->do_map)
2315 				return 0;
2316 			/* Everything mapped so far and we hit EOF */
2317 			break;
2318 		}
2319 	} while (lblk++, (bh = bh->b_this_page) != head);
2320 	/* So far everything mapped? Submit the page for IO. */
2321 	if (mpd->map.m_len == 0) {
2322 		mpage_submit_folio(mpd, head->b_folio);
2323 		mpage_folio_done(mpd, head->b_folio);
2324 	}
2325 	if (lblk >= blocks) {
2326 		mpd->scanned_until_end = 1;
2327 		return 0;
2328 	}
2329 	return 1;
2330 }
2331 
2332 /*
2333  * mpage_process_folio - update folio buffers corresponding to changed extent
2334  *			 and may submit fully mapped page for IO
2335  * @mpd: description of extent to map, on return next extent to map
2336  * @folio: Contains these buffers.
2337  * @m_lblk: logical block mapping.
2338  * @m_pblk: corresponding physical mapping.
2339  * @map_bh: determines on return whether this page requires any further
2340  *		  mapping or not.
2341  *
2342  * Scan given folio buffers corresponding to changed extent and update buffer
2343  * state according to new extent state.
2344  * We map delalloc buffers to their physical location, clear unwritten bits.
2345  * If the given folio is not fully mapped, we update @mpd to the next extent in
2346  * the given folio that needs mapping & return @map_bh as true.
2347  */
2348 static int mpage_process_folio(struct mpage_da_data *mpd, struct folio *folio,
2349 			      ext4_lblk_t *m_lblk, ext4_fsblk_t *m_pblk,
2350 			      bool *map_bh)
2351 {
2352 	struct buffer_head *head, *bh;
2353 	ext4_io_end_t *io_end = mpd->io_submit.io_end;
2354 	ext4_lblk_t lblk = *m_lblk;
2355 	ext4_fsblk_t pblock = *m_pblk;
2356 	int err = 0;
2357 	ssize_t io_end_size = 0;
2358 	struct ext4_io_end_vec *io_end_vec = ext4_last_io_end_vec(io_end);
2359 
2360 	bh = head = folio_buffers(folio);
2361 	do {
2362 		if (lblk < mpd->map.m_lblk)
2363 			continue;
2364 		if (lblk >= mpd->map.m_lblk + mpd->map.m_len) {
2365 			/*
2366 			 * Buffer after end of mapped extent.
2367 			 * Find next buffer in the folio to map.
2368 			 */
2369 			mpd->map.m_len = 0;
2370 			mpd->map.m_flags = 0;
2371 			io_end_vec->size += io_end_size;
2372 
2373 			err = mpage_process_page_bufs(mpd, head, bh, lblk);
2374 			if (err > 0)
2375 				err = 0;
2376 			if (!err && mpd->map.m_len && mpd->map.m_lblk > lblk) {
2377 				io_end_vec = ext4_alloc_io_end_vec(io_end);
2378 				if (IS_ERR(io_end_vec)) {
2379 					err = PTR_ERR(io_end_vec);
2380 					goto out;
2381 				}
2382 				io_end_vec->offset = EXT4_LBLK_TO_B(mpd->inode,
2383 								mpd->map.m_lblk);
2384 			}
2385 			*map_bh = true;
2386 			goto out;
2387 		}
2388 		if (buffer_delay(bh)) {
2389 			clear_buffer_delay(bh);
2390 			bh->b_blocknr = pblock++;
2391 		}
2392 		clear_buffer_unwritten(bh);
2393 		io_end_size += i_blocksize(mpd->inode);
2394 	} while (lblk++, (bh = bh->b_this_page) != head);
2395 
2396 	io_end_vec->size += io_end_size;
2397 	*map_bh = false;
2398 out:
2399 	*m_lblk = lblk;
2400 	*m_pblk = pblock;
2401 	return err;
2402 }
2403 
2404 /*
2405  * mpage_map_buffers - update buffers corresponding to changed extent and
2406  *		       submit fully mapped pages for IO
2407  *
2408  * @mpd - description of extent to map, on return next extent to map
2409  *
2410  * Scan buffers corresponding to changed extent (we expect corresponding pages
2411  * to be already locked) and update buffer state according to new extent state.
2412  * We map delalloc buffers to their physical location, clear unwritten bits,
2413  * and mark buffers as uninit when we perform writes to unwritten extents
2414  * and do extent conversion after IO is finished. If the last page is not fully
2415  * mapped, we update @map to the next extent in the last page that needs
2416  * mapping. Otherwise we submit the page for IO.
2417  */
2418 static int mpage_map_and_submit_buffers(struct mpage_da_data *mpd)
2419 {
2420 	struct folio_batch fbatch;
2421 	unsigned nr, i;
2422 	struct inode *inode = mpd->inode;
2423 	pgoff_t start, end;
2424 	ext4_lblk_t lblk;
2425 	ext4_fsblk_t pblock;
2426 	int err;
2427 	bool map_bh = false;
2428 
2429 	start = EXT4_LBLK_TO_PG(inode, mpd->map.m_lblk);
2430 	end = EXT4_LBLK_TO_PG(inode, mpd->map.m_lblk + mpd->map.m_len - 1);
2431 	pblock = mpd->map.m_pblk;
2432 
2433 	folio_batch_init(&fbatch);
2434 	while (start <= end) {
2435 		nr = filemap_get_folios(inode->i_mapping, &start, end, &fbatch);
2436 		if (nr == 0)
2437 			break;
2438 		for (i = 0; i < nr; i++) {
2439 			struct folio *folio = fbatch.folios[i];
2440 
2441 			lblk = EXT4_PG_TO_LBLK(inode, folio->index);
2442 			err = mpage_process_folio(mpd, folio, &lblk, &pblock,
2443 						 &map_bh);
2444 			/*
2445 			 * If map_bh is true, means page may require further bh
2446 			 * mapping, or maybe the page was submitted for IO.
2447 			 * So we return to call further extent mapping.
2448 			 */
2449 			if (err < 0 || map_bh)
2450 				goto out;
2451 			/* Page fully mapped - let IO run! */
2452 			mpage_submit_folio(mpd, folio);
2453 			mpage_folio_done(mpd, folio);
2454 		}
2455 		folio_batch_release(&fbatch);
2456 	}
2457 	/* Extent fully mapped and matches with page boundary. We are done. */
2458 	mpd->map.m_len = 0;
2459 	mpd->map.m_flags = 0;
2460 	return 0;
2461 out:
2462 	folio_batch_release(&fbatch);
2463 	return err;
2464 }
2465 
2466 static int mpage_map_one_extent(handle_t *handle, struct mpage_da_data *mpd)
2467 {
2468 	struct inode *inode = mpd->inode;
2469 	struct ext4_map_blocks *map = &mpd->map;
2470 	int get_blocks_flags;
2471 	int err, dioread_nolock;
2472 
2473 	/* Make sure transaction has enough credits for this extent */
2474 	err = ext4_journal_ensure_extent_credits(handle, inode);
2475 	if (err < 0)
2476 		return err;
2477 
2478 	trace_ext4_da_write_pages_extent(inode, map);
2479 	/*
2480 	 * Call ext4_map_blocks() to allocate any delayed allocation blocks, or
2481 	 * to convert an unwritten extent to be initialized (in the case
2482 	 * where we have written into one or more preallocated blocks).  It is
2483 	 * possible that we're going to need more metadata blocks than
2484 	 * previously reserved. However we must not fail because we're in
2485 	 * writeback and there is nothing we can do about it so it might result
2486 	 * in data loss.  So use reserved blocks to allocate metadata if
2487 	 * possible. In addition, do not cache any unrelated extents, as it
2488 	 * only holds the folio lock but does not hold the i_rwsem or
2489 	 * invalidate_lock, which could corrupt the extent status tree.
2490 	 */
2491 	get_blocks_flags = EXT4_GET_BLOCKS_CREATE |
2492 			   EXT4_GET_BLOCKS_METADATA_NOFAIL |
2493 			   EXT4_GET_BLOCKS_IO_SUBMIT |
2494 			   EXT4_EX_NOCACHE;
2495 
2496 	dioread_nolock = ext4_should_dioread_nolock(inode);
2497 	if (dioread_nolock)
2498 		get_blocks_flags |= EXT4_GET_BLOCKS_UNWRIT_EXT;
2499 
2500 	err = ext4_map_blocks(handle, inode, map, get_blocks_flags);
2501 	if (err < 0)
2502 		return err;
2503 	/*
2504 	 * A hole? This should never happen since mpage_add_bh_to_extent()
2505 	 * has filtered it out.
2506 	 */
2507 	if (WARN_ON_ONCE(!err))
2508 		return -EINVAL;
2509 	if (dioread_nolock && (map->m_flags & EXT4_MAP_UNWRITTEN)) {
2510 		if (!mpd->io_submit.io_end->handle &&
2511 		    ext4_handle_valid(handle)) {
2512 			mpd->io_submit.io_end->handle = handle->h_rsv_handle;
2513 			handle->h_rsv_handle = NULL;
2514 		}
2515 		ext4_set_io_unwritten_flag(mpd->io_submit.io_end);
2516 	}
2517 
2518 	BUG_ON(map->m_len == 0);
2519 	return 0;
2520 }
2521 
2522 /*
2523  * This is used to submit mapped buffers in a single folio that is not fully
2524  * mapped for various reasons, such as insufficient space or journal credits.
2525  */
2526 static int mpage_submit_partial_folio(struct mpage_da_data *mpd)
2527 {
2528 	struct inode *inode = mpd->inode;
2529 	struct folio *folio;
2530 	loff_t pos;
2531 
2532 	folio = filemap_get_folio(inode->i_mapping,
2533 				  mpd->start_pos >> PAGE_SHIFT);
2534 	if (IS_ERR(folio))
2535 		return PTR_ERR(folio);
2536 	/*
2537 	 * The mapped position should be within the current processing folio
2538 	 * but must not be the folio start position.
2539 	 */
2540 	pos = ((loff_t)mpd->map.m_lblk) << inode->i_blkbits;
2541 	if (WARN_ON_ONCE((folio_pos(folio) == pos) ||
2542 			 !folio_contains(folio, pos >> PAGE_SHIFT)))
2543 		return -EINVAL;
2544 
2545 	mpage_submit_folio(mpd, folio);
2546 	/*
2547 	 * Update start_pos to prevent this folio from being released in
2548 	 * mpage_release_unused_pages(), it will be reset to the aligned folio
2549 	 * pos when this folio is written again in the next round. Additionally,
2550 	 * do not update wbc->nr_to_write here, as it will be updated once the
2551 	 * entire folio has finished processing.
2552 	 */
2553 	mpd->start_pos = pos;
2554 	folio_unlock(folio);
2555 	folio_put(folio);
2556 	return 0;
2557 }
2558 
2559 /*
2560  * mpage_map_and_submit_extent - map extent starting at mpd->lblk of length
2561  *				 mpd->len and submit pages underlying it for IO
2562  *
2563  * @handle - handle for journal operations
2564  * @mpd - extent to map
2565  * @give_up_on_write - we set this to true iff there is a fatal error and there
2566  *                     is no hope of writing the data. The caller should discard
2567  *                     dirty pages to avoid infinite loops.
2568  *
2569  * The function maps extent starting at mpd->lblk of length mpd->len. If it is
2570  * delayed, blocks are allocated, if it is unwritten, we may need to convert
2571  * them to initialized or split the described range from larger unwritten
2572  * extent. Note that we need not map all the described range since allocation
2573  * can return less blocks or the range is covered by more unwritten extents. We
2574  * cannot map more because we are limited by reserved transaction credits. On
2575  * the other hand we always make sure that the last touched page is fully
2576  * mapped so that it can be written out (and thus forward progress is
2577  * guaranteed). After mapping we submit all mapped pages for IO.
2578  */
2579 static int mpage_map_and_submit_extent(handle_t *handle,
2580 				       struct mpage_da_data *mpd,
2581 				       bool *give_up_on_write)
2582 {
2583 	struct inode *inode = mpd->inode;
2584 	struct ext4_map_blocks *map = &mpd->map;
2585 	int err;
2586 	loff_t disksize;
2587 	int progress = 0;
2588 	ext4_io_end_t *io_end = mpd->io_submit.io_end;
2589 	struct ext4_io_end_vec *io_end_vec;
2590 
2591 	io_end_vec = ext4_alloc_io_end_vec(io_end);
2592 	if (IS_ERR(io_end_vec))
2593 		return PTR_ERR(io_end_vec);
2594 	io_end_vec->offset = EXT4_LBLK_TO_B(inode, map->m_lblk);
2595 	do {
2596 		err = mpage_map_one_extent(handle, mpd);
2597 		if (err < 0) {
2598 			struct super_block *sb = inode->i_sb;
2599 
2600 			if (ext4_emergency_state(sb))
2601 				goto invalidate_dirty_pages;
2602 			/*
2603 			 * Let the uper layers retry transient errors.
2604 			 * In the case of ENOSPC, if ext4_count_free_blocks()
2605 			 * is non-zero, a commit should free up blocks.
2606 			 */
2607 			if ((err == -ENOMEM) || (err == -EAGAIN) ||
2608 			    (err == -ENOSPC && ext4_count_free_clusters(sb))) {
2609 				/*
2610 				 * We may have already allocated extents for
2611 				 * some bhs inside the folio, issue the
2612 				 * corresponding data to prevent stale data.
2613 				 */
2614 				if (progress) {
2615 					if (mpage_submit_partial_folio(mpd))
2616 						goto invalidate_dirty_pages;
2617 					goto update_disksize;
2618 				}
2619 				return err;
2620 			}
2621 			ext4_msg(sb, KERN_CRIT,
2622 				 "Delayed block allocation failed for "
2623 				 "inode %llu at logical offset %llu with"
2624 				 " max blocks %u with error %d",
2625 				 inode->i_ino,
2626 				 (unsigned long long)map->m_lblk,
2627 				 (unsigned)map->m_len, -err);
2628 			ext4_msg(sb, KERN_CRIT,
2629 				 "This should not happen!! Data will "
2630 				 "be lost\n");
2631 			if (err == -ENOSPC)
2632 				ext4_print_free_blocks(inode);
2633 		invalidate_dirty_pages:
2634 			*give_up_on_write = true;
2635 			return err;
2636 		}
2637 		progress = 1;
2638 		/*
2639 		 * Update buffer state, submit mapped pages, and get us new
2640 		 * extent to map
2641 		 */
2642 		err = mpage_map_and_submit_buffers(mpd);
2643 		if (err < 0)
2644 			goto update_disksize;
2645 	} while (map->m_len);
2646 
2647 update_disksize:
2648 	/*
2649 	 * Update on-disk size after IO is submitted.  Races with
2650 	 * truncate are avoided by checking i_size under i_data_sem.
2651 	 */
2652 	disksize = mpd->start_pos;
2653 	if (disksize > READ_ONCE(EXT4_I(inode)->i_disksize)) {
2654 		int err2;
2655 		loff_t i_size;
2656 
2657 		down_write(&EXT4_I(inode)->i_data_sem);
2658 		i_size = i_size_read(inode);
2659 		if (disksize > i_size)
2660 			disksize = i_size;
2661 		if (disksize > EXT4_I(inode)->i_disksize)
2662 			EXT4_I(inode)->i_disksize = disksize;
2663 		up_write(&EXT4_I(inode)->i_data_sem);
2664 		err2 = ext4_mark_inode_dirty(handle, inode);
2665 		if (err2) {
2666 			ext4_error_err(inode->i_sb, -err2,
2667 				       "Failed to mark inode %llu dirty",
2668 				       inode->i_ino);
2669 		}
2670 		if (!err)
2671 			err = err2;
2672 	}
2673 	return err;
2674 }
2675 
2676 static int ext4_journal_folio_buffers(handle_t *handle, struct folio *folio,
2677 				     size_t len)
2678 {
2679 	struct buffer_head *page_bufs = folio_buffers(folio);
2680 	struct inode *inode = folio->mapping->host;
2681 	int ret, err;
2682 
2683 	ret = ext4_walk_page_buffers(handle, inode, page_bufs, 0, len,
2684 				     NULL, do_journal_get_write_access);
2685 	err = ext4_walk_page_buffers(handle, inode, page_bufs, 0, len,
2686 				     NULL, write_end_fn);
2687 	if (ret == 0)
2688 		ret = err;
2689 	err = ext4_jbd2_inode_add_write(handle, inode, folio_pos(folio), len);
2690 	if (ret == 0)
2691 		ret = err;
2692 	EXT4_I(inode)->i_datasync_tid = handle->h_transaction->t_tid;
2693 
2694 	return ret;
2695 }
2696 
2697 static int mpage_journal_page_buffers(handle_t *handle,
2698 				      struct mpage_da_data *mpd,
2699 				      struct folio *folio)
2700 {
2701 	struct inode *inode = mpd->inode;
2702 	loff_t size = i_size_read(inode);
2703 	size_t len = folio_size(folio);
2704 
2705 	folio_clear_checked(folio);
2706 	mpd->wbc->nr_to_write -= folio_nr_pages(folio);
2707 
2708 	if (folio_pos(folio) + len > size &&
2709 	    !ext4_verity_in_progress(inode))
2710 		len = size & (len - 1);
2711 
2712 	return ext4_journal_folio_buffers(handle, folio, len);
2713 }
2714 
2715 /*
2716  * mpage_prepare_extent_to_map - find & lock contiguous range of dirty pages
2717  * 				 needing mapping, submit mapped pages
2718  *
2719  * @mpd - where to look for pages
2720  *
2721  * Walk dirty pages in the mapping. If they are fully mapped, submit them for
2722  * IO immediately. If we cannot map blocks, we submit just already mapped
2723  * buffers in the page for IO and keep page dirty. When we can map blocks and
2724  * we find a page which isn't mapped we start accumulating extent of buffers
2725  * underlying these pages that needs mapping (formed by either delayed or
2726  * unwritten buffers). We also lock the pages containing these buffers. The
2727  * extent found is returned in @mpd structure (starting at mpd->lblk with
2728  * length mpd->len blocks).
2729  *
2730  * Note that this function can attach bios to one io_end structure which are
2731  * neither logically nor physically contiguous. Although it may seem as an
2732  * unnecessary complication, it is actually inevitable in blocksize < pagesize
2733  * case as we need to track IO to all buffers underlying a page in one io_end.
2734  */
2735 static int mpage_prepare_extent_to_map(struct mpage_da_data *mpd)
2736 {
2737 	struct address_space *mapping = mpd->inode->i_mapping;
2738 	struct folio_batch fbatch;
2739 	unsigned int nr_folios;
2740 	pgoff_t index = mpd->start_pos >> PAGE_SHIFT;
2741 	pgoff_t end = mpd->end_pos >> PAGE_SHIFT;
2742 	xa_mark_t tag;
2743 	int i, err = 0;
2744 	ext4_lblk_t lblk;
2745 	struct buffer_head *head;
2746 	handle_t *handle = NULL;
2747 	int bpp = ext4_journal_blocks_per_folio(mpd->inode);
2748 
2749 	tag = wbc_to_tag(mpd->wbc);
2750 
2751 	mpd->map.m_len = 0;
2752 	mpd->next_pos = mpd->start_pos;
2753 	if (ext4_should_journal_data(mpd->inode)) {
2754 		handle = ext4_journal_start(mpd->inode, EXT4_HT_WRITE_PAGE,
2755 					    bpp);
2756 		if (IS_ERR(handle))
2757 			return PTR_ERR(handle);
2758 	}
2759 	folio_batch_init(&fbatch);
2760 	while (index <= end) {
2761 		nr_folios = filemap_get_folios_tag(mapping, &index, end,
2762 				tag, &fbatch);
2763 		if (nr_folios == 0)
2764 			break;
2765 
2766 		for (i = 0; i < nr_folios; i++) {
2767 			struct folio *folio = fbatch.folios[i];
2768 
2769 			/*
2770 			 * Accumulated enough dirty pages? This doesn't apply
2771 			 * to WB_SYNC_ALL mode. For integrity sync we have to
2772 			 * keep going because someone may be concurrently
2773 			 * dirtying pages, and we might have synced a lot of
2774 			 * newly appeared dirty pages, but have not synced all
2775 			 * of the old dirty pages.
2776 			 */
2777 			if (mpd->wbc->sync_mode == WB_SYNC_NONE &&
2778 			    mpd->wbc->nr_to_write <=
2779 			    EXT4_LBLK_TO_PG(mpd->inode, mpd->map.m_len))
2780 				goto out;
2781 
2782 			/* If we can't merge this page, we are done. */
2783 			if (mpd->map.m_len > 0 &&
2784 			    mpd->next_pos != folio_pos(folio))
2785 				goto out;
2786 
2787 			if (handle) {
2788 				err = ext4_journal_ensure_credits(handle, bpp,
2789 								  0);
2790 				if (err < 0)
2791 					goto out;
2792 			}
2793 
2794 			folio_lock(folio);
2795 			/*
2796 			 * If the page is no longer dirty, or its mapping no
2797 			 * longer corresponds to inode we are writing (which
2798 			 * means it has been truncated or invalidated), or the
2799 			 * page is already under writeback and we are not doing
2800 			 * a data integrity writeback, skip the page
2801 			 */
2802 			if ((folio_test_writeback(folio) &&
2803 			    mpd->wbc->sync_mode == WB_SYNC_NONE) ||
2804 			    unlikely(folio->mapping != mapping)) {
2805 				folio_unlock(folio);
2806 				continue;
2807 			}
2808 			/*
2809 			 * If the folio is clean, skip writing it back.
2810 			 * Cycle the folio through the writeback state
2811 			 * though, to clear stale xarray tags.
2812 			 */
2813 			if (!folio_test_dirty(folio)) {
2814 				if (!folio_test_writeback(folio)) {
2815 					__folio_start_writeback(folio, false);
2816 					folio_end_writeback(folio);
2817 				}
2818 				folio_unlock(folio);
2819 				continue;
2820 			}
2821 
2822 			folio_wait_writeback(folio);
2823 			BUG_ON(folio_test_writeback(folio));
2824 
2825 			/*
2826 			 * Should never happen but for buggy code in
2827 			 * other subsystems that call
2828 			 * set_page_dirty() without properly warning
2829 			 * the file system first.  See [1] for more
2830 			 * information.
2831 			 *
2832 			 * [1] https://lore.kernel.org/linux-mm/20180103100430.GE4911@quack2.suse.cz
2833 			 */
2834 			if (!folio_buffers(folio)) {
2835 				ext4_warning_inode(mpd->inode, "page %lu does not have buffers attached", folio->index);
2836 				folio_clear_dirty(folio);
2837 				folio_unlock(folio);
2838 				continue;
2839 			}
2840 
2841 			if (mpd->map.m_len == 0)
2842 				mpd->start_pos = folio_pos(folio);
2843 			mpd->next_pos = folio_next_pos(folio);
2844 			/*
2845 			 * Writeout when we cannot modify metadata is simple.
2846 			 * Just submit the page. For data=journal mode we
2847 			 * first handle writeout of the page for checkpoint and
2848 			 * only after that handle delayed page dirtying. This
2849 			 * makes sure current data is checkpointed to the final
2850 			 * location before possibly journalling it again which
2851 			 * is desirable when the page is frequently dirtied
2852 			 * through a pin.
2853 			 */
2854 			if (!mpd->can_map) {
2855 				mpage_submit_folio(mpd, folio);
2856 				err = 0;
2857 				/* Pending dirtying of journalled data? */
2858 				if (folio_test_checked(folio)) {
2859 					err = mpage_journal_page_buffers(handle,
2860 						mpd, folio);
2861 					if (err < 0)
2862 						goto out;
2863 					mpd->journalled_more_data = 1;
2864 				}
2865 				mpage_folio_done(mpd, folio);
2866 			} else {
2867 				/* Add all dirty buffers to mpd */
2868 				lblk = EXT4_PG_TO_LBLK(mpd->inode, folio->index);
2869 				head = folio_buffers(folio);
2870 				err = mpage_process_page_bufs(mpd, head, head,
2871 						lblk);
2872 				if (err <= 0)
2873 					goto out;
2874 				err = 0;
2875 			}
2876 		}
2877 		folio_batch_release(&fbatch);
2878 		cond_resched();
2879 	}
2880 	mpd->scanned_until_end = 1;
2881 	if (handle)
2882 		ext4_journal_stop(handle);
2883 	return 0;
2884 out:
2885 	folio_batch_release(&fbatch);
2886 	if (handle)
2887 		ext4_journal_stop(handle);
2888 	return err;
2889 }
2890 
2891 static int ext4_do_writepages(struct mpage_da_data *mpd)
2892 {
2893 	struct writeback_control *wbc = mpd->wbc;
2894 	pgoff_t	writeback_index = 0;
2895 	long nr_to_write = wbc->nr_to_write;
2896 	int range_whole = 0;
2897 	int cycled = 1;
2898 	handle_t *handle = NULL;
2899 	struct inode *inode = mpd->inode;
2900 	struct address_space *mapping = inode->i_mapping;
2901 	int needed_blocks, rsv_blocks = 0, ret = 0;
2902 	struct ext4_sb_info *sbi = EXT4_SB(mapping->host->i_sb);
2903 	struct blk_plug plug;
2904 	bool give_up_on_write = false;
2905 
2906 	trace_ext4_writepages(inode, wbc);
2907 
2908 	/*
2909 	 * No pages to write? This is mainly a kludge to avoid starting
2910 	 * a transaction for special inodes like journal inode on last iput()
2911 	 * because that could violate lock ordering on umount
2912 	 */
2913 	if (!mapping->nrpages || !mapping_tagged(mapping, PAGECACHE_TAG_DIRTY))
2914 		goto out_writepages;
2915 
2916 	/*
2917 	 * Does ext4_change_inode_journal_flag() change the inode's
2918 	 * buffered I/O path?
2919 	 */
2920 	if (ext4_inode_buffered_iomap(inode))
2921 		goto out_writepages;
2922 
2923 	/*
2924 	 * If the filesystem has aborted, it is read-only, so return
2925 	 * right away instead of dumping stack traces later on that
2926 	 * will obscure the real source of the problem.  We test
2927 	 * fs shutdown state instead of sb->s_flag's SB_RDONLY because
2928 	 * the latter could be true if the filesystem is mounted
2929 	 * read-only, and in that case, ext4_writepages should
2930 	 * *never* be called, so if that ever happens, we would want
2931 	 * the stack trace.
2932 	 */
2933 	ret = ext4_emergency_state(mapping->host->i_sb);
2934 	if (unlikely(ret))
2935 		goto out_writepages;
2936 
2937 	/*
2938 	 * If we have inline data and arrive here, it means that
2939 	 * we will soon create the block for the 1st page, so
2940 	 * we'd better clear the inline data here.
2941 	 */
2942 	if (ext4_has_inline_data(inode)) {
2943 		/* Just inode will be modified... */
2944 		handle = ext4_journal_start(inode, EXT4_HT_INODE, 1);
2945 		if (IS_ERR(handle)) {
2946 			ret = PTR_ERR(handle);
2947 			goto out_writepages;
2948 		}
2949 		BUG_ON(ext4_test_inode_state(inode,
2950 				EXT4_STATE_MAY_INLINE_DATA));
2951 		ext4_destroy_inline_data(handle, inode);
2952 		ext4_journal_stop(handle);
2953 	}
2954 
2955 	/*
2956 	 * data=journal mode does not do delalloc so we just need to writeout /
2957 	 * journal already mapped buffers. On the other hand we need to commit
2958 	 * transaction to make data stable. We expect all the data to be
2959 	 * already in the journal (the only exception are DMA pinned pages
2960 	 * dirtied behind our back) so we commit transaction here and run the
2961 	 * writeback loop to checkpoint them. The checkpointing is not actually
2962 	 * necessary to make data persistent *but* quite a few places (extent
2963 	 * shifting operations, fsverity, ...) depend on being able to drop
2964 	 * pagecache pages after calling filemap_write_and_wait() and for that
2965 	 * checkpointing needs to happen.
2966 	 */
2967 	if (ext4_should_journal_data(inode)) {
2968 		mpd->can_map = 0;
2969 		if (wbc->sync_mode == WB_SYNC_ALL)
2970 			ext4_fc_commit(sbi->s_journal,
2971 				       EXT4_I(inode)->i_datasync_tid);
2972 	}
2973 	mpd->journalled_more_data = 0;
2974 
2975 	if (ext4_should_dioread_nolock(inode)) {
2976 		int bpf = ext4_journal_blocks_per_folio(inode);
2977 		/*
2978 		 * We may need to convert up to one extent per block in the
2979 		 * folio.
2980 		 */
2981 		rsv_blocks = ext4_meta_trans_blocks(inode, bpf, bpf, 0);
2982 	}
2983 
2984 	if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
2985 		range_whole = 1;
2986 
2987 	if (wbc->range_cyclic) {
2988 		writeback_index = mapping->writeback_index;
2989 		if (writeback_index)
2990 			cycled = 0;
2991 		mpd->start_pos = writeback_index << PAGE_SHIFT;
2992 		mpd->end_pos = LLONG_MAX;
2993 	} else {
2994 		mpd->start_pos = wbc->range_start;
2995 		mpd->end_pos = wbc->range_end;
2996 	}
2997 
2998 	ext4_io_submit_init(&mpd->io_submit, wbc);
2999 retry:
3000 	if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
3001 		tag_pages_for_writeback(mapping, mpd->start_pos >> PAGE_SHIFT,
3002 					mpd->end_pos >> PAGE_SHIFT);
3003 	blk_start_plug(&plug);
3004 
3005 	/*
3006 	 * First writeback pages that don't need mapping - we can avoid
3007 	 * starting a transaction unnecessarily and also avoid being blocked
3008 	 * in the block layer on device congestion while having transaction
3009 	 * started.
3010 	 */
3011 	mpd->do_map = 0;
3012 	mpd->scanned_until_end = 0;
3013 	mpd->io_submit.io_end = ext4_init_io_end(inode, GFP_KERNEL);
3014 	if (!mpd->io_submit.io_end) {
3015 		ret = -ENOMEM;
3016 		goto unplug;
3017 	}
3018 	ret = mpage_prepare_extent_to_map(mpd);
3019 	/* Unlock pages we didn't use */
3020 	mpage_release_unused_pages(mpd, false);
3021 	/* Submit prepared bio */
3022 	ext4_io_submit(&mpd->io_submit);
3023 	ext4_put_io_end_defer(mpd->io_submit.io_end);
3024 	mpd->io_submit.io_end = NULL;
3025 	if (ret < 0)
3026 		goto unplug;
3027 
3028 	while (!mpd->scanned_until_end && wbc->nr_to_write > 0) {
3029 		/* For each extent of pages we use new io_end */
3030 		mpd->io_submit.io_end = ext4_init_io_end(inode, GFP_KERNEL);
3031 		if (!mpd->io_submit.io_end) {
3032 			ret = -ENOMEM;
3033 			break;
3034 		}
3035 
3036 		WARN_ON_ONCE(!mpd->can_map);
3037 		/*
3038 		 * We have two constraints: We find one extent to map and we
3039 		 * must always write out whole page (makes a difference when
3040 		 * blocksize < pagesize) so that we don't block on IO when we
3041 		 * try to write out the rest of the page. Journalled mode is
3042 		 * not supported by delalloc.
3043 		 */
3044 		BUG_ON(ext4_should_journal_data(inode));
3045 		/*
3046 		 * Calculate the number of credits needed to reserve for one
3047 		 * extent of up to MAX_WRITEPAGES_EXTENT_LEN blocks. It will
3048 		 * attempt to extend the transaction or start a new iteration
3049 		 * if the reserved credits are insufficient.
3050 		 */
3051 		needed_blocks = ext4_chunk_trans_blocks(inode,
3052 						MAX_WRITEPAGES_EXTENT_LEN);
3053 		/* start a new transaction */
3054 		handle = ext4_journal_start_with_reserve(inode,
3055 				EXT4_HT_WRITE_PAGE, needed_blocks, rsv_blocks);
3056 		if (IS_ERR(handle)) {
3057 			ret = PTR_ERR(handle);
3058 			ext4_msg(inode->i_sb, KERN_CRIT, "%s: jbd2_start: "
3059 			       "%ld pages, ino %llu; err %d", __func__,
3060 				wbc->nr_to_write, inode->i_ino, ret);
3061 			/* Release allocated io_end */
3062 			ext4_put_io_end(mpd->io_submit.io_end);
3063 			mpd->io_submit.io_end = NULL;
3064 			break;
3065 		}
3066 		mpd->do_map = 1;
3067 
3068 		trace_ext4_da_write_folios_start(inode, mpd->start_pos,
3069 				mpd->next_pos, wbc);
3070 		ret = mpage_prepare_extent_to_map(mpd);
3071 		if (!ret && mpd->map.m_len)
3072 			ret = mpage_map_and_submit_extent(handle, mpd,
3073 					&give_up_on_write);
3074 		/*
3075 		 * Caution: If the handle is synchronous,
3076 		 * ext4_journal_stop() can wait for transaction commit
3077 		 * to finish which may depend on writeback of pages to
3078 		 * complete or on page lock to be released.  In that
3079 		 * case, we have to wait until after we have
3080 		 * submitted all the IO, released page locks we hold,
3081 		 * and dropped io_end reference (for extent conversion
3082 		 * to be able to complete) before stopping the handle.
3083 		 */
3084 		if (!ext4_handle_valid(handle) || handle->h_sync == 0) {
3085 			ext4_journal_stop(handle);
3086 			handle = NULL;
3087 			mpd->do_map = 0;
3088 		}
3089 		/* Unlock pages we didn't use */
3090 		mpage_release_unused_pages(mpd, give_up_on_write);
3091 		/* Submit prepared bio */
3092 		ext4_io_submit(&mpd->io_submit);
3093 
3094 		/*
3095 		 * Drop our io_end reference we got from init. We have
3096 		 * to be careful and use deferred io_end finishing if
3097 		 * we are still holding the transaction as we can
3098 		 * release the last reference to io_end which may end
3099 		 * up doing unwritten extent conversion.
3100 		 */
3101 		if (handle) {
3102 			ext4_put_io_end_defer(mpd->io_submit.io_end);
3103 			ext4_journal_stop(handle);
3104 		} else
3105 			ext4_put_io_end(mpd->io_submit.io_end);
3106 		mpd->io_submit.io_end = NULL;
3107 		trace_ext4_da_write_folios_end(inode, mpd->start_pos,
3108 				mpd->next_pos, wbc, ret);
3109 
3110 		if (ret == -ENOSPC && sbi->s_journal) {
3111 			/*
3112 			 * Commit the transaction which would
3113 			 * free blocks released in the transaction
3114 			 * and try again
3115 			 */
3116 			jbd2_journal_force_commit_nested(sbi->s_journal);
3117 			ret = 0;
3118 			continue;
3119 		}
3120 		if (ret == -EAGAIN)
3121 			ret = 0;
3122 		/* Fatal error - ENOMEM, EIO... */
3123 		if (ret)
3124 			break;
3125 	}
3126 unplug:
3127 	blk_finish_plug(&plug);
3128 	if (!ret && !cycled && wbc->nr_to_write > 0) {
3129 		cycled = 1;
3130 		mpd->end_pos = (writeback_index << PAGE_SHIFT) - 1;
3131 		mpd->start_pos = 0;
3132 		goto retry;
3133 	}
3134 
3135 	/* Update index */
3136 	if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0))
3137 		/*
3138 		 * Set the writeback_index so that range_cyclic
3139 		 * mode will write it back later
3140 		 */
3141 		mapping->writeback_index = mpd->start_pos >> PAGE_SHIFT;
3142 
3143 out_writepages:
3144 	trace_ext4_writepages_result(inode, wbc, ret,
3145 				     nr_to_write - wbc->nr_to_write);
3146 	return ret;
3147 }
3148 
3149 static int ext4_writepages(struct address_space *mapping,
3150 			   struct writeback_control *wbc)
3151 {
3152 	struct super_block *sb = mapping->host->i_sb;
3153 	struct mpage_da_data mpd = {
3154 		.inode = mapping->host,
3155 		.wbc = wbc,
3156 		.can_map = 1,
3157 	};
3158 	int ret;
3159 	int alloc_ctx;
3160 
3161 	ret = ext4_emergency_state(sb);
3162 	if (unlikely(ret))
3163 		return ret;
3164 
3165 	alloc_ctx = ext4_writepages_down_read(sb);
3166 	ret = ext4_do_writepages(&mpd);
3167 	/*
3168 	 * For data=journal writeback we could have come across pages marked
3169 	 * for delayed dirtying (PageChecked) which were just added to the
3170 	 * running transaction. Try once more to get them to stable storage.
3171 	 */
3172 	if (!ret && mpd.journalled_more_data)
3173 		ret = ext4_do_writepages(&mpd);
3174 	ext4_writepages_up_read(sb, alloc_ctx);
3175 
3176 	return ret;
3177 }
3178 
3179 int ext4_normal_submit_inode_data_buffers(struct jbd2_inode *jinode)
3180 {
3181 	loff_t range_start, range_end;
3182 	struct writeback_control wbc = {
3183 		.sync_mode = WB_SYNC_ALL,
3184 		.nr_to_write = LONG_MAX,
3185 	};
3186 	struct mpage_da_data mpd = {
3187 		.inode = jinode->i_vfs_inode,
3188 		.wbc = &wbc,
3189 		.can_map = 0,
3190 	};
3191 
3192 	if (!jbd2_jinode_get_dirty_range(jinode, &range_start, &range_end))
3193 		return 0;
3194 
3195 	wbc.range_start = range_start;
3196 	wbc.range_end = range_end;
3197 
3198 	return ext4_do_writepages(&mpd);
3199 }
3200 
3201 static int ext4_dax_writepages(struct address_space *mapping,
3202 			       struct writeback_control *wbc)
3203 {
3204 	int ret;
3205 	long nr_to_write = wbc->nr_to_write;
3206 	struct inode *inode = mapping->host;
3207 	int alloc_ctx;
3208 
3209 	ret = ext4_emergency_state(inode->i_sb);
3210 	if (unlikely(ret))
3211 		return ret;
3212 
3213 	alloc_ctx = ext4_writepages_down_read(inode->i_sb);
3214 	trace_ext4_writepages(inode, wbc);
3215 
3216 	ret = dax_writeback_mapping_range(mapping,
3217 					  EXT4_SB(inode->i_sb)->s_daxdev, wbc);
3218 	trace_ext4_writepages_result(inode, wbc, ret,
3219 				     nr_to_write - wbc->nr_to_write);
3220 	ext4_writepages_up_read(inode->i_sb, alloc_ctx);
3221 	return ret;
3222 }
3223 
3224 int ext4_nonda_switch(struct super_block *sb)
3225 {
3226 	s64 free_clusters, dirty_clusters;
3227 	struct ext4_sb_info *sbi = EXT4_SB(sb);
3228 
3229 	/*
3230 	 * switch to non delalloc mode if we are running low
3231 	 * on free block. The free block accounting via percpu
3232 	 * counters can get slightly wrong with percpu_counter_batch getting
3233 	 * accumulated on each CPU without updating global counters
3234 	 * Delalloc need an accurate free block accounting. So switch
3235 	 * to non delalloc when we are near to error range.
3236 	 */
3237 	free_clusters =
3238 		percpu_counter_read_positive(&sbi->s_freeclusters_counter);
3239 	dirty_clusters =
3240 		percpu_counter_read_positive(&sbi->s_dirtyclusters_counter);
3241 	/*
3242 	 * Start pushing delalloc when 1/2 of free blocks are dirty.
3243 	 */
3244 	if (dirty_clusters && (free_clusters < 2 * dirty_clusters))
3245 		try_to_writeback_inodes_sb(sb, WB_REASON_FS_FREE_SPACE);
3246 
3247 	if (2 * free_clusters < 3 * dirty_clusters ||
3248 	    free_clusters < (dirty_clusters + EXT4_FREECLUSTERS_WATERMARK)) {
3249 		/*
3250 		 * free block count is less than 150% of dirty blocks
3251 		 * or free blocks is less than watermark
3252 		 */
3253 		return 1;
3254 	}
3255 	return 0;
3256 }
3257 
3258 static int ext4_da_write_begin(const struct kiocb *iocb,
3259 			       struct address_space *mapping,
3260 			       loff_t pos, unsigned len,
3261 			       struct folio **foliop, void **fsdata)
3262 {
3263 	int ret, retries = 0;
3264 	struct folio *folio;
3265 	pgoff_t index;
3266 	struct inode *inode = mapping->host;
3267 
3268 	ret = ext4_emergency_state(inode->i_sb);
3269 	if (unlikely(ret))
3270 		return ret;
3271 
3272 	index = pos >> PAGE_SHIFT;
3273 
3274 	if (ext4_nonda_switch(inode->i_sb) || ext4_verity_in_progress(inode)) {
3275 		*fsdata = (void *)FALL_BACK_TO_NONDELALLOC;
3276 		return ext4_write_begin(iocb, mapping, pos,
3277 					len, foliop, fsdata);
3278 	}
3279 	*fsdata = (void *)0;
3280 	trace_ext4_da_write_begin(inode, pos, len);
3281 
3282 	if (ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) {
3283 		ret = ext4_generic_write_inline_data(mapping, inode, pos, len,
3284 						     foliop, true);
3285 		if (ret < 0)
3286 			return ret;
3287 		if (ret == 1) {
3288 			*fsdata = (void *)((unsigned long)*fsdata | EXT4_WRITE_DATA_INLINE);
3289 			return 0;
3290 		}
3291 	}
3292 
3293 retry:
3294 	folio = write_begin_get_folio(iocb, mapping, index, len);
3295 	if (IS_ERR(folio))
3296 		return PTR_ERR(folio);
3297 
3298 	if (len > folio_next_pos(folio) - pos)
3299 		len = folio_next_pos(folio) - pos;
3300 
3301 	ret = ext4_block_write_begin(NULL, folio, pos, len,
3302 				     ext4_da_get_block_prep);
3303 	if (ret < 0) {
3304 		folio_unlock(folio);
3305 		folio_put(folio);
3306 		/*
3307 		 * ext4_block_write_begin may have instantiated a few blocks
3308 		 * outside i_size.  Trim these off again. Don't need
3309 		 * i_size_read because we hold inode lock.
3310 		 */
3311 		if (pos + len > inode->i_size)
3312 			ext4_truncate_failed_write(inode);
3313 
3314 		if (ret == -ENOSPC &&
3315 		    ext4_should_retry_alloc(inode->i_sb, &retries))
3316 			goto retry;
3317 		return ret;
3318 	}
3319 
3320 	*foliop = folio;
3321 	return ret;
3322 }
3323 
3324 /*
3325  * Check if we should update i_disksize
3326  * when write to the end of file but not require block allocation
3327  */
3328 static int ext4_da_should_update_i_disksize(struct folio *folio,
3329 					    unsigned long offset)
3330 {
3331 	struct buffer_head *bh;
3332 	struct inode *inode = folio->mapping->host;
3333 	unsigned int idx;
3334 	int i;
3335 
3336 	bh = folio_buffers(folio);
3337 	idx = offset >> inode->i_blkbits;
3338 
3339 	for (i = 0; i < idx; i++)
3340 		bh = bh->b_this_page;
3341 
3342 	if (!buffer_mapped(bh) || (buffer_delay(bh)) || buffer_unwritten(bh))
3343 		return 0;
3344 	return 1;
3345 }
3346 
3347 static int ext4_da_do_write_end(struct address_space *mapping,
3348 			loff_t pos, unsigned len, unsigned copied,
3349 			struct folio *folio)
3350 {
3351 	struct inode *inode = mapping->host;
3352 	loff_t old_size = inode->i_size;
3353 	bool disksize_changed = false;
3354 	loff_t new_i_size;
3355 	handle_t *handle;
3356 
3357 	if (unlikely(!folio_buffers(folio))) {
3358 		folio_unlock(folio);
3359 		folio_put(folio);
3360 		return -EIO;
3361 	}
3362 	/*
3363 	 * block_write_end() will mark the inode as dirty with I_DIRTY_PAGES
3364 	 * flag, which all that's needed to trigger page writeback.
3365 	 */
3366 	copied = block_write_end(pos, len, copied, folio);
3367 	new_i_size = pos + copied;
3368 
3369 	/*
3370 	 * It's important to update i_size while still holding folio lock,
3371 	 * because folio writeout could otherwise come in and zero beyond
3372 	 * i_size.
3373 	 *
3374 	 * Since we are holding inode lock, we are sure i_disksize <=
3375 	 * i_size. We also know that if i_disksize < i_size, there are
3376 	 * delalloc writes pending in the range up to i_size. If the end of
3377 	 * the current write is <= i_size, there's no need to touch
3378 	 * i_disksize since writeback will push i_disksize up to i_size
3379 	 * eventually. If the end of the current write is > i_size and
3380 	 * inside an allocated block which ext4_da_should_update_i_disksize()
3381 	 * checked, we need to update i_disksize here as certain
3382 	 * ext4_writepages() paths not allocating blocks and update i_disksize.
3383 	 */
3384 	if (new_i_size > inode->i_size) {
3385 		unsigned long end;
3386 
3387 		i_size_write(inode, new_i_size);
3388 		end = offset_in_folio(folio, new_i_size - 1);
3389 		if (copied && ext4_da_should_update_i_disksize(folio, end)) {
3390 			ext4_update_i_disksize(inode, new_i_size);
3391 			disksize_changed = true;
3392 		}
3393 	}
3394 
3395 	folio_unlock(folio);
3396 	folio_put(folio);
3397 
3398 	if (pos > old_size)
3399 		pagecache_isize_extended(inode, old_size, pos);
3400 
3401 	if (!disksize_changed)
3402 		return copied;
3403 
3404 	handle = ext4_journal_start(inode, EXT4_HT_INODE, 1);
3405 	if (IS_ERR(handle))
3406 		return PTR_ERR(handle);
3407 	ext4_mark_inode_dirty(handle, inode);
3408 	ext4_journal_stop(handle);
3409 
3410 	return copied;
3411 }
3412 
3413 static int ext4_da_write_end(const struct kiocb *iocb,
3414 			     struct address_space *mapping,
3415 			     loff_t pos, unsigned len, unsigned copied,
3416 			     struct folio *folio, void *fsdata)
3417 {
3418 	struct inode *inode = mapping->host;
3419 	unsigned long write_mode = (unsigned long)fsdata;
3420 
3421 	if (write_mode & FALL_BACK_TO_NONDELALLOC)
3422 		return ext4_write_end(iocb, mapping, pos,
3423 				      len, copied, folio, fsdata);
3424 
3425 	trace_ext4_da_write_end(inode, pos, len, copied);
3426 
3427 	if (write_mode & EXT4_WRITE_DATA_INLINE)
3428 		return ext4_write_inline_data_end(inode, pos, len, copied,
3429 						  folio);
3430 
3431 	if (unlikely(copied < len) && !folio_test_uptodate(folio))
3432 		copied = 0;
3433 
3434 	return ext4_da_do_write_end(mapping, pos, len, copied, folio);
3435 }
3436 
3437 /*
3438  * Force all delayed allocation blocks to be allocated for a given inode.
3439  */
3440 int ext4_alloc_da_blocks(struct inode *inode)
3441 {
3442 	trace_ext4_alloc_da_blocks(inode);
3443 
3444 	if (!EXT4_I(inode)->i_reserved_data_blocks)
3445 		return 0;
3446 
3447 	/*
3448 	 * We do something simple for now.  The filemap_flush() will
3449 	 * also start triggering a write of the data blocks, which is
3450 	 * not strictly speaking necessary.  However, to do otherwise
3451 	 * would require replicating code paths in:
3452 	 *
3453 	 * ext4_writepages() ->
3454 	 *    write_cache_pages() ---> (via passed in callback function)
3455 	 *        __mpage_da_writepage() -->
3456 	 *           mpage_add_bh_to_extent()
3457 	 *           mpage_da_map_blocks()
3458 	 *
3459 	 * The problem is that write_cache_pages(), located in
3460 	 * mm/page-writeback.c, marks pages clean in preparation for
3461 	 * doing I/O, which is not desirable if we're not planning on
3462 	 * doing I/O at all.
3463 	 *
3464 	 * We could call write_cache_pages(), and then redirty all of
3465 	 * the pages by calling redirty_page_for_writepage() but that
3466 	 * would be ugly in the extreme.  So instead we would need to
3467 	 * replicate parts of the code in the above functions,
3468 	 * simplifying them because we wouldn't actually intend to
3469 	 * write out the pages, but rather only collect contiguous
3470 	 * logical block extents, call the multi-block allocator, and
3471 	 * then update the buffer heads with the block allocations.
3472 	 *
3473 	 * For now, though, we'll cheat by calling filemap_flush(),
3474 	 * which will map the blocks, and start the I/O, but not
3475 	 * actually wait for the I/O to complete.
3476 	 */
3477 	return filemap_flush(inode->i_mapping);
3478 }
3479 
3480 /*
3481  * bmap() is special.  It gets used by applications such as lilo and by
3482  * the swapper to find the on-disk block of a specific piece of data.
3483  *
3484  * Naturally, this is dangerous if the block concerned is still in the
3485  * journal.  If somebody makes a swapfile on an ext4 data-journaling
3486  * filesystem and enables swap, then they may get a nasty shock when the
3487  * data getting swapped to that swapfile suddenly gets overwritten by
3488  * the original zero's written out previously to the journal and
3489  * awaiting writeback in the kernel's buffer cache.
3490  *
3491  * So, if we see any bmap calls here on a modified, data-journaled file,
3492  * take extra steps to flush any blocks which might be in the cache.
3493  */
3494 static sector_t ext4_bmap(struct address_space *mapping, sector_t block)
3495 {
3496 	struct inode *inode = mapping->host;
3497 	sector_t ret = 0;
3498 
3499 	inode_lock_shared(inode);
3500 	/*
3501 	 * We can get here for an inline file via the FIBMAP ioctl
3502 	 */
3503 	if (ext4_has_inline_data(inode))
3504 		goto out;
3505 
3506 	if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY) &&
3507 	    (test_opt(inode->i_sb, DELALLOC) ||
3508 	     ext4_should_journal_data(inode))) {
3509 		/*
3510 		 * With delalloc or journalled data we want to sync the file so
3511 		 * that we can make sure we allocate blocks for file and data
3512 		 * is in place for the user to see it
3513 		 */
3514 		filemap_write_and_wait(mapping);
3515 	}
3516 
3517 	ret = iomap_bmap(mapping, block, ext4_iomap_next);
3518 
3519 out:
3520 	inode_unlock_shared(inode);
3521 	return ret;
3522 }
3523 
3524 static void ext4_invalidate_folio(struct folio *folio, size_t offset,
3525 				size_t length)
3526 {
3527 	trace_ext4_invalidate_folio(folio, offset, length);
3528 
3529 	/* No journalling happens on data buffers when this function is used */
3530 	WARN_ON(folio_buffers(folio) && buffer_jbd(folio_buffers(folio)));
3531 
3532 	block_invalidate_folio(folio, offset, length);
3533 }
3534 
3535 static int __ext4_journalled_invalidate_folio(struct folio *folio,
3536 					    size_t offset, size_t length)
3537 {
3538 	journal_t *journal = EXT4_JOURNAL(folio->mapping->host);
3539 
3540 	trace_ext4_journalled_invalidate_folio(folio, offset, length);
3541 
3542 	/*
3543 	 * If it's a full truncate we just forget about the pending dirtying
3544 	 */
3545 	if (offset == 0 && length == folio_size(folio))
3546 		folio_clear_checked(folio);
3547 
3548 	return jbd2_journal_invalidate_folio(journal, folio, offset, length);
3549 }
3550 
3551 /* Wrapper for aops... */
3552 static void ext4_journalled_invalidate_folio(struct folio *folio,
3553 					   size_t offset,
3554 					   size_t length)
3555 {
3556 	WARN_ON(__ext4_journalled_invalidate_folio(folio, offset, length) < 0);
3557 }
3558 
3559 static bool ext4_release_folio(struct folio *folio, gfp_t wait)
3560 {
3561 	struct inode *inode = folio->mapping->host;
3562 	journal_t *journal = EXT4_JOURNAL(inode);
3563 
3564 	trace_ext4_release_folio(inode, folio);
3565 
3566 	/* Page has dirty journalled data -> cannot release */
3567 	if (folio_test_checked(folio))
3568 		return false;
3569 	if (journal)
3570 		return jbd2_journal_try_to_free_buffers(journal, folio);
3571 	else
3572 		return try_to_free_buffers(folio);
3573 }
3574 
3575 static bool ext4_inode_datasync_dirty(struct inode *inode)
3576 {
3577 	journal_t *journal = EXT4_SB(inode->i_sb)->s_journal;
3578 	struct mapping_metadata_bhs *mmb;
3579 
3580 	if (journal) {
3581 		if (jbd2_transaction_committed(journal,
3582 			EXT4_I(inode)->i_datasync_tid))
3583 			return false;
3584 		if (test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT))
3585 			return !list_empty(&EXT4_I(inode)->i_fc_list);
3586 		return true;
3587 	}
3588 
3589 	mmb = ext4_i_metadata_bhs(inode);
3590 	/* Any metadata buffers to write? */
3591 	if (mmb && mmb_has_buffers(mmb))
3592 		return true;
3593 	return inode_state_read_once(inode) & I_DIRTY_DATASYNC;
3594 }
3595 
3596 static bool ext4_iomap_valid(struct inode *inode, const struct iomap *iomap)
3597 {
3598 	return iomap->validity_cookie == READ_ONCE(EXT4_I(inode)->i_es_seq);
3599 }
3600 
3601 const struct iomap_write_ops ext4_iomap_write_ops = {
3602 	.iomap_valid = ext4_iomap_valid,
3603 };
3604 
3605 static void ext4_set_iomap(struct inode *inode, struct iomap *iomap,
3606 			   struct ext4_map_blocks *map, loff_t offset,
3607 			   loff_t length, unsigned int flags)
3608 {
3609 	u8 blkbits = inode->i_blkbits;
3610 
3611 	/*
3612 	 * Writes that span EOF might trigger an I/O size update on completion,
3613 	 * so consider them to be dirty for the purpose of O_DSYNC, even if
3614 	 * there is no other metadata changes being made or are pending.
3615 	 */
3616 	iomap->flags = 0;
3617 	if (ext4_inode_datasync_dirty(inode) ||
3618 	    offset + length > i_size_read(inode))
3619 		iomap->flags |= IOMAP_F_DIRTY;
3620 
3621 	if (map->m_flags & EXT4_MAP_NEW)
3622 		iomap->flags |= IOMAP_F_NEW;
3623 
3624 	/* HW-offload atomics are always used */
3625 	if (flags & IOMAP_ATOMIC)
3626 		iomap->flags |= IOMAP_F_ATOMIC_BIO;
3627 
3628 	if (flags & IOMAP_DAX)
3629 		iomap->dax_dev = EXT4_SB(inode->i_sb)->s_daxdev;
3630 	else
3631 		iomap->bdev = inode->i_sb->s_bdev;
3632 	iomap->offset = EXT4_LBLK_TO_B(inode, map->m_lblk);
3633 	iomap->length = EXT4_LBLK_TO_B(inode, map->m_len);
3634 
3635 	if ((map->m_flags & EXT4_MAP_MAPPED) &&
3636 	    !ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
3637 		iomap->flags |= IOMAP_F_MERGED;
3638 
3639 	iomap->validity_cookie = map->m_seq;
3640 
3641 	/*
3642 	 * Flags passed to ext4_map_blocks() for direct I/O writes can result
3643 	 * in m_flags having both EXT4_MAP_MAPPED and EXT4_MAP_UNWRITTEN bits
3644 	 * set. In order for any allocated unwritten extents to be converted
3645 	 * into written extents correctly within the ->end_io() handler, we
3646 	 * need to ensure that the iomap->type is set appropriately. Hence, the
3647 	 * reason why we need to check whether the EXT4_MAP_UNWRITTEN bit has
3648 	 * been set first.
3649 	 */
3650 	if (map->m_flags & EXT4_MAP_UNWRITTEN) {
3651 		iomap->type = IOMAP_UNWRITTEN;
3652 		iomap->addr = (u64) map->m_pblk << blkbits;
3653 		if (flags & IOMAP_DAX)
3654 			iomap->addr += EXT4_SB(inode->i_sb)->s_dax_part_off;
3655 	} else if (map->m_flags & EXT4_MAP_MAPPED) {
3656 		iomap->type = IOMAP_MAPPED;
3657 		iomap->addr = (u64) map->m_pblk << blkbits;
3658 		if (flags & IOMAP_DAX)
3659 			iomap->addr += EXT4_SB(inode->i_sb)->s_dax_part_off;
3660 	} else if (map->m_flags & EXT4_MAP_DELAYED) {
3661 		iomap->type = IOMAP_DELALLOC;
3662 		iomap->addr = IOMAP_NULL_ADDR;
3663 	} else {
3664 		iomap->type = IOMAP_HOLE;
3665 		iomap->addr = IOMAP_NULL_ADDR;
3666 	}
3667 }
3668 
3669 static int ext4_map_blocks_atomic_write_slow(handle_t *handle,
3670 			struct inode *inode, struct ext4_map_blocks *map)
3671 {
3672 	ext4_lblk_t m_lblk = map->m_lblk;
3673 	unsigned int m_len = map->m_len;
3674 	unsigned int mapped_len = 0, m_flags = 0;
3675 	ext4_fsblk_t next_pblk = 0;
3676 	bool check_next_pblk = false;
3677 	int ret = 0;
3678 
3679 	WARN_ON_ONCE(!ext4_has_feature_bigalloc(inode->i_sb));
3680 
3681 	/*
3682 	 * This is a slow path in case of mixed mapping. We use
3683 	 * EXT4_GET_BLOCKS_CREATE_ZERO flag here to make sure we get a single
3684 	 * contiguous mapped mapping. This will ensure any unwritten or hole
3685 	 * regions within the requested range is zeroed out and we return
3686 	 * a single contiguous mapped extent.
3687 	 */
3688 	m_flags = EXT4_GET_BLOCKS_CREATE_ZERO;
3689 
3690 	do {
3691 		ret = ext4_map_blocks(handle, inode, map, m_flags);
3692 		if (ret < 0 && ret != -ENOSPC)
3693 			goto out_err;
3694 		/*
3695 		 * This should never happen, but let's return an error code to
3696 		 * avoid an infinite loop in here.
3697 		 */
3698 		if (ret == 0) {
3699 			ret = -EFSCORRUPTED;
3700 			ext4_warning_inode(inode,
3701 				"ext4_map_blocks() couldn't allocate blocks m_flags: 0x%x, ret:%d",
3702 				m_flags, ret);
3703 			goto out_err;
3704 		}
3705 		/*
3706 		 * With bigalloc we should never get ENOSPC nor discontiguous
3707 		 * physical extents.
3708 		 */
3709 		if ((check_next_pblk && next_pblk != map->m_pblk) ||
3710 				ret == -ENOSPC) {
3711 			ext4_warning_inode(inode,
3712 				"Non-contiguous allocation detected: expected %llu, got %llu, "
3713 				"or ext4_map_blocks() returned out of space ret: %d",
3714 				next_pblk, map->m_pblk, ret);
3715 			ret = -EFSCORRUPTED;
3716 			goto out_err;
3717 		}
3718 		next_pblk = map->m_pblk + map->m_len;
3719 		check_next_pblk = true;
3720 
3721 		mapped_len += map->m_len;
3722 		map->m_lblk += map->m_len;
3723 		map->m_len = m_len - mapped_len;
3724 	} while (mapped_len < m_len);
3725 
3726 	/*
3727 	 * We might have done some work in above loop, so we need to query the
3728 	 * start of the physical extent, based on the origin m_lblk and m_len.
3729 	 * Let's also ensure we were able to allocate the required range for
3730 	 * mixed mapping case.
3731 	 */
3732 	map->m_lblk = m_lblk;
3733 	map->m_len = m_len;
3734 	map->m_flags = 0;
3735 
3736 	ret = ext4_map_blocks(handle, inode, map,
3737 			      EXT4_GET_BLOCKS_QUERY_LAST_IN_LEAF);
3738 	if (ret != m_len) {
3739 		ext4_warning_inode(inode,
3740 			"allocation failed for atomic write request m_lblk:%u, m_len:%u, ret:%d\n",
3741 			m_lblk, m_len, ret);
3742 		ret = -EINVAL;
3743 	}
3744 	return ret;
3745 
3746 out_err:
3747 	/* reset map before returning an error */
3748 	map->m_lblk = m_lblk;
3749 	map->m_len = m_len;
3750 	map->m_flags = 0;
3751 	return ret;
3752 }
3753 
3754 /*
3755  * ext4_map_blocks_atomic: Helper routine to ensure the entire requested
3756  * range in @map [lblk, lblk + len) is one single contiguous extent with no
3757  * mixed mappings.
3758  *
3759  * We first use m_flags passed to us by our caller (ext4_iomap_alloc()).
3760  * We only call EXT4_GET_BLOCKS_ZERO in the slow path, when the underlying
3761  * physical extent for the requested range does not have a single contiguous
3762  * mapping type i.e. (Hole, Mapped, or Unwritten) throughout.
3763  * In that case we will loop over the requested range to allocate and zero out
3764  * the unwritten / holes in between, to get a single mapped extent from
3765  * [m_lblk, m_lblk +  m_len). Note that this is only possible because we know
3766  * this can be called only with bigalloc enabled filesystem where the underlying
3767  * cluster is already allocated. This avoids allocating discontiguous extents
3768  * in the slow path due to multiple calls to ext4_map_blocks().
3769  * The slow path is mostly non-performance critical path, so it should be ok to
3770  * loop using ext4_map_blocks() with appropriate flags to allocate & zero the
3771  * underlying short holes/unwritten extents within the requested range.
3772  */
3773 static int ext4_map_blocks_atomic_write(handle_t *handle, struct inode *inode,
3774 				struct ext4_map_blocks *map, int m_flags,
3775 				bool *force_commit)
3776 {
3777 	ext4_lblk_t m_lblk = map->m_lblk;
3778 	unsigned int m_len = map->m_len;
3779 	int ret = 0;
3780 
3781 	WARN_ON_ONCE(m_len > 1 && !ext4_has_feature_bigalloc(inode->i_sb));
3782 
3783 	ret = ext4_map_blocks(handle, inode, map, m_flags);
3784 	if (ret < 0 || ret == m_len)
3785 		goto out;
3786 	/*
3787 	 * This is a mixed mapping case where we were not able to allocate
3788 	 * a single contiguous extent. In that case let's reset requested
3789 	 * mapping and call the slow path.
3790 	 */
3791 	map->m_lblk = m_lblk;
3792 	map->m_len = m_len;
3793 	map->m_flags = 0;
3794 
3795 	/*
3796 	 * slow path means we have mixed mapping, that means we will need
3797 	 * to force txn commit.
3798 	 */
3799 	*force_commit = true;
3800 	return ext4_map_blocks_atomic_write_slow(handle, inode, map);
3801 out:
3802 	return ret;
3803 }
3804 
3805 static int ext4_iomap_alloc(struct inode *inode, struct ext4_map_blocks *map,
3806 			    unsigned int flags)
3807 {
3808 	handle_t *handle;
3809 	int ret, dio_credits, m_flags = 0, retries = 0;
3810 	bool force_commit = false;
3811 
3812 	if (flags & IOMAP_NOWAIT)
3813 		return -EAGAIN;
3814 
3815 	/*
3816 	 * Trim the mapping request to the maximum value that we can map at
3817 	 * once for direct I/O.
3818 	 */
3819 	if (map->m_len > DIO_MAX_BLOCKS)
3820 		map->m_len = DIO_MAX_BLOCKS;
3821 
3822 	/*
3823 	 * journal credits estimation for atomic writes. We call
3824 	 * ext4_map_blocks(), to find if there could be a mixed mapping. If yes,
3825 	 * then let's assume the no. of pextents required can be m_len i.e.
3826 	 * every alternate block can be unwritten and hole.
3827 	 */
3828 	if (flags & IOMAP_ATOMIC) {
3829 		unsigned int orig_mlen = map->m_len;
3830 
3831 		ret = ext4_map_blocks(NULL, inode, map, 0);
3832 		if (ret < 0)
3833 			return ret;
3834 		if (map->m_len < orig_mlen) {
3835 			map->m_len = orig_mlen;
3836 			dio_credits = ext4_meta_trans_blocks(inode, map->m_len,
3837 							     map->m_len, 0);
3838 		} else {
3839 			dio_credits = ext4_chunk_trans_blocks(inode,
3840 							      map->m_len);
3841 		}
3842 	} else {
3843 		dio_credits = ext4_chunk_trans_blocks(inode, map->m_len);
3844 	}
3845 
3846 retry:
3847 	/*
3848 	 * Either we allocate blocks and then don't get an unwritten extent, so
3849 	 * in that case we have reserved enough credits. Or, the blocks are
3850 	 * already allocated and unwritten. In that case, the extent conversion
3851 	 * fits into the credits as well.
3852 	 */
3853 	handle = ext4_journal_start(inode, EXT4_HT_MAP_BLOCKS, dio_credits);
3854 	if (IS_ERR(handle))
3855 		return PTR_ERR(handle);
3856 
3857 	/*
3858 	 * DAX and direct I/O are the only two operations that are currently
3859 	 * supported with IOMAP_WRITE.
3860 	 */
3861 	WARN_ON(!(flags & (IOMAP_DAX | IOMAP_DIRECT)));
3862 	if (flags & IOMAP_DAX)
3863 		m_flags = EXT4_GET_BLOCKS_CREATE_ZERO;
3864 	/*
3865 	 * We use i_size instead of i_disksize here because delalloc writeback
3866 	 * can complete at any point during the I/O and subsequently push the
3867 	 * i_disksize out to i_size. This could be beyond where direct I/O is
3868 	 * happening and thus expose allocated blocks to direct I/O reads.
3869 	 */
3870 	else if (EXT4_LBLK_TO_B(inode, map->m_lblk) >= i_size_read(inode))
3871 		m_flags = EXT4_GET_BLOCKS_CREATE;
3872 	else if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
3873 		m_flags = EXT4_GET_BLOCKS_CREATE_UNWRIT_EXT;
3874 
3875 	if (flags & IOMAP_ATOMIC)
3876 		ret = ext4_map_blocks_atomic_write(handle, inode, map, m_flags,
3877 						   &force_commit);
3878 	else
3879 		ret = ext4_map_blocks(handle, inode, map, m_flags);
3880 
3881 	/*
3882 	 * We cannot fill holes in indirect tree based inodes as that could
3883 	 * expose stale data in the case of a crash. Use the magic error code
3884 	 * to fallback to buffered I/O.
3885 	 */
3886 	if (!m_flags && !ret)
3887 		ret = -ENOTBLK;
3888 
3889 	ext4_journal_stop(handle);
3890 	if (ret == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries))
3891 		goto retry;
3892 
3893 	/*
3894 	 * Force commit the current transaction if the allocation spans a mixed
3895 	 * mapping range. This ensures any pending metadata updates (like
3896 	 * unwritten to written extents conversion) in this range are in
3897 	 * consistent state with the file data blocks, before performing the
3898 	 * actual write I/O. If the commit fails, the whole I/O must be aborted
3899 	 * to prevent any possible torn writes.
3900 	 */
3901 	if (ret > 0 && force_commit) {
3902 		int ret2;
3903 
3904 		ret2 = ext4_force_commit(inode->i_sb);
3905 		if (ret2)
3906 			return ret2;
3907 	}
3908 
3909 	return ret;
3910 }
3911 
3912 
3913 int ext4_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
3914 		unsigned flags, struct iomap *iomap, struct iomap *srcmap)
3915 {
3916 	int ret;
3917 	struct ext4_map_blocks map;
3918 	u8 blkbits = inode->i_blkbits;
3919 	unsigned int orig_mlen;
3920 
3921 	if ((offset >> blkbits) > EXT4_MAX_LOGICAL_BLOCK)
3922 		return -EINVAL;
3923 
3924 	if (WARN_ON_ONCE(ext4_has_inline_data(inode)))
3925 		return -ERANGE;
3926 
3927 	/*
3928 	 * Calculate the first and last logical blocks respectively.
3929 	 */
3930 	map.m_lblk = offset >> blkbits;
3931 	map.m_len = min_t(loff_t, (offset + length - 1) >> blkbits,
3932 			  EXT4_MAX_LOGICAL_BLOCK) - map.m_lblk + 1;
3933 	orig_mlen = map.m_len;
3934 
3935 	if (flags & IOMAP_WRITE) {
3936 		/*
3937 		 * We check here if the blocks are already allocated, then we
3938 		 * don't need to start a journal txn and we can directly return
3939 		 * the mapping information. This could boost performance
3940 		 * especially in multi-threaded overwrite requests.
3941 		 */
3942 		if (offset + length <= i_size_read(inode)) {
3943 			ret = ext4_map_blocks(NULL, inode, &map, 0);
3944 			/*
3945 			 * For DAX we convert extents to initialized ones before
3946 			 * copying the data, otherwise we do it after I/O so
3947 			 * there's no need to call into ext4_iomap_alloc().
3948 			 */
3949 			if ((map.m_flags & EXT4_MAP_MAPPED) ||
3950 			    (!(flags & IOMAP_DAX) &&
3951 			     (map.m_flags & EXT4_MAP_UNWRITTEN))) {
3952 				/*
3953 				 * For atomic writes the entire requested
3954 				 * length should be mapped.
3955 				 */
3956 				if (ret == orig_mlen ||
3957 				    (!(flags & IOMAP_ATOMIC) && ret > 0))
3958 					goto out;
3959 			}
3960 			map.m_len = orig_mlen;
3961 		}
3962 		ret = ext4_iomap_alloc(inode, &map, flags);
3963 	} else {
3964 		ret = ext4_map_blocks(NULL, inode, &map, 0);
3965 	}
3966 
3967 	if (ret < 0)
3968 		return ret;
3969 out:
3970 	/*
3971 	 * Sometimes I/O to an encrypted file has to be broken up to guarantee
3972 	 * DUN contiguity.  Handle this by limiting the length of the mapping
3973 	 * returned.
3974 	 */
3975 	map.m_len = fscrypt_limit_io_blocks(inode, map.m_lblk, map.m_len);
3976 
3977 	/*
3978 	 * Before returning to iomap, let's ensure the allocated mapping
3979 	 * covers the entire requested length for atomic writes.
3980 	 */
3981 	if (flags & IOMAP_ATOMIC) {
3982 		if (map.m_len < (length >> blkbits)) {
3983 			WARN_ON_ONCE(1);
3984 			return -EINVAL;
3985 		}
3986 	}
3987 	ext4_set_iomap(inode, iomap, &map, offset, length, flags);
3988 
3989 	return 0;
3990 }
3991 
3992 DEFINE_IOMAP_ITER_NEXT(ext4_iomap_next, ext4_iomap_begin);
3993 
3994 static int ext4_iomap_begin_report(struct inode *inode, loff_t offset,
3995 				   loff_t length, unsigned int flags,
3996 				   struct iomap *iomap, struct iomap *srcmap)
3997 {
3998 	int ret;
3999 	struct ext4_map_blocks map;
4000 	u8 blkbits = inode->i_blkbits;
4001 
4002 	if ((offset >> blkbits) > EXT4_MAX_LOGICAL_BLOCK)
4003 		return -EINVAL;
4004 
4005 	if (ext4_has_inline_data(inode)) {
4006 		ret = ext4_inline_data_iomap(inode, iomap);
4007 		if (ret != -EAGAIN) {
4008 			if (ret == 0 && offset >= iomap->length)
4009 				ret = -ENOENT;
4010 			return ret;
4011 		}
4012 	}
4013 
4014 	/*
4015 	 * Calculate the first and last logical block respectively.
4016 	 */
4017 	map.m_lblk = offset >> blkbits;
4018 	map.m_len = min_t(loff_t, (offset + length - 1) >> blkbits,
4019 			  EXT4_MAX_LOGICAL_BLOCK) - map.m_lblk + 1;
4020 
4021 	/*
4022 	 * Fiemap callers may call for offset beyond s_bitmap_maxbytes.
4023 	 * So handle it here itself instead of querying ext4_map_blocks().
4024 	 * Since ext4_map_blocks() will warn about it and will return
4025 	 * -EIO error.
4026 	 */
4027 	if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) {
4028 		struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
4029 
4030 		if (offset >= sbi->s_bitmap_maxbytes) {
4031 			map.m_flags = 0;
4032 			goto set_iomap;
4033 		}
4034 	}
4035 
4036 	ret = ext4_map_blocks(NULL, inode, &map, 0);
4037 	if (ret < 0)
4038 		return ret;
4039 set_iomap:
4040 	ext4_set_iomap(inode, iomap, &map, offset, length, flags);
4041 
4042 	return 0;
4043 }
4044 
4045 DEFINE_IOMAP_ITER_NEXT(ext4_iomap_next_report, ext4_iomap_begin_report);
4046 
4047 static int ext4_iomap_map_blocks(struct inode *inode, loff_t offset,
4048 				 loff_t length, struct ext4_map_blocks *map,
4049 				 int flags)
4050 {
4051 	u8 blkbits = inode->i_blkbits;
4052 
4053 	/* inode using the buffer_head buffered I/O path should not go here. */
4054 	if (WARN_ON_ONCE(!ext4_inode_buffered_iomap(inode)))
4055 		return -EINVAL;
4056 	if ((offset >> blkbits) > EXT4_MAX_LOGICAL_BLOCK)
4057 		return -EINVAL;
4058 
4059 	/* Calculate the first and last logical blocks respectively. */
4060 	map->m_lblk = offset >> blkbits;
4061 	map->m_len = min_t(loff_t, (offset + length - 1) >> blkbits,
4062 			   EXT4_MAX_LOGICAL_BLOCK) - map->m_lblk + 1;
4063 
4064 	if (flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE)
4065 		return ext4_da_map_blocks(inode, map);
4066 
4067 	return ext4_map_blocks(NULL, inode, map, flags);
4068 }
4069 
4070 static int ext4_iomap_buffered_read_begin(struct inode *inode, loff_t offset,
4071 		loff_t length, unsigned int flags, struct iomap *iomap,
4072 		struct iomap *srcmap)
4073 {
4074 	struct ext4_map_blocks map;
4075 	int ret;
4076 
4077 	if (unlikely(ext4_forced_shutdown(inode->i_sb)))
4078 		return -EIO;
4079 
4080 	/* Inline data support is not yet available. */
4081 	if (WARN_ON_ONCE(ext4_has_inline_data(inode)))
4082 		return -ERANGE;
4083 
4084 	ret = ext4_iomap_map_blocks(inode, offset, length, &map, 0);
4085 	if (ret < 0)
4086 		return ret;
4087 
4088 	trace_ext4_iomap_buffered_read_begin(inode, &map, offset, length,
4089 					     flags);
4090 	ext4_set_iomap(inode, iomap, &map, offset, length, flags);
4091 	return 0;
4092 }
4093 
4094 static int ext4_iomap_buffered_do_write_begin(struct inode *inode,
4095 		loff_t offset, loff_t length, unsigned int flags,
4096 		struct iomap *iomap, struct iomap *srcmap, bool delalloc)
4097 {
4098 	int ret, retries = 0;
4099 	struct ext4_map_blocks map;
4100 	int map_flags;
4101 
4102 	ret = ext4_emergency_state(inode->i_sb);
4103 	if (unlikely(ret))
4104 		return ret;
4105 
4106 	/* Inline data and non-extent are not supported. */
4107 	if (WARN_ON_ONCE(ext4_has_inline_data(inode)))
4108 		return -ERANGE;
4109 	if (WARN_ON_ONCE(!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)))
4110 		return -EINVAL;
4111 	if (WARN_ON_ONCE(!(flags & (IOMAP_WRITE | IOMAP_FAULT))))
4112 		return -EINVAL;
4113 
4114 	map_flags = delalloc ? EXT4_GET_BLOCKS_DELALLOC_RESERVE :
4115 			       EXT4_GET_BLOCKS_CREATE_UNWRIT_EXT;
4116 retry:
4117 	ret = ext4_iomap_map_blocks(inode, offset, length, &map, map_flags);
4118 	if (ret == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries))
4119 		goto retry;
4120 	if (ret < 0)
4121 		return ret;
4122 
4123 	trace_ext4_iomap_buffered_write_begin(inode, &map, offset, length,
4124 					      flags);
4125 	ext4_set_iomap(inode, iomap, &map, offset, length, flags);
4126 	return 0;
4127 }
4128 
4129 static int ext4_iomap_buffered_write_begin(struct inode *inode,
4130 		loff_t offset, loff_t length, unsigned int flags,
4131 		struct iomap *iomap, struct iomap *srcmap)
4132 {
4133 	return ext4_iomap_buffered_do_write_begin(inode, offset, length, flags,
4134 						  iomap, srcmap, false);
4135 }
4136 
4137 static int ext4_iomap_buffered_da_write_begin(struct inode *inode,
4138 		loff_t offset, loff_t length, unsigned int flags,
4139 		struct iomap *iomap, struct iomap *srcmap)
4140 {
4141 	return ext4_iomap_buffered_do_write_begin(inode, offset, length, flags,
4142 						  iomap, srcmap, true);
4143 }
4144 
4145 /*
4146  * On write failure, drop the stale delayed allocation range and release
4147  * its reserved space for both start and end blocks. Otherwise, we may
4148  * leave a range of delayed extents covered by a clean folio, which can
4149  * result in inaccurate space reservation accounting.
4150  */
4151 static void ext4_iomap_punch_delalloc(struct inode *inode, loff_t offset,
4152 				     loff_t length, struct iomap *iomap)
4153 {
4154 	down_write(&EXT4_I(inode)->i_data_sem);
4155 	ext4_es_remove_extent(inode, offset >> inode->i_blkbits,
4156 			DIV_ROUND_UP_ULL(length, EXT4_BLOCK_SIZE(inode->i_sb)));
4157 	up_write(&EXT4_I(inode)->i_data_sem);
4158 }
4159 
4160 static int ext4_iomap_buffered_da_write_end(struct inode *inode, loff_t offset,
4161 					    loff_t length, ssize_t written,
4162 					    unsigned int flags,
4163 					    struct iomap *iomap)
4164 {
4165 	loff_t start_byte, end_byte;
4166 
4167 	/* If we didn't reserve the blocks, we're not allowed to punch them. */
4168 	if (iomap->type != IOMAP_DELALLOC || !(iomap->flags & IOMAP_F_NEW))
4169 		return 0;
4170 
4171 	/*
4172 	 * iomap_page_mkwrite() will never fail in a way that requires delalloc
4173 	 * extents that it allocated to be revoked.  Hence never try to release
4174 	 * them here.
4175 	 */
4176 	if (flags & IOMAP_FAULT)
4177 		return 0;
4178 
4179 	/* Nothing to do if we've written the entire delalloc extent */
4180 	start_byte = iomap_last_written_block(inode, offset, written);
4181 	end_byte = round_up(offset + length, i_blocksize(inode));
4182 	if (start_byte >= end_byte)
4183 		return 0;
4184 
4185 	filemap_invalidate_lock(inode->i_mapping);
4186 	iomap_write_delalloc_release(inode, start_byte, end_byte, flags,
4187 				     iomap, ext4_iomap_punch_delalloc);
4188 	filemap_invalidate_unlock(inode->i_mapping);
4189 	return 0;
4190 }
4191 
4192 static int ext4_iomap_zero_begin(struct inode *inode,
4193 		loff_t offset, loff_t length, unsigned int flags,
4194 		struct iomap *iomap, struct iomap *srcmap)
4195 {
4196 	struct iomap_iter *iter = container_of(iomap, struct iomap_iter, iomap);
4197 	struct ext4_map_blocks map;
4198 	u8 blkbits = inode->i_blkbits;
4199 	unsigned int iomap_flags;
4200 	int ret;
4201 
4202 	ret = ext4_emergency_state(inode->i_sb);
4203 	if (unlikely(ret))
4204 		return ret;
4205 
4206 	if (WARN_ON_ONCE(!(flags & IOMAP_ZERO)))
4207 		return -EINVAL;
4208 
4209 again:
4210 	ret = ext4_iomap_map_blocks(inode, offset, length, &map, 0);
4211 	if (ret < 0)
4212 		return ret;
4213 
4214 	/*
4215 	 * Look up dirty folios for unwritten mappings within EOF. Providing
4216 	 * this bypasses the flush iomap uses to trigger extent conversion
4217 	 * when unwritten mappings have dirty pagecache in need of zeroing.
4218 	 */
4219 	iomap_flags = 0;
4220 	if (map.m_flags & EXT4_MAP_UNWRITTEN) {
4221 		loff_t start = ((loff_t)map.m_lblk) << blkbits;
4222 		loff_t end = ((loff_t)map.m_lblk + map.m_len) << blkbits;
4223 		unsigned int count;
4224 
4225 		count = iomap_fill_dirty_folios(iter, &start, end,
4226 						&iomap_flags);
4227 		if ((start >> blkbits) < map.m_lblk + map.m_len)
4228 			map.m_len = (start >> blkbits) - map.m_lblk;
4229 
4230 		/*
4231 		 * This can be raced by a concurrent writeback that cleans
4232 		 * the folio and converts the unwritten extent to written.
4233 		 * Recheck the mapping after a folio lock round in
4234 		 * iomap_fill_dirty_folios().
4235 		 */
4236 		if (count == 0 &&
4237 		    map.m_seq != READ_ONCE(EXT4_I(inode)->i_es_seq))
4238 			goto again;
4239 	}
4240 
4241 	trace_ext4_iomap_zero_begin(inode, &map, offset, length, flags);
4242 	ext4_set_iomap(inode, iomap, &map, offset, length, flags);
4243 	iomap->flags |= iomap_flags;
4244 
4245 	return 0;
4246 }
4247 
4248 static DEFINE_IOMAP_ITER_NEXT(ext4_iomap_zero_next, ext4_iomap_zero_begin);
4249 
4250 /*
4251  * Since we always allocate unwritten extents, there is no need for
4252  * iomap_end to clean up allocated blocks on a short write.
4253  */
4254 DEFINE_IOMAP_ITER_NEXT(ext4_iomap_buffered_write_next,
4255 		       ext4_iomap_buffered_write_begin);
4256 
4257 DEFINE_IOMAP_ITER_NEXT_END(ext4_iomap_buffered_da_write_next,
4258 			   ext4_iomap_buffered_da_write_begin,
4259 			   ext4_iomap_buffered_da_write_end);
4260 
4261 static DEFINE_IOMAP_ITER_NEXT(ext4_iomap_buffered_read_next,
4262 			      ext4_iomap_buffered_read_begin);
4263 
4264 static int ext4_iomap_read_folio(struct file *file, struct folio *folio)
4265 {
4266 	iomap_bio_read_folio(folio, ext4_iomap_buffered_read_next);
4267 	return 0;
4268 }
4269 
4270 static void ext4_iomap_readahead(struct readahead_control *rac)
4271 {
4272 	iomap_bio_readahead(rac, ext4_iomap_buffered_read_next);
4273 }
4274 
4275 
4276 static int ext4_iomap_map_writeback_range(struct iomap_writepage_ctx *wpc,
4277 					  loff_t offset, unsigned int dirty_len)
4278 {
4279 	struct inode *inode = wpc->inode;
4280 	struct super_block *sb = inode->i_sb;
4281 	struct journal_s *journal = EXT4_SB(sb)->s_journal;
4282 	struct ext4_map_blocks map;
4283 	ext4_lblk_t index = offset >> inode->i_blkbits;
4284 	unsigned int blk_len, blk_end;
4285 	int retries = 0;
4286 	int ret;
4287 
4288 	ret = ext4_emergency_state(sb);
4289 	if (unlikely(ret))
4290 		return ret;
4291 
4292 	/* Check validity of the cached writeback mapping. */
4293 	if (offset >= wpc->iomap.offset &&
4294 	    offset < wpc->iomap.offset + wpc->iomap.length &&
4295 	    ext4_iomap_valid(inode, &wpc->iomap))
4296 		return 0;
4297 
4298 	blk_len = dirty_len >> inode->i_blkbits;
4299 	blk_end = umin(wpc->wbc->range_end >> inode->i_blkbits, UINT_MAX - 1);
4300 	if (blk_end > index + blk_len)
4301 		blk_len = blk_end - index + 1;
4302 
4303 retry:
4304 	map.m_lblk = index;
4305 	map.m_len = min_t(unsigned int, MAX_WRITEPAGES_EXTENT_LEN, blk_len);
4306 	ret = ext4_map_blocks(NULL, inode, &map,
4307 			      EXT4_GET_BLOCKS_CREATE_UNWRIT_EXT |
4308 			      EXT4_GET_BLOCKS_METADATA_NOFAIL |
4309 			      EXT4_GET_BLOCKS_IO_SUBMIT |
4310 			      EXT4_EX_NOCACHE);
4311 	if (ret < 0) {
4312 		if (ext4_emergency_state(sb))
4313 			return ret;
4314 
4315 		/*
4316 		 * Retry transient ENOSPC errors, if
4317 		 * ext4_count_free_blocks() is non-zero, a commit
4318 		 * should free up blocks.
4319 		 */
4320 		if (ret == -ENOSPC && journal && ext4_count_free_clusters(sb) &&
4321 		    ++retries <= 3) {
4322 			jbd2_journal_force_commit_nested(journal);
4323 			goto retry;
4324 		}
4325 
4326 		ext4_msg(sb, KERN_CRIT,
4327 			 "Delayed block allocation failed for inode %llu at logical offset %llu with max blocks %u with error %d",
4328 			 inode->i_ino, (unsigned long long)map.m_lblk,
4329 			 (unsigned int)map.m_len, -ret);
4330 		ext4_msg(sb, KERN_CRIT,
4331 			 "This should not happen!! Data will be lost\n");
4332 		if (ret == -ENOSPC)
4333 			ext4_print_free_blocks(inode);
4334 		return ret;
4335 	}
4336 
4337 	trace_ext4_iomap_map_writeback_range(inode, &map, offset, dirty_len, 0);
4338 	ext4_set_iomap(inode, &wpc->iomap, &map, offset, dirty_len, 0);
4339 	return 0;
4340 }
4341 
4342 static void ext4_iomap_discard_folio(struct folio *folio, loff_t pos)
4343 {
4344 	struct inode *inode = folio->mapping->host;
4345 	loff_t length = folio_pos(folio) + folio_size(folio) - pos;
4346 	loff_t pstart, plen;
4347 
4348 	ext4_iomap_punch_delalloc(inode, pos, length, NULL);
4349 
4350 	/*
4351 	 * Clear the disksize-grow-pending state if the zeroed EOF block
4352 	 * fails to write back and is discarded.
4353 	 */
4354 	plen = ext4_iomap_get_disksize_pending_range(inode, &pstart);
4355 	if (plen && pos <= pstart && folio_next_pos(folio) >= pstart + plen)
4356 		ext4_iomap_clear_disksize_pending(inode);
4357 }
4358 
4359 static ssize_t ext4_iomap_writeback_range(struct iomap_writepage_ctx *wpc,
4360 					  struct folio *folio, u64 offset,
4361 					  unsigned int len, u64 end_pos)
4362 {
4363 	ssize_t ret;
4364 
4365 	ret = ext4_iomap_map_writeback_range(wpc, offset, len);
4366 	if (!ret)
4367 		ret = iomap_add_to_ioend(wpc, folio, offset, end_pos, len);
4368 	if (ret < 0)
4369 		ext4_iomap_discard_folio(folio, offset);
4370 	return ret;
4371 }
4372 
4373 static int ext4_iomap_writeback_submit(struct iomap_writepage_ctx *wpc,
4374 				       int error)
4375 {
4376 	struct iomap_ioend *ioend = wpc->wb_ctx;
4377 	struct inode *inode = ioend->io_inode;
4378 	struct ext4_inode_info *ei = EXT4_I(inode);
4379 	unsigned int blocksize = i_blocksize(inode);
4380 	loff_t pstart, plen;
4381 
4382 	/*
4383 	 * After I/O completion, a worker needs to be scheduled when:
4384 	 * 1) Unwritten extents require conversion.
4385 	 * 2) The file size needs to be extended.
4386 	 * 3) The journal needs to be aborted due to an I/O error.
4387 	 */
4388 	if ((ioend->io_flags & IOMAP_IOEND_UNWRITTEN) ||
4389 	    (ioend->io_offset + ioend->io_size > READ_ONCE(ei->i_disksize)) ||
4390 	    test_opt(ioend->io_inode->i_sb, DATA_ERR_ABORT))
4391 		ioend->io_bio.bi_end_io = ext4_iomap_end_bio;
4392 
4393 	/*
4394 	 * Mark the I/O as DISKSIZE_GROW_IO by setting io_private to
4395 	 * EXT4_IOMAP_IOEND_DISKSIZE_GROW_IO if it covers the pending range.
4396 	 * Such I/O will allow or trigger i_disksize advancement in the
4397 	 * ioend worker.
4398 	 */
4399 	plen = ext4_iomap_get_disksize_pending_range(inode, &pstart);
4400 	if (plen &&
4401 	    round_down(ioend->io_offset, blocksize) <= pstart &&
4402 	    round_up(ioend->io_offset + ioend->io_size, blocksize) >=
4403 			pstart + plen) {
4404 		trace_ext4_iomap_wb_disksize_pending_submit(inode,
4405 				ioend->io_offset, ioend->io_size);
4406 		ioend->io_bio.bi_end_io = ext4_iomap_end_bio;
4407 		ioend->io_private = (void *)EXT4_IOMAP_IOEND_DISKSIZE_GROW_IO;
4408 	}
4409 
4410 	/*
4411 	 * ext4_iomap_end_bio() always defers endio processing, disable
4412 	 * generic BIO in task to avoid double deferral since we will use
4413 	 * a private defer endio handler in process context.
4414 	 *
4415 	 * TODO: Switch all defer handlers to the generic bio complete
4416 	 * in task framework.
4417 	 */
4418 	if (ioend->io_bio.bi_end_io)
4419 		bio_clear_flag(&ioend->io_bio, BIO_COMPLETE_IN_TASK);
4420 
4421 	return iomap_ioend_writeback_submit(wpc, error);
4422 }
4423 
4424 static const struct iomap_writeback_ops ext4_writeback_ops = {
4425 	.writeback_range = ext4_iomap_writeback_range,
4426 	.writeback_submit = ext4_iomap_writeback_submit,
4427 };
4428 
4429 /*
4430  * If the current writeback range begins after the pending zeroed EOF
4431  * block range which straddles i_disksize, issue a separate writeback to
4432  * flush it first, so as to avoid prolonged waiting.
4433  */
4434 static void ext4_iomap_wb_submit_zeroed_eof(struct inode *inode,
4435 					    struct writeback_control *wbc)
4436 {
4437 	struct address_space *mapping = inode->i_mapping;
4438 	loff_t pstart, plen, range_start;
4439 
4440 	if (wbc->range_cyclic)
4441 		range_start = (loff_t)mapping->writeback_index << PAGE_SHIFT;
4442 	else
4443 		range_start = wbc->range_start;
4444 
4445 	plen = ext4_iomap_get_disksize_pending_range(inode, &pstart);
4446 	if (!plen)
4447 		return;
4448 	if (round_down(range_start, mapping_min_folio_nrbytes(mapping)) <
4449 	    pstart + plen)
4450 		return;
4451 
4452 	/* Keep the caller's sync mode to avoid stalling the background flusher. */
4453 	if (wbc->sync_mode == WB_SYNC_ALL)
4454 		filemap_fdatawrite_range(mapping, pstart, pstart + plen - 1);
4455 	else
4456 		filemap_flush_range(mapping, pstart, pstart + plen - 1);
4457 }
4458 
4459 static int ext4_iomap_writepages(struct address_space *mapping,
4460 				 struct writeback_control *wbc)
4461 {
4462 	struct inode *inode = mapping->host;
4463 	struct super_block *sb = inode->i_sb;
4464 	long nr = wbc->nr_to_write;
4465 	int alloc_ctx, ret;
4466 	struct iomap_writepage_ctx wpc = {
4467 		.inode = inode,
4468 		.wbc = wbc,
4469 		.ops = &ext4_writeback_ops,
4470 	};
4471 
4472 	ret = ext4_emergency_state(sb);
4473 	if (unlikely(ret))
4474 		return ret;
4475 
4476 	/*
4477 	 * Submit the pending zeroed EOF block range if the entire
4478 	 * writeback range lies beyond it.
4479 	 */
4480 	ext4_iomap_wb_submit_zeroed_eof(inode, wbc);
4481 
4482 	alloc_ctx = ext4_writepages_down_read(sb);
4483 	/*
4484 	 * Does ext4_change_inode_journal_flag() change the inode's
4485 	 * buffered I/O path?
4486 	 */
4487 	if (!ext4_inode_buffered_iomap(inode)) {
4488 		ext4_writepages_up_read(sb, alloc_ctx);
4489 		return ret;
4490 	}
4491 
4492 	trace_ext4_writepages(inode, wbc);
4493 	ret = iomap_writepages(&wpc);
4494 	trace_ext4_writepages_result(inode, wbc, ret, nr - wbc->nr_to_write);
4495 	ext4_writepages_up_read(sb, alloc_ctx);
4496 
4497 	return ret;
4498 }
4499 
4500 /*
4501  * For data=journal mode, folio should be marked dirty only when it was
4502  * writeably mapped. When that happens, it was already attached to the
4503  * transaction and marked as jbddirty (we take care of this in
4504  * ext4_page_mkwrite()). On transaction commit, we writeprotect page mappings
4505  * so we should have nothing to do here, except for the case when someone
4506  * had the page pinned and dirtied the page through this pin (e.g. by doing
4507  * direct IO to it). In that case we'd need to attach buffers here to the
4508  * transaction but we cannot due to lock ordering.  We cannot just dirty the
4509  * folio and leave attached buffers clean, because the buffers' dirty state is
4510  * "definitive".  We cannot just set the buffers dirty or jbddirty because all
4511  * the journalling code will explode.  So what we do is to mark the folio
4512  * "pending dirty" and next time ext4_writepages() is called, attach buffers
4513  * to the transaction appropriately.
4514  */
4515 static bool ext4_journalled_dirty_folio(struct address_space *mapping,
4516 		struct folio *folio)
4517 {
4518 	WARN_ON_ONCE(!folio_buffers(folio));
4519 	if (folio_maybe_dma_pinned(folio))
4520 		folio_set_checked(folio);
4521 	return filemap_dirty_folio(mapping, folio);
4522 }
4523 
4524 static bool ext4_dirty_folio(struct address_space *mapping, struct folio *folio)
4525 {
4526 	WARN_ON_ONCE(!folio_test_locked(folio) && !folio_test_dirty(folio));
4527 	WARN_ON_ONCE(!folio_buffers(folio));
4528 	return block_dirty_folio(mapping, folio);
4529 }
4530 
4531 static int ext4_iomap_swap_activate(struct swap_info_struct *sis,
4532 				    struct file *file, sector_t *span)
4533 {
4534 	return iomap_swapfile_activate(sis, file, span,
4535 				       ext4_iomap_next_report);
4536 }
4537 
4538 static const struct address_space_operations ext4_aops = {
4539 	.read_folio		= ext4_read_folio,
4540 	.readahead		= ext4_readahead,
4541 	.writepages		= ext4_writepages,
4542 	.write_begin		= ext4_write_begin,
4543 	.write_end		= ext4_write_end,
4544 	.dirty_folio		= ext4_dirty_folio,
4545 	.bmap			= ext4_bmap,
4546 	.invalidate_folio	= ext4_invalidate_folio,
4547 	.release_folio		= ext4_release_folio,
4548 	.migrate_folio		= buffer_migrate_folio,
4549 	.is_partially_uptodate  = block_is_partially_uptodate,
4550 	.error_remove_folio	= generic_error_remove_folio,
4551 	.swap_activate		= ext4_iomap_swap_activate,
4552 };
4553 
4554 static const struct address_space_operations ext4_journalled_aops = {
4555 	.read_folio		= ext4_read_folio,
4556 	.readahead		= ext4_readahead,
4557 	.writepages		= ext4_writepages,
4558 	.write_begin		= ext4_write_begin,
4559 	.write_end		= ext4_journalled_write_end,
4560 	.dirty_folio		= ext4_journalled_dirty_folio,
4561 	.bmap			= ext4_bmap,
4562 	.invalidate_folio	= ext4_journalled_invalidate_folio,
4563 	.release_folio		= ext4_release_folio,
4564 	.migrate_folio		= buffer_migrate_folio_norefs,
4565 	.is_partially_uptodate  = block_is_partially_uptodate,
4566 	.error_remove_folio	= generic_error_remove_folio,
4567 	.swap_activate		= ext4_iomap_swap_activate,
4568 };
4569 
4570 static const struct address_space_operations ext4_da_aops = {
4571 	.read_folio		= ext4_read_folio,
4572 	.readahead		= ext4_readahead,
4573 	.writepages		= ext4_writepages,
4574 	.write_begin		= ext4_da_write_begin,
4575 	.write_end		= ext4_da_write_end,
4576 	.dirty_folio		= ext4_dirty_folio,
4577 	.bmap			= ext4_bmap,
4578 	.invalidate_folio	= ext4_invalidate_folio,
4579 	.release_folio		= ext4_release_folio,
4580 	.migrate_folio		= buffer_migrate_folio,
4581 	.is_partially_uptodate  = block_is_partially_uptodate,
4582 	.error_remove_folio	= generic_error_remove_folio,
4583 	.swap_activate		= ext4_iomap_swap_activate,
4584 };
4585 
4586 static const struct address_space_operations ext4_iomap_aops = {
4587 	.read_folio		= ext4_iomap_read_folio,
4588 	.readahead		= ext4_iomap_readahead,
4589 	.writepages		= ext4_iomap_writepages,
4590 	.dirty_folio		= iomap_dirty_folio,
4591 	.bmap			= ext4_bmap,
4592 	.invalidate_folio	= iomap_invalidate_folio,
4593 	.release_folio		= iomap_release_folio,
4594 	.migrate_folio		= filemap_migrate_folio,
4595 	.is_partially_uptodate  = iomap_is_partially_uptodate,
4596 	.error_remove_folio	= generic_error_remove_folio,
4597 	.swap_activate		= ext4_iomap_swap_activate,
4598 };
4599 
4600 static const struct address_space_operations ext4_dax_aops = {
4601 	.writepages		= ext4_dax_writepages,
4602 	.dirty_folio		= noop_dirty_folio,
4603 	.bmap			= ext4_bmap,
4604 	.swap_activate		= ext4_iomap_swap_activate,
4605 };
4606 
4607 void ext4_set_aops(struct inode *inode)
4608 {
4609 	switch (ext4_inode_journal_mode(inode)) {
4610 	case EXT4_INODE_ORDERED_DATA_MODE:
4611 	case EXT4_INODE_WRITEBACK_DATA_MODE:
4612 		break;
4613 	case EXT4_INODE_JOURNAL_DATA_MODE:
4614 		inode->i_mapping->a_ops = &ext4_journalled_aops;
4615 		return;
4616 	default:
4617 		BUG();
4618 	}
4619 	if (IS_DAX(inode))
4620 		inode->i_mapping->a_ops = &ext4_dax_aops;
4621 	else if (ext4_inode_buffered_iomap(inode))
4622 		inode->i_mapping->a_ops = &ext4_iomap_aops;
4623 	else if (test_opt(inode->i_sb, DELALLOC))
4624 		inode->i_mapping->a_ops = &ext4_da_aops;
4625 	else
4626 		inode->i_mapping->a_ops = &ext4_aops;
4627 }
4628 
4629 /*
4630  * Here we can't skip an unwritten buffer even though it usually reads zero
4631  * because it might have data in pagecache (eg, if called from ext4_zero_range,
4632  * ext4_punch_hole, etc) which needs to be properly zeroed out. Otherwise a
4633  * racing writeback can come later and flush the stale pagecache to disk.
4634  *
4635  * Return the loaded bh if it actually needs zeroing - in written, dirty
4636  * unwritten, or delalloc state. Return NULL if it's clean (i.e., a hole or
4637  * a clean unwritten block).
4638  */
4639 static struct buffer_head *ext4_load_tail_bh(struct inode *inode, loff_t from)
4640 {
4641 	unsigned int offset, blocksize, pos;
4642 	ext4_lblk_t iblock;
4643 	struct address_space *mapping = inode->i_mapping;
4644 	struct buffer_head *bh;
4645 	struct folio *folio;
4646 	int err = 0;
4647 
4648 	folio = __filemap_get_folio(mapping, from >> PAGE_SHIFT,
4649 				    FGP_WRITEBEGIN | FGP_ACCESSED,
4650 				    mapping_gfp_constraint(mapping, ~__GFP_FS));
4651 	if (IS_ERR(folio))
4652 		return ERR_CAST(folio);
4653 
4654 	blocksize = inode->i_sb->s_blocksize;
4655 
4656 	iblock = EXT4_PG_TO_LBLK(inode, folio->index);
4657 
4658 	bh = folio_buffers(folio);
4659 	if (!bh)
4660 		bh = create_empty_buffers(folio, blocksize, 0);
4661 
4662 	/* Find the buffer that contains "offset" */
4663 	offset = offset_in_folio(folio, from);
4664 	pos = blocksize;
4665 	while (offset >= pos) {
4666 		bh = bh->b_this_page;
4667 		iblock++;
4668 		pos += blocksize;
4669 	}
4670 	if (buffer_freed(bh)) {
4671 		BUFFER_TRACE(bh, "freed: skip");
4672 		goto unlock;
4673 	}
4674 	if (!buffer_mapped(bh)) {
4675 		BUFFER_TRACE(bh, "unmapped");
4676 		err = ext4_get_block(inode, iblock, bh, 0);
4677 		if (err < 0)
4678 			goto unlock;
4679 		/*
4680 		 * It's a hole or a clean unwritten block - nothing to do.
4681 		 * Note that a lookup-only get_block (without
4682 		 * EXT4_GET_BLOCKS_CREATE) never sets BH_Mapped for clean
4683 		 * unwritten extents.
4684 		 */
4685 		if (!buffer_mapped(bh)) {
4686 			BUFFER_TRACE(bh, "still unmapped");
4687 			goto unlock;
4688 		}
4689 	}
4690 
4691 	/* Ok, it's mapped. Make sure it's up-to-date */
4692 	if (folio_test_uptodate(folio))
4693 		set_buffer_uptodate(bh);
4694 
4695 	if (!buffer_uptodate(bh)) {
4696 		err = ext4_read_bh_lock(bh, 0, true);
4697 		if (err)
4698 			goto unlock;
4699 	}
4700 	return bh;
4701 
4702 unlock:
4703 	folio_unlock(folio);
4704 	folio_put(folio);
4705 	return err ? ERR_PTR(err) : NULL;
4706 }
4707 
4708 static int ext4_block_do_zero_range(struct inode *inode, loff_t from,
4709 				    loff_t length, bool *did_zero,
4710 				    bool *zero_written)
4711 {
4712 	struct buffer_head *bh;
4713 	struct folio *folio;
4714 
4715 	bh = ext4_load_tail_bh(inode, from);
4716 	if (IS_ERR_OR_NULL(bh))
4717 		return PTR_ERR_OR_ZERO(bh);
4718 
4719 	folio = bh->b_folio;
4720 	folio_zero_range(folio, offset_in_folio(folio, from), length);
4721 	BUFFER_TRACE(bh, "zeroed end of block");
4722 
4723 	mark_buffer_dirty(bh);
4724 	if (did_zero)
4725 		*did_zero = true;
4726 	if (zero_written && !buffer_unwritten(bh) && !buffer_delay(bh))
4727 		*zero_written = true;
4728 
4729 	folio_unlock(folio);
4730 	folio_put(folio);
4731 	return 0;
4732 }
4733 
4734 static int ext4_block_journalled_zero_range(struct inode *inode, loff_t from,
4735 					    loff_t length, bool *did_zero)
4736 {
4737 	struct buffer_head *bh;
4738 	struct folio *folio;
4739 	handle_t *handle;
4740 	int err;
4741 
4742 	handle = ext4_journal_start(inode, EXT4_HT_MISC, 1);
4743 	if (IS_ERR(handle))
4744 		return PTR_ERR(handle);
4745 
4746 	bh = ext4_load_tail_bh(inode, from);
4747 	if (IS_ERR_OR_NULL(bh)) {
4748 		err = PTR_ERR_OR_ZERO(bh);
4749 		goto out_handle;
4750 	}
4751 	folio = bh->b_folio;
4752 
4753 	BUFFER_TRACE(bh, "get write access");
4754 	err = ext4_journal_get_write_access(handle, inode->i_sb, bh,
4755 					    EXT4_JTR_NONE);
4756 	if (err)
4757 		goto out;
4758 
4759 	folio_zero_range(folio, offset_in_folio(folio, from), length);
4760 	BUFFER_TRACE(bh, "zeroed end of block");
4761 
4762 	err = ext4_dirty_journalled_data(handle, bh);
4763 	if (err)
4764 		goto out;
4765 
4766 	if (did_zero)
4767 		*did_zero = true;
4768 out:
4769 	folio_unlock(folio);
4770 	folio_put(folio);
4771 out_handle:
4772 	ext4_journal_stop(handle);
4773 	return err;
4774 }
4775 
4776 static int ext4_block_iomap_zero_range(struct inode *inode, loff_t from,
4777 				       loff_t length, bool *did_zero,
4778 				       bool *zero_written)
4779 {
4780 	int ret;
4781 
4782 	/*
4783 	 * Zeroing out under an active handle can cause deadlock since
4784 	 * the order of acquiring the folio lock and starting a handle is
4785 	 * inconsistent with the iomap writeback procedure.
4786 	 */
4787 	if (WARN_ON_ONCE(ext4_handle_valid(journal_current_handle())))
4788 		return -EINVAL;
4789 
4790 	/* The zeroing scope should not extend across a block. */
4791 	if (WARN_ON_ONCE((from >> inode->i_blkbits) !=
4792 			 ((from + length - 1) >> inode->i_blkbits)))
4793 		return -EINVAL;
4794 
4795 	if (!(EXT4_SB(inode->i_sb)->s_mount_state & EXT4_ORPHAN_FS) &&
4796 	    !(inode_state_read_once(inode) & (I_NEW | I_FREEING)))
4797 		WARN_ON_ONCE(!inode_is_locked(inode) &&
4798 			!rwsem_is_locked(&inode->i_mapping->invalidate_lock));
4799 
4800 	ret = iomap_zero_range(inode, from, length, did_zero,
4801 			       ext4_iomap_zero_next, &ext4_iomap_write_ops,
4802 			       NULL);
4803 	if (ret)
4804 		return ret;
4805 
4806 	/*
4807 	 * TODO: The iomap does not distinguish between different types
4808 	 * of zeroing operations. So we always set zero_written whenever
4809 	 * zeroing is performed, which may cause unnecessary folio
4810 	 * flushing when zeroing occurs on delayed-allocated blocks.
4811 	 */
4812 	if (did_zero && zero_written)
4813 		*zero_written = *did_zero;
4814 
4815 	return 0;
4816 }
4817 
4818 /*
4819  * Zeros out a mapping of length 'length' starting from file offset
4820  * 'from'.  The range to be zero'd must be contained with in one block.
4821  * If the specified range exceeds the end of the block it will be
4822  * shortened to end of the block that corresponds to 'from'.
4823  */
4824 static int ext4_block_zero_range(struct inode *inode,
4825 				 loff_t from, loff_t length, bool *did_zero,
4826 				 bool *zero_written)
4827 {
4828 	unsigned blocksize = inode->i_sb->s_blocksize;
4829 	unsigned int max = blocksize - (from & (blocksize - 1));
4830 
4831 	/*
4832 	 * correct length if it does not fall between
4833 	 * 'from' and the end of the block
4834 	 */
4835 	if (length > max || length < 0)
4836 		length = max;
4837 
4838 	if (IS_DAX(inode)) {
4839 		return dax_zero_range(inode, from, length, did_zero,
4840 				      ext4_iomap_next);
4841 	} else if (ext4_should_journal_data(inode)) {
4842 		return ext4_block_journalled_zero_range(inode, from, length,
4843 							did_zero);
4844 	} else if (ext4_inode_buffered_iomap(inode)) {
4845 		return ext4_block_iomap_zero_range(inode, from, length,
4846 						   did_zero, zero_written);
4847 	}
4848 	return ext4_block_do_zero_range(inode, from, length, did_zero,
4849 					zero_written);
4850 }
4851 
4852 /*
4853  * Inodes using the iomap buffered I/O path do not use data=ordered mode.
4854  * Therefore, we mark the inode as disksize-grow-pending after zeroing the
4855  * EOF block. The zeroed block will be submitted before any subsequent
4856  * data.
4857  *
4858  * In the I/O completion path, ext4_iomap_wb_disksize_pending_wait() will
4859  * wait for I/O completion before advancing i_disksize if the write
4860  * extends beyond the zeroed boundary.
4861  *
4862  * When zeroed I/O is in progress, operations that extend i_disksize are
4863  * handled as follows:
4864  *
4865  *  - Truncate up, append fallocate and zero_range:
4866  *    Defer the update. The file size will be updated to i_size by the
4867  *    end_io handler once the ongoing pending I/O completes.
4868  *
4869  *  - Insert range and collapse range operations:
4870  *    Wait synchronously for the relevant I/O to complete before updating
4871  *    i_disksize.
4872  */
4873 static int ext4_iomap_mark_disksize_pending(struct inode *inode, loff_t from)
4874 {
4875 	struct folio *folio;
4876 
4877 	folio = filemap_lock_folio(inode->i_mapping, from >> PAGE_SHIFT);
4878 	if (IS_ERR(folio))
4879 		/* Already in writeback and cleared? */
4880 		return PTR_ERR(folio) == -ENOENT ? 0 : PTR_ERR(folio);
4881 
4882 	/*
4883 	 * Ensure that in-flight writeback, possibly started after
4884 	 * iomap_zero_range() unlocked the folio, has completed. Without
4885 	 * this wait folio_test_dirty() below may miss the zeroed data
4886 	 * (writeback clears PG_dirty), causing us to skip the
4887 	 * disksize-grow-pending tracking and potentially expose stale
4888 	 * on-disk data.
4889 	 */
4890 	folio_wait_writeback(folio);
4891 	WARN_ON_ONCE(folio_test_writeback(folio));
4892 
4893 	/*
4894 	 * If the zeroed range does not overlap the existing on-disk tail
4895 	 * block, the zeroed data lies beyond the currently on-disk data
4896 	 * and will be written back before i_disksize is advanced past it,
4897 	 * so no stale data can be exposed.
4898 	 *
4899 	 * Checking i_disksize here (after folio_wait_writeback()) is
4900 	 * necessary and safe. If a delalloc writeback of this folio was
4901 	 * in-flight, it could be raced by a concurrent mmap write which
4902 	 * corrupts the tail block beyond i_size but the i_disksize is not
4903 	 * advanced. folio_wait_writeback() has waited for its completion
4904 	 * and the ioend has advanced i_disksize accordingly. If writeback
4905 	 * had not started, we don't need to mark any pending state because
4906 	 * any future writeback will carry the pagecache content that now
4907 	 * includes the zeroed data, so no stale data can appear on disk
4908 	 * even without the pending tracking.
4909 	 */
4910 	if (from >= round_up(READ_ONCE(EXT4_I(inode)->i_disksize),
4911 			     i_blocksize(inode)))
4912 		goto out;
4913 
4914 	/*
4915 	 * Mark the inode as disksize-grow-pending. The zeroed block will
4916 	 * be written out by the generic writepages cycle or any other
4917 	 * syncing operation.
4918 	 *
4919 	 * Multiple overlapping unaligned EOF writes should not happen,
4920 	 * because we only mark the pending state after zeroing the on-disk
4921 	 * EOF block, and i_disksize can only be updated after the previous
4922 	 * zeroed pending block has been written back or the dirty folio
4923 	 * has been discarded.
4924 	 */
4925 	if (likely(folio_test_dirty(folio) &&
4926 		   !ext4_test_inode_state(inode,
4927 					  EXT4_STATE_DISKSIZE_GROW_PENDING))) {
4928 		trace_ext4_iomap_mark_disksize_pending(inode);
4929 		ext4_set_inode_state(inode, EXT4_STATE_DISKSIZE_GROW_PENDING);
4930 	}
4931 out:
4932 	folio_unlock(folio);
4933 	folio_put(folio);
4934 	return 0;
4935 }
4936 
4937 /*
4938  * Submit and wait for the pending zeroed EOF block range to complete
4939  * if the given range [@offset, @end) fully covers it.  Must be called
4940  * outside the context of an active journal handle and hold the i_rwsem.
4941  */
4942 int ext4_iomap_sync_zeroed_eof(struct inode *inode, loff_t offset, loff_t end)
4943 {
4944 	loff_t pstart, plen;
4945 	int ret;
4946 
4947 	if (!ext4_inode_buffered_iomap(inode))
4948 		return 0;
4949 
4950 	plen = ext4_iomap_get_disksize_pending_range(inode, &pstart);
4951 	if (!plen || offset > pstart || end < pstart + plen)
4952 		return 0;
4953 
4954 	ret = filemap_write_and_wait_range(inode->i_mapping, pstart,
4955 					   pstart + plen - 1);
4956 	if (ret)
4957 		return ret;
4958 
4959 	/*
4960 	 * The pending bit should have been cleared by the ioend worker,
4961 	 * which runs before the folio writeback flag is cleared.  The
4962 	 * caller holds i_rwsem, so no concurrent ext4_block_zero_eof()
4963 	 * can re-set it.
4964 	 */
4965 	WARN_ON_ONCE(ext4_test_inode_state(inode,
4966 				EXT4_STATE_DISKSIZE_GROW_PENDING));
4967 	return 0;
4968 }
4969 
4970 /*
4971  * Zero out a mapping from file offset 'from' up to the end of the block
4972  * which corresponds to 'from' or to the given 'end' inside this block.
4973  * This required during truncate up and performing append writes. We need
4974  * to physically zero the tail end of that block so it doesn't yield old
4975  * data if the file is grown.
4976  */
4977 int ext4_block_zero_eof(struct inode *inode, loff_t from, loff_t end)
4978 {
4979 	unsigned int blocksize = i_blocksize(inode);
4980 	unsigned int offset;
4981 	loff_t length = end - from;
4982 	bool did_zero = false;
4983 	bool zero_written = false;
4984 	int err;
4985 
4986 	offset = from & (blocksize - 1);
4987 	if (!offset || from >= end)
4988 		return 0;
4989 	/*
4990 	 * Inline data has no tail block to zero out.  Note that a race with
4991 	 * ext4_page_mkwrite() converting inline data to an extent without
4992 	 * holding i_rwsem is safe, as that path zeroes the full block before
4993 	 * copying in the inline data.
4994 	 */
4995 	if (ext4_has_inline_data(inode))
4996 		return 0;
4997 	/* If we are processing an encrypted inode during orphan list handling */
4998 	if (IS_ENCRYPTED(inode) && !fscrypt_has_encryption_key(inode))
4999 		return 0;
5000 
5001 	if (length > blocksize - offset)
5002 		length = blocksize - offset;
5003 
5004 	err = ext4_block_zero_range(inode, from, length,
5005 				    &did_zero, &zero_written);
5006 	if (err)
5007 		return err;
5008 	/*
5009 	 * It's necessary to order zeroed data before update i_disksize when
5010 	 * truncating up or performing an append write, because there might be
5011 	 * exposing stale on-disk data which may caused by concurrent post-EOF
5012 	 * mmap write during folio writeback.
5013 	 */
5014 	if (did_zero && zero_written && !IS_DAX(inode)) {
5015 		if (ext4_should_order_data(inode)) {
5016 			handle_t *handle;
5017 
5018 			handle = ext4_journal_start(inode, EXT4_HT_MISC, 1);
5019 			if (IS_ERR(handle))
5020 				return PTR_ERR(handle);
5021 
5022 			err = ext4_jbd2_inode_add_write(handle, inode, from,
5023 							length);
5024 			ext4_journal_stop(handle);
5025 		} else if (ext4_inode_buffered_iomap(inode))
5026 			err = ext4_iomap_mark_disksize_pending(inode, from);
5027 		if (err)
5028 			return err;
5029 	}
5030 
5031 	trace_ext4_block_zero_eof(inode, from, length, did_zero, zero_written);
5032 	return 0;
5033 }
5034 
5035 /*
5036  * Zero out the unaligned head and tail of the [lstart, lstart+length)
5037  * range.
5038  *
5039  * On return, @partial_zeroed records which edges actually got
5040  * partial-zeroed.  Set EXT4_PARTIAL_ZERO_START/EXT4_PARTIAL_ZERO_END if
5041  * the head/tail block got actually partially zeroed (in written, dirty
5042  * unwritten or delalloc state).  Cleared if the head/tail block is a
5043  * hole or a clean unwritten block, in which case there is nothing that
5044  * needs zeroing.  When the head and tail land in the same block, both
5045  * bits are set together on a successful zeroing.
5046  */
5047 int ext4_zero_partial_blocks(struct inode *inode, loff_t lstart, loff_t length,
5048 			     unsigned int *partial_zeroed)
5049 {
5050 	struct super_block *sb = inode->i_sb;
5051 	unsigned partial_start, partial_end;
5052 	ext4_fsblk_t start, end;
5053 	loff_t byte_end = (lstart + length - 1);
5054 	bool did_zero = false;
5055 	int err = 0;
5056 
5057 	partial_start = lstart & (sb->s_blocksize - 1);
5058 	partial_end = byte_end & (sb->s_blocksize - 1);
5059 
5060 	start = lstart >> sb->s_blocksize_bits;
5061 	end = byte_end >> sb->s_blocksize_bits;
5062 
5063 	/* Handle partial zero within the single block */
5064 	if (start == end &&
5065 	    (partial_start || (partial_end != sb->s_blocksize - 1))) {
5066 		err = ext4_block_zero_range(inode, lstart, length, &did_zero,
5067 					    NULL);
5068 		if (did_zero)
5069 			*partial_zeroed |= (EXT4_PARTIAL_ZERO_START |
5070 					    EXT4_PARTIAL_ZERO_END);
5071 		return err;
5072 	}
5073 	/* Handle partial zero out on the start of the range */
5074 	if (partial_start) {
5075 		err = ext4_block_zero_range(inode, lstart, sb->s_blocksize,
5076 					    &did_zero, NULL);
5077 		if (err)
5078 			return err;
5079 		if (did_zero)
5080 			*partial_zeroed |= EXT4_PARTIAL_ZERO_START;
5081 	}
5082 	/* Handle partial zero out on the end of the range */
5083 	if (partial_end != sb->s_blocksize - 1) {
5084 		did_zero = false;
5085 		err = ext4_block_zero_range(inode, byte_end - partial_end,
5086 					    partial_end + 1, &did_zero, NULL);
5087 		if (err)
5088 			return err;
5089 		if (did_zero)
5090 			*partial_zeroed |= EXT4_PARTIAL_ZERO_END;
5091 	}
5092 	return err;
5093 }
5094 
5095 int ext4_can_truncate(struct inode *inode)
5096 {
5097 	if (S_ISREG(inode->i_mode))
5098 		return 1;
5099 	if (S_ISDIR(inode->i_mode))
5100 		return 1;
5101 	if (S_ISLNK(inode->i_mode))
5102 		return !ext4_inode_is_fast_symlink(inode);
5103 	return 0;
5104 }
5105 
5106 /*
5107  * We have to make sure i_disksize gets properly updated before we truncate
5108  * page cache due to hole punching or zero range. Otherwise i_disksize update
5109  * can get lost as it may have been postponed to submission of writeback but
5110  * that will never happen if we remove the folio containing i_size from the
5111  * page cache. Also if we punch hole within i_size but above i_disksize,
5112  * following ext4_page_mkwrite() may mistakenly allocate written blocks over
5113  * the hole and thus introduce allocated blocks beyond i_disksize which is
5114  * not allowed (e2fsck would complain in case of crash).
5115  */
5116 int ext4_update_disksize_before_punch(struct inode *inode, loff_t offset,
5117 				      loff_t len)
5118 {
5119 	handle_t *handle;
5120 	int ret;
5121 
5122 	loff_t size = i_size_read(inode);
5123 
5124 	WARN_ON(!inode_is_locked(inode));
5125 	if (offset > size)
5126 		return 0;
5127 
5128 	/*
5129 	 * We are going to punch the pending zeroed EOF block, persist
5130 	 * it to ensure i_disksize can be safely updated thereafter.
5131 	 */
5132 	ret = ext4_iomap_sync_zeroed_eof(inode, offset, offset + len);
5133 	if (ret)
5134 		return ret;
5135 
5136 	if (offset + len < size)
5137 		size = offset + len;
5138 	if (EXT4_I(inode)->i_disksize >= size)
5139 		return 0;
5140 
5141 	handle = ext4_journal_start(inode, EXT4_HT_MISC, 1);
5142 	if (IS_ERR(handle))
5143 		return PTR_ERR(handle);
5144 	ext4_update_i_disksize(inode, size);
5145 	ret = ext4_mark_inode_dirty(handle, inode);
5146 	ext4_journal_stop(handle);
5147 
5148 	return ret;
5149 }
5150 
5151 static inline void ext4_truncate_folio(struct inode *inode,
5152 				       loff_t start, loff_t end)
5153 {
5154 	unsigned long blocksize = i_blocksize(inode);
5155 	struct folio *folio;
5156 
5157 	/* Nothing to be done if no complete block needs to be truncated. */
5158 	if (round_up(start, blocksize) >= round_down(end, blocksize))
5159 		return;
5160 
5161 	folio = filemap_lock_folio(inode->i_mapping, start >> PAGE_SHIFT);
5162 	if (IS_ERR(folio))
5163 		return;
5164 
5165 	if (folio_mkclean(folio))
5166 		folio_mark_dirty(folio);
5167 	folio_unlock(folio);
5168 	folio_put(folio);
5169 }
5170 
5171 int ext4_truncate_page_cache_block_range(struct inode *inode,
5172 					 loff_t start, loff_t end)
5173 {
5174 	unsigned long blocksize = i_blocksize(inode);
5175 	int ret;
5176 
5177 	/*
5178 	 * For journalled data we need to write (and checkpoint) pages
5179 	 * before discarding page cache to avoid inconsitent data on disk
5180 	 * in case of crash before freeing or unwritten converting trans
5181 	 * is committed.
5182 	 */
5183 	if (ext4_should_journal_data(inode)) {
5184 		ret = filemap_write_and_wait_range(inode->i_mapping, start,
5185 						   end - 1);
5186 		if (ret)
5187 			return ret;
5188 		goto truncate_pagecache;
5189 	}
5190 
5191 	/*
5192 	 * If the block size is less than the page size, the file's mapped
5193 	 * blocks within one page could be freed or converted to unwritten.
5194 	 * So it's necessary to remove writable userspace mappings, and then
5195 	 * ext4_page_mkwrite() can be called during subsequent write access
5196 	 * to these partial folios.
5197 	 */
5198 	if (!IS_ALIGNED(start | end, PAGE_SIZE) &&
5199 	    blocksize < PAGE_SIZE && start < inode->i_size) {
5200 		loff_t page_boundary = round_up(start, PAGE_SIZE);
5201 
5202 		ext4_truncate_folio(inode, start, min(page_boundary, end));
5203 		if (end > page_boundary)
5204 			ext4_truncate_folio(inode,
5205 					    round_down(end, PAGE_SIZE), end);
5206 	}
5207 
5208 truncate_pagecache:
5209 	truncate_pagecache_range(inode, start, end - 1);
5210 	return 0;
5211 }
5212 
5213 static void ext4_wait_dax_page(struct inode *inode)
5214 {
5215 	filemap_invalidate_unlock(inode->i_mapping);
5216 	schedule();
5217 	filemap_invalidate_lock(inode->i_mapping);
5218 }
5219 
5220 int ext4_break_layouts(struct inode *inode)
5221 {
5222 	if (WARN_ON_ONCE(!rwsem_is_locked(&inode->i_mapping->invalidate_lock)))
5223 		return -EINVAL;
5224 
5225 	return dax_break_layout_inode(inode, ext4_wait_dax_page);
5226 }
5227 
5228 /*
5229  * ext4_punch_hole: punches a hole in a file by releasing the blocks
5230  * associated with the given offset and length
5231  *
5232  * @inode:  File inode
5233  * @offset: The offset where the hole will begin
5234  * @len:    The length of the hole
5235  *
5236  * Returns: 0 on success or negative on failure
5237  */
5238 
5239 int ext4_punch_hole(struct file *file, loff_t offset, loff_t length)
5240 {
5241 	struct inode *inode = file_inode(file);
5242 	struct super_block *sb = inode->i_sb;
5243 	ext4_lblk_t start_lblk, end_lblk;
5244 	loff_t max_end = sb->s_maxbytes;
5245 	loff_t end = offset + length;
5246 	handle_t *handle;
5247 	unsigned int credits;
5248 	unsigned int partial_zeroed = 0;
5249 	int ret;
5250 
5251 	trace_ext4_punch_hole(inode, offset, length, 0);
5252 	WARN_ON_ONCE(!inode_is_locked(inode));
5253 
5254 	/*
5255 	 * For indirect-block based inodes, make sure that the hole within
5256 	 * one block before last range.
5257 	 */
5258 	if (!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
5259 		max_end = EXT4_SB(sb)->s_bitmap_maxbytes - sb->s_blocksize;
5260 
5261 	/* No need to punch hole beyond i_size */
5262 	if (offset >= inode->i_size || offset >= max_end)
5263 		return 0;
5264 
5265 	/*
5266 	 * If the hole extends beyond i_size, set the hole to end after
5267 	 * the block that contains i_size to save pointless tail block zeroing.
5268 	 */
5269 	if (end >= inode->i_size)
5270 		end = round_up(inode->i_size, sb->s_blocksize);
5271 	if (end > max_end)
5272 		end = max_end;
5273 	length = end - offset;
5274 
5275 	ret = ext4_update_disksize_before_punch(inode, offset, length);
5276 	if (ret)
5277 		return ret;
5278 
5279 	/* Now release the pages and zero block aligned part of pages*/
5280 	ret = ext4_truncate_page_cache_block_range(inode, offset, end);
5281 	if (ret)
5282 		return ret;
5283 
5284 	ret = ext4_zero_partial_blocks(inode, offset, length, &partial_zeroed);
5285 	if (ret)
5286 		return ret;
5287 	/*
5288 	 * On the iomap path, partial invalidate of a folio without an ifs
5289 	 * leaves sub-block dirty bits uncleared.  Drain writeback before
5290 	 * removing extents so that any bio in flight on a partial folio
5291 	 * completes against blocks still owned by this inode.
5292 	 */
5293 	if (ext4_inode_buffered_iomap(inode) ||
5294 	    (((file->f_flags & O_SYNC) || IS_SYNC(inode)) && partial_zeroed)) {
5295 		ret = filemap_write_and_wait_range(inode->i_mapping, offset,
5296 						   end - 1);
5297 		if (ret)
5298 			return ret;
5299 	}
5300 
5301 	if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
5302 		credits = ext4_chunk_trans_extent(inode, 0);
5303 	else
5304 		credits = ext4_blocks_for_truncate(inode);
5305 	handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE, credits);
5306 	if (IS_ERR(handle)) {
5307 		ret = PTR_ERR(handle);
5308 		ext4_std_error(sb, ret);
5309 		return ret;
5310 	}
5311 
5312 	/* If there are blocks to remove, do it */
5313 	start_lblk = EXT4_B_TO_LBLK(inode, offset);
5314 	end_lblk = end >> inode->i_blkbits;
5315 
5316 	if (end_lblk > start_lblk) {
5317 		ext4_lblk_t hole_len = end_lblk - start_lblk;
5318 
5319 		ext4_fc_track_inode(handle, inode);
5320 		ext4_check_map_extents_env(inode);
5321 		down_write(&EXT4_I(inode)->i_data_sem);
5322 		ext4_discard_preallocations(inode);
5323 
5324 		ext4_es_remove_extent(inode, start_lblk, hole_len);
5325 
5326 		if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
5327 			ret = ext4_ext_remove_space(inode, start_lblk,
5328 						    end_lblk - 1);
5329 		else
5330 			ret = ext4_ind_remove_space(handle, inode, start_lblk,
5331 						    end_lblk);
5332 		if (ret) {
5333 			up_write(&EXT4_I(inode)->i_data_sem);
5334 			goto out_handle;
5335 		}
5336 
5337 		ext4_es_insert_extent(inode, start_lblk, hole_len, ~0,
5338 				      EXTENT_STATUS_HOLE, 0);
5339 		up_write(&EXT4_I(inode)->i_data_sem);
5340 	}
5341 	ext4_fc_track_range(handle, inode, start_lblk, end_lblk);
5342 
5343 	ret = ext4_mark_inode_dirty(handle, inode);
5344 	if (unlikely(ret))
5345 		goto out_handle;
5346 
5347 	ext4_update_inode_fsync_trans(handle, inode, 1);
5348 	if ((file->f_flags & O_SYNC) || IS_SYNC(inode))
5349 		ext4_handle_sync(handle);
5350 out_handle:
5351 	ext4_journal_stop(handle);
5352 	return ret;
5353 }
5354 
5355 int ext4_inode_attach_jinode(struct inode *inode)
5356 {
5357 	struct ext4_inode_info *ei = EXT4_I(inode);
5358 	struct jbd2_inode *jinode;
5359 
5360 	if (ei->jinode || !EXT4_SB(inode->i_sb)->s_journal)
5361 		return 0;
5362 
5363 	jinode = jbd2_alloc_inode(GFP_KERNEL);
5364 	spin_lock(&inode->i_lock);
5365 	if (!ei->jinode) {
5366 		if (!jinode) {
5367 			spin_unlock(&inode->i_lock);
5368 			return -ENOMEM;
5369 		}
5370 		jbd2_journal_init_jbd_inode(jinode, inode);
5371 		/*
5372 		 * Publish ->jinode only after it is fully initialized so that
5373 		 * readers never observe a partially initialized jbd2_inode.
5374 		 */
5375 		smp_wmb();
5376 		WRITE_ONCE(ei->jinode, jinode);
5377 		jinode = NULL;
5378 	}
5379 	spin_unlock(&inode->i_lock);
5380 	if (unlikely(jinode != NULL))
5381 		jbd2_free_inode(jinode);
5382 	return 0;
5383 }
5384 
5385 /*
5386  * ext4_truncate()
5387  *
5388  * We block out ext4_get_block() block instantiations across the entire
5389  * transaction, and VFS/VM ensures that ext4_truncate() cannot run
5390  * simultaneously on behalf of the same inode.
5391  *
5392  * As we work through the truncate and commit bits of it to the journal there
5393  * is one core, guiding principle: the file's tree must always be consistent on
5394  * disk.  We must be able to restart the truncate after a crash.
5395  *
5396  * The file's tree may be transiently inconsistent in memory (although it
5397  * probably isn't), but whenever we close off and commit a journal transaction,
5398  * the contents of (the filesystem + the journal) must be consistent and
5399  * restartable.  It's pretty simple, really: bottom up, right to left (although
5400  * left-to-right works OK too).
5401  *
5402  * Note that at recovery time, journal replay occurs *before* the restart of
5403  * truncate against the orphan inode list.
5404  *
5405  * The committed inode has the new, desired i_size (which is the same as
5406  * i_disksize in this case).  After a crash, ext4_orphan_cleanup() will see
5407  * that this inode's truncate did not complete and it will again call
5408  * ext4_truncate() to have another go.  So there will be instantiated blocks
5409  * to the right of the truncation point in a crashed ext4 filesystem.  But
5410  * that's fine - as long as they are linked from the inode, the post-crash
5411  * ext4_truncate() run will find them and release them.
5412  */
5413 int ext4_truncate(struct inode *inode)
5414 {
5415 	struct ext4_inode_info *ei = EXT4_I(inode);
5416 	unsigned int credits;
5417 	int err = 0, err2;
5418 	handle_t *handle;
5419 
5420 	/*
5421 	 * There is a possibility that we're either freeing the inode
5422 	 * or it's a completely new inode. In those cases we might not
5423 	 * have i_rwsem locked because it's not necessary.
5424 	 */
5425 	if (!(inode_state_read_once(inode) & (I_NEW | I_FREEING)))
5426 		WARN_ON(!inode_is_locked(inode));
5427 	trace_ext4_truncate_enter(inode);
5428 
5429 	if (!ext4_can_truncate(inode))
5430 		goto out_trace;
5431 
5432 	if (inode->i_size == 0 && !test_opt(inode->i_sb, NO_AUTO_DA_ALLOC))
5433 		ext4_set_inode_state(inode, EXT4_STATE_DA_ALLOC_CLOSE);
5434 
5435 	if (ext4_has_inline_data(inode)) {
5436 		int has_inline = 1;
5437 
5438 		err = ext4_inline_data_truncate(inode, &has_inline);
5439 		if (err || has_inline)
5440 			goto out_trace;
5441 	}
5442 
5443 	/* If we zero-out tail of the page, we have to create jinode for jbd2 */
5444 	if (inode->i_size & (inode->i_sb->s_blocksize - 1)) {
5445 		err = ext4_inode_attach_jinode(inode);
5446 		if (err)
5447 			goto out_trace;
5448 
5449 		/* Zero to the end of the block containing i_size */
5450 		err = ext4_block_zero_eof(inode, inode->i_size, LLONG_MAX);
5451 		if (err)
5452 			goto out_trace;
5453 	}
5454 
5455 	if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
5456 		credits = ext4_chunk_trans_extent(inode, 1);
5457 	else
5458 		credits = ext4_blocks_for_truncate(inode);
5459 
5460 	handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE, credits);
5461 	if (IS_ERR(handle)) {
5462 		err = PTR_ERR(handle);
5463 		goto out_trace;
5464 	}
5465 
5466 	/*
5467 	 * We add the inode to the orphan list, so that if this
5468 	 * truncate spans multiple transactions, and we crash, we will
5469 	 * resume the truncate when the filesystem recovers.  It also
5470 	 * marks the inode dirty, to catch the new size.
5471 	 *
5472 	 * Implication: the file must always be in a sane, consistent
5473 	 * truncatable state while each transaction commits.
5474 	 */
5475 	err = ext4_orphan_add(handle, inode);
5476 	if (err)
5477 		goto out_stop;
5478 
5479 	ext4_fc_track_inode(handle, inode);
5480 	ext4_check_map_extents_env(inode);
5481 
5482 	down_write(&EXT4_I(inode)->i_data_sem);
5483 	ext4_discard_preallocations(inode);
5484 
5485 	if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
5486 		err = ext4_ext_truncate(handle, inode);
5487 	else
5488 		ext4_ind_truncate(handle, inode);
5489 
5490 	up_write(&ei->i_data_sem);
5491 	if (err)
5492 		goto out_stop;
5493 
5494 	if (IS_SYNC(inode))
5495 		ext4_handle_sync(handle);
5496 
5497 out_stop:
5498 	/*
5499 	 * If this was a simple ftruncate() and the file will remain alive,
5500 	 * then we need to clear up the orphan record which we created above.
5501 	 * However, if this was a real unlink then we were called by
5502 	 * ext4_evict_inode(), and we allow that function to clean up the
5503 	 * orphan info for us.
5504 	 */
5505 	if (inode->i_nlink)
5506 		ext4_orphan_del(handle, inode);
5507 
5508 	inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode));
5509 	err2 = ext4_mark_inode_dirty(handle, inode);
5510 	if (unlikely(err2 && !err))
5511 		err = err2;
5512 	ext4_journal_stop(handle);
5513 
5514 out_trace:
5515 	trace_ext4_truncate_exit(inode);
5516 	return err;
5517 }
5518 
5519 static inline u64 ext4_inode_peek_iversion(const struct inode *inode)
5520 {
5521 	if (unlikely(EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL))
5522 		return inode_peek_iversion_raw(inode);
5523 	else
5524 		return inode_peek_iversion(inode);
5525 }
5526 
5527 static int ext4_inode_blocks_set(struct ext4_inode *raw_inode,
5528 				 struct ext4_inode_info *ei)
5529 {
5530 	struct inode *inode = &(ei->vfs_inode);
5531 	u64 i_blocks = READ_ONCE(inode->i_blocks);
5532 	struct super_block *sb = inode->i_sb;
5533 
5534 	if (i_blocks <= ~0U) {
5535 		/*
5536 		 * i_blocks can be represented in a 32 bit variable
5537 		 * as multiple of 512 bytes
5538 		 */
5539 		raw_inode->i_blocks_lo   = cpu_to_le32(i_blocks);
5540 		raw_inode->i_blocks_high = 0;
5541 		ext4_clear_inode_flag(inode, EXT4_INODE_HUGE_FILE);
5542 		return 0;
5543 	}
5544 
5545 	/*
5546 	 * This should never happen since sb->s_maxbytes should not have
5547 	 * allowed this, sb->s_maxbytes was set according to the huge_file
5548 	 * feature in ext4_fill_super().
5549 	 */
5550 	if (!ext4_has_feature_huge_file(sb))
5551 		return -EFSCORRUPTED;
5552 
5553 	if (i_blocks <= 0xffffffffffffULL) {
5554 		/*
5555 		 * i_blocks can be represented in a 48 bit variable
5556 		 * as multiple of 512 bytes
5557 		 */
5558 		raw_inode->i_blocks_lo   = cpu_to_le32(i_blocks);
5559 		raw_inode->i_blocks_high = cpu_to_le16(i_blocks >> 32);
5560 		ext4_clear_inode_flag(inode, EXT4_INODE_HUGE_FILE);
5561 	} else {
5562 		ext4_set_inode_flag(inode, EXT4_INODE_HUGE_FILE);
5563 		/* i_block is stored in file system block size */
5564 		i_blocks = i_blocks >> (inode->i_blkbits - 9);
5565 		raw_inode->i_blocks_lo   = cpu_to_le32(i_blocks);
5566 		raw_inode->i_blocks_high = cpu_to_le16(i_blocks >> 32);
5567 	}
5568 	return 0;
5569 }
5570 
5571 static int ext4_fill_raw_inode(struct inode *inode, struct ext4_inode *raw_inode)
5572 {
5573 	struct ext4_inode_info *ei = EXT4_I(inode);
5574 	uid_t i_uid;
5575 	gid_t i_gid;
5576 	projid_t i_projid;
5577 	int block;
5578 	int err;
5579 
5580 	err = ext4_inode_blocks_set(raw_inode, ei);
5581 
5582 	raw_inode->i_mode = cpu_to_le16(inode->i_mode);
5583 	i_uid = i_uid_read(inode);
5584 	i_gid = i_gid_read(inode);
5585 	i_projid = from_kprojid(&init_user_ns, ei->i_projid);
5586 	if (!(test_opt(inode->i_sb, NO_UID32))) {
5587 		raw_inode->i_uid_low = cpu_to_le16(low_16_bits(i_uid));
5588 		raw_inode->i_gid_low = cpu_to_le16(low_16_bits(i_gid));
5589 		/*
5590 		 * Fix up interoperability with old kernels. Otherwise,
5591 		 * old inodes get re-used with the upper 16 bits of the
5592 		 * uid/gid intact.
5593 		 */
5594 		if (ei->i_dtime && !ext4_inode_orphan_tracked(inode)) {
5595 			raw_inode->i_uid_high = 0;
5596 			raw_inode->i_gid_high = 0;
5597 		} else {
5598 			raw_inode->i_uid_high =
5599 				cpu_to_le16(high_16_bits(i_uid));
5600 			raw_inode->i_gid_high =
5601 				cpu_to_le16(high_16_bits(i_gid));
5602 		}
5603 	} else {
5604 		raw_inode->i_uid_low = cpu_to_le16(fs_high2lowuid(i_uid));
5605 		raw_inode->i_gid_low = cpu_to_le16(fs_high2lowgid(i_gid));
5606 		raw_inode->i_uid_high = 0;
5607 		raw_inode->i_gid_high = 0;
5608 	}
5609 	raw_inode->i_links_count = cpu_to_le16(inode->i_nlink);
5610 
5611 	EXT4_INODE_SET_CTIME(inode, raw_inode);
5612 	EXT4_INODE_SET_MTIME(inode, raw_inode);
5613 	EXT4_INODE_SET_ATIME(inode, raw_inode);
5614 	EXT4_EINODE_SET_XTIME(i_crtime, ei, raw_inode);
5615 
5616 	raw_inode->i_dtime = cpu_to_le32(ei->i_dtime);
5617 	raw_inode->i_flags = cpu_to_le32(ei->i_flags & 0xFFFFFFFF);
5618 	if (likely(!test_opt2(inode->i_sb, HURD_COMPAT)))
5619 		raw_inode->i_file_acl_high =
5620 			cpu_to_le16(ei->i_file_acl >> 32);
5621 	raw_inode->i_file_acl_lo = cpu_to_le32(ei->i_file_acl);
5622 	ext4_isize_set(raw_inode, ei->i_disksize);
5623 
5624 	raw_inode->i_generation = cpu_to_le32(inode->i_generation);
5625 	if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode)) {
5626 		if (old_valid_dev(inode->i_rdev)) {
5627 			raw_inode->i_block[0] =
5628 				cpu_to_le32(old_encode_dev(inode->i_rdev));
5629 			raw_inode->i_block[1] = 0;
5630 		} else {
5631 			raw_inode->i_block[0] = 0;
5632 			raw_inode->i_block[1] =
5633 				cpu_to_le32(new_encode_dev(inode->i_rdev));
5634 			raw_inode->i_block[2] = 0;
5635 		}
5636 	} else if (!ext4_has_inline_data(inode)) {
5637 		for (block = 0; block < EXT4_N_BLOCKS; block++)
5638 			raw_inode->i_block[block] = ei->i_data[block];
5639 	}
5640 
5641 	if (likely(!test_opt2(inode->i_sb, HURD_COMPAT))) {
5642 		u64 ivers = ext4_inode_peek_iversion(inode);
5643 
5644 		raw_inode->i_disk_version = cpu_to_le32(ivers);
5645 		if (ei->i_extra_isize) {
5646 			if (EXT4_FITS_IN_INODE(raw_inode, ei, i_version_hi))
5647 				raw_inode->i_version_hi =
5648 					cpu_to_le32(ivers >> 32);
5649 			raw_inode->i_extra_isize =
5650 				cpu_to_le16(ei->i_extra_isize);
5651 		}
5652 	}
5653 
5654 	if (i_projid != EXT4_DEF_PROJID &&
5655 	    !ext4_has_feature_project(inode->i_sb))
5656 		err = err ?: -EFSCORRUPTED;
5657 
5658 	if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE &&
5659 	    EXT4_FITS_IN_INODE(raw_inode, ei, i_projid))
5660 		raw_inode->i_projid = cpu_to_le32(i_projid);
5661 
5662 	ext4_inode_csum_set(inode, raw_inode, ei);
5663 	return err;
5664 }
5665 
5666 /*
5667  * ext4_get_inode_loc returns with an extra refcount against the inode's
5668  * underlying buffer_head on success. If we pass 'inode' and it does not
5669  * have in-inode xattr, we have all inode data in memory that is needed
5670  * to recreate the on-disk version of this inode.
5671  */
5672 static int __ext4_get_inode_loc(struct super_block *sb, unsigned long ino,
5673 				struct inode *inode, struct ext4_iloc *iloc,
5674 				ext4_fsblk_t *ret_block)
5675 {
5676 	struct ext4_group_desc	*gdp;
5677 	struct buffer_head	*bh;
5678 	ext4_fsblk_t		block;
5679 	struct blk_plug		plug;
5680 	int			inodes_per_block, inode_offset;
5681 
5682 	iloc->bh = NULL;
5683 	if (ino < EXT4_ROOT_INO ||
5684 	    ino > le32_to_cpu(EXT4_SB(sb)->s_es->s_inodes_count))
5685 		return -EFSCORRUPTED;
5686 
5687 	iloc->block_group = (ino - 1) / EXT4_INODES_PER_GROUP(sb);
5688 	gdp = ext4_get_group_desc(sb, iloc->block_group, NULL);
5689 	if (!gdp)
5690 		return -EIO;
5691 
5692 	/*
5693 	 * Figure out the offset within the block group inode table
5694 	 */
5695 	inodes_per_block = EXT4_SB(sb)->s_inodes_per_block;
5696 	inode_offset = ((ino - 1) %
5697 			EXT4_INODES_PER_GROUP(sb));
5698 	iloc->offset = (inode_offset % inodes_per_block) * EXT4_INODE_SIZE(sb);
5699 
5700 	block = ext4_inode_table(sb, gdp);
5701 	if ((block <= le32_to_cpu(EXT4_SB(sb)->s_es->s_first_data_block)) ||
5702 	    (block >= ext4_blocks_count(EXT4_SB(sb)->s_es))) {
5703 		ext4_error(sb, "Invalid inode table block %llu in "
5704 			   "block_group %u", block, iloc->block_group);
5705 		return -EFSCORRUPTED;
5706 	}
5707 	block += (inode_offset / inodes_per_block);
5708 
5709 	bh = sb_getblk(sb, block);
5710 	if (unlikely(!bh))
5711 		return -ENOMEM;
5712 	if (ext4_buffer_uptodate(bh))
5713 		goto has_buffer;
5714 
5715 	lock_buffer(bh);
5716 	if (ext4_buffer_uptodate(bh)) {
5717 		/* Someone brought it uptodate while we waited */
5718 		unlock_buffer(bh);
5719 		goto has_buffer;
5720 	}
5721 
5722 	/*
5723 	 * If we have all information of the inode in memory and this
5724 	 * is the only valid inode in the block, we need not read the
5725 	 * block.
5726 	 */
5727 	if (inode && !ext4_test_inode_state(inode, EXT4_STATE_XATTR)) {
5728 		struct buffer_head *bitmap_bh;
5729 		int i, start;
5730 
5731 		start = inode_offset & ~(inodes_per_block - 1);
5732 
5733 		/* Is the inode bitmap in cache? */
5734 		bitmap_bh = sb_getblk(sb, ext4_inode_bitmap(sb, gdp));
5735 		if (unlikely(!bitmap_bh))
5736 			goto make_io;
5737 
5738 		/*
5739 		 * If the inode bitmap isn't in cache then the
5740 		 * optimisation may end up performing two reads instead
5741 		 * of one, so skip it.
5742 		 */
5743 		if (!buffer_uptodate(bitmap_bh)) {
5744 			brelse(bitmap_bh);
5745 			goto make_io;
5746 		}
5747 		for (i = start; i < start + inodes_per_block; i++) {
5748 			if (i == inode_offset)
5749 				continue;
5750 			if (ext4_test_bit(i, bitmap_bh->b_data))
5751 				break;
5752 		}
5753 		brelse(bitmap_bh);
5754 		if (i == start + inodes_per_block) {
5755 			struct ext4_inode *raw_inode =
5756 				(struct ext4_inode *) (bh->b_data + iloc->offset);
5757 
5758 			/* all other inodes are free, so skip I/O */
5759 			memset(bh->b_data, 0, bh->b_size);
5760 			if (!ext4_test_inode_state(inode, EXT4_STATE_NEW))
5761 				ext4_fill_raw_inode(inode, raw_inode);
5762 			set_buffer_uptodate(bh);
5763 			unlock_buffer(bh);
5764 			goto has_buffer;
5765 		}
5766 	}
5767 
5768 make_io:
5769 	/*
5770 	 * If we need to do any I/O, try to pre-readahead extra
5771 	 * blocks from the inode table.
5772 	 */
5773 	blk_start_plug(&plug);
5774 	if (EXT4_SB(sb)->s_inode_readahead_blks) {
5775 		ext4_fsblk_t b, end, table;
5776 		unsigned num;
5777 		__u32 ra_blks = EXT4_SB(sb)->s_inode_readahead_blks;
5778 
5779 		table = ext4_inode_table(sb, gdp);
5780 		/* s_inode_readahead_blks is always a power of 2 */
5781 		b = block & ~((ext4_fsblk_t) ra_blks - 1);
5782 		if (table > b)
5783 			b = table;
5784 		end = b + ra_blks;
5785 		num = EXT4_INODES_PER_GROUP(sb);
5786 		if (ext4_has_group_desc_csum(sb))
5787 			num -= ext4_itable_unused_count(sb, gdp);
5788 		table += num / inodes_per_block;
5789 		if (end > table)
5790 			end = table;
5791 		while (b <= end)
5792 			ext4_sb_breadahead_unmovable(sb, b++);
5793 	}
5794 
5795 	/*
5796 	 * There are other valid inodes in the buffer, this inode
5797 	 * has in-inode xattrs, or we don't have this inode in memory.
5798 	 * Read the block from disk.
5799 	 */
5800 	trace_ext4_load_inode(sb, ino);
5801 	ext4_read_bh_nowait(bh, REQ_META | REQ_PRIO, NULL,
5802 			    ext4_simulate_fail(sb, EXT4_SIM_INODE_EIO));
5803 	blk_finish_plug(&plug);
5804 	wait_on_buffer(bh);
5805 	if (!buffer_uptodate(bh)) {
5806 		if (ret_block)
5807 			*ret_block = block;
5808 		brelse(bh);
5809 		return -EIO;
5810 	}
5811 has_buffer:
5812 	iloc->bh = bh;
5813 	return 0;
5814 }
5815 
5816 static int __ext4_get_inode_loc_noinmem(struct inode *inode,
5817 					struct ext4_iloc *iloc)
5818 {
5819 	ext4_fsblk_t err_blk = 0;
5820 	int ret;
5821 
5822 	ret = __ext4_get_inode_loc(inode->i_sb, inode->i_ino, NULL, iloc,
5823 					&err_blk);
5824 
5825 	if (ret == -EIO)
5826 		ext4_error_inode_block(inode, err_blk, EIO,
5827 					"unable to read itable block");
5828 
5829 	return ret;
5830 }
5831 
5832 int ext4_get_inode_loc(struct inode *inode, struct ext4_iloc *iloc)
5833 {
5834 	ext4_fsblk_t err_blk = 0;
5835 	int ret;
5836 
5837 	ret = __ext4_get_inode_loc(inode->i_sb, inode->i_ino, inode, iloc,
5838 					&err_blk);
5839 
5840 	if (ret == -EIO)
5841 		ext4_error_inode_block(inode, err_blk, EIO,
5842 					"unable to read itable block");
5843 
5844 	return ret;
5845 }
5846 
5847 /*
5848  * ext4_get_inode_loc_noio() is a best-effort variant of ext4_get_inode_loc().
5849  * It looks up the inode table block in the buffer cache and returns -EAGAIN if
5850  * the block is not present or not uptodate, without starting any I/O. It may
5851  * sleep on the folio lock of the block.
5852  */
5853 int ext4_get_inode_loc_noio(struct inode *inode, struct ext4_iloc *iloc)
5854 {
5855 	struct super_block *sb = inode->i_sb;
5856 	struct ext4_group_desc *gdp;
5857 	struct buffer_head *bh;
5858 	ext4_fsblk_t block;
5859 	int inodes_per_block, inode_offset;
5860 	unsigned long ino = inode->i_ino;
5861 
5862 	iloc->bh = NULL;
5863 	if (ino < EXT4_ROOT_INO ||
5864 	    ino > le32_to_cpu(EXT4_SB(sb)->s_es->s_inodes_count))
5865 		return -EFSCORRUPTED;
5866 
5867 	iloc->block_group = (ino - 1) / EXT4_INODES_PER_GROUP(sb);
5868 	gdp = ext4_get_group_desc(sb, iloc->block_group, NULL);
5869 	if (!gdp)
5870 		return -EIO;
5871 
5872 	/* Figure out the offset within the block group inode table. */
5873 	inodes_per_block = EXT4_SB(sb)->s_inodes_per_block;
5874 	inode_offset = ((ino - 1) % EXT4_INODES_PER_GROUP(sb));
5875 	iloc->offset = (inode_offset % inodes_per_block) * EXT4_INODE_SIZE(sb);
5876 
5877 	block = ext4_inode_table(sb, gdp);
5878 	if (block <= le32_to_cpu(EXT4_SB(sb)->s_es->s_first_data_block) ||
5879 	    block >= ext4_blocks_count(EXT4_SB(sb)->s_es)) {
5880 		ext4_error(sb,
5881 			   "Invalid inode table block %llu in block_group %u",
5882 			   block, iloc->block_group);
5883 		return -EFSCORRUPTED;
5884 	}
5885 	block += inode_offset / inodes_per_block;
5886 
5887 	bh = sb_find_get_block_nonatomic(sb, block);
5888 	if (!bh)
5889 		return -EAGAIN;
5890 	if (!ext4_buffer_uptodate(bh)) {
5891 		brelse(bh);
5892 		return -EAGAIN;
5893 	}
5894 
5895 	iloc->bh = bh;
5896 	return 0;
5897 }
5898 
5899 
5900 int ext4_get_fc_inode_loc(struct super_block *sb, unsigned long ino,
5901 			  struct ext4_iloc *iloc)
5902 {
5903 	return __ext4_get_inode_loc(sb, ino, NULL, iloc, NULL);
5904 }
5905 
5906 static bool ext4_should_enable_dax(struct inode *inode)
5907 {
5908 	struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
5909 
5910 	if (test_opt2(inode->i_sb, DAX_NEVER))
5911 		return false;
5912 	if (!S_ISREG(inode->i_mode))
5913 		return false;
5914 	if (ext4_should_journal_data(inode))
5915 		return false;
5916 	if (ext4_has_inline_data(inode))
5917 		return false;
5918 	if (ext4_test_inode_flag(inode, EXT4_INODE_ENCRYPT))
5919 		return false;
5920 	if (ext4_test_inode_flag(inode, EXT4_INODE_VERITY))
5921 		return false;
5922 	if (!test_bit(EXT4_FLAGS_BDEV_IS_DAX, &sbi->s_ext4_flags))
5923 		return false;
5924 	if (test_opt(inode->i_sb, DAX_ALWAYS))
5925 		return true;
5926 
5927 	return ext4_test_inode_flag(inode, EXT4_INODE_DAX);
5928 }
5929 
5930 void ext4_set_inode_flags(struct inode *inode, bool init)
5931 {
5932 	unsigned int flags = EXT4_I(inode)->i_flags;
5933 	unsigned int new_fl = 0;
5934 
5935 	WARN_ON_ONCE(IS_DAX(inode) && init);
5936 
5937 	if (flags & EXT4_SYNC_FL)
5938 		new_fl |= S_SYNC;
5939 	if (flags & EXT4_APPEND_FL)
5940 		new_fl |= S_APPEND;
5941 	if (flags & EXT4_IMMUTABLE_FL)
5942 		new_fl |= S_IMMUTABLE;
5943 	if (flags & EXT4_NOATIME_FL)
5944 		new_fl |= S_NOATIME;
5945 	if (flags & EXT4_DIRSYNC_FL)
5946 		new_fl |= S_DIRSYNC;
5947 
5948 	/* Because of the way inode_set_flags() works we must preserve S_DAX
5949 	 * here if already set. */
5950 	new_fl |= (inode->i_flags & S_DAX);
5951 	if (init && ext4_should_enable_dax(inode))
5952 		new_fl |= S_DAX;
5953 
5954 	if (flags & EXT4_ENCRYPT_FL)
5955 		new_fl |= S_ENCRYPTED;
5956 	if (flags & EXT4_CASEFOLD_FL)
5957 		new_fl |= S_CASEFOLD;
5958 	if (flags & EXT4_VERITY_FL)
5959 		new_fl |= S_VERITY;
5960 	inode_set_flags(inode, new_fl,
5961 			S_SYNC|S_APPEND|S_IMMUTABLE|S_NOATIME|S_DIRSYNC|S_DAX|
5962 			S_ENCRYPTED|S_CASEFOLD|S_VERITY);
5963 }
5964 
5965 static blkcnt_t ext4_inode_blocks(struct ext4_inode *raw_inode,
5966 				  struct ext4_inode_info *ei)
5967 {
5968 	blkcnt_t i_blocks ;
5969 	struct inode *inode = &(ei->vfs_inode);
5970 	struct super_block *sb = inode->i_sb;
5971 
5972 	if (ext4_has_feature_huge_file(sb)) {
5973 		/* we are using combined 48 bit field */
5974 		i_blocks = ((u64)le16_to_cpu(raw_inode->i_blocks_high)) << 32 |
5975 					le32_to_cpu(raw_inode->i_blocks_lo);
5976 		if (ext4_test_inode_flag(inode, EXT4_INODE_HUGE_FILE)) {
5977 			/* i_blocks represent file system block size */
5978 			return i_blocks  << (inode->i_blkbits - 9);
5979 		} else {
5980 			return i_blocks;
5981 		}
5982 	} else {
5983 		return le32_to_cpu(raw_inode->i_blocks_lo);
5984 	}
5985 }
5986 
5987 static inline int ext4_iget_extra_inode(struct inode *inode,
5988 					 struct ext4_inode *raw_inode,
5989 					 struct ext4_inode_info *ei)
5990 {
5991 	__le32 *magic = (void *)raw_inode +
5992 			EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize;
5993 
5994 	if (EXT4_INODE_HAS_XATTR_SPACE(inode)  &&
5995 	    *magic == cpu_to_le32(EXT4_XATTR_MAGIC)) {
5996 		int err;
5997 
5998 		err = xattr_check_inode(inode, IHDR(inode, raw_inode),
5999 					ITAIL(inode, raw_inode));
6000 		if (err)
6001 			return err;
6002 
6003 		ext4_set_inode_state(inode, EXT4_STATE_XATTR);
6004 		err = ext4_find_inline_data_nolock(inode);
6005 		if (!err && ext4_has_inline_data(inode))
6006 			ext4_set_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA);
6007 		return err;
6008 	} else
6009 		EXT4_I(inode)->i_inline_off = 0;
6010 	return 0;
6011 }
6012 
6013 int ext4_get_projid(struct inode *inode, kprojid_t *projid)
6014 {
6015 	if (!ext4_has_feature_project(inode->i_sb))
6016 		return -EOPNOTSUPP;
6017 	*projid = EXT4_I(inode)->i_projid;
6018 	return 0;
6019 }
6020 
6021 /*
6022  * ext4 has self-managed i_version for ea inodes, it stores the lower 32bit of
6023  * refcount in i_version, so use raw values if inode has EXT4_EA_INODE_FL flag
6024  * set.
6025  */
6026 static inline void ext4_inode_set_iversion_queried(struct inode *inode, u64 val)
6027 {
6028 	if (unlikely(EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL))
6029 		inode_set_iversion_raw(inode, val);
6030 	else
6031 		inode_set_iversion_queried(inode, val);
6032 }
6033 
6034 static int check_igot_inode(struct inode *inode, ext4_iget_flags flags,
6035 			    const char *function, unsigned int line)
6036 {
6037 	const char *err_str;
6038 
6039 	if (flags & EXT4_IGET_EA_INODE) {
6040 		if (!(EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL)) {
6041 			err_str = "missing EA_INODE flag";
6042 			goto error;
6043 		}
6044 		if (ext4_test_inode_state(inode, EXT4_STATE_XATTR) ||
6045 		    EXT4_I(inode)->i_file_acl) {
6046 			err_str = "ea_inode with extended attributes";
6047 			goto error;
6048 		}
6049 	} else {
6050 		if ((EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL)) {
6051 			/*
6052 			 * open_by_handle_at() could provide an old inode number
6053 			 * that has since been reused for an ea_inode; this does
6054 			 * not indicate filesystem corruption
6055 			 */
6056 			if (flags & EXT4_IGET_HANDLE)
6057 				return -ESTALE;
6058 			err_str = "unexpected EA_INODE flag";
6059 			goto error;
6060 		}
6061 	}
6062 	if (is_bad_inode(inode) && !(flags & EXT4_IGET_BAD)) {
6063 		err_str = "unexpected bad inode w/o EXT4_IGET_BAD";
6064 		goto error;
6065 	}
6066 	return 0;
6067 
6068 error:
6069 	ext4_error_inode(inode, function, line, 0, "%s", err_str);
6070 	return -EFSCORRUPTED;
6071 }
6072 
6073 /*
6074  * Determine whether an inode should use the iomap buffered I/O path.
6075  * EXT4_STATE_BUFFERED_IOMAP is generally set at inode initialization
6076  * time. Online switching of the buffered I/O path on an active inode is
6077  * NOT supported, with the exception of changing a per-inode journal
6078  * flag.
6079  *
6080  * For features like inline data, fsverity, and encryption that can be
6081  * dynamically enabled or disabled, we check the superblock-level
6082  * feature flags. If any of these is globally enabled, no inode is
6083  * allowed into the iomap buffered I/O path. This avoids the complexity
6084  * of dynamic toggling.
6085  *
6086  * For the global data journal mode (EXT4_MOUNT_JOURNAL_DATA), dynamic
6087  * change through remount is deferred. It will only become available
6088  * after the inode is re-initialized (i.e., after the last reference
6089  * drops and the inode is re-read from disk with the journal flag
6090  * cleared).
6091  *
6092  * For the per-inode data journal mode (EXT4_INODE_JOURNAL_DATA),
6093  * dynamic changes take effect immediately. This is safe because
6094  * address_space operations can be switched and all page cache can be
6095  * dropped under i_rwsem and filemap_invalidate_lock.
6096  *
6097  * For extent-to-indirect block migration (via EXT4_IOC_SETFLAGS
6098  * clearing EXT4_EXTENTS_FL), this operation is directly rejected for
6099  * inodes using the iomap path.
6100  *
6101  * When remounting to toggle the buffered_iomap mount option, the change
6102  * of I/O path is deferred as well, it will be available after the inode
6103  * is re-initialized.
6104  */
6105 void ext4_enable_buffered_iomap(struct inode *inode)
6106 {
6107 	struct super_block *sb = inode->i_sb;
6108 
6109 	if (!test_opt2(sb, BUFFERED_IOMAP))
6110 		return;
6111 	if (!S_ISREG(inode->i_mode))
6112 		return;
6113 	if (ext4_test_inode_flag(inode, EXT4_INODE_EA_INODE))
6114 		return;
6115 
6116 	/* Unsupported Features */
6117 	if (ext4_has_feature_inline_data(sb))
6118 		return;
6119 	if (ext4_has_feature_verity(sb))
6120 		return;
6121 	if (ext4_has_feature_encrypt(sb))
6122 		return;
6123 	if (test_opt(sb, DATA_FLAGS) == EXT4_MOUNT_JOURNAL_DATA ||
6124 	    ext4_test_inode_flag(inode, EXT4_INODE_JOURNAL_DATA))
6125 		return;
6126 	if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)))
6127 		return;
6128 
6129 	ext4_set_inode_state(inode, EXT4_STATE_BUFFERED_IOMAP);
6130 
6131 	/*
6132 	 * Install the iomap end_io handler on the shared conversion
6133 	 * work.  This is safe at inode initialization and during the
6134 	 * buffered I/O path changes where we flush all pending
6135 	 * writebacks and drop page cache under i_rwsem and
6136 	 * filemap_invalidate_lock.
6137 	 */
6138 	INIT_WORK(&EXT4_I(inode)->i_rsv_conversion_work, ext4_iomap_end_io);
6139 }
6140 
6141 static void ext4_disable_buffered_iomap(struct inode *inode)
6142 {
6143 	ext4_clear_inode_state(inode, EXT4_STATE_BUFFERED_IOMAP);
6144 
6145 	/*
6146 	 * Reinstall the buffer_head end_io handler on the shared
6147 	 * conversion work.  This is safe during the buffered I/O path
6148 	 * changes where we flush all pending writebacks and drop page
6149 	 * cache under i_rwsem and filemap_invalidate_lock.
6150 	 */
6151 	INIT_WORK(&EXT4_I(inode)->i_rsv_conversion_work, ext4_end_io_rsv_work);
6152 }
6153 
6154 void ext4_set_inode_mapping_order(struct inode *inode)
6155 {
6156 	struct super_block *sb = inode->i_sb;
6157 	u16 min_order, max_order;
6158 
6159 	max_order = EXT4_SB(sb)->s_max_folio_order;
6160 	if (!max_order)
6161 		return;
6162 
6163 	min_order = EXT4_SB(sb)->s_min_folio_order;
6164 	if (!min_order && !S_ISREG(inode->i_mode))
6165 		return;
6166 
6167 	if (ext4_test_inode_flag(inode, EXT4_INODE_JOURNAL_DATA))
6168 		max_order = min_order;
6169 
6170 	mapping_set_folio_order_range(inode->i_mapping, min_order, max_order);
6171 }
6172 
6173 static int ext4_iget_match(struct inode *inode, u64 ino, void *data)
6174 {
6175 	if (inode->i_ino != ino)
6176 		return 0;
6177 	spin_lock(&inode->i_lock);
6178 	if (inode_state_read(inode) & (I_FREEING | I_WILL_FREE | I_CREATING)) {
6179 		spin_unlock(&inode->i_lock);
6180 		return -1;
6181 	}
6182 	__iget(inode);
6183 	spin_unlock(&inode->i_lock);
6184 	return 1;
6185 }
6186 
6187 struct inode *__ext4_iget(struct super_block *sb, unsigned long ino,
6188 			  ext4_iget_flags flags, const char *function,
6189 			  unsigned int line)
6190 {
6191 	struct ext4_iloc iloc;
6192 	struct ext4_inode *raw_inode;
6193 	struct ext4_inode_info *ei;
6194 	struct ext4_super_block *es = EXT4_SB(sb)->s_es;
6195 	struct inode *inode;
6196 	journal_t *journal = EXT4_SB(sb)->s_journal;
6197 	long ret;
6198 	loff_t size;
6199 	int block;
6200 	uid_t i_uid;
6201 	gid_t i_gid;
6202 	projid_t i_projid;
6203 
6204 	if ((!(flags & EXT4_IGET_SPECIAL) && is_special_ino(sb, ino)) ||
6205 	    (ino < EXT4_ROOT_INO) ||
6206 	    (ino > le32_to_cpu(es->s_inodes_count))) {
6207 		if (flags & EXT4_IGET_HANDLE)
6208 			return ERR_PTR(-ESTALE);
6209 		__ext4_error(sb, function, line, false, EFSCORRUPTED, 0,
6210 			     "inode #%lu: comm %s: iget: illegal inode #",
6211 			     ino, current->comm);
6212 		return ERR_PTR(-EFSCORRUPTED);
6213 	}
6214 
6215 	if (flags & EXT4_IGET_NOWAIT) {
6216 		inode = find_inode_nowait(sb, ino, ext4_iget_match, NULL);
6217 		if (!inode)
6218 			return ERR_PTR(-ENOENT);
6219 
6220 		if (inode_state_read_once(inode) & I_NEW)
6221 			wait_on_new_inode(inode);
6222 
6223 		if (unlikely(inode_unhashed(inode))) {
6224 			iput(inode);
6225 			return ERR_PTR(-ENOENT);
6226 		}
6227 	} else {
6228 		inode = iget_locked(sb, ino);
6229 		if (!inode)
6230 			return ERR_PTR(-ENOMEM);
6231 	}
6232 
6233 	if (!(inode_state_read_once(inode) & I_NEW)) {
6234 		ret = check_igot_inode(inode, flags, function, line);
6235 		if (ret) {
6236 			iput(inode);
6237 			return ERR_PTR(ret);
6238 		}
6239 		return inode;
6240 	}
6241 
6242 	ei = EXT4_I(inode);
6243 	iloc.bh = NULL;
6244 
6245 	ret = __ext4_get_inode_loc_noinmem(inode, &iloc);
6246 	if (ret < 0)
6247 		goto bad_inode;
6248 	raw_inode = ext4_raw_inode(&iloc);
6249 
6250 	if ((flags & EXT4_IGET_HANDLE) &&
6251 	    (raw_inode->i_links_count == 0) && (raw_inode->i_mode == 0)) {
6252 		ret = -ESTALE;
6253 		goto bad_inode;
6254 	}
6255 
6256 	if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) {
6257 		ei->i_extra_isize = le16_to_cpu(raw_inode->i_extra_isize);
6258 		if (EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize >
6259 			EXT4_INODE_SIZE(inode->i_sb) ||
6260 		    (ei->i_extra_isize & 3)) {
6261 			ext4_error_inode(inode, function, line, 0,
6262 					 "iget: bad extra_isize %u "
6263 					 "(inode size %u)",
6264 					 ei->i_extra_isize,
6265 					 EXT4_INODE_SIZE(inode->i_sb));
6266 			ret = -EFSCORRUPTED;
6267 			goto bad_inode;
6268 		}
6269 	} else
6270 		ei->i_extra_isize = 0;
6271 
6272 	/* Precompute checksum seed for inode metadata */
6273 	if (ext4_has_feature_metadata_csum(sb)) {
6274 		struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
6275 		__u32 csum;
6276 		__le32 inum = cpu_to_le32(inode->i_ino);
6277 		__le32 gen = raw_inode->i_generation;
6278 		csum = ext4_chksum(sbi->s_csum_seed, (__u8 *)&inum,
6279 				   sizeof(inum));
6280 		ei->i_csum_seed = ext4_chksum(csum, (__u8 *)&gen, sizeof(gen));
6281 	}
6282 
6283 	if ((!ext4_inode_csum_verify(inode, raw_inode, ei) ||
6284 	    ext4_simulate_fail(sb, EXT4_SIM_INODE_CRC)) &&
6285 	     (!(EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY))) {
6286 		ext4_error_inode_err(inode, function, line, 0,
6287 				EFSBADCRC, "iget: checksum invalid");
6288 		ret = -EFSBADCRC;
6289 		goto bad_inode;
6290 	}
6291 
6292 	inode->i_mode = le16_to_cpu(raw_inode->i_mode);
6293 	i_uid = (uid_t)le16_to_cpu(raw_inode->i_uid_low);
6294 	i_gid = (gid_t)le16_to_cpu(raw_inode->i_gid_low);
6295 	if (ext4_has_feature_project(sb) &&
6296 	    EXT4_INODE_SIZE(sb) > EXT4_GOOD_OLD_INODE_SIZE &&
6297 	    EXT4_FITS_IN_INODE(raw_inode, ei, i_projid))
6298 		i_projid = (projid_t)le32_to_cpu(raw_inode->i_projid);
6299 	else
6300 		i_projid = EXT4_DEF_PROJID;
6301 
6302 	if (!(test_opt(inode->i_sb, NO_UID32))) {
6303 		i_uid |= le16_to_cpu(raw_inode->i_uid_high) << 16;
6304 		i_gid |= le16_to_cpu(raw_inode->i_gid_high) << 16;
6305 	}
6306 	i_uid_write(inode, i_uid);
6307 	i_gid_write(inode, i_gid);
6308 	ei->i_projid = make_kprojid(&init_user_ns, i_projid);
6309 	set_nlink(inode, le16_to_cpu(raw_inode->i_links_count));
6310 
6311 	ei->i_inline_off = 0;
6312 	ei->i_dir_start_lookup = 0;
6313 	ei->i_dtime = le32_to_cpu(raw_inode->i_dtime);
6314 	/* We now have enough fields to check if the inode was active or not.
6315 	 * This is needed because nfsd might try to access dead inodes
6316 	 * the test is that same one that e2fsck uses
6317 	 * NeilBrown 1999oct15
6318 	 */
6319 	if (inode->i_nlink == 0) {
6320 		if ((inode->i_mode == 0 || flags & EXT4_IGET_SPECIAL ||
6321 		     !(EXT4_SB(inode->i_sb)->s_mount_state & EXT4_ORPHAN_FS)) &&
6322 		    ino != EXT4_BOOT_LOADER_INO) {
6323 			/* this inode is deleted or unallocated */
6324 			if (flags & EXT4_IGET_SPECIAL) {
6325 				ext4_error_inode(inode, function, line, 0,
6326 						 "iget: special inode unallocated");
6327 				ret = -EFSCORRUPTED;
6328 			} else
6329 				ret = -ESTALE;
6330 			goto bad_inode;
6331 		}
6332 		/* The only unlinked inodes we let through here have
6333 		 * valid i_mode and are being read by the orphan
6334 		 * recovery code: that's fine, we're about to complete
6335 		 * the process of deleting those.
6336 		 * OR it is the EXT4_BOOT_LOADER_INO which is
6337 		 * not initialized on a new filesystem. */
6338 	}
6339 	ei->i_flags = le32_to_cpu(raw_inode->i_flags);
6340 	ext4_set_inode_flags(inode, true);
6341 	/* Detect invalid flag combination - can't have both inline data and extents */
6342 	if (ext4_test_inode_flag(inode, EXT4_INODE_INLINE_DATA) &&
6343 	    ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) {
6344 		ext4_error_inode(inode, function, line, 0,
6345 			"inode has both inline data and extents flags");
6346 		ret = -EFSCORRUPTED;
6347 		goto bad_inode;
6348 	}
6349 	inode->i_blocks = ext4_inode_blocks(raw_inode, ei);
6350 	ei->i_file_acl = le32_to_cpu(raw_inode->i_file_acl_lo);
6351 	if (ext4_has_feature_64bit(sb))
6352 		ei->i_file_acl |=
6353 			((__u64)le16_to_cpu(raw_inode->i_file_acl_high)) << 32;
6354 	inode->i_size = ext4_isize(sb, raw_inode);
6355 	size = i_size_read(inode);
6356 	if (size < 0 || size > ext4_get_maxbytes(inode)) {
6357 		ext4_error_inode(inode, function, line, 0,
6358 				 "iget: bad i_size value: %lld", size);
6359 		ret = -EFSCORRUPTED;
6360 		goto bad_inode;
6361 	}
6362 	/*
6363 	 * If dir_index is not enabled but there's dir with INDEX flag set,
6364 	 * we'd normally treat htree data as empty space. But with metadata
6365 	 * checksumming that corrupts checksums so forbid that.
6366 	 */
6367 	if (!ext4_has_feature_dir_index(sb) &&
6368 	    ext4_has_feature_metadata_csum(sb) &&
6369 	    ext4_test_inode_flag(inode, EXT4_INODE_INDEX)) {
6370 		ext4_error_inode(inode, function, line, 0,
6371 			 "iget: Dir with htree data on filesystem without dir_index feature.");
6372 		ret = -EFSCORRUPTED;
6373 		goto bad_inode;
6374 	}
6375 	ei->i_disksize = inode->i_size;
6376 #ifdef CONFIG_QUOTA
6377 	ei->i_reserved_quota = 0;
6378 #endif
6379 	inode->i_generation = le32_to_cpu(raw_inode->i_generation);
6380 	ei->i_block_group = iloc.block_group;
6381 	ei->i_last_alloc_group = ~0;
6382 	/*
6383 	 * NOTE! The in-memory inode i_data array is in little-endian order
6384 	 * even on big-endian machines: we do NOT byteswap the block numbers!
6385 	 */
6386 	for (block = 0; block < EXT4_N_BLOCKS; block++)
6387 		ei->i_data[block] = raw_inode->i_block[block];
6388 	INIT_LIST_HEAD(&ei->i_orphan);
6389 	ext4_fc_init_inode(&ei->vfs_inode);
6390 
6391 	/*
6392 	 * Set transaction id's of transactions that have to be committed
6393 	 * to finish f[data]sync. We set them to currently running transaction
6394 	 * as we cannot be sure that the inode or some of its metadata isn't
6395 	 * part of the transaction - the inode could have been reclaimed and
6396 	 * now it is reread from disk.
6397 	 */
6398 	if (journal) {
6399 		transaction_t *transaction;
6400 		tid_t tid;
6401 
6402 		read_lock(&journal->j_state_lock);
6403 		if (journal->j_running_transaction)
6404 			transaction = journal->j_running_transaction;
6405 		else
6406 			transaction = journal->j_committing_transaction;
6407 		if (transaction)
6408 			tid = transaction->t_tid;
6409 		else
6410 			tid = journal->j_commit_sequence;
6411 		read_unlock(&journal->j_state_lock);
6412 		ei->i_sync_tid = tid;
6413 		ei->i_datasync_tid = tid;
6414 	}
6415 
6416 	if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) {
6417 		if (ei->i_extra_isize == 0) {
6418 			/* The extra space is currently unused. Use it. */
6419 			BUILD_BUG_ON(sizeof(struct ext4_inode) & 3);
6420 			ei->i_extra_isize = sizeof(struct ext4_inode) -
6421 					    EXT4_GOOD_OLD_INODE_SIZE;
6422 		} else {
6423 			ret = ext4_iget_extra_inode(inode, raw_inode, ei);
6424 			if (ret)
6425 				goto bad_inode;
6426 		}
6427 	}
6428 
6429 	EXT4_INODE_GET_CTIME(inode, raw_inode);
6430 	EXT4_INODE_GET_ATIME(inode, raw_inode);
6431 	EXT4_INODE_GET_MTIME(inode, raw_inode);
6432 	EXT4_EINODE_GET_XTIME(i_crtime, ei, raw_inode);
6433 
6434 	if (likely(!test_opt2(inode->i_sb, HURD_COMPAT))) {
6435 		u64 ivers = le32_to_cpu(raw_inode->i_disk_version);
6436 
6437 		if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) {
6438 			if (EXT4_FITS_IN_INODE(raw_inode, ei, i_version_hi))
6439 				ivers |=
6440 		    (__u64)(le32_to_cpu(raw_inode->i_version_hi)) << 32;
6441 		}
6442 		ext4_inode_set_iversion_queried(inode, ivers);
6443 	}
6444 
6445 	ret = 0;
6446 	if (ei->i_file_acl &&
6447 	    !ext4_inode_block_valid(inode, ei->i_file_acl, 1)) {
6448 		ext4_error_inode(inode, function, line, 0,
6449 				 "iget: bad extended attribute block %llu",
6450 				 ei->i_file_acl);
6451 		ret = -EFSCORRUPTED;
6452 		goto bad_inode;
6453 	} else if (!ext4_has_inline_data(inode)) {
6454 		/* validate the block references in the inode */
6455 		if (!(EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY) &&
6456 			(S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode) ||
6457 			(S_ISLNK(inode->i_mode) &&
6458 			!ext4_inode_is_fast_symlink(inode)))) {
6459 			if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
6460 				ret = ext4_ext_check_inode(inode);
6461 			else
6462 				ret = ext4_ind_check_inode(inode);
6463 		}
6464 	}
6465 	if (ret)
6466 		goto bad_inode;
6467 
6468 	ext4_enable_buffered_iomap(inode);
6469 
6470 	if (S_ISREG(inode->i_mode)) {
6471 		inode->i_op = &ext4_file_inode_operations;
6472 		inode->i_fop = &ext4_file_operations;
6473 		ext4_set_aops(inode);
6474 	} else if (S_ISDIR(inode->i_mode)) {
6475 		inode->i_op = &ext4_dir_inode_operations;
6476 		inode->i_fop = &ext4_dir_operations;
6477 	} else if (S_ISLNK(inode->i_mode)) {
6478 		/* VFS does not allow setting these so must be corruption */
6479 		if (IS_APPEND(inode) || IS_IMMUTABLE(inode)) {
6480 			ext4_error_inode(inode, function, line, 0,
6481 					 "iget: immutable or append flags "
6482 					 "not allowed on symlinks");
6483 			ret = -EFSCORRUPTED;
6484 			goto bad_inode;
6485 		}
6486 		if (IS_ENCRYPTED(inode)) {
6487 			inode->i_op = &ext4_encrypted_symlink_inode_operations;
6488 		} else if (ext4_inode_is_fast_symlink(inode)) {
6489 			inode->i_op = &ext4_fast_symlink_inode_operations;
6490 
6491 			/*
6492 			 * Orphan cleanup can see inodes with i_size == 0
6493 			 * and i_data uninitialized. Skip size checks in
6494 			 * that case. This is safe because the first thing
6495 			 * ext4_evict_inode() does for fast symlinks is
6496 			 * clearing of i_data and i_size.
6497 			 */
6498 			if ((EXT4_SB(sb)->s_mount_state & EXT4_ORPHAN_FS)) {
6499 				if (inode->i_nlink != 0) {
6500 					ext4_error_inode(inode, function, line, 0,
6501 						"invalid orphan symlink nlink %d",
6502 						inode->i_nlink);
6503 					ret = -EFSCORRUPTED;
6504 					goto bad_inode;
6505 				}
6506 			} else {
6507 				if (inode->i_size == 0 ||
6508 				    inode->i_size >= sizeof(ei->i_data) ||
6509 				    strnlen((char *)ei->i_data, inode->i_size + 1) !=
6510 						inode->i_size) {
6511 					ext4_error_inode(inode, function, line, 0,
6512 						"invalid fast symlink length %llu",
6513 						(unsigned long long)inode->i_size);
6514 					ret = -EFSCORRUPTED;
6515 					goto bad_inode;
6516 				}
6517 				inode_set_cached_link(inode, (char *)ei->i_data,
6518 						      inode->i_size);
6519 			}
6520 		} else {
6521 			inode->i_op = &ext4_symlink_inode_operations;
6522 		}
6523 	} else if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode) ||
6524 	      S_ISFIFO(inode->i_mode) || S_ISSOCK(inode->i_mode)) {
6525 		inode->i_op = &ext4_special_inode_operations;
6526 		if (raw_inode->i_block[0])
6527 			init_special_inode(inode, inode->i_mode,
6528 			   old_decode_dev(le32_to_cpu(raw_inode->i_block[0])));
6529 		else
6530 			init_special_inode(inode, inode->i_mode,
6531 			   new_decode_dev(le32_to_cpu(raw_inode->i_block[1])));
6532 	} else if (ino == EXT4_BOOT_LOADER_INO) {
6533 		make_bad_inode(inode);
6534 	} else {
6535 		ret = -EFSCORRUPTED;
6536 		ext4_error_inode(inode, function, line, 0,
6537 				 "iget: bogus i_mode (%o)", inode->i_mode);
6538 		goto bad_inode;
6539 	}
6540 	if (IS_CASEFOLDED(inode) && !ext4_has_feature_casefold(inode->i_sb)) {
6541 		ext4_error_inode(inode, function, line, 0,
6542 				 "casefold flag without casefold feature");
6543 		ret = -EFSCORRUPTED;
6544 		goto bad_inode;
6545 	}
6546 	if (IS_ENCRYPTED(inode) && sb->s_blocksize > PAGE_SIZE) {
6547 		ext4_error_inode(inode, function, line, 0,
6548 				 "encrypted inode with block size larger than page size");
6549 		ret = -EFSCORRUPTED;
6550 		goto bad_inode;
6551 	}
6552 
6553 	ext4_set_inode_mapping_order(inode);
6554 
6555 	ret = check_igot_inode(inode, flags, function, line);
6556 	/*
6557 	 * -ESTALE here means there is nothing inherently wrong with the inode,
6558 	 * it's just not an inode we can return for an fhandle lookup.
6559 	 */
6560 	if (ret == -ESTALE) {
6561 		brelse(iloc.bh);
6562 		unlock_new_inode(inode);
6563 		iput(inode);
6564 		return ERR_PTR(-ESTALE);
6565 	}
6566 	if (ret)
6567 		goto bad_inode;
6568 	brelse(iloc.bh);
6569 	/* Initialize the "no ACL's" state for the simple cases */
6570 	if (!ext4_test_inode_state(inode, EXT4_STATE_XATTR) && !ei->i_file_acl)
6571 		cache_no_acl(inode);
6572 	unlock_new_inode(inode);
6573 	return inode;
6574 
6575 bad_inode:
6576 	brelse(iloc.bh);
6577 	iget_failed(inode);
6578 	return ERR_PTR(ret);
6579 }
6580 
6581 static void __ext4_update_other_inode_time(struct super_block *sb,
6582 					   unsigned long orig_ino,
6583 					   unsigned long ino,
6584 					   struct ext4_inode *raw_inode)
6585 {
6586 	struct inode *inode;
6587 
6588 	inode = find_inode_by_ino_rcu(sb, ino);
6589 	if (!inode)
6590 		return;
6591 
6592 	if (!inode_is_dirtytime_only(inode))
6593 		return;
6594 
6595 	spin_lock(&inode->i_lock);
6596 	if (inode_is_dirtytime_only(inode)) {
6597 		struct ext4_inode_info	*ei = EXT4_I(inode);
6598 
6599 		inode_state_clear(inode, I_DIRTY_TIME);
6600 		spin_unlock(&inode->i_lock);
6601 
6602 		spin_lock(&ei->i_raw_lock);
6603 		EXT4_INODE_SET_CTIME(inode, raw_inode);
6604 		EXT4_INODE_SET_MTIME(inode, raw_inode);
6605 		EXT4_INODE_SET_ATIME(inode, raw_inode);
6606 		ext4_inode_csum_set(inode, raw_inode, ei);
6607 		spin_unlock(&ei->i_raw_lock);
6608 		trace_ext4_other_inode_update_time(inode, orig_ino);
6609 		return;
6610 	}
6611 	spin_unlock(&inode->i_lock);
6612 }
6613 
6614 /*
6615  * Opportunistically update the other time fields for other inodes in
6616  * the same inode table block.
6617  */
6618 static void ext4_update_other_inodes_time(struct super_block *sb,
6619 					  unsigned long orig_ino, char *buf)
6620 {
6621 	unsigned long ino;
6622 	int i, inodes_per_block = EXT4_SB(sb)->s_inodes_per_block;
6623 	int inode_size = EXT4_INODE_SIZE(sb);
6624 
6625 	/*
6626 	 * Calculate the first inode in the inode table block.  Inode
6627 	 * numbers are one-based.  That is, the first inode in a block
6628 	 * (assuming 4k blocks and 256 byte inodes) is (n*16 + 1).
6629 	 */
6630 	ino = ((orig_ino - 1) & ~(inodes_per_block - 1)) + 1;
6631 	rcu_read_lock();
6632 	for (i = 0; i < inodes_per_block; i++, ino++, buf += inode_size) {
6633 		if (ino == orig_ino)
6634 			continue;
6635 		__ext4_update_other_inode_time(sb, orig_ino, ino,
6636 					       (struct ext4_inode *)buf);
6637 	}
6638 	rcu_read_unlock();
6639 }
6640 
6641 /*
6642  * Post the struct inode info into an on-disk inode location in the
6643  * buffer-cache.  This gobbles the caller's reference to the
6644  * buffer_head in the inode location struct.
6645  *
6646  * The caller must have write access to iloc->bh.
6647  */
6648 static int ext4_do_update_inode(handle_t *handle,
6649 				struct inode *inode,
6650 				struct ext4_iloc *iloc)
6651 {
6652 	struct ext4_inode *raw_inode = ext4_raw_inode(iloc);
6653 	struct ext4_inode_info *ei = EXT4_I(inode);
6654 	struct buffer_head *bh = iloc->bh;
6655 	struct super_block *sb = inode->i_sb;
6656 	int err;
6657 	int need_datasync = 0, set_large_file = 0;
6658 
6659 	spin_lock(&ei->i_raw_lock);
6660 
6661 	/*
6662 	 * For fields not tracked in the in-memory inode, initialise them
6663 	 * to zero for new inodes.
6664 	 */
6665 	if (ext4_test_inode_state(inode, EXT4_STATE_NEW))
6666 		memset(raw_inode, 0, EXT4_SB(inode->i_sb)->s_inode_size);
6667 
6668 	if (READ_ONCE(ei->i_disksize) != ext4_isize(inode->i_sb, raw_inode))
6669 		need_datasync = 1;
6670 	if (ei->i_disksize > 0x7fffffffULL) {
6671 		if (!ext4_has_feature_large_file(sb) ||
6672 		    EXT4_SB(sb)->s_es->s_rev_level == cpu_to_le32(EXT4_GOOD_OLD_REV))
6673 			set_large_file = 1;
6674 	}
6675 
6676 	err = ext4_fill_raw_inode(inode, raw_inode);
6677 	spin_unlock(&ei->i_raw_lock);
6678 	if (err) {
6679 		EXT4_ERROR_INODE(inode, "corrupted inode contents");
6680 		goto out_brelse;
6681 	}
6682 
6683 	if (inode->i_sb->s_flags & SB_LAZYTIME)
6684 		ext4_update_other_inodes_time(inode->i_sb, inode->i_ino,
6685 					      bh->b_data);
6686 
6687 	BUFFER_TRACE(bh, "call ext4_handle_dirty_metadata");
6688 	err = ext4_handle_dirty_metadata(handle, NULL, bh);
6689 	if (err)
6690 		goto out_error;
6691 	ext4_clear_inode_state(inode, EXT4_STATE_NEW);
6692 	if (set_large_file) {
6693 		BUFFER_TRACE(EXT4_SB(sb)->s_sbh, "get write access");
6694 		err = ext4_journal_get_write_access(handle, sb,
6695 						    EXT4_SB(sb)->s_sbh,
6696 						    EXT4_JTR_NONE);
6697 		if (err)
6698 			goto out_error;
6699 		lock_buffer(EXT4_SB(sb)->s_sbh);
6700 		ext4_set_feature_large_file(sb);
6701 		ext4_superblock_csum_set(sb);
6702 		unlock_buffer(EXT4_SB(sb)->s_sbh);
6703 		ext4_handle_sync(handle);
6704 		err = ext4_handle_dirty_metadata(handle, NULL,
6705 						 EXT4_SB(sb)->s_sbh);
6706 	}
6707 	ext4_update_inode_fsync_trans(handle, inode, need_datasync);
6708 out_error:
6709 	ext4_std_error(inode->i_sb, err);
6710 out_brelse:
6711 	brelse(bh);
6712 	return err;
6713 }
6714 
6715 /*
6716  * ext4_write_inode()
6717  *
6718  * We are called from a few places:
6719  *
6720  * - Within generic_file_aio_write() -> generic_write_sync() for O_SYNC files.
6721  *   Here, there will be no transaction running. We wait for any running
6722  *   transaction to commit.
6723  *
6724  * - Within flush work (sys_sync(), kupdate and such).
6725  *   We wait on commit, if told to.
6726  *
6727  * - Within iput_final() -> write_inode_now()
6728  *   We wait on commit, if told to.
6729  *
6730  * In all cases it is actually safe for us to return without doing anything,
6731  * because the inode has been copied into a raw inode buffer in
6732  * ext4_mark_inode_dirty().  This is a correctness thing for WB_SYNC_ALL
6733  * writeback.
6734  *
6735  * For nojournal mode all the work is done in ext4_sync_inode_metadata()
6736  * because inode content is already copied into raw inode buffer and inode
6737  * is marked with I_METADATA_WRITEBACK.
6738  *
6739  * Note that we are absolutely dependent upon all inode dirtiers doing the
6740  * right thing: they *must* call mark_inode_dirty() after dirtying info in
6741  * which we are interested.
6742  *
6743  * It would be a bug for them to not do this.  The code:
6744  *
6745  *	mark_inode_dirty(inode)
6746  *	stuff();
6747  *	inode->i_size = expr;
6748  *
6749  * is in error because write_inode() could occur while `stuff()' is running,
6750  * and the new i_size will be lost.  Plus the inode will no longer be on the
6751  * superblock's dirty inode list.
6752  */
6753 int ext4_write_inode(struct inode *inode, struct writeback_control *wbc)
6754 {
6755 	int err;
6756 
6757 	if (WARN_ON_ONCE(current->flags & PF_MEMALLOC))
6758 		return 0;
6759 
6760 	err = ext4_emergency_state(inode->i_sb);
6761 	if (unlikely(err))
6762 		return err;
6763 
6764 	if (!EXT4_SB(inode->i_sb)->s_journal)
6765 		return 0;
6766 
6767 	if (ext4_journal_current_handle()) {
6768 		ext4_debug("called recursively, non-PF_MEMALLOC!\n");
6769 		dump_stack();
6770 		return -EIO;
6771 	}
6772 
6773 	/*
6774 	 * No need to force transaction in WB_SYNC_NONE mode. Also
6775 	 * ext4_sync_fs() will force the commit after everything is
6776 	 * written.
6777 	 */
6778 	if (wbc->sync_mode != WB_SYNC_ALL || wbc->for_sync)
6779 		return 0;
6780 
6781 	return ext4_fc_commit(EXT4_SB(inode->i_sb)->s_journal,
6782 						EXT4_I(inode)->i_sync_tid);
6783 }
6784 
6785 int ext4_sync_inode_metadata(struct inode *inode, struct writeback_control *wbc)
6786 {
6787 	struct ext4_iloc iloc;
6788 	struct mapping_metadata_bhs *mmb;
6789 	int err;
6790 
6791 	/* We should only get here in nojournal mode */
6792 	if (WARN_ON_ONCE(EXT4_SB(inode->i_sb)->s_journal))
6793 		return -EFSCORRUPTED;
6794 
6795 	mmb = READ_ONCE(EXT4_I(inode)->i_metadata_bhs);
6796 	if (mmb) {
6797 		err = mmb_sync(mmb);
6798 		if (err)
6799 			return err;
6800 	}
6801 	/* A block that isn't cached is neither dirty nor under writeback. */
6802 	err = ext4_get_inode_loc_noio(inode, &iloc);
6803 	if (err == -EAGAIN)
6804 		return 0;
6805 	if (err)
6806 		return err;
6807 	sync_dirty_buffer(iloc.bh);
6808 	if (buffer_write_io_error(iloc.bh)) {
6809 		ext4_error_inode_block(inode, iloc.bh->b_blocknr, EIO,
6810 				       "IO error syncing inode");
6811 		err = -EIO;
6812 	}
6813 	brelse(iloc.bh);
6814 	return err;
6815 }
6816 
6817 /*
6818  * In data=journal mode ext4_journalled_invalidate_folio() may fail to invalidate
6819  * buffers that are attached to a folio straddling i_size and are undergoing
6820  * commit. In that case we have to wait for commit to finish and try again.
6821  */
6822 static void ext4_wait_for_tail_page_commit(struct inode *inode)
6823 {
6824 	unsigned offset;
6825 	journal_t *journal = EXT4_SB(inode->i_sb)->s_journal;
6826 	tid_t commit_tid;
6827 	int ret;
6828 	bool has_transaction;
6829 
6830 	offset = inode->i_size & (PAGE_SIZE - 1);
6831 	/*
6832 	 * If the folio is fully truncated, we don't need to wait for any commit
6833 	 * (and we even should not as __ext4_journalled_invalidate_folio() may
6834 	 * strip all buffers from the folio but keep the folio dirty which can then
6835 	 * confuse e.g. concurrent ext4_writepages() seeing dirty folio without
6836 	 * buffers). Also we don't need to wait for any commit if all buffers in
6837 	 * the folio remain valid. This is most beneficial for the common case of
6838 	 * blocksize == PAGESIZE.
6839 	 */
6840 	if (!offset || offset > (PAGE_SIZE - i_blocksize(inode)))
6841 		return;
6842 	while (1) {
6843 		struct folio *folio = filemap_lock_folio(inode->i_mapping,
6844 				      inode->i_size >> PAGE_SHIFT);
6845 		if (IS_ERR(folio))
6846 			return;
6847 		ret = __ext4_journalled_invalidate_folio(folio, offset,
6848 						folio_size(folio) - offset);
6849 		folio_unlock(folio);
6850 		folio_put(folio);
6851 		if (ret != -EBUSY)
6852 			return;
6853 		has_transaction = false;
6854 		read_lock(&journal->j_state_lock);
6855 		if (journal->j_committing_transaction) {
6856 			commit_tid = journal->j_committing_transaction->t_tid;
6857 			has_transaction = true;
6858 		}
6859 		read_unlock(&journal->j_state_lock);
6860 		if (has_transaction)
6861 			jbd2_log_wait_commit(journal, commit_tid);
6862 	}
6863 }
6864 
6865 /*
6866  * Set i_size and i_disksize to 'newsize'.
6867  *
6868  * Both i_rwsem and i_data_sem are required here to avoid races between
6869  * generic append writeback (or zeroed pending I/O writeback) and
6870  * concurrent operations (e.g., fallocate, truncate) that also modify
6871  * i_size and i_disksize. This also ensures that the writeback ioend worker
6872  * observes the latest i_size under the same lock protection.
6873  */
6874 static inline void ext4_set_inode_size(struct inode *inode, loff_t newsize)
6875 {
6876 	WARN_ON_ONCE(S_ISREG(inode->i_mode) && !inode_is_locked(inode));
6877 
6878 	down_write(&EXT4_I(inode)->i_data_sem);
6879 	i_size_write(inode, newsize);
6880 	__ext4_set_i_disksize(inode, newsize);
6881 	up_write(&EXT4_I(inode)->i_data_sem);
6882 }
6883 
6884 static int ext4_truncate_up(struct inode *inode, loff_t oldsize, loff_t newsize)
6885 {
6886 	ext4_lblk_t old_lblk, new_lblk;
6887 	handle_t *handle;
6888 	int ret;
6889 
6890 	if (!IS_ALIGNED(oldsize | newsize, i_blocksize(inode))) {
6891 		ret = ext4_inode_attach_jinode(inode);
6892 		if (ret)
6893 			return ret;
6894 	}
6895 
6896 	inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode));
6897 	if (!IS_ALIGNED(oldsize, i_blocksize(inode))) {
6898 		ret = ext4_block_zero_eof(inode, oldsize, LLONG_MAX);
6899 		if (ret)
6900 			return ret;
6901 	}
6902 
6903 	handle = ext4_journal_start(inode, EXT4_HT_INODE, 3);
6904 	if (IS_ERR(handle))
6905 		return PTR_ERR(handle);
6906 
6907 	old_lblk = oldsize > 0 ? (oldsize - 1) >> inode->i_blkbits : 0;
6908 	new_lblk = newsize > 0 ? (newsize - 1) >> inode->i_blkbits : 0;
6909 	ext4_fc_track_range(handle, inode, old_lblk, new_lblk);
6910 
6911 	ext4_set_inode_size(inode, newsize);
6912 
6913 	ret = ext4_mark_inode_dirty(handle, inode);
6914 	ext4_journal_stop(handle);
6915 	if (ret)
6916 		return ret;
6917 	/*
6918 	 * isize extend must be called outside an active handle due to
6919 	 * the lock ordering of transaction start and folio lock in the
6920 	 * iomap buffered I/O path (folio lock -> transaction start).
6921 	 */
6922 	pagecache_isize_extended(inode, oldsize, newsize);
6923 	return 0;
6924 }
6925 
6926 static int ext4_truncate_down(struct inode *inode, loff_t oldsize,
6927 			      loff_t newsize, int *orphan)
6928 {
6929 	ext4_lblk_t start_lblk;
6930 	handle_t *handle;
6931 	int ret;
6932 
6933 	/* Do not change i_size. */
6934 	if (newsize == oldsize)
6935 		goto truncate;
6936 
6937 	/* Shrink. */
6938 	handle = ext4_journal_start(inode, EXT4_HT_INODE, 3);
6939 	if (IS_ERR(handle))
6940 		return PTR_ERR(handle);
6941 
6942 	if (ext4_handle_valid(handle)) {
6943 		ret = ext4_orphan_add(handle, inode);
6944 		*orphan = 1;
6945 		if (ret) {
6946 			ext4_journal_stop(handle);
6947 			return ret;
6948 		}
6949 	}
6950 
6951 	start_lblk = newsize > 0 ? (newsize - 1) >> inode->i_blkbits : 0;
6952 	ext4_fc_track_range(handle, inode, start_lblk, EXT_MAX_BLOCKS - 1);
6953 
6954 	down_write(&EXT4_I(inode)->i_data_sem);
6955 	/*
6956 	 * Truncate the zeroed EOF block invalidates the pending disksize
6957 	 * update, so clear the disksize-grow-pending state.
6958 	 */
6959 	if (ext4_test_inode_state(inode, EXT4_STATE_DISKSIZE_GROW_PENDING) &&
6960 	    (newsize <= EXT4_I(inode)->i_disksize))
6961 		ext4_iomap_clear_disksize_pending(inode);
6962 
6963 	i_size_write(inode, newsize);
6964 	__ext4_set_i_disksize(inode, newsize);
6965 	up_write(&EXT4_I(inode)->i_data_sem);
6966 
6967 	ret = ext4_mark_inode_dirty(handle, inode);
6968 	ext4_journal_stop(handle);
6969 	if (ret)
6970 		return ret;
6971 
6972 	if (ext4_should_journal_data(inode))
6973 		ext4_wait_for_tail_page_commit(inode);
6974 truncate:
6975 	/*
6976 	 * Truncate pagecache after we've waited for commit in data=journal
6977 	 * mode to make pages freeable.  Call ext4_truncate() even if
6978 	 * i_size didn't change to truncate possible preallocated blocks.
6979 	 */
6980 	truncate_pagecache(inode, newsize);
6981 	return ext4_truncate(inode);
6982 }
6983 
6984 /*
6985  * ext4_setattr()
6986  *
6987  * Called from notify_change.
6988  *
6989  * We want to trap VFS attempts to truncate the file as soon as
6990  * possible.  In particular, we want to make sure that when the VFS
6991  * shrinks i_size, we put the inode on the orphan list and modify
6992  * i_disksize immediately, so that during the subsequent flushing of
6993  * dirty pages and freeing of disk blocks, we can guarantee that any
6994  * commit will leave the blocks being flushed in an unused state on
6995  * disk.  (On recovery, the inode will get truncated and the blocks will
6996  * be freed, so we have a strong guarantee that no future commit will
6997  * leave these blocks visible to the user.)
6998  *
6999  * Another thing we have to assure is that if we are in ordered mode
7000  * and inode is still attached to the committing transaction, we must
7001  * we start writeout of all the dirty pages which are being truncated.
7002  * This way we are sure that all the data written in the previous
7003  * transaction are already on disk (truncate waits for pages under
7004  * writeback).
7005  *
7006  * Called with inode->i_rwsem down.
7007  */
7008 int ext4_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
7009 		 struct iattr *attr)
7010 {
7011 	struct inode *inode = d_inode(dentry);
7012 	int error, rc = 0;
7013 	int orphan = 0;
7014 	const unsigned int ia_valid = attr->ia_valid;
7015 	bool inc_ivers = true;
7016 
7017 	error = ext4_emergency_state(inode->i_sb);
7018 	if (unlikely(error))
7019 		return error;
7020 
7021 	if (unlikely(IS_IMMUTABLE(inode)))
7022 		return -EPERM;
7023 
7024 	if (unlikely(IS_APPEND(inode) &&
7025 		     (ia_valid & (ATTR_MODE | ATTR_UID |
7026 				  ATTR_GID | ATTR_TIMES_SET))))
7027 		return -EPERM;
7028 
7029 	error = setattr_prepare(idmap, dentry, attr);
7030 	if (error)
7031 		return error;
7032 
7033 	error = fscrypt_prepare_setattr(dentry, attr);
7034 	if (error)
7035 		return error;
7036 
7037 	if (is_quota_modification(idmap, inode, attr)) {
7038 		error = dquot_initialize(inode);
7039 		if (error)
7040 			return error;
7041 	}
7042 
7043 	if (i_uid_needs_update(idmap, attr, inode) ||
7044 	    i_gid_needs_update(idmap, attr, inode)) {
7045 		handle_t *handle;
7046 
7047 		/* (user+group)*(old+new) structure, inode write (sb,
7048 		 * inode block, ? - but truncate inode update has it) */
7049 		handle = ext4_journal_start(inode, EXT4_HT_QUOTA,
7050 			(EXT4_MAXQUOTAS_INIT_BLOCKS(inode->i_sb) +
7051 			 EXT4_MAXQUOTAS_DEL_BLOCKS(inode->i_sb)) + 3);
7052 		if (IS_ERR(handle)) {
7053 			error = PTR_ERR(handle);
7054 			goto err_out;
7055 		}
7056 
7057 		/* dquot_transfer() calls back ext4_get_inode_usage() which
7058 		 * counts xattr inode references.
7059 		 */
7060 		down_read(&EXT4_I(inode)->xattr_sem);
7061 		error = dquot_transfer(idmap, inode, attr);
7062 		up_read(&EXT4_I(inode)->xattr_sem);
7063 
7064 		if (error) {
7065 			ext4_journal_stop(handle);
7066 			return error;
7067 		}
7068 		/* Update corresponding info in inode so that everything is in
7069 		 * one transaction */
7070 		i_uid_update(idmap, attr, inode);
7071 		i_gid_update(idmap, attr, inode);
7072 		error = ext4_mark_inode_dirty(handle, inode);
7073 		ext4_journal_stop(handle);
7074 		if (unlikely(error)) {
7075 			return error;
7076 		}
7077 	}
7078 
7079 	if (attr->ia_valid & ATTR_SIZE) {
7080 		loff_t oldsize = inode->i_size;
7081 		int shrink = (attr->ia_size < inode->i_size);
7082 
7083 		if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) {
7084 			struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
7085 
7086 			if (attr->ia_size > sbi->s_bitmap_maxbytes) {
7087 				return -EFBIG;
7088 			}
7089 		}
7090 		if (!S_ISREG(inode->i_mode)) {
7091 			return -EINVAL;
7092 		}
7093 
7094 		if (attr->ia_size == inode->i_size)
7095 			inc_ivers = false;
7096 
7097 		/*
7098 		 * If file has inline data but new size exceeds inline capacity,
7099 		 * convert to extent-based storage first to prevent inconsistent
7100 		 * state (inline flag set but size exceeds inline capacity).
7101 		 */
7102 		if (ext4_has_inline_data(inode) &&
7103 		    attr->ia_size > EXT4_I(inode)->i_inline_size) {
7104 			error = ext4_convert_inline_data(inode);
7105 			if (error)
7106 				goto err_out;
7107 		}
7108 
7109 		if (shrink) {
7110 			if (ext4_should_order_data(inode)) {
7111 				error = ext4_begin_ordered_truncate(inode,
7112 							    attr->ia_size);
7113 				if (error)
7114 					goto err_out;
7115 			}
7116 			/*
7117 			 * Blocks are going to be removed from the inode. Wait
7118 			 * for dio in flight.
7119 			 */
7120 			inode_dio_wait(inode);
7121 		}
7122 
7123 		filemap_invalidate_lock(inode->i_mapping);
7124 
7125 		rc = ext4_break_layouts(inode);
7126 		if (rc) {
7127 			filemap_invalidate_unlock(inode->i_mapping);
7128 			goto err_out;
7129 		}
7130 
7131 		if (attr->ia_size > oldsize)
7132 			error = ext4_truncate_up(inode, oldsize, attr->ia_size);
7133 		else {
7134 			/* Shrink or do not change i_size. */
7135 			error = ext4_truncate_down(inode, oldsize,
7136 						   attr->ia_size, &orphan);
7137 		}
7138 
7139 		filemap_invalidate_unlock(inode->i_mapping);
7140 	}
7141 
7142 	if (!error) {
7143 		if (inc_ivers)
7144 			inode_inc_iversion(inode);
7145 		setattr_copy(idmap, inode, attr);
7146 		mark_inode_dirty(inode);
7147 	}
7148 
7149 	/*
7150 	 * If the call to ext4_truncate failed to get a transaction handle at
7151 	 * all, we need to clean up the in-core orphan list manually.
7152 	 */
7153 	if (orphan && inode->i_nlink)
7154 		ext4_orphan_del(NULL, inode);
7155 
7156 	if (!error && (ia_valid & ATTR_MODE))
7157 		rc = posix_acl_chmod(idmap, dentry, inode->i_mode);
7158 
7159 err_out:
7160 	if  (error)
7161 		ext4_std_error(inode->i_sb, error);
7162 	if (!error)
7163 		error = rc;
7164 	return error;
7165 }
7166 
7167 u32 ext4_dio_alignment(struct inode *inode)
7168 {
7169 	if (fsverity_active(inode))
7170 		return 0;
7171 	if (ext4_should_journal_data(inode))
7172 		return 0;
7173 	if (ext4_has_inline_data(inode))
7174 		return 0;
7175 	if (IS_ENCRYPTED(inode))
7176 		return i_blocksize(inode);
7177 	return 1; /* use the iomap defaults */
7178 }
7179 
7180 int ext4_getattr(const struct mnt_idmap *idmap, const struct path *path,
7181 		 struct kstat *stat, u32 request_mask, unsigned int query_flags)
7182 {
7183 	struct inode *inode = d_inode(path->dentry);
7184 	struct ext4_inode *raw_inode;
7185 	struct ext4_inode_info *ei = EXT4_I(inode);
7186 	unsigned int flags;
7187 
7188 	if ((request_mask & STATX_BTIME) &&
7189 	    EXT4_FITS_IN_INODE(raw_inode, ei, i_crtime)) {
7190 		stat->result_mask |= STATX_BTIME;
7191 		stat->btime.tv_sec = ei->i_crtime.tv_sec;
7192 		stat->btime.tv_nsec = ei->i_crtime.tv_nsec;
7193 	}
7194 
7195 	/* Return the DIO alignment restrictions if requested. */
7196 	if ((request_mask & STATX_DIOALIGN) && S_ISREG(inode->i_mode)) {
7197 		u32 dio_align = ext4_dio_alignment(inode);
7198 
7199 		stat->result_mask |= STATX_DIOALIGN;
7200 		if (dio_align == 1) {
7201 			struct block_device *bdev = inode->i_sb->s_bdev;
7202 
7203 			/* iomap defaults */
7204 			stat->dio_mem_align = bdev_dma_alignment(bdev) + 1;
7205 			stat->dio_offset_align = bdev_logical_block_size(bdev);
7206 		} else {
7207 			stat->dio_mem_align = dio_align;
7208 			stat->dio_offset_align = dio_align;
7209 		}
7210 	}
7211 
7212 	if ((request_mask & STATX_WRITE_ATOMIC) && S_ISREG(inode->i_mode)) {
7213 		struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
7214 		unsigned int awu_min = 0, awu_max = 0;
7215 
7216 		if (ext4_inode_can_atomic_write(inode)) {
7217 			awu_min = sbi->s_awu_min;
7218 			awu_max = sbi->s_awu_max;
7219 		}
7220 
7221 		generic_fill_statx_atomic_writes(stat, awu_min, awu_max, 0);
7222 	}
7223 
7224 	flags = ei->i_flags & EXT4_FL_USER_VISIBLE;
7225 	if (flags & EXT4_APPEND_FL)
7226 		stat->attributes |= STATX_ATTR_APPEND;
7227 	if (flags & EXT4_COMPR_FL)
7228 		stat->attributes |= STATX_ATTR_COMPRESSED;
7229 	if (flags & EXT4_ENCRYPT_FL)
7230 		stat->attributes |= STATX_ATTR_ENCRYPTED;
7231 	if (flags & EXT4_IMMUTABLE_FL)
7232 		stat->attributes |= STATX_ATTR_IMMUTABLE;
7233 	if (flags & EXT4_NODUMP_FL)
7234 		stat->attributes |= STATX_ATTR_NODUMP;
7235 
7236 	stat->attributes_mask |= (STATX_ATTR_APPEND |
7237 				  STATX_ATTR_COMPRESSED |
7238 				  STATX_ATTR_ENCRYPTED |
7239 				  STATX_ATTR_IMMUTABLE |
7240 				  STATX_ATTR_NODUMP);
7241 
7242 	generic_fillattr(idmap, request_mask, inode, stat);
7243 	return 0;
7244 }
7245 
7246 int ext4_file_getattr(const struct mnt_idmap *idmap,
7247 		      const struct path *path, struct kstat *stat,
7248 		      u32 request_mask, unsigned int query_flags)
7249 {
7250 	struct inode *inode = d_inode(path->dentry);
7251 	u64 delalloc_blocks;
7252 
7253 	ext4_getattr(idmap, path, stat, request_mask, query_flags);
7254 
7255 	/*
7256 	 * If there is inline data in the inode, the inode will normally not
7257 	 * have data blocks allocated (it may have an external xattr block).
7258 	 * Report at least one sector for such files, so tools like tar, rsync,
7259 	 * others don't incorrectly think the file is completely sparse.
7260 	 */
7261 	if (unlikely(ext4_has_inline_data(inode)))
7262 		stat->blocks += (stat->size + 511) >> 9;
7263 
7264 	/*
7265 	 * We can't update i_blocks if the block allocation is delayed
7266 	 * otherwise in the case of system crash before the real block
7267 	 * allocation is done, we will have i_blocks inconsistent with
7268 	 * on-disk file blocks.
7269 	 * We always keep i_blocks updated together with real
7270 	 * allocation. But to not confuse with user, stat
7271 	 * will return the blocks that include the delayed allocation
7272 	 * blocks for this file.
7273 	 */
7274 	delalloc_blocks = EXT4_C2B(EXT4_SB(inode->i_sb),
7275 				   EXT4_I(inode)->i_reserved_data_blocks);
7276 	stat->blocks += delalloc_blocks << (inode->i_sb->s_blocksize_bits - 9);
7277 	return 0;
7278 }
7279 
7280 static int ext4_index_trans_blocks(struct inode *inode, int lblocks,
7281 				   int pextents)
7282 {
7283 	if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)))
7284 		return ext4_ind_trans_blocks(inode, lblocks);
7285 	return ext4_ext_index_trans_blocks(inode, pextents);
7286 }
7287 
7288 /*
7289  * Calculate number of credits needed in a transaction to:
7290  *   * Allocate data blocks from @alloc_extents different groups - note that
7291  *     with flexbg a single physical extent can span multiple groups but
7292  *     single mballoc request only returns extent within one group.
7293  *   * Allocate metatadata (extent tree blocks, indirect blocks) to store
7294  *     pointers to @pextents data extents having @lblocks in total.
7295  *   * Modify extent tree / indirect block tree, inode, superblock, quota
7296  *     tracking, xattr blocks
7297  */
7298 int ext4_meta_trans_blocks(struct inode *inode, int lblocks, int pextents,
7299 			   int alloc_extents)
7300 {
7301 	ext4_group_t groups, ngroups = ext4_get_groups_count(inode->i_sb);
7302 	int gdpblocks;
7303 	int idxblocks;
7304 	int ret;
7305 
7306 	/*
7307 	 * How many index and leaf blocks need to touch to map @lblocks
7308 	 * logical blocks to @pextents physical extents?
7309 	 */
7310 	idxblocks = ext4_index_trans_blocks(inode, lblocks, pextents);
7311 
7312 	/*
7313 	 * Now let's see how many group bitmaps and group descriptors need
7314 	 * to account
7315 	 */
7316 	groups = idxblocks + alloc_extents;
7317 	gdpblocks = groups;
7318 	if (groups > ngroups)
7319 		groups = ngroups;
7320 	if (groups > EXT4_SB(inode->i_sb)->s_gdb_count)
7321 		gdpblocks = EXT4_SB(inode->i_sb)->s_gdb_count;
7322 
7323 	/* bitmaps and block group descriptor blocks */
7324 	ret = idxblocks + groups + gdpblocks;
7325 
7326 	/* Blocks for super block, inode, quota and xattr blocks */
7327 	ret += EXT4_META_TRANS_BLOCKS(inode->i_sb);
7328 
7329 	return ret;
7330 }
7331 
7332 /*
7333  * Calculate the journal credits for modifying the number of blocks
7334  * in a single extent within one transaction. 'nrblocks' is used only
7335  * for non-extent inodes. For extent type inodes, 'nrblocks' can be
7336  * zero if the exact number of blocks is unknown.
7337  */
7338 int ext4_chunk_trans_extent(struct inode *inode, int nrblocks)
7339 {
7340 	int ret;
7341 
7342 	ret = ext4_meta_trans_blocks(inode, nrblocks, 1, 1);
7343 	/* Account for data blocks for journalled mode */
7344 	if (ext4_should_journal_data(inode))
7345 		ret += nrblocks;
7346 	return ret;
7347 }
7348 
7349 /*
7350  * Calculate the journal credits for a chunk of data modification.
7351  *
7352  * This is called from DIO, fallocate or whoever calling
7353  * ext4_map_blocks() to map/allocate a chunk of contiguous disk blocks.
7354  *
7355  * journal buffers for data blocks are not included here, as DIO
7356  * and fallocate do no need to journal data buffers.
7357  */
7358 int ext4_chunk_trans_blocks(struct inode *inode, int nrblocks)
7359 {
7360 	return ext4_meta_trans_blocks(inode, nrblocks, 1, 1);
7361 }
7362 
7363 /*
7364  * The caller must have previously called ext4_reserve_inode_write().
7365  * Give this, we know that the caller already has write access to iloc->bh.
7366  */
7367 int ext4_mark_iloc_dirty(handle_t *handle,
7368 			 struct inode *inode, struct ext4_iloc *iloc)
7369 {
7370 	struct super_block *sb = inode->i_sb;
7371 	int err = 0;
7372 
7373 	err = ext4_emergency_state(sb);
7374 	if (unlikely(err)) {
7375 		put_bh(iloc->bh);
7376 		return err;
7377 	}
7378 	ext4_fc_track_inode(handle, inode);
7379 
7380 	/* the do_update_inode consumes one bh->b_count */
7381 	get_bh(iloc->bh);
7382 
7383 	/* ext4_do_update_inode() does jbd2_journal_dirty_metadata */
7384 	err = ext4_do_update_inode(handle, inode, iloc);
7385 	put_bh(iloc->bh);
7386 	/*
7387 	 * Mark that there's metadata writeout pending for the inode so that it
7388 	 * gets properly flushed on fsync(2) and similar. We don't bother for
7389 	 * fastcommit replay as that flushes the whole bdev afterwards anyway.
7390 	 * It is faster this way and we avoid entering fs writeback paths which
7391 	 * aren't fully initialized yet.
7392 	 */
7393 	if (!ext4_handle_valid(handle) &&
7394 	    !(EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY)) {
7395 		/*
7396 		 * Inode didn't need to go through dirtying, make sure it is
7397 		 * attached to wb so that writeback can handle it.
7398 		 */
7399 		spin_lock(&inode->i_lock);
7400 		inode_attach_wb(inode, NULL);
7401 		spin_unlock(&inode->i_lock);
7402 		set_inode_metadata_writeback(inode);
7403 	}
7404 	return err;
7405 }
7406 
7407 /*
7408  * On success, We end up with an outstanding reference count against
7409  * iloc->bh.  This _must_ be cleaned up later.
7410  */
7411 
7412 int
7413 ext4_reserve_inode_write(handle_t *handle, struct inode *inode,
7414 			 struct ext4_iloc *iloc)
7415 {
7416 	int err;
7417 
7418 	err = ext4_emergency_state(inode->i_sb);
7419 	if (unlikely(err))
7420 		return err;
7421 
7422 	err = ext4_get_inode_loc(inode, iloc);
7423 	if (!err) {
7424 		BUFFER_TRACE(iloc->bh, "get_write_access");
7425 		err = ext4_journal_get_write_access(handle, inode->i_sb,
7426 						    iloc->bh, EXT4_JTR_NONE);
7427 		if (err) {
7428 			brelse(iloc->bh);
7429 			iloc->bh = NULL;
7430 		}
7431 		ext4_fc_track_inode(handle, inode);
7432 	}
7433 	ext4_std_error(inode->i_sb, err);
7434 	return err;
7435 }
7436 
7437 static int __ext4_expand_extra_isize(struct inode *inode,
7438 				     unsigned int new_extra_isize,
7439 				     struct ext4_iloc *iloc,
7440 				     handle_t *handle, int *no_expand)
7441 {
7442 	struct ext4_inode *raw_inode;
7443 	struct ext4_xattr_ibody_header *header;
7444 	unsigned int inode_size = EXT4_INODE_SIZE(inode->i_sb);
7445 	struct ext4_inode_info *ei = EXT4_I(inode);
7446 	int error;
7447 
7448 	/* this was checked at iget time, but double check for good measure */
7449 	if ((EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize > inode_size) ||
7450 	    (ei->i_extra_isize & 3)) {
7451 		EXT4_ERROR_INODE(inode, "bad extra_isize %u (inode size %u)",
7452 				 ei->i_extra_isize,
7453 				 EXT4_INODE_SIZE(inode->i_sb));
7454 		return -EFSCORRUPTED;
7455 	}
7456 	if ((new_extra_isize < ei->i_extra_isize) ||
7457 	    (new_extra_isize < 4) ||
7458 	    (new_extra_isize > inode_size - EXT4_GOOD_OLD_INODE_SIZE))
7459 		return -EINVAL;	/* Should never happen */
7460 
7461 	raw_inode = ext4_raw_inode(iloc);
7462 
7463 	header = IHDR(inode, raw_inode);
7464 
7465 	/* No extended attributes present */
7466 	if (!ext4_test_inode_state(inode, EXT4_STATE_XATTR) ||
7467 	    header->h_magic != cpu_to_le32(EXT4_XATTR_MAGIC)) {
7468 		memset((void *)raw_inode + EXT4_GOOD_OLD_INODE_SIZE +
7469 		       EXT4_I(inode)->i_extra_isize, 0,
7470 		       new_extra_isize - EXT4_I(inode)->i_extra_isize);
7471 		EXT4_I(inode)->i_extra_isize = new_extra_isize;
7472 		return 0;
7473 	}
7474 
7475 	/*
7476 	 * We may need to allocate external xattr block so we need quotas
7477 	 * initialized. Here we can be called with various locks held so we
7478 	 * cannot affort to initialize quotas ourselves. So just bail.
7479 	 */
7480 	if (dquot_initialize_needed(inode))
7481 		return -EAGAIN;
7482 
7483 	/* try to expand with EAs present */
7484 	error = ext4_expand_extra_isize_ea(inode, new_extra_isize,
7485 					   raw_inode, handle);
7486 	if (error) {
7487 		/*
7488 		 * Inode size expansion failed; don't try again
7489 		 */
7490 		*no_expand = 1;
7491 	}
7492 
7493 	return error;
7494 }
7495 
7496 /*
7497  * Expand an inode by new_extra_isize bytes.
7498  * Returns 0 on success or negative error number on failure.
7499  */
7500 static int ext4_try_to_expand_extra_isize(struct inode *inode,
7501 					  unsigned int new_extra_isize,
7502 					  struct ext4_iloc iloc,
7503 					  handle_t *handle)
7504 {
7505 	int no_expand;
7506 	int error;
7507 
7508 	if (ext4_test_inode_state(inode, EXT4_STATE_NO_EXPAND))
7509 		return -EOVERFLOW;
7510 
7511 	/*
7512 	 * Skip expansion during mount (!SB_ACTIVE).  Expanding extra isize
7513 	 * may move xattrs to external blocks and release ea_inodes via iput.
7514 	 * When !SB_ACTIVE, iput triggers write_inode_now() which acquires
7515 	 * s_writepages_rwsem, causing a deadlock with the caller's active
7516 	 * jbd2 handle (lock order: s_writepages_rwsem -> jbd2_handle).
7517 	 *
7518 	 * Skip it while unmounting as well.  shrink_dcache_for_umount()
7519 	 * clears s_root before generic_shutdown_super() clears SB_ACTIVE, and
7520 	 * the last iput() of a lazytime inode in that window redirties it and
7521 	 * lands here.  Moving xattrs out to a block then needs a new EA inode,
7522 	 * which ext4_xattr_inode_create() refuses without s_root.
7523 	 */
7524 	if (unlikely(!(inode->i_sb->s_flags & SB_ACTIVE) || !inode->i_sb->s_root))
7525 		return -EBUSY;
7526 
7527 	/*
7528 	 * In nojournal mode, we can immediately attempt to expand
7529 	 * the inode.  When journaled, we first need to obtain extra
7530 	 * buffer credits since we may write into the EA block
7531 	 * with this same handle. If journal_extend fails, then it will
7532 	 * only result in a minor loss of functionality for that inode.
7533 	 * If this is felt to be critical, then e2fsck should be run to
7534 	 * force a large enough s_min_extra_isize.
7535 	 */
7536 	if (ext4_journal_extend(handle,
7537 				EXT4_DATA_TRANS_BLOCKS(inode->i_sb), 0) != 0)
7538 		return -ENOSPC;
7539 
7540 	if (ext4_write_trylock_xattr(inode, &no_expand) == 0)
7541 		return -EBUSY;
7542 
7543 	error = __ext4_expand_extra_isize(inode, new_extra_isize, &iloc,
7544 					  handle, &no_expand);
7545 	ext4_write_unlock_xattr(inode, &no_expand);
7546 
7547 	return error;
7548 }
7549 
7550 int ext4_expand_extra_isize(struct inode *inode,
7551 			    unsigned int new_extra_isize,
7552 			    struct ext4_iloc *iloc)
7553 {
7554 	handle_t *handle;
7555 	int no_expand;
7556 	int error, rc;
7557 
7558 	if (ext4_test_inode_state(inode, EXT4_STATE_NO_EXPAND)) {
7559 		brelse(iloc->bh);
7560 		return -EOVERFLOW;
7561 	}
7562 
7563 	handle = ext4_journal_start(inode, EXT4_HT_INODE,
7564 				    EXT4_DATA_TRANS_BLOCKS(inode->i_sb));
7565 	if (IS_ERR(handle)) {
7566 		error = PTR_ERR(handle);
7567 		brelse(iloc->bh);
7568 		return error;
7569 	}
7570 
7571 	ext4_write_lock_xattr(inode, &no_expand);
7572 
7573 	BUFFER_TRACE(iloc->bh, "get_write_access");
7574 	error = ext4_journal_get_write_access(handle, inode->i_sb, iloc->bh,
7575 					      EXT4_JTR_NONE);
7576 	if (error) {
7577 		brelse(iloc->bh);
7578 		goto out_unlock;
7579 	}
7580 
7581 	error = __ext4_expand_extra_isize(inode, new_extra_isize, iloc,
7582 					  handle, &no_expand);
7583 
7584 	rc = ext4_mark_iloc_dirty(handle, inode, iloc);
7585 	if (!error)
7586 		error = rc;
7587 
7588 out_unlock:
7589 	ext4_write_unlock_xattr(inode, &no_expand);
7590 	ext4_journal_stop(handle);
7591 	return error;
7592 }
7593 
7594 /*
7595  * What we do here is to mark the in-core inode as clean with respect to inode
7596  * dirtiness (it may still be data-dirty).
7597  * This means that the in-core inode may be reaped by prune_icache
7598  * without having to perform any I/O.  This is a very good thing,
7599  * because *any* task may call prune_icache - even ones which
7600  * have a transaction open against a different journal.
7601  *
7602  * Is this cheating?  Not really.  Sure, we haven't written the
7603  * inode out, but prune_icache isn't a user-visible syncing function.
7604  * Whenever the user wants stuff synced (sys_sync, sys_msync, sys_fsync)
7605  * we start and wait on commits.
7606  */
7607 int __ext4_mark_inode_dirty(handle_t *handle, struct inode *inode,
7608 				const char *func, unsigned int line)
7609 {
7610 	struct ext4_iloc iloc;
7611 	struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
7612 	int err;
7613 
7614 	might_sleep();
7615 	trace_ext4_mark_inode_dirty(inode, _RET_IP_);
7616 	err = ext4_reserve_inode_write(handle, inode, &iloc);
7617 	if (err)
7618 		goto out;
7619 
7620 	if (EXT4_I(inode)->i_extra_isize < sbi->s_want_extra_isize)
7621 		ext4_try_to_expand_extra_isize(inode, sbi->s_want_extra_isize,
7622 					       iloc, handle);
7623 
7624 	err = ext4_mark_iloc_dirty(handle, inode, &iloc);
7625 out:
7626 	if (unlikely(err))
7627 		ext4_error_inode_err(inode, func, line, 0, err,
7628 					"mark_inode_dirty error");
7629 	return err;
7630 }
7631 
7632 /*
7633  * ext4_dirty_inode() is called from __mark_inode_dirty()
7634  *
7635  * We're really interested in the case where a file is being extended.
7636  * i_size has been changed by generic_commit_write() and we thus need
7637  * to include the updated inode in the current transaction.
7638  *
7639  * Also, dquot_alloc_block() will always dirty the inode when blocks
7640  * are allocated to the file.
7641  *
7642  * If the inode is marked synchronous, we don't honour that here - doing
7643  * so would cause a commit on atime updates, which we don't bother doing.
7644  * We handle synchronous inodes at the highest possible level.
7645  */
7646 void ext4_dirty_inode(struct inode *inode, int flags)
7647 {
7648 	handle_t *handle;
7649 
7650 	handle = ext4_journal_start(inode, EXT4_HT_INODE, 2);
7651 	if (IS_ERR(handle))
7652 		return;
7653 	ext4_mark_inode_dirty(handle, inode);
7654 	ext4_journal_stop(handle);
7655 }
7656 
7657 int ext4_change_inode_journal_flag(struct inode *inode, int val)
7658 {
7659 	journal_t *journal;
7660 	handle_t *handle;
7661 	int err;
7662 	int alloc_ctx;
7663 
7664 	/*
7665 	 * We have to be very careful here: changing a data block's
7666 	 * journaling status dynamically is dangerous.  If we write a
7667 	 * data block to the journal, change the status and then delete
7668 	 * that block, we risk forgetting to revoke the old log record
7669 	 * from the journal and so a subsequent replay can corrupt data.
7670 	 * So, first we make sure that the journal is empty and that
7671 	 * nobody is changing anything.
7672 	 */
7673 
7674 	journal = EXT4_JOURNAL(inode);
7675 	if (!journal)
7676 		return 0;
7677 	if (is_journal_aborted(journal))
7678 		return -EROFS;
7679 
7680 	/* Wait for all existing dio workers */
7681 	inode_dio_wait(inode);
7682 
7683 	/*
7684 	 * Before flushing the journal and switching inode's aops, we have
7685 	 * to flush all dirty data the inode has. There can be outstanding
7686 	 * delayed allocations, there can be unwritten extents created by
7687 	 * fallocate or buffered writes in dioread_nolock mode covered by
7688 	 * dirty data which can be converted only after flushing the dirty
7689 	 * data (and journalled aops don't know how to handle these cases).
7690 	 */
7691 	filemap_invalidate_lock(inode->i_mapping);
7692 	err = filemap_write_and_wait(inode->i_mapping);
7693 	if (err < 0) {
7694 		filemap_invalidate_unlock(inode->i_mapping);
7695 		return err;
7696 	}
7697 	/* Before switch the inode journalling mode evict all the page cache. */
7698 	truncate_pagecache(inode, 0);
7699 
7700 	alloc_ctx = ext4_writepages_down_write(inode->i_sb);
7701 	jbd2_journal_lock_updates(journal);
7702 
7703 	/*
7704 	 * OK, there are no updates running now, and all cached data is
7705 	 * synced to disk.  We are now in a completely consistent state
7706 	 * which doesn't have anything in the journal, and we know that
7707 	 * no filesystem updates are running, so it is safe to modify
7708 	 * the inode's in-core data-journaling state flag now.
7709 	 */
7710 
7711 	if (val) {
7712 		ext4_set_inode_flag(inode, EXT4_INODE_JOURNAL_DATA);
7713 		ext4_disable_buffered_iomap(inode);
7714 	} else {
7715 		err = jbd2_journal_flush(journal, 0);
7716 		if (err < 0) {
7717 			jbd2_journal_unlock_updates(journal);
7718 			ext4_writepages_up_write(inode->i_sb, alloc_ctx);
7719 			filemap_invalidate_unlock(inode->i_mapping);
7720 			return err;
7721 		}
7722 		ext4_clear_inode_flag(inode, EXT4_INODE_JOURNAL_DATA);
7723 		ext4_enable_buffered_iomap(inode);
7724 	}
7725 	ext4_set_aops(inode);
7726 	ext4_set_inode_mapping_order(inode);
7727 
7728 	jbd2_journal_unlock_updates(journal);
7729 	ext4_writepages_up_write(inode->i_sb, alloc_ctx);
7730 	filemap_invalidate_unlock(inode->i_mapping);
7731 
7732 	/* Finally we can mark the inode as dirty. */
7733 
7734 	handle = ext4_journal_start(inode, EXT4_HT_INODE, 1);
7735 	if (IS_ERR(handle))
7736 		return PTR_ERR(handle);
7737 
7738 	ext4_fc_mark_ineligible(inode->i_sb,
7739 		EXT4_FC_REASON_JOURNAL_FLAG_CHANGE, handle);
7740 	err = ext4_mark_inode_dirty(handle, inode);
7741 	ext4_handle_sync(handle);
7742 	ext4_journal_stop(handle);
7743 	ext4_std_error(inode->i_sb, err);
7744 
7745 	return err;
7746 }
7747 
7748 static int ext4_bh_unmapped(handle_t *handle, struct inode *inode,
7749 			    struct buffer_head *bh)
7750 {
7751 	return !buffer_mapped(bh);
7752 }
7753 
7754 static int ext4_block_page_mkwrite(struct inode *inode, struct folio *folio,
7755 				   get_block_t get_block)
7756 {
7757 	handle_t *handle;
7758 	loff_t size;
7759 	unsigned long len;
7760 	int credits;
7761 	int ret;
7762 
7763 	credits = ext4_chunk_trans_extent(inode,
7764 			ext4_journal_blocks_per_folio(inode));
7765 	handle = ext4_journal_start(inode, EXT4_HT_WRITE_PAGE, credits);
7766 	if (IS_ERR(handle))
7767 		return PTR_ERR(handle);
7768 
7769 	folio_lock(folio);
7770 	size = i_size_read(inode);
7771 	/* Page got truncated from under us? */
7772 	if (folio->mapping != inode->i_mapping || folio_pos(folio) > size) {
7773 		ret = -EFAULT;
7774 		goto out_error;
7775 	}
7776 
7777 	len = folio_size(folio);
7778 	if (folio_pos(folio) + len > size)
7779 		len = size - folio_pos(folio);
7780 
7781 	ret = ext4_block_write_begin(handle, folio, 0, len, get_block);
7782 	if (ret)
7783 		goto out_error;
7784 
7785 	if (!ext4_should_journal_data(inode)) {
7786 		block_commit_write(folio, 0, len);
7787 		folio_mark_dirty(folio);
7788 	} else {
7789 		ret = ext4_journal_folio_buffers(handle, folio, len);
7790 		if (ret)
7791 			goto out_error;
7792 	}
7793 	ext4_journal_stop(handle);
7794 	folio_wait_stable(folio);
7795 	return ret;
7796 
7797 out_error:
7798 	folio_unlock(folio);
7799 	ext4_journal_stop(handle);
7800 	return ret;
7801 }
7802 
7803 static vm_fault_t ext4_iomap_page_mkwrite(struct vm_fault *vmf)
7804 {
7805 	struct inode *inode = file_inode(vmf->vma->vm_file);
7806 	iomap_iter_next_fn next;
7807 
7808 	/*
7809 	 * ext4_nonda_switch() could writeback this folio, so have to
7810 	 * call it before lock folio.
7811 	 */
7812 	if (test_opt(inode->i_sb, DELALLOC) && !ext4_nonda_switch(inode->i_sb))
7813 		next = ext4_iomap_buffered_da_write_next;
7814 	else
7815 		next = ext4_iomap_buffered_write_next;
7816 
7817 	return iomap_page_mkwrite(vmf, next, NULL);
7818 }
7819 
7820 vm_fault_t ext4_page_mkwrite(struct vm_fault *vmf)
7821 {
7822 	struct vm_area_struct *vma = vmf->vma;
7823 	struct folio *folio = page_folio(vmf->page);
7824 	loff_t size;
7825 	unsigned long len;
7826 	int err;
7827 	vm_fault_t ret;
7828 	struct file *file = vma->vm_file;
7829 	struct inode *inode = file_inode(file);
7830 	struct address_space *mapping = inode->i_mapping;
7831 	get_block_t *get_block = ext4_get_block;
7832 	int retries = 0;
7833 
7834 	if (unlikely(IS_IMMUTABLE(inode)))
7835 		return VM_FAULT_SIGBUS;
7836 
7837 	sb_start_pagefault(inode->i_sb);
7838 	file_update_time(vma->vm_file);
7839 
7840 	filemap_invalidate_lock_shared(mapping);
7841 
7842 	err = ext4_convert_inline_data(inode);
7843 	if (err)
7844 		goto out_ret;
7845 
7846 	if (ext4_inode_buffered_iomap(inode)) {
7847 		ret = ext4_iomap_page_mkwrite(vmf);
7848 		goto out;
7849 	}
7850 
7851 	/*
7852 	 * On data journalling we skip straight to the transaction handle:
7853 	 * there's no delalloc; page truncated will be checked later; the
7854 	 * early return w/ all buffers mapped (calculates size/len) can't
7855 	 * be used; and there's no dioread_nolock, so only ext4_get_block.
7856 	 */
7857 	if (ext4_should_journal_data(inode))
7858 		goto retry_alloc;
7859 
7860 	/* Delalloc case is easy... */
7861 	if (test_opt(inode->i_sb, DELALLOC) &&
7862 	    !ext4_nonda_switch(inode->i_sb)) {
7863 		do {
7864 			err = block_page_mkwrite(vma, vmf,
7865 						   ext4_da_get_block_prep);
7866 		} while (err == -ENOSPC &&
7867 		       ext4_should_retry_alloc(inode->i_sb, &retries));
7868 		goto out_ret;
7869 	}
7870 
7871 	folio_lock(folio);
7872 	size = i_size_read(inode);
7873 	/* Page got truncated from under us? */
7874 	if (folio->mapping != mapping || folio_pos(folio) > size) {
7875 		folio_unlock(folio);
7876 		ret = VM_FAULT_NOPAGE;
7877 		goto out;
7878 	}
7879 
7880 	len = folio_size(folio);
7881 	if (folio_pos(folio) + len > size)
7882 		len = size - folio_pos(folio);
7883 	/*
7884 	 * Return if we have all the buffers mapped. This avoids the need to do
7885 	 * journal_start/journal_stop which can block and take a long time
7886 	 *
7887 	 * This cannot be done for data journalling, as we have to add the
7888 	 * inode to the transaction's list to writeprotect pages on commit.
7889 	 */
7890 	if (folio_buffers(folio)) {
7891 		if (!ext4_walk_page_buffers(NULL, inode, folio_buffers(folio),
7892 					    0, len, NULL,
7893 					    ext4_bh_unmapped)) {
7894 			/* Wait so that we don't change page under IO */
7895 			folio_wait_stable(folio);
7896 			ret = VM_FAULT_LOCKED;
7897 			goto out;
7898 		}
7899 	}
7900 	folio_unlock(folio);
7901 	/* OK, we need to fill the hole... */
7902 	if (ext4_should_dioread_nolock(inode))
7903 		get_block = ext4_get_block_unwritten;
7904 retry_alloc:
7905 	/* Start journal and allocate blocks */
7906 	err = ext4_block_page_mkwrite(inode, folio, get_block);
7907 	if (err == -EAGAIN ||
7908 	    (err == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries)))
7909 		goto retry_alloc;
7910 out_ret:
7911 	ret = vmf_fs_error(err);
7912 out:
7913 	filemap_invalidate_unlock_shared(mapping);
7914 	sb_end_pagefault(inode->i_sb);
7915 	return ret;
7916 }
7917