xref: /linux/fs/f2fs/data.c (revision 0974f486f3dde9df1ad979d4ff341dc9c2d545f5)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * fs/f2fs/data.c
4  *
5  * Copyright (c) 2012 Samsung Electronics Co., Ltd.
6  *             http://www.samsung.com/
7  */
8 #include <linux/fs.h>
9 #include <linux/f2fs_fs.h>
10 #include <linux/sched/mm.h>
11 #include <linux/mpage.h>
12 #include <linux/writeback.h>
13 #include <linux/pagevec.h>
14 #include <linux/blkdev.h>
15 #include <linux/bio.h>
16 #include <linux/blk-crypto.h>
17 #include <linux/swap.h>
18 #include <linux/prefetch.h>
19 #include <linux/uio.h>
20 #include <linux/sched/signal.h>
21 #include <linux/fiemap.h>
22 #include <linux/iomap.h>
23 
24 #include "f2fs.h"
25 #include "node.h"
26 #include "segment.h"
27 #include "iostat.h"
28 #include <trace/events/f2fs.h>
29 
30 #define NUM_PREALLOC_POST_READ_CTXS	128
31 
32 static struct kmem_cache *bio_post_read_ctx_cache;
33 static struct kmem_cache *bio_entry_slab;
34 static mempool_t *bio_post_read_ctx_pool;
35 static struct bio_set f2fs_bioset;
36 
37 #define	F2FS_BIO_POOL_SIZE	NR_CURSEG_TYPE
38 
f2fs_init_bioset(void)39 int __init f2fs_init_bioset(void)
40 {
41 	return bioset_init(&f2fs_bioset, F2FS_BIO_POOL_SIZE,
42 					0, BIOSET_NEED_BVECS);
43 }
44 
f2fs_destroy_bioset(void)45 void f2fs_destroy_bioset(void)
46 {
47 	bioset_exit(&f2fs_bioset);
48 }
49 
f2fs_is_cp_guaranteed(const struct folio * folio)50 bool f2fs_is_cp_guaranteed(const struct folio *folio)
51 {
52 	struct address_space *mapping = folio->mapping;
53 	struct inode *inode;
54 	struct f2fs_sb_info *sbi;
55 
56 	if (fscrypt_is_bounce_folio(folio))
57 		return folio_test_f2fs_gcing(fscrypt_pagecache_folio(folio));
58 
59 	inode = mapping->host;
60 	sbi = F2FS_I_SB(inode);
61 
62 	if (inode->i_ino == F2FS_META_INO(sbi) ||
63 			inode->i_ino == F2FS_NODE_INO(sbi) ||
64 			S_ISDIR(inode->i_mode))
65 		return true;
66 
67 	if ((S_ISREG(inode->i_mode) && IS_NOQUOTA(inode)) ||
68 			folio_test_f2fs_gcing(folio))
69 		return true;
70 	return false;
71 }
72 
__read_io_type(struct folio * folio)73 static enum count_type __read_io_type(struct folio *folio)
74 {
75 	struct address_space *mapping = folio->mapping;
76 
77 	if (mapping) {
78 		struct inode *inode = mapping->host;
79 		struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
80 
81 		if (inode->i_ino == F2FS_META_INO(sbi))
82 			return F2FS_RD_META;
83 
84 		if (inode->i_ino == F2FS_NODE_INO(sbi))
85 			return F2FS_RD_NODE;
86 	}
87 	return F2FS_RD_DATA;
88 }
89 
90 /* postprocessing steps for read bios */
91 enum bio_post_read_step {
92 #ifdef CONFIG_FS_ENCRYPTION
93 	STEP_DECRYPT	= BIT(0),
94 #else
95 	STEP_DECRYPT	= 0,	/* compile out the decryption-related code */
96 #endif
97 #ifdef CONFIG_F2FS_FS_COMPRESSION
98 	STEP_DECOMPRESS	= BIT(1),
99 #else
100 	STEP_DECOMPRESS	= 0,	/* compile out the decompression-related code */
101 #endif
102 #ifdef CONFIG_FS_VERITY
103 	STEP_VERITY	= BIT(2),
104 #else
105 	STEP_VERITY	= 0,	/* compile out the verity-related code */
106 #endif
107 };
108 
109 struct bio_post_read_ctx {
110 	struct bio *bio;
111 	struct f2fs_sb_info *sbi;
112 	struct work_struct work;
113 	unsigned int enabled_steps;
114 	/*
115 	 * decompression_attempted keeps track of whether
116 	 * f2fs_end_read_compressed_page() has been called on the pages in the
117 	 * bio that belong to a compressed cluster yet.
118 	 */
119 	bool decompression_attempted;
120 	block_t fs_blkaddr;
121 };
122 
123 /*
124  * Update and unlock a bio's pages, and free the bio.
125  *
126  * This marks pages up-to-date only if there was no error in the bio (I/O error,
127  * decryption error, or verity error), as indicated by bio->bi_status.
128  *
129  * "Compressed pages" (pagecache pages backed by a compressed cluster on-disk)
130  * aren't marked up-to-date here, as decompression is done on a per-compression-
131  * cluster basis rather than a per-bio basis.  Instead, we only must do two
132  * things for each compressed page here: call f2fs_end_read_compressed_page()
133  * with failed=true if an error occurred before it would have normally gotten
134  * called (i.e., I/O error or decryption error, but *not* verity error), and
135  * release the bio's reference to the decompress_io_ctx of the page's cluster.
136  */
f2fs_finish_read_bio(struct bio * bio,bool in_task)137 static void f2fs_finish_read_bio(struct bio *bio, bool in_task)
138 {
139 	struct folio_iter fi;
140 	struct bio_post_read_ctx *ctx = bio->bi_private;
141 
142 	bio_for_each_folio_all(fi, bio) {
143 		struct folio *folio = fi.folio;
144 
145 		if (f2fs_is_compressed_page(folio)) {
146 			if (ctx && !ctx->decompression_attempted)
147 				f2fs_end_read_compressed_page(folio, true, 0,
148 							in_task);
149 			f2fs_put_folio_dic(folio, in_task);
150 			continue;
151 		}
152 
153 		dec_page_count(F2FS_F_SB(folio), __read_io_type(folio));
154 		folio_end_read(folio, bio->bi_status == BLK_STS_OK);
155 	}
156 
157 	if (ctx)
158 		mempool_free(ctx, bio_post_read_ctx_pool);
159 	bio_put(bio);
160 }
161 
f2fs_verify_bio(struct work_struct * work)162 static void f2fs_verify_bio(struct work_struct *work)
163 {
164 	struct bio_post_read_ctx *ctx =
165 		container_of(work, struct bio_post_read_ctx, work);
166 	struct bio *bio = ctx->bio;
167 	bool may_have_compressed_pages = (ctx->enabled_steps & STEP_DECOMPRESS);
168 
169 	/*
170 	 * fsverity_verify_bio() may call readahead() again, and while verity
171 	 * will be disabled for this, decryption and/or decompression may still
172 	 * be needed, resulting in another bio_post_read_ctx being allocated.
173 	 * So to prevent deadlocks we need to release the current ctx to the
174 	 * mempool first.  This assumes that verity is the last post-read step.
175 	 */
176 	mempool_free(ctx, bio_post_read_ctx_pool);
177 	bio->bi_private = NULL;
178 
179 	/*
180 	 * Verify the bio's pages with fs-verity.  Exclude compressed pages,
181 	 * as those were handled separately by f2fs_end_read_compressed_page().
182 	 */
183 	if (may_have_compressed_pages) {
184 		struct folio_iter fi;
185 
186 		bio_for_each_folio_all(fi, bio) {
187 			struct folio *folio = fi.folio;
188 
189 			if (!f2fs_is_compressed_page(folio) &&
190 			    !fsverity_verify_page(&folio->page)) {
191 				bio->bi_status = BLK_STS_IOERR;
192 				break;
193 			}
194 		}
195 	} else {
196 		fsverity_verify_bio(bio);
197 	}
198 
199 	f2fs_finish_read_bio(bio, true);
200 }
201 
202 /*
203  * If the bio's data needs to be verified with fs-verity, then enqueue the
204  * verity work for the bio.  Otherwise finish the bio now.
205  *
206  * Note that to avoid deadlocks, the verity work can't be done on the
207  * decryption/decompression workqueue.  This is because verifying the data pages
208  * can involve reading verity metadata pages from the file, and these verity
209  * metadata pages may be encrypted and/or compressed.
210  */
f2fs_verify_and_finish_bio(struct bio * bio,bool in_task)211 static void f2fs_verify_and_finish_bio(struct bio *bio, bool in_task)
212 {
213 	struct bio_post_read_ctx *ctx = bio->bi_private;
214 
215 	if (ctx && (ctx->enabled_steps & STEP_VERITY)) {
216 		INIT_WORK(&ctx->work, f2fs_verify_bio);
217 		fsverity_enqueue_verify_work(&ctx->work);
218 	} else {
219 		f2fs_finish_read_bio(bio, in_task);
220 	}
221 }
222 
223 /*
224  * Handle STEP_DECOMPRESS by decompressing any compressed clusters whose last
225  * remaining page was read by @ctx->bio.
226  *
227  * Note that a bio may span clusters (even a mix of compressed and uncompressed
228  * clusters) or be for just part of a cluster.  STEP_DECOMPRESS just indicates
229  * that the bio includes at least one compressed page.  The actual decompression
230  * is done on a per-cluster basis, not a per-bio basis.
231  */
f2fs_handle_step_decompress(struct bio_post_read_ctx * ctx,bool in_task)232 static void f2fs_handle_step_decompress(struct bio_post_read_ctx *ctx,
233 		bool in_task)
234 {
235 	struct folio_iter fi;
236 	bool all_compressed = true;
237 	block_t blkaddr = ctx->fs_blkaddr;
238 
239 	bio_for_each_folio_all(fi, ctx->bio) {
240 		struct folio *folio = fi.folio;
241 
242 		if (f2fs_is_compressed_page(folio))
243 			f2fs_end_read_compressed_page(folio, false, blkaddr,
244 						      in_task);
245 		else
246 			all_compressed = false;
247 
248 		blkaddr++;
249 	}
250 
251 	ctx->decompression_attempted = true;
252 
253 	/*
254 	 * Optimization: if all the bio's pages are compressed, then scheduling
255 	 * the per-bio verity work is unnecessary, as verity will be fully
256 	 * handled at the compression cluster level.
257 	 */
258 	if (all_compressed)
259 		ctx->enabled_steps &= ~STEP_VERITY;
260 }
261 
f2fs_post_read_work(struct work_struct * work)262 static void f2fs_post_read_work(struct work_struct *work)
263 {
264 	struct bio_post_read_ctx *ctx =
265 		container_of(work, struct bio_post_read_ctx, work);
266 	struct bio *bio = ctx->bio;
267 
268 	if ((ctx->enabled_steps & STEP_DECRYPT) && !fscrypt_decrypt_bio(bio)) {
269 		f2fs_finish_read_bio(bio, true);
270 		return;
271 	}
272 
273 	if (ctx->enabled_steps & STEP_DECOMPRESS)
274 		f2fs_handle_step_decompress(ctx, true);
275 
276 	f2fs_verify_and_finish_bio(bio, true);
277 }
278 
f2fs_read_end_io(struct bio * bio)279 static void f2fs_read_end_io(struct bio *bio)
280 {
281 	struct f2fs_sb_info *sbi = F2FS_F_SB(bio_first_folio_all(bio));
282 	struct bio_post_read_ctx *ctx;
283 	bool intask = in_task() && !irqs_disabled();
284 
285 	iostat_update_and_unbind_ctx(bio);
286 	ctx = bio->bi_private;
287 
288 	if (time_to_inject(sbi, FAULT_READ_IO))
289 		bio->bi_status = BLK_STS_IOERR;
290 
291 	if (bio->bi_status != BLK_STS_OK) {
292 		f2fs_finish_read_bio(bio, intask);
293 		return;
294 	}
295 
296 	if (ctx) {
297 		unsigned int enabled_steps = ctx->enabled_steps &
298 					(STEP_DECRYPT | STEP_DECOMPRESS);
299 
300 		/*
301 		 * If we have only decompression step between decompression and
302 		 * decrypt, we don't need post processing for this.
303 		 */
304 		if (enabled_steps == STEP_DECOMPRESS &&
305 				!f2fs_low_mem_mode(sbi)) {
306 			f2fs_handle_step_decompress(ctx, intask);
307 		} else if (enabled_steps) {
308 			INIT_WORK(&ctx->work, f2fs_post_read_work);
309 			queue_work(ctx->sbi->post_read_wq, &ctx->work);
310 			return;
311 		}
312 	}
313 
314 	f2fs_verify_and_finish_bio(bio, intask);
315 }
316 
f2fs_write_end_io(struct bio * bio)317 static void f2fs_write_end_io(struct bio *bio)
318 {
319 	struct f2fs_sb_info *sbi;
320 	struct folio_iter fi;
321 
322 	iostat_update_and_unbind_ctx(bio);
323 	sbi = bio->bi_private;
324 
325 	if (time_to_inject(sbi, FAULT_WRITE_IO))
326 		bio->bi_status = BLK_STS_IOERR;
327 
328 	bio_for_each_folio_all(fi, bio) {
329 		struct folio *folio = fi.folio;
330 		enum count_type type;
331 
332 		if (fscrypt_is_bounce_folio(folio)) {
333 			struct folio *io_folio = folio;
334 
335 			folio = fscrypt_pagecache_folio(io_folio);
336 			fscrypt_free_bounce_page(&io_folio->page);
337 		}
338 
339 #ifdef CONFIG_F2FS_FS_COMPRESSION
340 		if (f2fs_is_compressed_page(folio)) {
341 			f2fs_compress_write_end_io(bio, folio);
342 			continue;
343 		}
344 #endif
345 
346 		type = WB_DATA_TYPE(folio, false);
347 
348 		if (unlikely(bio->bi_status != BLK_STS_OK)) {
349 			mapping_set_error(folio->mapping, -EIO);
350 			if (type == F2FS_WB_CP_DATA)
351 				f2fs_stop_checkpoint(sbi, true,
352 						STOP_CP_REASON_WRITE_FAIL);
353 		}
354 
355 		f2fs_bug_on(sbi, is_node_folio(folio) &&
356 				folio->index != nid_of_node(folio));
357 
358 		dec_page_count(sbi, type);
359 		if (f2fs_in_warm_node_list(sbi, folio))
360 			f2fs_del_fsync_node_entry(sbi, folio);
361 		folio_clear_f2fs_gcing(folio);
362 		folio_end_writeback(folio);
363 	}
364 	if (!get_pages(sbi, F2FS_WB_CP_DATA) &&
365 				wq_has_sleeper(&sbi->cp_wait))
366 		wake_up(&sbi->cp_wait);
367 
368 	bio_put(bio);
369 }
370 
371 #ifdef CONFIG_BLK_DEV_ZONED
f2fs_zone_write_end_io(struct bio * bio)372 static void f2fs_zone_write_end_io(struct bio *bio)
373 {
374 	struct f2fs_bio_info *io = (struct f2fs_bio_info *)bio->bi_private;
375 
376 	bio->bi_private = io->bi_private;
377 	complete(&io->zone_wait);
378 	f2fs_write_end_io(bio);
379 }
380 #endif
381 
f2fs_target_device(struct f2fs_sb_info * sbi,block_t blk_addr,sector_t * sector)382 struct block_device *f2fs_target_device(struct f2fs_sb_info *sbi,
383 		block_t blk_addr, sector_t *sector)
384 {
385 	struct block_device *bdev = sbi->sb->s_bdev;
386 	int i;
387 
388 	if (f2fs_is_multi_device(sbi)) {
389 		for (i = 0; i < sbi->s_ndevs; i++) {
390 			if (FDEV(i).start_blk <= blk_addr &&
391 			    FDEV(i).end_blk >= blk_addr) {
392 				blk_addr -= FDEV(i).start_blk;
393 				bdev = FDEV(i).bdev;
394 				break;
395 			}
396 		}
397 	}
398 
399 	if (sector)
400 		*sector = SECTOR_FROM_BLOCK(blk_addr);
401 	return bdev;
402 }
403 
f2fs_target_device_index(struct f2fs_sb_info * sbi,block_t blkaddr)404 int f2fs_target_device_index(struct f2fs_sb_info *sbi, block_t blkaddr)
405 {
406 	int i;
407 
408 	if (!f2fs_is_multi_device(sbi))
409 		return 0;
410 
411 	for (i = 0; i < sbi->s_ndevs; i++)
412 		if (FDEV(i).start_blk <= blkaddr && FDEV(i).end_blk >= blkaddr)
413 			return i;
414 	return 0;
415 }
416 
f2fs_io_flags(struct f2fs_io_info * fio)417 static blk_opf_t f2fs_io_flags(struct f2fs_io_info *fio)
418 {
419 	unsigned int temp_mask = GENMASK(NR_TEMP_TYPE - 1, 0);
420 	unsigned int fua_flag, meta_flag, io_flag;
421 	blk_opf_t op_flags = 0;
422 
423 	if (fio->op != REQ_OP_WRITE)
424 		return 0;
425 	if (fio->type == DATA)
426 		io_flag = fio->sbi->data_io_flag;
427 	else if (fio->type == NODE)
428 		io_flag = fio->sbi->node_io_flag;
429 	else
430 		return 0;
431 
432 	fua_flag = io_flag & temp_mask;
433 	meta_flag = (io_flag >> NR_TEMP_TYPE) & temp_mask;
434 
435 	/*
436 	 * data/node io flag bits per temp:
437 	 *      REQ_META     |      REQ_FUA      |
438 	 *    5 |    4 |   3 |    2 |    1 |   0 |
439 	 * Cold | Warm | Hot | Cold | Warm | Hot |
440 	 */
441 	if (BIT(fio->temp) & meta_flag)
442 		op_flags |= REQ_META;
443 	if (BIT(fio->temp) & fua_flag)
444 		op_flags |= REQ_FUA;
445 
446 	if (fio->type == DATA &&
447 	    F2FS_I(fio->folio->mapping->host)->ioprio_hint == F2FS_IOPRIO_WRITE)
448 		op_flags |= REQ_PRIO;
449 
450 	return op_flags;
451 }
452 
__bio_alloc(struct f2fs_io_info * fio,int npages)453 static struct bio *__bio_alloc(struct f2fs_io_info *fio, int npages)
454 {
455 	struct f2fs_sb_info *sbi = fio->sbi;
456 	struct block_device *bdev;
457 	sector_t sector;
458 	struct bio *bio;
459 
460 	bdev = f2fs_target_device(sbi, fio->new_blkaddr, &sector);
461 	bio = bio_alloc_bioset(bdev, npages,
462 				fio->op | fio->op_flags | f2fs_io_flags(fio),
463 				GFP_NOIO, &f2fs_bioset);
464 	bio->bi_iter.bi_sector = sector;
465 	if (is_read_io(fio->op)) {
466 		bio->bi_end_io = f2fs_read_end_io;
467 		bio->bi_private = NULL;
468 	} else {
469 		bio->bi_end_io = f2fs_write_end_io;
470 		bio->bi_private = sbi;
471 		bio->bi_write_hint = f2fs_io_type_to_rw_hint(sbi,
472 						fio->type, fio->temp);
473 	}
474 	iostat_alloc_and_bind_ctx(sbi, bio, NULL);
475 
476 	if (fio->io_wbc)
477 		wbc_init_bio(fio->io_wbc, bio);
478 
479 	return bio;
480 }
481 
f2fs_set_bio_crypt_ctx(struct bio * bio,const struct inode * inode,pgoff_t first_idx,const struct f2fs_io_info * fio,gfp_t gfp_mask)482 static void f2fs_set_bio_crypt_ctx(struct bio *bio, const struct inode *inode,
483 				  pgoff_t first_idx,
484 				  const struct f2fs_io_info *fio,
485 				  gfp_t gfp_mask)
486 {
487 	/*
488 	 * The f2fs garbage collector sets ->encrypted_page when it wants to
489 	 * read/write raw data without encryption.
490 	 */
491 	if (!fio || !fio->encrypted_page)
492 		fscrypt_set_bio_crypt_ctx(bio, inode, first_idx, gfp_mask);
493 }
494 
f2fs_crypt_mergeable_bio(struct bio * bio,const struct inode * inode,pgoff_t next_idx,const struct f2fs_io_info * fio)495 static bool f2fs_crypt_mergeable_bio(struct bio *bio, const struct inode *inode,
496 				     pgoff_t next_idx,
497 				     const struct f2fs_io_info *fio)
498 {
499 	/*
500 	 * The f2fs garbage collector sets ->encrypted_page when it wants to
501 	 * read/write raw data without encryption.
502 	 */
503 	if (fio && fio->encrypted_page)
504 		return !bio_has_crypt_ctx(bio);
505 
506 	return fscrypt_mergeable_bio(bio, inode, next_idx);
507 }
508 
f2fs_submit_read_bio(struct f2fs_sb_info * sbi,struct bio * bio,enum page_type type)509 void f2fs_submit_read_bio(struct f2fs_sb_info *sbi, struct bio *bio,
510 				 enum page_type type)
511 {
512 	WARN_ON_ONCE(!is_read_io(bio_op(bio)));
513 	trace_f2fs_submit_read_bio(sbi->sb, type, bio);
514 
515 	iostat_update_submit_ctx(bio, type);
516 	submit_bio(bio);
517 }
518 
f2fs_submit_write_bio(struct f2fs_sb_info * sbi,struct bio * bio,enum page_type type)519 static void f2fs_submit_write_bio(struct f2fs_sb_info *sbi, struct bio *bio,
520 				  enum page_type type)
521 {
522 	WARN_ON_ONCE(is_read_io(bio_op(bio)));
523 	trace_f2fs_submit_write_bio(sbi->sb, type, bio);
524 	iostat_update_submit_ctx(bio, type);
525 	submit_bio(bio);
526 }
527 
__submit_merged_bio(struct f2fs_bio_info * io)528 static void __submit_merged_bio(struct f2fs_bio_info *io)
529 {
530 	struct f2fs_io_info *fio = &io->fio;
531 
532 	if (!io->bio)
533 		return;
534 
535 	if (is_read_io(fio->op)) {
536 		trace_f2fs_prepare_read_bio(io->sbi->sb, fio->type, io->bio);
537 		f2fs_submit_read_bio(io->sbi, io->bio, fio->type);
538 	} else {
539 		trace_f2fs_prepare_write_bio(io->sbi->sb, fio->type, io->bio);
540 		f2fs_submit_write_bio(io->sbi, io->bio, fio->type);
541 	}
542 	io->bio = NULL;
543 }
544 
__has_merged_page(struct bio * bio,struct inode * inode,struct folio * folio,nid_t ino)545 static bool __has_merged_page(struct bio *bio, struct inode *inode,
546 						struct folio *folio, nid_t ino)
547 {
548 	struct folio_iter fi;
549 
550 	if (!bio)
551 		return false;
552 
553 	if (!inode && !folio && !ino)
554 		return true;
555 
556 	bio_for_each_folio_all(fi, bio) {
557 		struct folio *target = fi.folio;
558 
559 		if (fscrypt_is_bounce_folio(target)) {
560 			target = fscrypt_pagecache_folio(target);
561 			if (IS_ERR(target))
562 				continue;
563 		}
564 		if (f2fs_is_compressed_page(target)) {
565 			target = f2fs_compress_control_folio(target);
566 			if (IS_ERR(target))
567 				continue;
568 		}
569 
570 		if (inode && inode == target->mapping->host)
571 			return true;
572 		if (folio && folio == target)
573 			return true;
574 		if (ino && ino == ino_of_node(target))
575 			return true;
576 	}
577 
578 	return false;
579 }
580 
f2fs_init_write_merge_io(struct f2fs_sb_info * sbi)581 int f2fs_init_write_merge_io(struct f2fs_sb_info *sbi)
582 {
583 	int i;
584 
585 	for (i = 0; i < NR_PAGE_TYPE; i++) {
586 		int n = (i == META) ? 1 : NR_TEMP_TYPE;
587 		int j;
588 
589 		sbi->write_io[i] = f2fs_kmalloc(sbi,
590 				array_size(n, sizeof(struct f2fs_bio_info)),
591 				GFP_KERNEL);
592 		if (!sbi->write_io[i])
593 			return -ENOMEM;
594 
595 		for (j = HOT; j < n; j++) {
596 			struct f2fs_bio_info *io = &sbi->write_io[i][j];
597 
598 			init_f2fs_rwsem(&io->io_rwsem);
599 			io->sbi = sbi;
600 			io->bio = NULL;
601 			io->last_block_in_bio = 0;
602 			spin_lock_init(&io->io_lock);
603 			INIT_LIST_HEAD(&io->io_list);
604 			INIT_LIST_HEAD(&io->bio_list);
605 			init_f2fs_rwsem(&io->bio_list_lock);
606 #ifdef CONFIG_BLK_DEV_ZONED
607 			init_completion(&io->zone_wait);
608 			io->zone_pending_bio = NULL;
609 			io->bi_private = NULL;
610 #endif
611 		}
612 	}
613 
614 	return 0;
615 }
616 
__f2fs_submit_merged_write(struct f2fs_sb_info * sbi,enum page_type type,enum temp_type temp)617 static void __f2fs_submit_merged_write(struct f2fs_sb_info *sbi,
618 				enum page_type type, enum temp_type temp)
619 {
620 	enum page_type btype = PAGE_TYPE_OF_BIO(type);
621 	struct f2fs_bio_info *io = sbi->write_io[btype] + temp;
622 
623 	f2fs_down_write(&io->io_rwsem);
624 
625 	if (!io->bio)
626 		goto unlock_out;
627 
628 	/* change META to META_FLUSH in the checkpoint procedure */
629 	if (type >= META_FLUSH) {
630 		io->fio.type = META_FLUSH;
631 		io->bio->bi_opf |= REQ_META | REQ_PRIO | REQ_SYNC;
632 		if (!test_opt(sbi, NOBARRIER))
633 			io->bio->bi_opf |= REQ_PREFLUSH | REQ_FUA;
634 	}
635 	__submit_merged_bio(io);
636 unlock_out:
637 	f2fs_up_write(&io->io_rwsem);
638 }
639 
__submit_merged_write_cond(struct f2fs_sb_info * sbi,struct inode * inode,struct folio * folio,nid_t ino,enum page_type type,bool force)640 static void __submit_merged_write_cond(struct f2fs_sb_info *sbi,
641 				struct inode *inode, struct folio *folio,
642 				nid_t ino, enum page_type type, bool force)
643 {
644 	enum temp_type temp;
645 	bool ret = true;
646 
647 	for (temp = HOT; temp < NR_TEMP_TYPE; temp++) {
648 		if (!force)	{
649 			enum page_type btype = PAGE_TYPE_OF_BIO(type);
650 			struct f2fs_bio_info *io = sbi->write_io[btype] + temp;
651 
652 			f2fs_down_read(&io->io_rwsem);
653 			ret = __has_merged_page(io->bio, inode, folio, ino);
654 			f2fs_up_read(&io->io_rwsem);
655 		}
656 		if (ret)
657 			__f2fs_submit_merged_write(sbi, type, temp);
658 
659 		/* TODO: use HOT temp only for meta pages now. */
660 		if (type >= META)
661 			break;
662 	}
663 }
664 
f2fs_submit_merged_write(struct f2fs_sb_info * sbi,enum page_type type)665 void f2fs_submit_merged_write(struct f2fs_sb_info *sbi, enum page_type type)
666 {
667 	__submit_merged_write_cond(sbi, NULL, NULL, 0, type, true);
668 }
669 
f2fs_submit_merged_write_cond(struct f2fs_sb_info * sbi,struct inode * inode,struct folio * folio,nid_t ino,enum page_type type)670 void f2fs_submit_merged_write_cond(struct f2fs_sb_info *sbi,
671 				struct inode *inode, struct folio *folio,
672 				nid_t ino, enum page_type type)
673 {
674 	__submit_merged_write_cond(sbi, inode, folio, ino, type, false);
675 }
676 
f2fs_flush_merged_writes(struct f2fs_sb_info * sbi)677 void f2fs_flush_merged_writes(struct f2fs_sb_info *sbi)
678 {
679 	f2fs_submit_merged_write(sbi, DATA);
680 	f2fs_submit_merged_write(sbi, NODE);
681 	f2fs_submit_merged_write(sbi, META);
682 }
683 
684 /*
685  * Fill the locked page with data located in the block address.
686  * A caller needs to unlock the page on failure.
687  */
f2fs_submit_page_bio(struct f2fs_io_info * fio)688 int f2fs_submit_page_bio(struct f2fs_io_info *fio)
689 {
690 	struct bio *bio;
691 	struct folio *fio_folio = fio->folio;
692 	struct folio *data_folio = fio->encrypted_page ?
693 			page_folio(fio->encrypted_page) : fio_folio;
694 
695 	if (!f2fs_is_valid_blkaddr(fio->sbi, fio->new_blkaddr,
696 			fio->is_por ? META_POR : (__is_meta_io(fio) ?
697 			META_GENERIC : DATA_GENERIC_ENHANCE)))
698 		return -EFSCORRUPTED;
699 
700 	trace_f2fs_submit_folio_bio(data_folio, fio);
701 
702 	/* Allocate a new bio */
703 	bio = __bio_alloc(fio, 1);
704 
705 	f2fs_set_bio_crypt_ctx(bio, fio_folio->mapping->host,
706 			fio_folio->index, fio, GFP_NOIO);
707 	bio_add_folio_nofail(bio, data_folio, folio_size(data_folio), 0);
708 
709 	if (fio->io_wbc && !is_read_io(fio->op))
710 		wbc_account_cgroup_owner(fio->io_wbc, fio_folio, PAGE_SIZE);
711 
712 	inc_page_count(fio->sbi, is_read_io(fio->op) ?
713 			__read_io_type(data_folio) : WB_DATA_TYPE(fio->folio, false));
714 
715 	if (is_read_io(bio_op(bio)))
716 		f2fs_submit_read_bio(fio->sbi, bio, fio->type);
717 	else
718 		f2fs_submit_write_bio(fio->sbi, bio, fio->type);
719 	return 0;
720 }
721 
page_is_mergeable(struct f2fs_sb_info * sbi,struct bio * bio,block_t last_blkaddr,block_t cur_blkaddr)722 static bool page_is_mergeable(struct f2fs_sb_info *sbi, struct bio *bio,
723 				block_t last_blkaddr, block_t cur_blkaddr)
724 {
725 	if (unlikely(sbi->max_io_bytes &&
726 			bio->bi_iter.bi_size >= sbi->max_io_bytes))
727 		return false;
728 	if (last_blkaddr + 1 != cur_blkaddr)
729 		return false;
730 	return bio->bi_bdev == f2fs_target_device(sbi, cur_blkaddr, NULL);
731 }
732 
io_type_is_mergeable(struct f2fs_bio_info * io,struct f2fs_io_info * fio)733 static bool io_type_is_mergeable(struct f2fs_bio_info *io,
734 						struct f2fs_io_info *fio)
735 {
736 	if (io->fio.op != fio->op)
737 		return false;
738 	return io->fio.op_flags == fio->op_flags;
739 }
740 
io_is_mergeable(struct f2fs_sb_info * sbi,struct bio * bio,struct f2fs_bio_info * io,struct f2fs_io_info * fio,block_t last_blkaddr,block_t cur_blkaddr)741 static bool io_is_mergeable(struct f2fs_sb_info *sbi, struct bio *bio,
742 					struct f2fs_bio_info *io,
743 					struct f2fs_io_info *fio,
744 					block_t last_blkaddr,
745 					block_t cur_blkaddr)
746 {
747 	if (!page_is_mergeable(sbi, bio, last_blkaddr, cur_blkaddr))
748 		return false;
749 	return io_type_is_mergeable(io, fio);
750 }
751 
add_bio_entry(struct f2fs_sb_info * sbi,struct bio * bio,struct page * page,enum temp_type temp)752 static void add_bio_entry(struct f2fs_sb_info *sbi, struct bio *bio,
753 				struct page *page, enum temp_type temp)
754 {
755 	struct f2fs_bio_info *io = sbi->write_io[DATA] + temp;
756 	struct bio_entry *be;
757 
758 	be = f2fs_kmem_cache_alloc(bio_entry_slab, GFP_NOFS, true, NULL);
759 	be->bio = bio;
760 	bio_get(bio);
761 
762 	if (bio_add_page(bio, page, PAGE_SIZE, 0) != PAGE_SIZE)
763 		f2fs_bug_on(sbi, 1);
764 
765 	f2fs_down_write(&io->bio_list_lock);
766 	list_add_tail(&be->list, &io->bio_list);
767 	f2fs_up_write(&io->bio_list_lock);
768 }
769 
del_bio_entry(struct bio_entry * be)770 static void del_bio_entry(struct bio_entry *be)
771 {
772 	list_del(&be->list);
773 	kmem_cache_free(bio_entry_slab, be);
774 }
775 
add_ipu_page(struct f2fs_io_info * fio,struct bio ** bio,struct page * page)776 static int add_ipu_page(struct f2fs_io_info *fio, struct bio **bio,
777 							struct page *page)
778 {
779 	struct folio *fio_folio = fio->folio;
780 	struct f2fs_sb_info *sbi = fio->sbi;
781 	enum temp_type temp;
782 	bool found = false;
783 	int ret = -EAGAIN;
784 
785 	for (temp = HOT; temp < NR_TEMP_TYPE && !found; temp++) {
786 		struct f2fs_bio_info *io = sbi->write_io[DATA] + temp;
787 		struct list_head *head = &io->bio_list;
788 		struct bio_entry *be;
789 
790 		f2fs_down_write(&io->bio_list_lock);
791 		list_for_each_entry(be, head, list) {
792 			if (be->bio != *bio)
793 				continue;
794 
795 			found = true;
796 
797 			f2fs_bug_on(sbi, !page_is_mergeable(sbi, *bio,
798 							    *fio->last_block,
799 							    fio->new_blkaddr));
800 			if (f2fs_crypt_mergeable_bio(*bio,
801 					fio_folio->mapping->host,
802 					fio_folio->index, fio) &&
803 			    bio_add_page(*bio, page, PAGE_SIZE, 0) ==
804 					PAGE_SIZE) {
805 				ret = 0;
806 				break;
807 			}
808 
809 			/* page can't be merged into bio; submit the bio */
810 			del_bio_entry(be);
811 			f2fs_submit_write_bio(sbi, *bio, DATA);
812 			break;
813 		}
814 		f2fs_up_write(&io->bio_list_lock);
815 	}
816 
817 	if (ret) {
818 		bio_put(*bio);
819 		*bio = NULL;
820 	}
821 
822 	return ret;
823 }
824 
f2fs_submit_merged_ipu_write(struct f2fs_sb_info * sbi,struct bio ** bio,struct folio * folio)825 void f2fs_submit_merged_ipu_write(struct f2fs_sb_info *sbi,
826 					struct bio **bio, struct folio *folio)
827 {
828 	enum temp_type temp;
829 	bool found = false;
830 	struct bio *target = bio ? *bio : NULL;
831 
832 	f2fs_bug_on(sbi, !target && !folio);
833 
834 	for (temp = HOT; temp < NR_TEMP_TYPE && !found; temp++) {
835 		struct f2fs_bio_info *io = sbi->write_io[DATA] + temp;
836 		struct list_head *head = &io->bio_list;
837 		struct bio_entry *be;
838 
839 		if (list_empty(head))
840 			continue;
841 
842 		f2fs_down_read(&io->bio_list_lock);
843 		list_for_each_entry(be, head, list) {
844 			if (target)
845 				found = (target == be->bio);
846 			else
847 				found = __has_merged_page(be->bio, NULL,
848 							folio, 0);
849 			if (found)
850 				break;
851 		}
852 		f2fs_up_read(&io->bio_list_lock);
853 
854 		if (!found)
855 			continue;
856 
857 		found = false;
858 
859 		f2fs_down_write(&io->bio_list_lock);
860 		list_for_each_entry(be, head, list) {
861 			if (target)
862 				found = (target == be->bio);
863 			else
864 				found = __has_merged_page(be->bio, NULL,
865 							folio, 0);
866 			if (found) {
867 				target = be->bio;
868 				del_bio_entry(be);
869 				break;
870 			}
871 		}
872 		f2fs_up_write(&io->bio_list_lock);
873 	}
874 
875 	if (found)
876 		f2fs_submit_write_bio(sbi, target, DATA);
877 	if (bio && *bio) {
878 		bio_put(*bio);
879 		*bio = NULL;
880 	}
881 }
882 
f2fs_merge_page_bio(struct f2fs_io_info * fio)883 int f2fs_merge_page_bio(struct f2fs_io_info *fio)
884 {
885 	struct bio *bio = *fio->bio;
886 	struct folio *data_folio = fio->encrypted_page ?
887 			page_folio(fio->encrypted_page) : fio->folio;
888 	struct folio *folio = fio->folio;
889 
890 	if (!f2fs_is_valid_blkaddr(fio->sbi, fio->new_blkaddr,
891 			__is_meta_io(fio) ? META_GENERIC : DATA_GENERIC))
892 		return -EFSCORRUPTED;
893 
894 	trace_f2fs_submit_folio_bio(data_folio, fio);
895 
896 	if (bio && !page_is_mergeable(fio->sbi, bio, *fio->last_block,
897 						fio->new_blkaddr))
898 		f2fs_submit_merged_ipu_write(fio->sbi, &bio, NULL);
899 alloc_new:
900 	if (!bio) {
901 		bio = __bio_alloc(fio, BIO_MAX_VECS);
902 		f2fs_set_bio_crypt_ctx(bio, folio->mapping->host,
903 				folio->index, fio, GFP_NOIO);
904 
905 		add_bio_entry(fio->sbi, bio, &data_folio->page, fio->temp);
906 	} else {
907 		if (add_ipu_page(fio, &bio, &data_folio->page))
908 			goto alloc_new;
909 	}
910 
911 	if (fio->io_wbc)
912 		wbc_account_cgroup_owner(fio->io_wbc, folio, folio_size(folio));
913 
914 	inc_page_count(fio->sbi, WB_DATA_TYPE(data_folio, false));
915 
916 	*fio->last_block = fio->new_blkaddr;
917 	*fio->bio = bio;
918 
919 	return 0;
920 }
921 
922 #ifdef CONFIG_BLK_DEV_ZONED
is_end_zone_blkaddr(struct f2fs_sb_info * sbi,block_t blkaddr)923 static bool is_end_zone_blkaddr(struct f2fs_sb_info *sbi, block_t blkaddr)
924 {
925 	struct block_device *bdev = sbi->sb->s_bdev;
926 	int devi = 0;
927 
928 	if (f2fs_is_multi_device(sbi)) {
929 		devi = f2fs_target_device_index(sbi, blkaddr);
930 		if (blkaddr < FDEV(devi).start_blk ||
931 		    blkaddr > FDEV(devi).end_blk) {
932 			f2fs_err(sbi, "Invalid block %x", blkaddr);
933 			return false;
934 		}
935 		blkaddr -= FDEV(devi).start_blk;
936 		bdev = FDEV(devi).bdev;
937 	}
938 	return bdev_is_zoned(bdev) &&
939 		f2fs_blkz_is_seq(sbi, devi, blkaddr) &&
940 		(blkaddr % sbi->blocks_per_blkz == sbi->blocks_per_blkz - 1);
941 }
942 #endif
943 
f2fs_submit_page_write(struct f2fs_io_info * fio)944 void f2fs_submit_page_write(struct f2fs_io_info *fio)
945 {
946 	struct f2fs_sb_info *sbi = fio->sbi;
947 	enum page_type btype = PAGE_TYPE_OF_BIO(fio->type);
948 	struct f2fs_bio_info *io = sbi->write_io[btype] + fio->temp;
949 	struct folio *bio_folio;
950 	enum count_type type;
951 
952 	f2fs_bug_on(sbi, is_read_io(fio->op));
953 
954 	f2fs_down_write(&io->io_rwsem);
955 next:
956 #ifdef CONFIG_BLK_DEV_ZONED
957 	if (f2fs_sb_has_blkzoned(sbi) && btype < META && io->zone_pending_bio) {
958 		wait_for_completion_io(&io->zone_wait);
959 		bio_put(io->zone_pending_bio);
960 		io->zone_pending_bio = NULL;
961 		io->bi_private = NULL;
962 	}
963 #endif
964 
965 	if (fio->in_list) {
966 		spin_lock(&io->io_lock);
967 		if (list_empty(&io->io_list)) {
968 			spin_unlock(&io->io_lock);
969 			goto out;
970 		}
971 		fio = list_first_entry(&io->io_list,
972 						struct f2fs_io_info, list);
973 		list_del(&fio->list);
974 		spin_unlock(&io->io_lock);
975 	}
976 
977 	verify_fio_blkaddr(fio);
978 
979 	if (fio->encrypted_page)
980 		bio_folio = page_folio(fio->encrypted_page);
981 	else if (fio->compressed_page)
982 		bio_folio = page_folio(fio->compressed_page);
983 	else
984 		bio_folio = fio->folio;
985 
986 	/* set submitted = true as a return value */
987 	fio->submitted = 1;
988 
989 	type = WB_DATA_TYPE(bio_folio, fio->compressed_page);
990 	inc_page_count(sbi, type);
991 
992 	if (io->bio &&
993 	    (!io_is_mergeable(sbi, io->bio, io, fio, io->last_block_in_bio,
994 			      fio->new_blkaddr) ||
995 	     !f2fs_crypt_mergeable_bio(io->bio, fio_inode(fio),
996 				bio_folio->index, fio)))
997 		__submit_merged_bio(io);
998 alloc_new:
999 	if (io->bio == NULL) {
1000 		io->bio = __bio_alloc(fio, BIO_MAX_VECS);
1001 		f2fs_set_bio_crypt_ctx(io->bio, fio_inode(fio),
1002 				bio_folio->index, fio, GFP_NOIO);
1003 		io->fio = *fio;
1004 	}
1005 
1006 	if (!bio_add_folio(io->bio, bio_folio, folio_size(bio_folio), 0)) {
1007 		__submit_merged_bio(io);
1008 		goto alloc_new;
1009 	}
1010 
1011 	if (fio->io_wbc)
1012 		wbc_account_cgroup_owner(fio->io_wbc, fio->folio,
1013 				folio_size(fio->folio));
1014 
1015 	io->last_block_in_bio = fio->new_blkaddr;
1016 
1017 	trace_f2fs_submit_folio_write(fio->folio, fio);
1018 #ifdef CONFIG_BLK_DEV_ZONED
1019 	if (f2fs_sb_has_blkzoned(sbi) && btype < META &&
1020 			is_end_zone_blkaddr(sbi, fio->new_blkaddr)) {
1021 		bio_get(io->bio);
1022 		reinit_completion(&io->zone_wait);
1023 		io->bi_private = io->bio->bi_private;
1024 		io->bio->bi_private = io;
1025 		io->bio->bi_end_io = f2fs_zone_write_end_io;
1026 		io->zone_pending_bio = io->bio;
1027 		__submit_merged_bio(io);
1028 	}
1029 #endif
1030 	if (fio->in_list)
1031 		goto next;
1032 out:
1033 	if (is_sbi_flag_set(sbi, SBI_IS_SHUTDOWN) ||
1034 				!f2fs_is_checkpoint_ready(sbi))
1035 		__submit_merged_bio(io);
1036 	f2fs_up_write(&io->io_rwsem);
1037 }
1038 
f2fs_grab_read_bio(struct inode * inode,block_t blkaddr,unsigned nr_pages,blk_opf_t op_flag,pgoff_t first_idx,bool for_write)1039 static struct bio *f2fs_grab_read_bio(struct inode *inode, block_t blkaddr,
1040 				      unsigned nr_pages, blk_opf_t op_flag,
1041 				      pgoff_t first_idx, bool for_write)
1042 {
1043 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1044 	struct bio *bio;
1045 	struct bio_post_read_ctx *ctx = NULL;
1046 	unsigned int post_read_steps = 0;
1047 	sector_t sector;
1048 	struct block_device *bdev = f2fs_target_device(sbi, blkaddr, &sector);
1049 
1050 	bio = bio_alloc_bioset(bdev, bio_max_segs(nr_pages),
1051 			       REQ_OP_READ | op_flag,
1052 			       for_write ? GFP_NOIO : GFP_KERNEL, &f2fs_bioset);
1053 	bio->bi_iter.bi_sector = sector;
1054 	f2fs_set_bio_crypt_ctx(bio, inode, first_idx, NULL, GFP_NOFS);
1055 	bio->bi_end_io = f2fs_read_end_io;
1056 
1057 	if (fscrypt_inode_uses_fs_layer_crypto(inode))
1058 		post_read_steps |= STEP_DECRYPT;
1059 
1060 	if (f2fs_need_verity(inode, first_idx))
1061 		post_read_steps |= STEP_VERITY;
1062 
1063 	/*
1064 	 * STEP_DECOMPRESS is handled specially, since a compressed file might
1065 	 * contain both compressed and uncompressed clusters.  We'll allocate a
1066 	 * bio_post_read_ctx if the file is compressed, but the caller is
1067 	 * responsible for enabling STEP_DECOMPRESS if it's actually needed.
1068 	 */
1069 
1070 	if (post_read_steps || f2fs_compressed_file(inode)) {
1071 		/* Due to the mempool, this never fails. */
1072 		ctx = mempool_alloc(bio_post_read_ctx_pool, GFP_NOFS);
1073 		ctx->bio = bio;
1074 		ctx->sbi = sbi;
1075 		ctx->enabled_steps = post_read_steps;
1076 		ctx->fs_blkaddr = blkaddr;
1077 		ctx->decompression_attempted = false;
1078 		bio->bi_private = ctx;
1079 	}
1080 	iostat_alloc_and_bind_ctx(sbi, bio, ctx);
1081 
1082 	return bio;
1083 }
1084 
1085 /* This can handle encryption stuffs */
f2fs_submit_page_read(struct inode * inode,struct folio * folio,block_t blkaddr,blk_opf_t op_flags,bool for_write)1086 static int f2fs_submit_page_read(struct inode *inode, struct folio *folio,
1087 				 block_t blkaddr, blk_opf_t op_flags,
1088 				 bool for_write)
1089 {
1090 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1091 	struct bio *bio;
1092 
1093 	bio = f2fs_grab_read_bio(inode, blkaddr, 1, op_flags,
1094 					folio->index, for_write);
1095 	if (IS_ERR(bio))
1096 		return PTR_ERR(bio);
1097 
1098 	/* wait for GCed page writeback via META_MAPPING */
1099 	f2fs_wait_on_block_writeback(inode, blkaddr);
1100 
1101 	if (!bio_add_folio(bio, folio, PAGE_SIZE, 0)) {
1102 		iostat_update_and_unbind_ctx(bio);
1103 		if (bio->bi_private)
1104 			mempool_free(bio->bi_private, bio_post_read_ctx_pool);
1105 		bio_put(bio);
1106 		return -EFAULT;
1107 	}
1108 	inc_page_count(sbi, F2FS_RD_DATA);
1109 	f2fs_update_iostat(sbi, NULL, FS_DATA_READ_IO, F2FS_BLKSIZE);
1110 	f2fs_submit_read_bio(sbi, bio, DATA);
1111 	return 0;
1112 }
1113 
__set_data_blkaddr(struct dnode_of_data * dn,block_t blkaddr)1114 static void __set_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr)
1115 {
1116 	__le32 *addr = get_dnode_addr(dn->inode, dn->node_folio);
1117 
1118 	dn->data_blkaddr = blkaddr;
1119 	addr[dn->ofs_in_node] = cpu_to_le32(dn->data_blkaddr);
1120 }
1121 
1122 /*
1123  * Lock ordering for the change of data block address:
1124  * ->data_page
1125  *  ->node_folio
1126  *    update block addresses in the node page
1127  */
f2fs_set_data_blkaddr(struct dnode_of_data * dn,block_t blkaddr)1128 void f2fs_set_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr)
1129 {
1130 	f2fs_folio_wait_writeback(dn->node_folio, NODE, true, true);
1131 	__set_data_blkaddr(dn, blkaddr);
1132 	if (folio_mark_dirty(dn->node_folio))
1133 		dn->node_changed = true;
1134 }
1135 
f2fs_update_data_blkaddr(struct dnode_of_data * dn,block_t blkaddr)1136 void f2fs_update_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr)
1137 {
1138 	f2fs_set_data_blkaddr(dn, blkaddr);
1139 	f2fs_update_read_extent_cache(dn);
1140 }
1141 
1142 /* dn->ofs_in_node will be returned with up-to-date last block pointer */
f2fs_reserve_new_blocks(struct dnode_of_data * dn,blkcnt_t count)1143 int f2fs_reserve_new_blocks(struct dnode_of_data *dn, blkcnt_t count)
1144 {
1145 	struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
1146 	int err;
1147 
1148 	if (!count)
1149 		return 0;
1150 
1151 	if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC)))
1152 		return -EPERM;
1153 	err = inc_valid_block_count(sbi, dn->inode, &count, true);
1154 	if (unlikely(err))
1155 		return err;
1156 
1157 	trace_f2fs_reserve_new_blocks(dn->inode, dn->nid,
1158 						dn->ofs_in_node, count);
1159 
1160 	f2fs_folio_wait_writeback(dn->node_folio, NODE, true, true);
1161 
1162 	for (; count > 0; dn->ofs_in_node++) {
1163 		block_t blkaddr = f2fs_data_blkaddr(dn);
1164 
1165 		if (blkaddr == NULL_ADDR) {
1166 			__set_data_blkaddr(dn, NEW_ADDR);
1167 			count--;
1168 		}
1169 	}
1170 
1171 	if (folio_mark_dirty(dn->node_folio))
1172 		dn->node_changed = true;
1173 	return 0;
1174 }
1175 
1176 /* Should keep dn->ofs_in_node unchanged */
f2fs_reserve_new_block(struct dnode_of_data * dn)1177 int f2fs_reserve_new_block(struct dnode_of_data *dn)
1178 {
1179 	unsigned int ofs_in_node = dn->ofs_in_node;
1180 	int ret;
1181 
1182 	ret = f2fs_reserve_new_blocks(dn, 1);
1183 	dn->ofs_in_node = ofs_in_node;
1184 	return ret;
1185 }
1186 
f2fs_reserve_block(struct dnode_of_data * dn,pgoff_t index)1187 int f2fs_reserve_block(struct dnode_of_data *dn, pgoff_t index)
1188 {
1189 	bool need_put = dn->inode_folio ? false : true;
1190 	int err;
1191 
1192 	err = f2fs_get_dnode_of_data(dn, index, ALLOC_NODE);
1193 	if (err)
1194 		return err;
1195 
1196 	if (dn->data_blkaddr == NULL_ADDR)
1197 		err = f2fs_reserve_new_block(dn);
1198 	if (err || need_put)
1199 		f2fs_put_dnode(dn);
1200 	return err;
1201 }
1202 
f2fs_get_read_data_folio(struct inode * inode,pgoff_t index,blk_opf_t op_flags,bool for_write,pgoff_t * next_pgofs)1203 struct folio *f2fs_get_read_data_folio(struct inode *inode, pgoff_t index,
1204 		blk_opf_t op_flags, bool for_write, pgoff_t *next_pgofs)
1205 {
1206 	struct address_space *mapping = inode->i_mapping;
1207 	struct dnode_of_data dn;
1208 	struct folio *folio;
1209 	int err;
1210 
1211 	folio = f2fs_grab_cache_folio(mapping, index, for_write);
1212 	if (IS_ERR(folio))
1213 		return folio;
1214 
1215 	if (f2fs_lookup_read_extent_cache_block(inode, index,
1216 						&dn.data_blkaddr)) {
1217 		if (!f2fs_is_valid_blkaddr(F2FS_I_SB(inode), dn.data_blkaddr,
1218 						DATA_GENERIC_ENHANCE_READ)) {
1219 			err = -EFSCORRUPTED;
1220 			goto put_err;
1221 		}
1222 		goto got_it;
1223 	}
1224 
1225 	set_new_dnode(&dn, inode, NULL, NULL, 0);
1226 	err = f2fs_get_dnode_of_data(&dn, index, LOOKUP_NODE);
1227 	if (err) {
1228 		if (err == -ENOENT && next_pgofs)
1229 			*next_pgofs = f2fs_get_next_page_offset(&dn, index);
1230 		goto put_err;
1231 	}
1232 	f2fs_put_dnode(&dn);
1233 
1234 	if (unlikely(dn.data_blkaddr == NULL_ADDR)) {
1235 		err = -ENOENT;
1236 		if (next_pgofs)
1237 			*next_pgofs = index + 1;
1238 		goto put_err;
1239 	}
1240 	if (dn.data_blkaddr != NEW_ADDR &&
1241 			!f2fs_is_valid_blkaddr(F2FS_I_SB(inode),
1242 						dn.data_blkaddr,
1243 						DATA_GENERIC_ENHANCE)) {
1244 		err = -EFSCORRUPTED;
1245 		goto put_err;
1246 	}
1247 got_it:
1248 	if (folio_test_uptodate(folio)) {
1249 		folio_unlock(folio);
1250 		return folio;
1251 	}
1252 
1253 	/*
1254 	 * A new dentry page is allocated but not able to be written, since its
1255 	 * new inode page couldn't be allocated due to -ENOSPC.
1256 	 * In such the case, its blkaddr can be remained as NEW_ADDR.
1257 	 * see, f2fs_add_link -> f2fs_get_new_data_folio ->
1258 	 * f2fs_init_inode_metadata.
1259 	 */
1260 	if (dn.data_blkaddr == NEW_ADDR) {
1261 		folio_zero_segment(folio, 0, folio_size(folio));
1262 		if (!folio_test_uptodate(folio))
1263 			folio_mark_uptodate(folio);
1264 		folio_unlock(folio);
1265 		return folio;
1266 	}
1267 
1268 	err = f2fs_submit_page_read(inode, folio, dn.data_blkaddr,
1269 						op_flags, for_write);
1270 	if (err)
1271 		goto put_err;
1272 	return folio;
1273 
1274 put_err:
1275 	f2fs_folio_put(folio, true);
1276 	return ERR_PTR(err);
1277 }
1278 
f2fs_find_data_folio(struct inode * inode,pgoff_t index,pgoff_t * next_pgofs)1279 struct folio *f2fs_find_data_folio(struct inode *inode, pgoff_t index,
1280 					pgoff_t *next_pgofs)
1281 {
1282 	struct address_space *mapping = inode->i_mapping;
1283 	struct folio *folio;
1284 
1285 	folio = __filemap_get_folio(mapping, index, FGP_ACCESSED, 0);
1286 	if (IS_ERR(folio))
1287 		goto read;
1288 	if (folio_test_uptodate(folio))
1289 		return folio;
1290 	f2fs_folio_put(folio, false);
1291 
1292 read:
1293 	folio = f2fs_get_read_data_folio(inode, index, 0, false, next_pgofs);
1294 	if (IS_ERR(folio))
1295 		return folio;
1296 
1297 	if (folio_test_uptodate(folio))
1298 		return folio;
1299 
1300 	folio_wait_locked(folio);
1301 	if (unlikely(!folio_test_uptodate(folio))) {
1302 		f2fs_folio_put(folio, false);
1303 		return ERR_PTR(-EIO);
1304 	}
1305 	return folio;
1306 }
1307 
1308 /*
1309  * If it tries to access a hole, return an error.
1310  * Because, the callers, functions in dir.c and GC, should be able to know
1311  * whether this page exists or not.
1312  */
f2fs_get_lock_data_folio(struct inode * inode,pgoff_t index,bool for_write)1313 struct folio *f2fs_get_lock_data_folio(struct inode *inode, pgoff_t index,
1314 							bool for_write)
1315 {
1316 	struct address_space *mapping = inode->i_mapping;
1317 	struct folio *folio;
1318 
1319 	folio = f2fs_get_read_data_folio(inode, index, 0, for_write, NULL);
1320 	if (IS_ERR(folio))
1321 		return folio;
1322 
1323 	/* wait for read completion */
1324 	folio_lock(folio);
1325 	if (unlikely(folio->mapping != mapping || !folio_test_uptodate(folio))) {
1326 		f2fs_folio_put(folio, true);
1327 		return ERR_PTR(-EIO);
1328 	}
1329 	return folio;
1330 }
1331 
1332 /*
1333  * Caller ensures that this data page is never allocated.
1334  * A new zero-filled data page is allocated in the page cache.
1335  *
1336  * Also, caller should grab and release a rwsem by calling f2fs_lock_op() and
1337  * f2fs_unlock_op().
1338  * Note that, ifolio is set only by make_empty_dir, and if any error occur,
1339  * ifolio should be released by this function.
1340  */
f2fs_get_new_data_folio(struct inode * inode,struct folio * ifolio,pgoff_t index,bool new_i_size)1341 struct folio *f2fs_get_new_data_folio(struct inode *inode,
1342 		struct folio *ifolio, pgoff_t index, bool new_i_size)
1343 {
1344 	struct address_space *mapping = inode->i_mapping;
1345 	struct folio *folio;
1346 	struct dnode_of_data dn;
1347 	int err;
1348 
1349 	folio = f2fs_grab_cache_folio(mapping, index, true);
1350 	if (IS_ERR(folio)) {
1351 		/*
1352 		 * before exiting, we should make sure ifolio will be released
1353 		 * if any error occur.
1354 		 */
1355 		f2fs_folio_put(ifolio, true);
1356 		return ERR_PTR(-ENOMEM);
1357 	}
1358 
1359 	set_new_dnode(&dn, inode, ifolio, NULL, 0);
1360 	err = f2fs_reserve_block(&dn, index);
1361 	if (err) {
1362 		f2fs_folio_put(folio, true);
1363 		return ERR_PTR(err);
1364 	}
1365 	if (!ifolio)
1366 		f2fs_put_dnode(&dn);
1367 
1368 	if (folio_test_uptodate(folio))
1369 		goto got_it;
1370 
1371 	if (dn.data_blkaddr == NEW_ADDR) {
1372 		folio_zero_segment(folio, 0, folio_size(folio));
1373 		if (!folio_test_uptodate(folio))
1374 			folio_mark_uptodate(folio);
1375 	} else {
1376 		f2fs_folio_put(folio, true);
1377 
1378 		/* if ifolio exists, blkaddr should be NEW_ADDR */
1379 		f2fs_bug_on(F2FS_I_SB(inode), ifolio);
1380 		folio = f2fs_get_lock_data_folio(inode, index, true);
1381 		if (IS_ERR(folio))
1382 			return folio;
1383 	}
1384 got_it:
1385 	if (new_i_size && i_size_read(inode) <
1386 				((loff_t)(index + 1) << PAGE_SHIFT))
1387 		f2fs_i_size_write(inode, ((loff_t)(index + 1) << PAGE_SHIFT));
1388 	return folio;
1389 }
1390 
__allocate_data_block(struct dnode_of_data * dn,int seg_type)1391 static int __allocate_data_block(struct dnode_of_data *dn, int seg_type)
1392 {
1393 	struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
1394 	struct f2fs_summary sum;
1395 	struct node_info ni;
1396 	block_t old_blkaddr;
1397 	blkcnt_t count = 1;
1398 	int err;
1399 
1400 	if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC)))
1401 		return -EPERM;
1402 
1403 	err = f2fs_get_node_info(sbi, dn->nid, &ni, false);
1404 	if (err)
1405 		return err;
1406 
1407 	dn->data_blkaddr = f2fs_data_blkaddr(dn);
1408 	if (dn->data_blkaddr == NULL_ADDR) {
1409 		err = inc_valid_block_count(sbi, dn->inode, &count, true);
1410 		if (unlikely(err))
1411 			return err;
1412 	}
1413 
1414 	set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version);
1415 	old_blkaddr = dn->data_blkaddr;
1416 	err = f2fs_allocate_data_block(sbi, NULL, old_blkaddr,
1417 				&dn->data_blkaddr, &sum, seg_type, NULL);
1418 	if (err)
1419 		return err;
1420 
1421 	if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO)
1422 		f2fs_invalidate_internal_cache(sbi, old_blkaddr, 1);
1423 
1424 	f2fs_update_data_blkaddr(dn, dn->data_blkaddr);
1425 	return 0;
1426 }
1427 
f2fs_map_lock(struct f2fs_sb_info * sbi,int flag)1428 static void f2fs_map_lock(struct f2fs_sb_info *sbi, int flag)
1429 {
1430 	if (flag == F2FS_GET_BLOCK_PRE_AIO)
1431 		f2fs_down_read(&sbi->node_change);
1432 	else
1433 		f2fs_lock_op(sbi);
1434 }
1435 
f2fs_map_unlock(struct f2fs_sb_info * sbi,int flag)1436 static void f2fs_map_unlock(struct f2fs_sb_info *sbi, int flag)
1437 {
1438 	if (flag == F2FS_GET_BLOCK_PRE_AIO)
1439 		f2fs_up_read(&sbi->node_change);
1440 	else
1441 		f2fs_unlock_op(sbi);
1442 }
1443 
f2fs_get_block_locked(struct dnode_of_data * dn,pgoff_t index)1444 int f2fs_get_block_locked(struct dnode_of_data *dn, pgoff_t index)
1445 {
1446 	struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
1447 	int err = 0;
1448 
1449 	f2fs_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO);
1450 	if (!f2fs_lookup_read_extent_cache_block(dn->inode, index,
1451 						&dn->data_blkaddr))
1452 		err = f2fs_reserve_block(dn, index);
1453 	f2fs_map_unlock(sbi, F2FS_GET_BLOCK_PRE_AIO);
1454 
1455 	return err;
1456 }
1457 
f2fs_map_no_dnode(struct inode * inode,struct f2fs_map_blocks * map,struct dnode_of_data * dn,pgoff_t pgoff)1458 static int f2fs_map_no_dnode(struct inode *inode,
1459 		struct f2fs_map_blocks *map, struct dnode_of_data *dn,
1460 		pgoff_t pgoff)
1461 {
1462 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1463 
1464 	/*
1465 	 * There is one exceptional case that read_node_page() may return
1466 	 * -ENOENT due to filesystem has been shutdown or cp_error, return
1467 	 * -EIO in that case.
1468 	 */
1469 	if (map->m_may_create &&
1470 	    (is_sbi_flag_set(sbi, SBI_IS_SHUTDOWN) || f2fs_cp_error(sbi)))
1471 		return -EIO;
1472 
1473 	if (map->m_next_pgofs)
1474 		*map->m_next_pgofs = f2fs_get_next_page_offset(dn, pgoff);
1475 	if (map->m_next_extent)
1476 		*map->m_next_extent = f2fs_get_next_page_offset(dn, pgoff);
1477 	return 0;
1478 }
1479 
f2fs_map_blocks_cached(struct inode * inode,struct f2fs_map_blocks * map,int flag)1480 static bool f2fs_map_blocks_cached(struct inode *inode,
1481 		struct f2fs_map_blocks *map, int flag)
1482 {
1483 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1484 	unsigned int maxblocks = map->m_len;
1485 	pgoff_t pgoff = (pgoff_t)map->m_lblk;
1486 	struct extent_info ei = {};
1487 
1488 	if (!f2fs_lookup_read_extent_cache(inode, pgoff, &ei))
1489 		return false;
1490 
1491 	map->m_pblk = ei.blk + pgoff - ei.fofs;
1492 	map->m_len = min((pgoff_t)maxblocks, ei.fofs + ei.len - pgoff);
1493 	map->m_flags = F2FS_MAP_MAPPED;
1494 	if (map->m_next_extent)
1495 		*map->m_next_extent = pgoff + map->m_len;
1496 
1497 	/* for hardware encryption, but to avoid potential issue in future */
1498 	if (flag == F2FS_GET_BLOCK_DIO)
1499 		f2fs_wait_on_block_writeback_range(inode,
1500 					map->m_pblk, map->m_len);
1501 
1502 	if (f2fs_allow_multi_device_dio(sbi, flag)) {
1503 		int bidx = f2fs_target_device_index(sbi, map->m_pblk);
1504 		struct f2fs_dev_info *dev = &sbi->devs[bidx];
1505 
1506 		map->m_bdev = dev->bdev;
1507 		map->m_pblk -= dev->start_blk;
1508 		map->m_len = min(map->m_len, dev->end_blk + 1 - map->m_pblk);
1509 	} else {
1510 		map->m_bdev = inode->i_sb->s_bdev;
1511 	}
1512 	return true;
1513 }
1514 
map_is_mergeable(struct f2fs_sb_info * sbi,struct f2fs_map_blocks * map,block_t blkaddr,int flag,int bidx,int ofs)1515 static bool map_is_mergeable(struct f2fs_sb_info *sbi,
1516 				struct f2fs_map_blocks *map,
1517 				block_t blkaddr, int flag, int bidx,
1518 				int ofs)
1519 {
1520 	if (map->m_multidev_dio && map->m_bdev != FDEV(bidx).bdev)
1521 		return false;
1522 	if (map->m_pblk != NEW_ADDR && blkaddr == (map->m_pblk + ofs))
1523 		return true;
1524 	if (map->m_pblk == NEW_ADDR && blkaddr == NEW_ADDR)
1525 		return true;
1526 	if (flag == F2FS_GET_BLOCK_PRE_DIO)
1527 		return true;
1528 	if (flag == F2FS_GET_BLOCK_DIO &&
1529 		map->m_pblk == NULL_ADDR && blkaddr == NULL_ADDR)
1530 		return true;
1531 	return false;
1532 }
1533 
1534 /*
1535  * f2fs_map_blocks() tries to find or build mapping relationship which
1536  * maps continuous logical blocks to physical blocks, and return such
1537  * info via f2fs_map_blocks structure.
1538  */
f2fs_map_blocks(struct inode * inode,struct f2fs_map_blocks * map,int flag)1539 int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map, int flag)
1540 {
1541 	unsigned int maxblocks = map->m_len;
1542 	struct dnode_of_data dn;
1543 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1544 	int mode = map->m_may_create ? ALLOC_NODE : LOOKUP_NODE;
1545 	pgoff_t pgofs, end_offset, end;
1546 	int err = 0, ofs = 1;
1547 	unsigned int ofs_in_node, last_ofs_in_node;
1548 	blkcnt_t prealloc;
1549 	block_t blkaddr;
1550 	unsigned int start_pgofs;
1551 	int bidx = 0;
1552 	bool is_hole;
1553 	bool lfs_dio_write;
1554 
1555 	if (!maxblocks)
1556 		return 0;
1557 
1558 	lfs_dio_write = (flag == F2FS_GET_BLOCK_DIO && f2fs_lfs_mode(sbi) &&
1559 				map->m_may_create);
1560 
1561 	if (!map->m_may_create && f2fs_map_blocks_cached(inode, map, flag))
1562 		goto out;
1563 
1564 	map->m_bdev = inode->i_sb->s_bdev;
1565 	map->m_multidev_dio =
1566 		f2fs_allow_multi_device_dio(F2FS_I_SB(inode), flag);
1567 
1568 	map->m_len = 0;
1569 	map->m_flags = 0;
1570 
1571 	/* it only supports block size == page size */
1572 	pgofs =	(pgoff_t)map->m_lblk;
1573 	end = pgofs + maxblocks;
1574 
1575 next_dnode:
1576 	if (map->m_may_create) {
1577 		if (f2fs_lfs_mode(sbi))
1578 			f2fs_balance_fs(sbi, true);
1579 		f2fs_map_lock(sbi, flag);
1580 	}
1581 
1582 	/* When reading holes, we need its node page */
1583 	set_new_dnode(&dn, inode, NULL, NULL, 0);
1584 	err = f2fs_get_dnode_of_data(&dn, pgofs, mode);
1585 	if (err) {
1586 		if (flag == F2FS_GET_BLOCK_BMAP)
1587 			map->m_pblk = 0;
1588 		if (err == -ENOENT)
1589 			err = f2fs_map_no_dnode(inode, map, &dn, pgofs);
1590 		goto unlock_out;
1591 	}
1592 
1593 	start_pgofs = pgofs;
1594 	prealloc = 0;
1595 	last_ofs_in_node = ofs_in_node = dn.ofs_in_node;
1596 	end_offset = ADDRS_PER_PAGE(dn.node_folio, inode);
1597 
1598 next_block:
1599 	blkaddr = f2fs_data_blkaddr(&dn);
1600 	is_hole = !__is_valid_data_blkaddr(blkaddr);
1601 	if (!is_hole &&
1602 	    !f2fs_is_valid_blkaddr(sbi, blkaddr, DATA_GENERIC_ENHANCE)) {
1603 		err = -EFSCORRUPTED;
1604 		goto sync_out;
1605 	}
1606 
1607 	/* use out-place-update for direct IO under LFS mode */
1608 	if (map->m_may_create && (is_hole ||
1609 		(flag == F2FS_GET_BLOCK_DIO && f2fs_lfs_mode(sbi) &&
1610 		!f2fs_is_pinned_file(inode) && map->m_last_pblk != blkaddr))) {
1611 		if (unlikely(f2fs_cp_error(sbi))) {
1612 			err = -EIO;
1613 			goto sync_out;
1614 		}
1615 
1616 		switch (flag) {
1617 		case F2FS_GET_BLOCK_PRE_AIO:
1618 			if (blkaddr == NULL_ADDR) {
1619 				prealloc++;
1620 				last_ofs_in_node = dn.ofs_in_node;
1621 			}
1622 			break;
1623 		case F2FS_GET_BLOCK_PRE_DIO:
1624 		case F2FS_GET_BLOCK_DIO:
1625 			err = __allocate_data_block(&dn, map->m_seg_type);
1626 			if (err)
1627 				goto sync_out;
1628 			if (flag == F2FS_GET_BLOCK_PRE_DIO)
1629 				file_need_truncate(inode);
1630 			set_inode_flag(inode, FI_APPEND_WRITE);
1631 			break;
1632 		default:
1633 			WARN_ON_ONCE(1);
1634 			err = -EIO;
1635 			goto sync_out;
1636 		}
1637 
1638 		blkaddr = dn.data_blkaddr;
1639 		if (is_hole)
1640 			map->m_flags |= F2FS_MAP_NEW;
1641 	} else if (is_hole) {
1642 		if (f2fs_compressed_file(inode) &&
1643 		    f2fs_sanity_check_cluster(&dn)) {
1644 			err = -EFSCORRUPTED;
1645 			f2fs_handle_error(sbi,
1646 					ERROR_CORRUPTED_CLUSTER);
1647 			goto sync_out;
1648 		}
1649 
1650 		switch (flag) {
1651 		case F2FS_GET_BLOCK_PRECACHE:
1652 			goto sync_out;
1653 		case F2FS_GET_BLOCK_BMAP:
1654 			map->m_pblk = 0;
1655 			goto sync_out;
1656 		case F2FS_GET_BLOCK_FIEMAP:
1657 			if (blkaddr == NULL_ADDR) {
1658 				if (map->m_next_pgofs)
1659 					*map->m_next_pgofs = pgofs + 1;
1660 				goto sync_out;
1661 			}
1662 			break;
1663 		case F2FS_GET_BLOCK_DIO:
1664 			if (map->m_next_pgofs)
1665 				*map->m_next_pgofs = pgofs + 1;
1666 			break;
1667 		default:
1668 			/* for defragment case */
1669 			if (map->m_next_pgofs)
1670 				*map->m_next_pgofs = pgofs + 1;
1671 			goto sync_out;
1672 		}
1673 	}
1674 
1675 	if (flag == F2FS_GET_BLOCK_PRE_AIO)
1676 		goto skip;
1677 
1678 	if (map->m_multidev_dio)
1679 		bidx = f2fs_target_device_index(sbi, blkaddr);
1680 
1681 	if (map->m_len == 0) {
1682 		/* reserved delalloc block should be mapped for fiemap. */
1683 		if (blkaddr == NEW_ADDR)
1684 			map->m_flags |= F2FS_MAP_DELALLOC;
1685 		/* DIO READ and hole case, should not map the blocks. */
1686 		if (!(flag == F2FS_GET_BLOCK_DIO && is_hole && !map->m_may_create))
1687 			map->m_flags |= F2FS_MAP_MAPPED;
1688 
1689 		map->m_pblk = blkaddr;
1690 		map->m_len = 1;
1691 
1692 		if (map->m_multidev_dio)
1693 			map->m_bdev = FDEV(bidx).bdev;
1694 
1695 		if (lfs_dio_write)
1696 			map->m_last_pblk = NULL_ADDR;
1697 	} else if (map_is_mergeable(sbi, map, blkaddr, flag, bidx, ofs)) {
1698 		ofs++;
1699 		map->m_len++;
1700 	} else {
1701 		if (lfs_dio_write && !f2fs_is_pinned_file(inode))
1702 			map->m_last_pblk = blkaddr;
1703 		goto sync_out;
1704 	}
1705 
1706 skip:
1707 	dn.ofs_in_node++;
1708 	pgofs++;
1709 
1710 	/* preallocate blocks in batch for one dnode page */
1711 	if (flag == F2FS_GET_BLOCK_PRE_AIO &&
1712 			(pgofs == end || dn.ofs_in_node == end_offset)) {
1713 
1714 		dn.ofs_in_node = ofs_in_node;
1715 		err = f2fs_reserve_new_blocks(&dn, prealloc);
1716 		if (err)
1717 			goto sync_out;
1718 
1719 		map->m_len += dn.ofs_in_node - ofs_in_node;
1720 		if (prealloc && dn.ofs_in_node != last_ofs_in_node + 1) {
1721 			err = -ENOSPC;
1722 			goto sync_out;
1723 		}
1724 		dn.ofs_in_node = end_offset;
1725 	}
1726 
1727 	if (pgofs >= end)
1728 		goto sync_out;
1729 	else if (dn.ofs_in_node < end_offset)
1730 		goto next_block;
1731 
1732 	if (flag == F2FS_GET_BLOCK_PRECACHE) {
1733 		if (map->m_flags & F2FS_MAP_MAPPED) {
1734 			unsigned int ofs = start_pgofs - map->m_lblk;
1735 
1736 			f2fs_update_read_extent_cache_range(&dn,
1737 				start_pgofs, map->m_pblk + ofs,
1738 				map->m_len - ofs);
1739 		}
1740 	}
1741 
1742 	f2fs_put_dnode(&dn);
1743 
1744 	if (map->m_may_create) {
1745 		f2fs_map_unlock(sbi, flag);
1746 		f2fs_balance_fs(sbi, dn.node_changed);
1747 	}
1748 	goto next_dnode;
1749 
1750 sync_out:
1751 
1752 	if (flag == F2FS_GET_BLOCK_DIO && map->m_flags & F2FS_MAP_MAPPED) {
1753 		/*
1754 		 * for hardware encryption, but to avoid potential issue
1755 		 * in future
1756 		 */
1757 		f2fs_wait_on_block_writeback_range(inode,
1758 						map->m_pblk, map->m_len);
1759 
1760 		if (map->m_multidev_dio) {
1761 			block_t blk_addr = map->m_pblk;
1762 
1763 			bidx = f2fs_target_device_index(sbi, map->m_pblk);
1764 
1765 			map->m_bdev = FDEV(bidx).bdev;
1766 			map->m_pblk -= FDEV(bidx).start_blk;
1767 
1768 			if (map->m_may_create)
1769 				f2fs_update_device_state(sbi, inode->i_ino,
1770 							blk_addr, map->m_len);
1771 
1772 			f2fs_bug_on(sbi, blk_addr + map->m_len >
1773 						FDEV(bidx).end_blk + 1);
1774 		}
1775 	}
1776 
1777 	if (flag == F2FS_GET_BLOCK_PRECACHE) {
1778 		if (map->m_flags & F2FS_MAP_MAPPED) {
1779 			unsigned int ofs = start_pgofs - map->m_lblk;
1780 
1781 			f2fs_update_read_extent_cache_range(&dn,
1782 				start_pgofs, map->m_pblk + ofs,
1783 				map->m_len - ofs);
1784 		}
1785 		if (map->m_next_extent)
1786 			*map->m_next_extent = pgofs + 1;
1787 	}
1788 	f2fs_put_dnode(&dn);
1789 unlock_out:
1790 	if (map->m_may_create) {
1791 		f2fs_map_unlock(sbi, flag);
1792 		f2fs_balance_fs(sbi, dn.node_changed);
1793 	}
1794 out:
1795 	trace_f2fs_map_blocks(inode, map, flag, err);
1796 	return err;
1797 }
1798 
f2fs_overwrite_io(struct inode * inode,loff_t pos,size_t len)1799 bool f2fs_overwrite_io(struct inode *inode, loff_t pos, size_t len)
1800 {
1801 	struct f2fs_map_blocks map;
1802 	block_t last_lblk;
1803 	int err;
1804 
1805 	if (pos + len > i_size_read(inode))
1806 		return false;
1807 
1808 	map.m_lblk = F2FS_BYTES_TO_BLK(pos);
1809 	map.m_next_pgofs = NULL;
1810 	map.m_next_extent = NULL;
1811 	map.m_seg_type = NO_CHECK_TYPE;
1812 	map.m_may_create = false;
1813 	last_lblk = F2FS_BLK_ALIGN(pos + len);
1814 
1815 	while (map.m_lblk < last_lblk) {
1816 		map.m_len = last_lblk - map.m_lblk;
1817 		err = f2fs_map_blocks(inode, &map, F2FS_GET_BLOCK_DEFAULT);
1818 		if (err || map.m_len == 0)
1819 			return false;
1820 		map.m_lblk += map.m_len;
1821 	}
1822 	return true;
1823 }
1824 
f2fs_xattr_fiemap(struct inode * inode,struct fiemap_extent_info * fieinfo)1825 static int f2fs_xattr_fiemap(struct inode *inode,
1826 				struct fiemap_extent_info *fieinfo)
1827 {
1828 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1829 	struct node_info ni;
1830 	__u64 phys = 0, len;
1831 	__u32 flags;
1832 	nid_t xnid = F2FS_I(inode)->i_xattr_nid;
1833 	int err = 0;
1834 
1835 	if (f2fs_has_inline_xattr(inode)) {
1836 		int offset;
1837 		struct folio *folio = f2fs_grab_cache_folio(NODE_MAPPING(sbi),
1838 				inode->i_ino, false);
1839 
1840 		if (IS_ERR(folio))
1841 			return PTR_ERR(folio);
1842 
1843 		err = f2fs_get_node_info(sbi, inode->i_ino, &ni, false);
1844 		if (err) {
1845 			f2fs_folio_put(folio, true);
1846 			return err;
1847 		}
1848 
1849 		phys = F2FS_BLK_TO_BYTES(ni.blk_addr);
1850 		offset = offsetof(struct f2fs_inode, i_addr) +
1851 					sizeof(__le32) * (DEF_ADDRS_PER_INODE -
1852 					get_inline_xattr_addrs(inode));
1853 
1854 		phys += offset;
1855 		len = inline_xattr_size(inode);
1856 
1857 		f2fs_folio_put(folio, true);
1858 
1859 		flags = FIEMAP_EXTENT_DATA_INLINE | FIEMAP_EXTENT_NOT_ALIGNED;
1860 
1861 		if (!xnid)
1862 			flags |= FIEMAP_EXTENT_LAST;
1863 
1864 		err = fiemap_fill_next_extent(fieinfo, 0, phys, len, flags);
1865 		trace_f2fs_fiemap(inode, 0, phys, len, flags, err);
1866 		if (err)
1867 			return err;
1868 	}
1869 
1870 	if (xnid) {
1871 		struct folio *folio = f2fs_grab_cache_folio(NODE_MAPPING(sbi),
1872 				xnid, false);
1873 
1874 		if (IS_ERR(folio))
1875 			return PTR_ERR(folio);
1876 
1877 		err = f2fs_get_node_info(sbi, xnid, &ni, false);
1878 		if (err) {
1879 			f2fs_folio_put(folio, true);
1880 			return err;
1881 		}
1882 
1883 		phys = F2FS_BLK_TO_BYTES(ni.blk_addr);
1884 		len = inode->i_sb->s_blocksize;
1885 
1886 		f2fs_folio_put(folio, true);
1887 
1888 		flags = FIEMAP_EXTENT_LAST;
1889 	}
1890 
1891 	if (phys) {
1892 		err = fiemap_fill_next_extent(fieinfo, 0, phys, len, flags);
1893 		trace_f2fs_fiemap(inode, 0, phys, len, flags, err);
1894 	}
1895 
1896 	return (err < 0 ? err : 0);
1897 }
1898 
f2fs_fiemap(struct inode * inode,struct fiemap_extent_info * fieinfo,u64 start,u64 len)1899 int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
1900 		u64 start, u64 len)
1901 {
1902 	struct f2fs_map_blocks map;
1903 	sector_t start_blk, last_blk, blk_len, max_len;
1904 	pgoff_t next_pgofs;
1905 	u64 logical = 0, phys = 0, size = 0;
1906 	u32 flags = 0;
1907 	int ret = 0;
1908 	bool compr_cluster = false, compr_appended;
1909 	unsigned int cluster_size = F2FS_I(inode)->i_cluster_size;
1910 	unsigned int count_in_cluster = 0;
1911 	loff_t maxbytes;
1912 
1913 	if (fieinfo->fi_flags & FIEMAP_FLAG_CACHE) {
1914 		ret = f2fs_precache_extents(inode);
1915 		if (ret)
1916 			return ret;
1917 	}
1918 
1919 	ret = fiemap_prep(inode, fieinfo, start, &len, FIEMAP_FLAG_XATTR);
1920 	if (ret)
1921 		return ret;
1922 
1923 	inode_lock_shared(inode);
1924 
1925 	maxbytes = F2FS_BLK_TO_BYTES(max_file_blocks(inode));
1926 	if (start > maxbytes) {
1927 		ret = -EFBIG;
1928 		goto out;
1929 	}
1930 
1931 	if (len > maxbytes || (maxbytes - len) < start)
1932 		len = maxbytes - start;
1933 
1934 	if (fieinfo->fi_flags & FIEMAP_FLAG_XATTR) {
1935 		ret = f2fs_xattr_fiemap(inode, fieinfo);
1936 		goto out;
1937 	}
1938 
1939 	if (f2fs_has_inline_data(inode) || f2fs_has_inline_dentry(inode)) {
1940 		ret = f2fs_inline_data_fiemap(inode, fieinfo, start, len);
1941 		if (ret != -EAGAIN)
1942 			goto out;
1943 	}
1944 
1945 	start_blk = F2FS_BYTES_TO_BLK(start);
1946 	last_blk = F2FS_BYTES_TO_BLK(start + len - 1);
1947 	blk_len = last_blk - start_blk + 1;
1948 	max_len = F2FS_BYTES_TO_BLK(maxbytes) - start_blk;
1949 
1950 next:
1951 	memset(&map, 0, sizeof(map));
1952 	map.m_lblk = start_blk;
1953 	map.m_len = blk_len;
1954 	map.m_next_pgofs = &next_pgofs;
1955 	map.m_seg_type = NO_CHECK_TYPE;
1956 
1957 	if (compr_cluster) {
1958 		map.m_lblk += 1;
1959 		map.m_len = cluster_size - count_in_cluster;
1960 	}
1961 
1962 	ret = f2fs_map_blocks(inode, &map, F2FS_GET_BLOCK_FIEMAP);
1963 	if (ret)
1964 		goto out;
1965 
1966 	/* HOLE */
1967 	if (!compr_cluster && !(map.m_flags & F2FS_MAP_FLAGS)) {
1968 		start_blk = next_pgofs;
1969 
1970 		if (F2FS_BLK_TO_BYTES(start_blk) < maxbytes)
1971 			goto prep_next;
1972 
1973 		flags |= FIEMAP_EXTENT_LAST;
1974 	}
1975 
1976 	/*
1977 	 * current extent may cross boundary of inquiry, increase len to
1978 	 * requery.
1979 	 */
1980 	if (!compr_cluster && (map.m_flags & F2FS_MAP_MAPPED) &&
1981 				map.m_lblk + map.m_len - 1 == last_blk &&
1982 				blk_len != max_len) {
1983 		blk_len = max_len;
1984 		goto next;
1985 	}
1986 
1987 	compr_appended = false;
1988 	/* In a case of compressed cluster, append this to the last extent */
1989 	if (compr_cluster && ((map.m_flags & F2FS_MAP_DELALLOC) ||
1990 			!(map.m_flags & F2FS_MAP_FLAGS))) {
1991 		compr_appended = true;
1992 		goto skip_fill;
1993 	}
1994 
1995 	if (size) {
1996 		flags |= FIEMAP_EXTENT_MERGED;
1997 		if (IS_ENCRYPTED(inode))
1998 			flags |= FIEMAP_EXTENT_DATA_ENCRYPTED;
1999 
2000 		ret = fiemap_fill_next_extent(fieinfo, logical,
2001 				phys, size, flags);
2002 		trace_f2fs_fiemap(inode, logical, phys, size, flags, ret);
2003 		if (ret)
2004 			goto out;
2005 		size = 0;
2006 	}
2007 
2008 	if (start_blk > last_blk)
2009 		goto out;
2010 
2011 skip_fill:
2012 	if (map.m_pblk == COMPRESS_ADDR) {
2013 		compr_cluster = true;
2014 		count_in_cluster = 1;
2015 	} else if (compr_appended) {
2016 		unsigned int appended_blks = cluster_size -
2017 						count_in_cluster + 1;
2018 		size += F2FS_BLK_TO_BYTES(appended_blks);
2019 		start_blk += appended_blks;
2020 		compr_cluster = false;
2021 	} else {
2022 		logical = F2FS_BLK_TO_BYTES(start_blk);
2023 		phys = __is_valid_data_blkaddr(map.m_pblk) ?
2024 			F2FS_BLK_TO_BYTES(map.m_pblk) : 0;
2025 		size = F2FS_BLK_TO_BYTES(map.m_len);
2026 		flags = 0;
2027 
2028 		if (compr_cluster) {
2029 			flags = FIEMAP_EXTENT_ENCODED;
2030 			count_in_cluster += map.m_len;
2031 			if (count_in_cluster == cluster_size) {
2032 				compr_cluster = false;
2033 				size += F2FS_BLKSIZE;
2034 			}
2035 		} else if (map.m_flags & F2FS_MAP_DELALLOC) {
2036 			flags = FIEMAP_EXTENT_UNWRITTEN;
2037 		}
2038 
2039 		start_blk += F2FS_BYTES_TO_BLK(size);
2040 	}
2041 
2042 prep_next:
2043 	cond_resched();
2044 	if (fatal_signal_pending(current))
2045 		ret = -EINTR;
2046 	else
2047 		goto next;
2048 out:
2049 	if (ret == 1)
2050 		ret = 0;
2051 
2052 	inode_unlock_shared(inode);
2053 	return ret;
2054 }
2055 
f2fs_readpage_limit(struct inode * inode)2056 static inline loff_t f2fs_readpage_limit(struct inode *inode)
2057 {
2058 	if (IS_ENABLED(CONFIG_FS_VERITY) && IS_VERITY(inode))
2059 		return F2FS_BLK_TO_BYTES(max_file_blocks(inode));
2060 
2061 	return i_size_read(inode);
2062 }
2063 
f2fs_ra_op_flags(struct readahead_control * rac)2064 static inline blk_opf_t f2fs_ra_op_flags(struct readahead_control *rac)
2065 {
2066 	return rac ? REQ_RAHEAD : 0;
2067 }
2068 
f2fs_read_single_page(struct inode * inode,struct folio * folio,unsigned nr_pages,struct f2fs_map_blocks * map,struct bio ** bio_ret,sector_t * last_block_in_bio,struct readahead_control * rac)2069 static int f2fs_read_single_page(struct inode *inode, struct folio *folio,
2070 					unsigned nr_pages,
2071 					struct f2fs_map_blocks *map,
2072 					struct bio **bio_ret,
2073 					sector_t *last_block_in_bio,
2074 					struct readahead_control *rac)
2075 {
2076 	struct bio *bio = *bio_ret;
2077 	const unsigned int blocksize = F2FS_BLKSIZE;
2078 	sector_t block_in_file;
2079 	sector_t last_block;
2080 	sector_t last_block_in_file;
2081 	sector_t block_nr;
2082 	pgoff_t index = folio->index;
2083 	int ret = 0;
2084 
2085 	block_in_file = (sector_t)index;
2086 	last_block = block_in_file + nr_pages;
2087 	last_block_in_file = F2FS_BYTES_TO_BLK(f2fs_readpage_limit(inode) +
2088 							blocksize - 1);
2089 	if (last_block > last_block_in_file)
2090 		last_block = last_block_in_file;
2091 
2092 	/* just zeroing out page which is beyond EOF */
2093 	if (block_in_file >= last_block)
2094 		goto zero_out;
2095 	/*
2096 	 * Map blocks using the previous result first.
2097 	 */
2098 	if ((map->m_flags & F2FS_MAP_MAPPED) &&
2099 			block_in_file > map->m_lblk &&
2100 			block_in_file < (map->m_lblk + map->m_len))
2101 		goto got_it;
2102 
2103 	/*
2104 	 * Then do more f2fs_map_blocks() calls until we are
2105 	 * done with this page.
2106 	 */
2107 	map->m_lblk = block_in_file;
2108 	map->m_len = last_block - block_in_file;
2109 
2110 	ret = f2fs_map_blocks(inode, map, F2FS_GET_BLOCK_DEFAULT);
2111 	if (ret)
2112 		goto out;
2113 got_it:
2114 	if ((map->m_flags & F2FS_MAP_MAPPED)) {
2115 		block_nr = map->m_pblk + block_in_file - map->m_lblk;
2116 		folio_set_mappedtodisk(folio);
2117 
2118 		if (!f2fs_is_valid_blkaddr(F2FS_I_SB(inode), block_nr,
2119 						DATA_GENERIC_ENHANCE_READ)) {
2120 			ret = -EFSCORRUPTED;
2121 			goto out;
2122 		}
2123 	} else {
2124 zero_out:
2125 		folio_zero_segment(folio, 0, folio_size(folio));
2126 		if (f2fs_need_verity(inode, index) &&
2127 		    !fsverity_verify_folio(folio)) {
2128 			ret = -EIO;
2129 			goto out;
2130 		}
2131 		if (!folio_test_uptodate(folio))
2132 			folio_mark_uptodate(folio);
2133 		folio_unlock(folio);
2134 		goto out;
2135 	}
2136 
2137 	/*
2138 	 * This page will go to BIO.  Do we need to send this
2139 	 * BIO off first?
2140 	 */
2141 	if (bio && (!page_is_mergeable(F2FS_I_SB(inode), bio,
2142 				       *last_block_in_bio, block_nr) ||
2143 		    !f2fs_crypt_mergeable_bio(bio, inode, index, NULL))) {
2144 submit_and_realloc:
2145 		f2fs_submit_read_bio(F2FS_I_SB(inode), bio, DATA);
2146 		bio = NULL;
2147 	}
2148 	if (bio == NULL) {
2149 		bio = f2fs_grab_read_bio(inode, block_nr, nr_pages,
2150 				f2fs_ra_op_flags(rac), index,
2151 				false);
2152 		if (IS_ERR(bio)) {
2153 			ret = PTR_ERR(bio);
2154 			bio = NULL;
2155 			goto out;
2156 		}
2157 	}
2158 
2159 	/*
2160 	 * If the page is under writeback, we need to wait for
2161 	 * its completion to see the correct decrypted data.
2162 	 */
2163 	f2fs_wait_on_block_writeback(inode, block_nr);
2164 
2165 	if (!bio_add_folio(bio, folio, blocksize, 0))
2166 		goto submit_and_realloc;
2167 
2168 	inc_page_count(F2FS_I_SB(inode), F2FS_RD_DATA);
2169 	f2fs_update_iostat(F2FS_I_SB(inode), NULL, FS_DATA_READ_IO,
2170 							F2FS_BLKSIZE);
2171 	*last_block_in_bio = block_nr;
2172 out:
2173 	*bio_ret = bio;
2174 	return ret;
2175 }
2176 
2177 #ifdef CONFIG_F2FS_FS_COMPRESSION
f2fs_read_multi_pages(struct compress_ctx * cc,struct bio ** bio_ret,unsigned nr_pages,sector_t * last_block_in_bio,struct readahead_control * rac,bool for_write)2178 int f2fs_read_multi_pages(struct compress_ctx *cc, struct bio **bio_ret,
2179 				unsigned nr_pages, sector_t *last_block_in_bio,
2180 				struct readahead_control *rac, bool for_write)
2181 {
2182 	struct dnode_of_data dn;
2183 	struct inode *inode = cc->inode;
2184 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
2185 	struct bio *bio = *bio_ret;
2186 	unsigned int start_idx = cc->cluster_idx << cc->log_cluster_size;
2187 	sector_t last_block_in_file;
2188 	const unsigned int blocksize = F2FS_BLKSIZE;
2189 	struct decompress_io_ctx *dic = NULL;
2190 	struct extent_info ei = {};
2191 	bool from_dnode = true;
2192 	int i;
2193 	int ret = 0;
2194 
2195 	if (unlikely(f2fs_cp_error(sbi))) {
2196 		ret = -EIO;
2197 		from_dnode = false;
2198 		goto out_put_dnode;
2199 	}
2200 
2201 	f2fs_bug_on(sbi, f2fs_cluster_is_empty(cc));
2202 
2203 	last_block_in_file = F2FS_BYTES_TO_BLK(f2fs_readpage_limit(inode) +
2204 							blocksize - 1);
2205 
2206 	/* get rid of pages beyond EOF */
2207 	for (i = 0; i < cc->cluster_size; i++) {
2208 		struct page *page = cc->rpages[i];
2209 		struct folio *folio;
2210 
2211 		if (!page)
2212 			continue;
2213 
2214 		folio = page_folio(page);
2215 		if ((sector_t)folio->index >= last_block_in_file) {
2216 			folio_zero_segment(folio, 0, folio_size(folio));
2217 			if (!folio_test_uptodate(folio))
2218 				folio_mark_uptodate(folio);
2219 		} else if (!folio_test_uptodate(folio)) {
2220 			continue;
2221 		}
2222 		folio_unlock(folio);
2223 		if (for_write)
2224 			folio_put(folio);
2225 		cc->rpages[i] = NULL;
2226 		cc->nr_rpages--;
2227 	}
2228 
2229 	/* we are done since all pages are beyond EOF */
2230 	if (f2fs_cluster_is_empty(cc))
2231 		goto out;
2232 
2233 	if (f2fs_lookup_read_extent_cache(inode, start_idx, &ei))
2234 		from_dnode = false;
2235 
2236 	if (!from_dnode)
2237 		goto skip_reading_dnode;
2238 
2239 	set_new_dnode(&dn, inode, NULL, NULL, 0);
2240 	ret = f2fs_get_dnode_of_data(&dn, start_idx, LOOKUP_NODE);
2241 	if (ret)
2242 		goto out;
2243 
2244 	f2fs_bug_on(sbi, dn.data_blkaddr != COMPRESS_ADDR);
2245 
2246 skip_reading_dnode:
2247 	for (i = 1; i < cc->cluster_size; i++) {
2248 		block_t blkaddr;
2249 
2250 		blkaddr = from_dnode ? data_blkaddr(dn.inode, dn.node_folio,
2251 					dn.ofs_in_node + i) :
2252 					ei.blk + i - 1;
2253 
2254 		if (!__is_valid_data_blkaddr(blkaddr))
2255 			break;
2256 
2257 		if (!f2fs_is_valid_blkaddr(sbi, blkaddr, DATA_GENERIC)) {
2258 			ret = -EFAULT;
2259 			goto out_put_dnode;
2260 		}
2261 		cc->nr_cpages++;
2262 
2263 		if (!from_dnode && i >= ei.c_len)
2264 			break;
2265 	}
2266 
2267 	/* nothing to decompress */
2268 	if (cc->nr_cpages == 0) {
2269 		ret = 0;
2270 		goto out_put_dnode;
2271 	}
2272 
2273 	dic = f2fs_alloc_dic(cc);
2274 	if (IS_ERR(dic)) {
2275 		ret = PTR_ERR(dic);
2276 		goto out_put_dnode;
2277 	}
2278 
2279 	for (i = 0; i < cc->nr_cpages; i++) {
2280 		struct folio *folio = page_folio(dic->cpages[i]);
2281 		block_t blkaddr;
2282 		struct bio_post_read_ctx *ctx;
2283 
2284 		blkaddr = from_dnode ? data_blkaddr(dn.inode, dn.node_folio,
2285 					dn.ofs_in_node + i + 1) :
2286 					ei.blk + i;
2287 
2288 		f2fs_wait_on_block_writeback(inode, blkaddr);
2289 
2290 		if (f2fs_load_compressed_folio(sbi, folio, blkaddr)) {
2291 			if (atomic_dec_and_test(&dic->remaining_pages)) {
2292 				f2fs_decompress_cluster(dic, true);
2293 				break;
2294 			}
2295 			continue;
2296 		}
2297 
2298 		if (bio && (!page_is_mergeable(sbi, bio,
2299 					*last_block_in_bio, blkaddr) ||
2300 		    !f2fs_crypt_mergeable_bio(bio, inode, folio->index, NULL))) {
2301 submit_and_realloc:
2302 			f2fs_submit_read_bio(sbi, bio, DATA);
2303 			bio = NULL;
2304 		}
2305 
2306 		if (!bio) {
2307 			bio = f2fs_grab_read_bio(inode, blkaddr, nr_pages - i,
2308 					f2fs_ra_op_flags(rac),
2309 					folio->index, for_write);
2310 			if (IS_ERR(bio)) {
2311 				ret = PTR_ERR(bio);
2312 				f2fs_decompress_end_io(dic, ret, true);
2313 				f2fs_put_dnode(&dn);
2314 				*bio_ret = NULL;
2315 				return ret;
2316 			}
2317 		}
2318 
2319 		if (!bio_add_folio(bio, folio, blocksize, 0))
2320 			goto submit_and_realloc;
2321 
2322 		ctx = get_post_read_ctx(bio);
2323 		ctx->enabled_steps |= STEP_DECOMPRESS;
2324 		refcount_inc(&dic->refcnt);
2325 
2326 		inc_page_count(sbi, F2FS_RD_DATA);
2327 		f2fs_update_iostat(sbi, inode, FS_DATA_READ_IO, F2FS_BLKSIZE);
2328 		*last_block_in_bio = blkaddr;
2329 	}
2330 
2331 	if (from_dnode)
2332 		f2fs_put_dnode(&dn);
2333 
2334 	*bio_ret = bio;
2335 	return 0;
2336 
2337 out_put_dnode:
2338 	if (from_dnode)
2339 		f2fs_put_dnode(&dn);
2340 out:
2341 	for (i = 0; i < cc->cluster_size; i++) {
2342 		if (cc->rpages[i]) {
2343 			ClearPageUptodate(cc->rpages[i]);
2344 			unlock_page(cc->rpages[i]);
2345 		}
2346 	}
2347 	*bio_ret = bio;
2348 	return ret;
2349 }
2350 #endif
2351 
2352 /*
2353  * This function was originally taken from fs/mpage.c, and customized for f2fs.
2354  * Major change was from block_size == page_size in f2fs by default.
2355  */
f2fs_mpage_readpages(struct inode * inode,struct readahead_control * rac,struct folio * folio)2356 static int f2fs_mpage_readpages(struct inode *inode,
2357 		struct readahead_control *rac, struct folio *folio)
2358 {
2359 	struct bio *bio = NULL;
2360 	sector_t last_block_in_bio = 0;
2361 	struct f2fs_map_blocks map;
2362 #ifdef CONFIG_F2FS_FS_COMPRESSION
2363 	struct compress_ctx cc = {
2364 		.inode = inode,
2365 		.log_cluster_size = F2FS_I(inode)->i_log_cluster_size,
2366 		.cluster_size = F2FS_I(inode)->i_cluster_size,
2367 		.cluster_idx = NULL_CLUSTER,
2368 		.rpages = NULL,
2369 		.cpages = NULL,
2370 		.nr_rpages = 0,
2371 		.nr_cpages = 0,
2372 	};
2373 	pgoff_t nc_cluster_idx = NULL_CLUSTER;
2374 	pgoff_t index;
2375 #endif
2376 	unsigned nr_pages = rac ? readahead_count(rac) : 1;
2377 	unsigned max_nr_pages = nr_pages;
2378 	int ret = 0;
2379 
2380 #ifdef CONFIG_F2FS_FS_COMPRESSION
2381 	if (f2fs_compressed_file(inode)) {
2382 		index = rac ? readahead_index(rac) : folio->index;
2383 		max_nr_pages = round_up(index + nr_pages, cc.cluster_size) -
2384 				round_down(index, cc.cluster_size);
2385 	}
2386 #endif
2387 
2388 	map.m_pblk = 0;
2389 	map.m_lblk = 0;
2390 	map.m_len = 0;
2391 	map.m_flags = 0;
2392 	map.m_next_pgofs = NULL;
2393 	map.m_next_extent = NULL;
2394 	map.m_seg_type = NO_CHECK_TYPE;
2395 	map.m_may_create = false;
2396 
2397 	for (; nr_pages; nr_pages--) {
2398 		if (rac) {
2399 			folio = readahead_folio(rac);
2400 			prefetchw(&folio->flags);
2401 		}
2402 
2403 #ifdef CONFIG_F2FS_FS_COMPRESSION
2404 		index = folio->index;
2405 
2406 		if (!f2fs_compressed_file(inode))
2407 			goto read_single_page;
2408 
2409 		/* there are remained compressed pages, submit them */
2410 		if (!f2fs_cluster_can_merge_page(&cc, index)) {
2411 			ret = f2fs_read_multi_pages(&cc, &bio,
2412 						max_nr_pages,
2413 						&last_block_in_bio,
2414 						rac, false);
2415 			f2fs_destroy_compress_ctx(&cc, false);
2416 			if (ret)
2417 				goto set_error_page;
2418 		}
2419 		if (cc.cluster_idx == NULL_CLUSTER) {
2420 			if (nc_cluster_idx == index >> cc.log_cluster_size)
2421 				goto read_single_page;
2422 
2423 			ret = f2fs_is_compressed_cluster(inode, index);
2424 			if (ret < 0)
2425 				goto set_error_page;
2426 			else if (!ret) {
2427 				nc_cluster_idx =
2428 					index >> cc.log_cluster_size;
2429 				goto read_single_page;
2430 			}
2431 
2432 			nc_cluster_idx = NULL_CLUSTER;
2433 		}
2434 		ret = f2fs_init_compress_ctx(&cc);
2435 		if (ret)
2436 			goto set_error_page;
2437 
2438 		f2fs_compress_ctx_add_page(&cc, folio);
2439 
2440 		goto next_page;
2441 read_single_page:
2442 #endif
2443 
2444 		ret = f2fs_read_single_page(inode, folio, max_nr_pages, &map,
2445 					&bio, &last_block_in_bio, rac);
2446 		if (ret) {
2447 #ifdef CONFIG_F2FS_FS_COMPRESSION
2448 set_error_page:
2449 #endif
2450 			folio_zero_segment(folio, 0, folio_size(folio));
2451 			folio_unlock(folio);
2452 		}
2453 #ifdef CONFIG_F2FS_FS_COMPRESSION
2454 next_page:
2455 #endif
2456 
2457 #ifdef CONFIG_F2FS_FS_COMPRESSION
2458 		if (f2fs_compressed_file(inode)) {
2459 			/* last page */
2460 			if (nr_pages == 1 && !f2fs_cluster_is_empty(&cc)) {
2461 				ret = f2fs_read_multi_pages(&cc, &bio,
2462 							max_nr_pages,
2463 							&last_block_in_bio,
2464 							rac, false);
2465 				f2fs_destroy_compress_ctx(&cc, false);
2466 			}
2467 		}
2468 #endif
2469 	}
2470 	if (bio)
2471 		f2fs_submit_read_bio(F2FS_I_SB(inode), bio, DATA);
2472 	return ret;
2473 }
2474 
f2fs_read_data_folio(struct file * file,struct folio * folio)2475 static int f2fs_read_data_folio(struct file *file, struct folio *folio)
2476 {
2477 	struct inode *inode = folio->mapping->host;
2478 	int ret = -EAGAIN;
2479 
2480 	trace_f2fs_readpage(folio, DATA);
2481 
2482 	if (!f2fs_is_compress_backend_ready(inode)) {
2483 		folio_unlock(folio);
2484 		return -EOPNOTSUPP;
2485 	}
2486 
2487 	/* If the file has inline data, try to read it directly */
2488 	if (f2fs_has_inline_data(inode))
2489 		ret = f2fs_read_inline_data(inode, folio);
2490 	if (ret == -EAGAIN)
2491 		ret = f2fs_mpage_readpages(inode, NULL, folio);
2492 	return ret;
2493 }
2494 
f2fs_readahead(struct readahead_control * rac)2495 static void f2fs_readahead(struct readahead_control *rac)
2496 {
2497 	struct inode *inode = rac->mapping->host;
2498 
2499 	trace_f2fs_readpages(inode, readahead_index(rac), readahead_count(rac));
2500 
2501 	if (!f2fs_is_compress_backend_ready(inode))
2502 		return;
2503 
2504 	/* If the file has inline data, skip readahead */
2505 	if (f2fs_has_inline_data(inode))
2506 		return;
2507 
2508 	f2fs_mpage_readpages(inode, rac, NULL);
2509 }
2510 
f2fs_encrypt_one_page(struct f2fs_io_info * fio)2511 int f2fs_encrypt_one_page(struct f2fs_io_info *fio)
2512 {
2513 	struct inode *inode = fio_inode(fio);
2514 	struct folio *mfolio;
2515 	struct page *page;
2516 	gfp_t gfp_flags = GFP_NOFS;
2517 
2518 	if (!f2fs_encrypted_file(inode))
2519 		return 0;
2520 
2521 	page = fio->compressed_page ? fio->compressed_page : fio->page;
2522 
2523 	if (fscrypt_inode_uses_inline_crypto(inode))
2524 		return 0;
2525 
2526 retry_encrypt:
2527 	fio->encrypted_page = fscrypt_encrypt_pagecache_blocks(page_folio(page),
2528 					PAGE_SIZE, 0, gfp_flags);
2529 	if (IS_ERR(fio->encrypted_page)) {
2530 		/* flush pending IOs and wait for a while in the ENOMEM case */
2531 		if (PTR_ERR(fio->encrypted_page) == -ENOMEM) {
2532 			f2fs_flush_merged_writes(fio->sbi);
2533 			memalloc_retry_wait(GFP_NOFS);
2534 			gfp_flags |= __GFP_NOFAIL;
2535 			goto retry_encrypt;
2536 		}
2537 		return PTR_ERR(fio->encrypted_page);
2538 	}
2539 
2540 	mfolio = filemap_lock_folio(META_MAPPING(fio->sbi), fio->old_blkaddr);
2541 	if (!IS_ERR(mfolio)) {
2542 		if (folio_test_uptodate(mfolio))
2543 			memcpy(folio_address(mfolio),
2544 				page_address(fio->encrypted_page), PAGE_SIZE);
2545 		f2fs_folio_put(mfolio, true);
2546 	}
2547 	return 0;
2548 }
2549 
check_inplace_update_policy(struct inode * inode,struct f2fs_io_info * fio)2550 static inline bool check_inplace_update_policy(struct inode *inode,
2551 				struct f2fs_io_info *fio)
2552 {
2553 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
2554 
2555 	if (IS_F2FS_IPU_HONOR_OPU_WRITE(sbi) &&
2556 	    is_inode_flag_set(inode, FI_OPU_WRITE))
2557 		return false;
2558 	if (IS_F2FS_IPU_FORCE(sbi))
2559 		return true;
2560 	if (IS_F2FS_IPU_SSR(sbi) && f2fs_need_SSR(sbi))
2561 		return true;
2562 	if (IS_F2FS_IPU_UTIL(sbi) && utilization(sbi) > SM_I(sbi)->min_ipu_util)
2563 		return true;
2564 	if (IS_F2FS_IPU_SSR_UTIL(sbi) && f2fs_need_SSR(sbi) &&
2565 	    utilization(sbi) > SM_I(sbi)->min_ipu_util)
2566 		return true;
2567 
2568 	/*
2569 	 * IPU for rewrite async pages
2570 	 */
2571 	if (IS_F2FS_IPU_ASYNC(sbi) && fio && fio->op == REQ_OP_WRITE &&
2572 	    !(fio->op_flags & REQ_SYNC) && !IS_ENCRYPTED(inode))
2573 		return true;
2574 
2575 	/* this is only set during fdatasync */
2576 	if (IS_F2FS_IPU_FSYNC(sbi) && is_inode_flag_set(inode, FI_NEED_IPU))
2577 		return true;
2578 
2579 	if (unlikely(fio && is_sbi_flag_set(sbi, SBI_CP_DISABLED) &&
2580 			!f2fs_is_checkpointed_data(sbi, fio->old_blkaddr)))
2581 		return true;
2582 
2583 	return false;
2584 }
2585 
f2fs_should_update_inplace(struct inode * inode,struct f2fs_io_info * fio)2586 bool f2fs_should_update_inplace(struct inode *inode, struct f2fs_io_info *fio)
2587 {
2588 	/* swap file is migrating in aligned write mode */
2589 	if (is_inode_flag_set(inode, FI_ALIGNED_WRITE))
2590 		return false;
2591 
2592 	if (f2fs_is_pinned_file(inode))
2593 		return true;
2594 
2595 	/* if this is cold file, we should overwrite to avoid fragmentation */
2596 	if (file_is_cold(inode) && !is_inode_flag_set(inode, FI_OPU_WRITE))
2597 		return true;
2598 
2599 	return check_inplace_update_policy(inode, fio);
2600 }
2601 
f2fs_should_update_outplace(struct inode * inode,struct f2fs_io_info * fio)2602 bool f2fs_should_update_outplace(struct inode *inode, struct f2fs_io_info *fio)
2603 {
2604 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
2605 
2606 	/* The below cases were checked when setting it. */
2607 	if (f2fs_is_pinned_file(inode))
2608 		return false;
2609 	if (fio && is_sbi_flag_set(sbi, SBI_NEED_FSCK))
2610 		return true;
2611 	if (f2fs_lfs_mode(sbi))
2612 		return true;
2613 	if (S_ISDIR(inode->i_mode))
2614 		return true;
2615 	if (IS_NOQUOTA(inode))
2616 		return true;
2617 	if (f2fs_used_in_atomic_write(inode))
2618 		return true;
2619 	/* rewrite low ratio compress data w/ OPU mode to avoid fragmentation */
2620 	if (f2fs_compressed_file(inode) &&
2621 		F2FS_OPTION(sbi).compress_mode == COMPR_MODE_USER &&
2622 		is_inode_flag_set(inode, FI_ENABLE_COMPRESS))
2623 		return true;
2624 
2625 	/* swap file is migrating in aligned write mode */
2626 	if (is_inode_flag_set(inode, FI_ALIGNED_WRITE))
2627 		return true;
2628 
2629 	if (is_inode_flag_set(inode, FI_OPU_WRITE))
2630 		return true;
2631 
2632 	if (fio) {
2633 		if (page_private_gcing(fio->page))
2634 			return true;
2635 		if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED) &&
2636 			f2fs_is_checkpointed_data(sbi, fio->old_blkaddr)))
2637 			return true;
2638 	}
2639 	return false;
2640 }
2641 
need_inplace_update(struct f2fs_io_info * fio)2642 static inline bool need_inplace_update(struct f2fs_io_info *fio)
2643 {
2644 	struct inode *inode = fio_inode(fio);
2645 
2646 	if (f2fs_should_update_outplace(inode, fio))
2647 		return false;
2648 
2649 	return f2fs_should_update_inplace(inode, fio);
2650 }
2651 
f2fs_do_write_data_page(struct f2fs_io_info * fio)2652 int f2fs_do_write_data_page(struct f2fs_io_info *fio)
2653 {
2654 	struct folio *folio = fio->folio;
2655 	struct inode *inode = folio->mapping->host;
2656 	struct dnode_of_data dn;
2657 	struct node_info ni;
2658 	bool ipu_force = false;
2659 	bool atomic_commit;
2660 	int err = 0;
2661 
2662 	/* Use COW inode to make dnode_of_data for atomic write */
2663 	atomic_commit = f2fs_is_atomic_file(inode) &&
2664 				folio_test_f2fs_atomic(folio);
2665 	if (atomic_commit)
2666 		set_new_dnode(&dn, F2FS_I(inode)->cow_inode, NULL, NULL, 0);
2667 	else
2668 		set_new_dnode(&dn, inode, NULL, NULL, 0);
2669 
2670 	if (need_inplace_update(fio) &&
2671 	    f2fs_lookup_read_extent_cache_block(inode, folio->index,
2672 						&fio->old_blkaddr)) {
2673 		if (!f2fs_is_valid_blkaddr(fio->sbi, fio->old_blkaddr,
2674 						DATA_GENERIC_ENHANCE))
2675 			return -EFSCORRUPTED;
2676 
2677 		ipu_force = true;
2678 		fio->need_lock = LOCK_DONE;
2679 		goto got_it;
2680 	}
2681 
2682 	/* Deadlock due to between page->lock and f2fs_lock_op */
2683 	if (fio->need_lock == LOCK_REQ && !f2fs_trylock_op(fio->sbi))
2684 		return -EAGAIN;
2685 
2686 	err = f2fs_get_dnode_of_data(&dn, folio->index, LOOKUP_NODE);
2687 	if (err)
2688 		goto out;
2689 
2690 	fio->old_blkaddr = dn.data_blkaddr;
2691 
2692 	/* This page is already truncated */
2693 	if (fio->old_blkaddr == NULL_ADDR) {
2694 		folio_clear_uptodate(folio);
2695 		folio_clear_f2fs_gcing(folio);
2696 		goto out_writepage;
2697 	}
2698 got_it:
2699 	if (__is_valid_data_blkaddr(fio->old_blkaddr) &&
2700 		!f2fs_is_valid_blkaddr(fio->sbi, fio->old_blkaddr,
2701 						DATA_GENERIC_ENHANCE)) {
2702 		err = -EFSCORRUPTED;
2703 		goto out_writepage;
2704 	}
2705 
2706 	/* wait for GCed page writeback via META_MAPPING */
2707 	if (fio->meta_gc)
2708 		f2fs_wait_on_block_writeback(inode, fio->old_blkaddr);
2709 
2710 	/*
2711 	 * If current allocation needs SSR,
2712 	 * it had better in-place writes for updated data.
2713 	 */
2714 	if (ipu_force ||
2715 		(__is_valid_data_blkaddr(fio->old_blkaddr) &&
2716 					need_inplace_update(fio))) {
2717 		err = f2fs_encrypt_one_page(fio);
2718 		if (err)
2719 			goto out_writepage;
2720 
2721 		folio_start_writeback(folio);
2722 		f2fs_put_dnode(&dn);
2723 		if (fio->need_lock == LOCK_REQ)
2724 			f2fs_unlock_op(fio->sbi);
2725 		err = f2fs_inplace_write_data(fio);
2726 		if (err) {
2727 			if (fscrypt_inode_uses_fs_layer_crypto(inode))
2728 				fscrypt_finalize_bounce_page(&fio->encrypted_page);
2729 			folio_end_writeback(folio);
2730 		} else {
2731 			set_inode_flag(inode, FI_UPDATE_WRITE);
2732 		}
2733 		trace_f2fs_do_write_data_page(folio, IPU);
2734 		return err;
2735 	}
2736 
2737 	if (fio->need_lock == LOCK_RETRY) {
2738 		if (!f2fs_trylock_op(fio->sbi)) {
2739 			err = -EAGAIN;
2740 			goto out_writepage;
2741 		}
2742 		fio->need_lock = LOCK_REQ;
2743 	}
2744 
2745 	err = f2fs_get_node_info(fio->sbi, dn.nid, &ni, false);
2746 	if (err)
2747 		goto out_writepage;
2748 
2749 	fio->version = ni.version;
2750 
2751 	err = f2fs_encrypt_one_page(fio);
2752 	if (err)
2753 		goto out_writepage;
2754 
2755 	folio_start_writeback(folio);
2756 
2757 	if (fio->compr_blocks && fio->old_blkaddr == COMPRESS_ADDR)
2758 		f2fs_i_compr_blocks_update(inode, fio->compr_blocks - 1, false);
2759 
2760 	/* LFS mode write path */
2761 	f2fs_outplace_write_data(&dn, fio);
2762 	trace_f2fs_do_write_data_page(folio, OPU);
2763 	set_inode_flag(inode, FI_APPEND_WRITE);
2764 	if (atomic_commit)
2765 		folio_clear_f2fs_atomic(folio);
2766 out_writepage:
2767 	f2fs_put_dnode(&dn);
2768 out:
2769 	if (fio->need_lock == LOCK_REQ)
2770 		f2fs_unlock_op(fio->sbi);
2771 	return err;
2772 }
2773 
f2fs_write_single_data_page(struct folio * folio,int * submitted,struct bio ** bio,sector_t * last_block,struct writeback_control * wbc,enum iostat_type io_type,int compr_blocks,bool allow_balance)2774 int f2fs_write_single_data_page(struct folio *folio, int *submitted,
2775 				struct bio **bio,
2776 				sector_t *last_block,
2777 				struct writeback_control *wbc,
2778 				enum iostat_type io_type,
2779 				int compr_blocks,
2780 				bool allow_balance)
2781 {
2782 	struct inode *inode = folio->mapping->host;
2783 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
2784 	loff_t i_size = i_size_read(inode);
2785 	const pgoff_t end_index = ((unsigned long long)i_size)
2786 							>> PAGE_SHIFT;
2787 	loff_t psize = (loff_t)(folio->index + 1) << PAGE_SHIFT;
2788 	unsigned offset = 0;
2789 	bool need_balance_fs = false;
2790 	bool quota_inode = IS_NOQUOTA(inode);
2791 	int err = 0;
2792 	struct f2fs_io_info fio = {
2793 		.sbi = sbi,
2794 		.ino = inode->i_ino,
2795 		.type = DATA,
2796 		.op = REQ_OP_WRITE,
2797 		.op_flags = wbc_to_write_flags(wbc),
2798 		.old_blkaddr = NULL_ADDR,
2799 		.folio = folio,
2800 		.encrypted_page = NULL,
2801 		.submitted = 0,
2802 		.compr_blocks = compr_blocks,
2803 		.need_lock = compr_blocks ? LOCK_DONE : LOCK_RETRY,
2804 		.meta_gc = f2fs_meta_inode_gc_required(inode) ? 1 : 0,
2805 		.io_type = io_type,
2806 		.io_wbc = wbc,
2807 		.bio = bio,
2808 		.last_block = last_block,
2809 	};
2810 
2811 	trace_f2fs_writepage(folio, DATA);
2812 
2813 	/* we should bypass data pages to proceed the kworker jobs */
2814 	if (unlikely(f2fs_cp_error(sbi))) {
2815 		mapping_set_error(folio->mapping, -EIO);
2816 		/*
2817 		 * don't drop any dirty dentry pages for keeping lastest
2818 		 * directory structure.
2819 		 */
2820 		if (S_ISDIR(inode->i_mode) &&
2821 				!is_sbi_flag_set(sbi, SBI_IS_CLOSE))
2822 			goto redirty_out;
2823 
2824 		/* keep data pages in remount-ro mode */
2825 		if (F2FS_OPTION(sbi).errors == MOUNT_ERRORS_READONLY)
2826 			goto redirty_out;
2827 		goto out;
2828 	}
2829 
2830 	if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
2831 		goto redirty_out;
2832 
2833 	if (folio->index < end_index ||
2834 			f2fs_verity_in_progress(inode) ||
2835 			compr_blocks)
2836 		goto write;
2837 
2838 	/*
2839 	 * If the offset is out-of-range of file size,
2840 	 * this page does not have to be written to disk.
2841 	 */
2842 	offset = i_size & (PAGE_SIZE - 1);
2843 	if ((folio->index >= end_index + 1) || !offset)
2844 		goto out;
2845 
2846 	folio_zero_segment(folio, offset, folio_size(folio));
2847 write:
2848 	/* Dentry/quota blocks are controlled by checkpoint */
2849 	if (S_ISDIR(inode->i_mode) || quota_inode) {
2850 		/*
2851 		 * We need to wait for node_write to avoid block allocation during
2852 		 * checkpoint. This can only happen to quota writes which can cause
2853 		 * the below discard race condition.
2854 		 */
2855 		if (quota_inode)
2856 			f2fs_down_read(&sbi->node_write);
2857 
2858 		fio.need_lock = LOCK_DONE;
2859 		err = f2fs_do_write_data_page(&fio);
2860 
2861 		if (quota_inode)
2862 			f2fs_up_read(&sbi->node_write);
2863 
2864 		goto done;
2865 	}
2866 
2867 	need_balance_fs = true;
2868 	err = -EAGAIN;
2869 	if (f2fs_has_inline_data(inode)) {
2870 		err = f2fs_write_inline_data(inode, folio);
2871 		if (!err)
2872 			goto out;
2873 	}
2874 
2875 	if (err == -EAGAIN) {
2876 		err = f2fs_do_write_data_page(&fio);
2877 		if (err == -EAGAIN) {
2878 			f2fs_bug_on(sbi, compr_blocks);
2879 			fio.need_lock = LOCK_REQ;
2880 			err = f2fs_do_write_data_page(&fio);
2881 		}
2882 	}
2883 
2884 	if (err) {
2885 		file_set_keep_isize(inode);
2886 	} else {
2887 		spin_lock(&F2FS_I(inode)->i_size_lock);
2888 		if (F2FS_I(inode)->last_disk_size < psize)
2889 			F2FS_I(inode)->last_disk_size = psize;
2890 		spin_unlock(&F2FS_I(inode)->i_size_lock);
2891 	}
2892 
2893 done:
2894 	if (err && err != -ENOENT)
2895 		goto redirty_out;
2896 
2897 out:
2898 	inode_dec_dirty_pages(inode);
2899 	if (err) {
2900 		folio_clear_uptodate(folio);
2901 		folio_clear_f2fs_gcing(folio);
2902 	}
2903 	folio_unlock(folio);
2904 	if (!S_ISDIR(inode->i_mode) && !IS_NOQUOTA(inode) &&
2905 			!F2FS_I(inode)->wb_task && allow_balance)
2906 		f2fs_balance_fs(sbi, need_balance_fs);
2907 
2908 	if (unlikely(f2fs_cp_error(sbi))) {
2909 		f2fs_submit_merged_write(sbi, DATA);
2910 		if (bio && *bio)
2911 			f2fs_submit_merged_ipu_write(sbi, bio, NULL);
2912 		submitted = NULL;
2913 	}
2914 
2915 	if (submitted)
2916 		*submitted = fio.submitted;
2917 
2918 	return 0;
2919 
2920 redirty_out:
2921 	folio_redirty_for_writepage(wbc, folio);
2922 	/*
2923 	 * pageout() in MM translates EAGAIN, so calls handle_write_error()
2924 	 * -> mapping_set_error() -> set_bit(AS_EIO, ...).
2925 	 * file_write_and_wait_range() will see EIO error, which is critical
2926 	 * to return value of fsync() followed by atomic_write failure to user.
2927 	 */
2928 	folio_unlock(folio);
2929 	if (!err)
2930 		return 1;
2931 	return err;
2932 }
2933 
2934 /*
2935  * This function was copied from write_cache_pages from mm/page-writeback.c.
2936  * The major change is making write step of cold data page separately from
2937  * warm/hot data page.
2938  */
f2fs_write_cache_pages(struct address_space * mapping,struct writeback_control * wbc,enum iostat_type io_type)2939 static int f2fs_write_cache_pages(struct address_space *mapping,
2940 					struct writeback_control *wbc,
2941 					enum iostat_type io_type)
2942 {
2943 	int ret = 0;
2944 	int done = 0, retry = 0;
2945 	struct page *pages_local[F2FS_ONSTACK_PAGES];
2946 	struct page **pages = pages_local;
2947 	struct folio_batch fbatch;
2948 	struct f2fs_sb_info *sbi = F2FS_M_SB(mapping);
2949 	struct bio *bio = NULL;
2950 	sector_t last_block;
2951 #ifdef CONFIG_F2FS_FS_COMPRESSION
2952 	struct inode *inode = mapping->host;
2953 	struct compress_ctx cc = {
2954 		.inode = inode,
2955 		.log_cluster_size = F2FS_I(inode)->i_log_cluster_size,
2956 		.cluster_size = F2FS_I(inode)->i_cluster_size,
2957 		.cluster_idx = NULL_CLUSTER,
2958 		.rpages = NULL,
2959 		.nr_rpages = 0,
2960 		.cpages = NULL,
2961 		.valid_nr_cpages = 0,
2962 		.rbuf = NULL,
2963 		.cbuf = NULL,
2964 		.rlen = PAGE_SIZE * F2FS_I(inode)->i_cluster_size,
2965 		.private = NULL,
2966 	};
2967 #endif
2968 	int nr_folios, p, idx;
2969 	int nr_pages;
2970 	unsigned int max_pages = F2FS_ONSTACK_PAGES;
2971 	pgoff_t index;
2972 	pgoff_t end;		/* Inclusive */
2973 	pgoff_t done_index;
2974 	int range_whole = 0;
2975 	xa_mark_t tag;
2976 	int nwritten = 0;
2977 	int submitted = 0;
2978 	int i;
2979 
2980 #ifdef CONFIG_F2FS_FS_COMPRESSION
2981 	if (f2fs_compressed_file(inode) &&
2982 		1 << cc.log_cluster_size > F2FS_ONSTACK_PAGES) {
2983 		pages = f2fs_kzalloc(sbi, sizeof(struct page *) <<
2984 				cc.log_cluster_size, GFP_NOFS | __GFP_NOFAIL);
2985 		max_pages = 1 << cc.log_cluster_size;
2986 	}
2987 #endif
2988 
2989 	folio_batch_init(&fbatch);
2990 
2991 	if (get_dirty_pages(mapping->host) <=
2992 				SM_I(F2FS_M_SB(mapping))->min_hot_blocks)
2993 		set_inode_flag(mapping->host, FI_HOT_DATA);
2994 	else
2995 		clear_inode_flag(mapping->host, FI_HOT_DATA);
2996 
2997 	if (wbc->range_cyclic) {
2998 		index = mapping->writeback_index; /* prev offset */
2999 		end = -1;
3000 	} else {
3001 		index = wbc->range_start >> PAGE_SHIFT;
3002 		end = wbc->range_end >> PAGE_SHIFT;
3003 		if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
3004 			range_whole = 1;
3005 	}
3006 	if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
3007 		tag = PAGECACHE_TAG_TOWRITE;
3008 	else
3009 		tag = PAGECACHE_TAG_DIRTY;
3010 retry:
3011 	retry = 0;
3012 	if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
3013 		tag_pages_for_writeback(mapping, index, end);
3014 	done_index = index;
3015 	while (!done && !retry && (index <= end)) {
3016 		nr_pages = 0;
3017 again:
3018 		nr_folios = filemap_get_folios_tag(mapping, &index, end,
3019 				tag, &fbatch);
3020 		if (nr_folios == 0) {
3021 			if (nr_pages)
3022 				goto write;
3023 			break;
3024 		}
3025 
3026 		for (i = 0; i < nr_folios; i++) {
3027 			struct folio *folio = fbatch.folios[i];
3028 
3029 			idx = 0;
3030 			p = folio_nr_pages(folio);
3031 add_more:
3032 			pages[nr_pages] = folio_page(folio, idx);
3033 			folio_get(folio);
3034 			if (++nr_pages == max_pages) {
3035 				index = folio->index + idx + 1;
3036 				folio_batch_release(&fbatch);
3037 				goto write;
3038 			}
3039 			if (++idx < p)
3040 				goto add_more;
3041 		}
3042 		folio_batch_release(&fbatch);
3043 		goto again;
3044 write:
3045 		for (i = 0; i < nr_pages; i++) {
3046 			struct page *page = pages[i];
3047 			struct folio *folio = page_folio(page);
3048 			bool need_readd;
3049 readd:
3050 			need_readd = false;
3051 #ifdef CONFIG_F2FS_FS_COMPRESSION
3052 			if (f2fs_compressed_file(inode)) {
3053 				void *fsdata = NULL;
3054 				struct page *pagep;
3055 				int ret2;
3056 
3057 				ret = f2fs_init_compress_ctx(&cc);
3058 				if (ret) {
3059 					done = 1;
3060 					break;
3061 				}
3062 
3063 				if (!f2fs_cluster_can_merge_page(&cc,
3064 								folio->index)) {
3065 					ret = f2fs_write_multi_pages(&cc,
3066 						&submitted, wbc, io_type);
3067 					if (!ret)
3068 						need_readd = true;
3069 					goto result;
3070 				}
3071 
3072 				if (unlikely(f2fs_cp_error(sbi)))
3073 					goto lock_folio;
3074 
3075 				if (!f2fs_cluster_is_empty(&cc))
3076 					goto lock_folio;
3077 
3078 				if (f2fs_all_cluster_page_ready(&cc,
3079 					pages, i, nr_pages, true))
3080 					goto lock_folio;
3081 
3082 				ret2 = f2fs_prepare_compress_overwrite(
3083 							inode, &pagep,
3084 							folio->index, &fsdata);
3085 				if (ret2 < 0) {
3086 					ret = ret2;
3087 					done = 1;
3088 					break;
3089 				} else if (ret2 &&
3090 					(!f2fs_compress_write_end(inode,
3091 						fsdata, folio->index, 1) ||
3092 					 !f2fs_all_cluster_page_ready(&cc,
3093 						pages, i, nr_pages,
3094 						false))) {
3095 					retry = 1;
3096 					break;
3097 				}
3098 			}
3099 #endif
3100 			/* give a priority to WB_SYNC threads */
3101 			if (atomic_read(&sbi->wb_sync_req[DATA]) &&
3102 					wbc->sync_mode == WB_SYNC_NONE) {
3103 				done = 1;
3104 				break;
3105 			}
3106 #ifdef CONFIG_F2FS_FS_COMPRESSION
3107 lock_folio:
3108 #endif
3109 			done_index = folio->index;
3110 retry_write:
3111 			folio_lock(folio);
3112 
3113 			if (unlikely(folio->mapping != mapping)) {
3114 continue_unlock:
3115 				folio_unlock(folio);
3116 				continue;
3117 			}
3118 
3119 			if (!folio_test_dirty(folio)) {
3120 				/* someone wrote it for us */
3121 				goto continue_unlock;
3122 			}
3123 
3124 			if (folio_test_writeback(folio)) {
3125 				if (wbc->sync_mode == WB_SYNC_NONE)
3126 					goto continue_unlock;
3127 				f2fs_folio_wait_writeback(folio, DATA, true, true);
3128 			}
3129 
3130 			if (!folio_clear_dirty_for_io(folio))
3131 				goto continue_unlock;
3132 
3133 #ifdef CONFIG_F2FS_FS_COMPRESSION
3134 			if (f2fs_compressed_file(inode)) {
3135 				folio_get(folio);
3136 				f2fs_compress_ctx_add_page(&cc, folio);
3137 				continue;
3138 			}
3139 #endif
3140 			submitted = 0;
3141 			ret = f2fs_write_single_data_page(folio,
3142 					&submitted, &bio, &last_block,
3143 					wbc, io_type, 0, true);
3144 #ifdef CONFIG_F2FS_FS_COMPRESSION
3145 result:
3146 #endif
3147 			nwritten += submitted;
3148 			wbc->nr_to_write -= submitted;
3149 
3150 			if (unlikely(ret)) {
3151 				/*
3152 				 * keep nr_to_write, since vfs uses this to
3153 				 * get # of written pages.
3154 				 */
3155 				if (ret == 1) {
3156 					ret = 0;
3157 					goto next;
3158 				} else if (ret == -EAGAIN) {
3159 					ret = 0;
3160 					if (wbc->sync_mode == WB_SYNC_ALL) {
3161 						f2fs_io_schedule_timeout(
3162 							DEFAULT_IO_TIMEOUT);
3163 						goto retry_write;
3164 					}
3165 					goto next;
3166 				}
3167 				done_index = folio_next_index(folio);
3168 				done = 1;
3169 				break;
3170 			}
3171 
3172 			if (wbc->nr_to_write <= 0 &&
3173 					wbc->sync_mode == WB_SYNC_NONE) {
3174 				done = 1;
3175 				break;
3176 			}
3177 next:
3178 			if (need_readd)
3179 				goto readd;
3180 		}
3181 		release_pages(pages, nr_pages);
3182 		cond_resched();
3183 	}
3184 #ifdef CONFIG_F2FS_FS_COMPRESSION
3185 	/* flush remained pages in compress cluster */
3186 	if (f2fs_compressed_file(inode) && !f2fs_cluster_is_empty(&cc)) {
3187 		ret = f2fs_write_multi_pages(&cc, &submitted, wbc, io_type);
3188 		nwritten += submitted;
3189 		wbc->nr_to_write -= submitted;
3190 		if (ret) {
3191 			done = 1;
3192 			retry = 0;
3193 		}
3194 	}
3195 	if (f2fs_compressed_file(inode))
3196 		f2fs_destroy_compress_ctx(&cc, false);
3197 #endif
3198 	if (retry) {
3199 		index = 0;
3200 		end = -1;
3201 		goto retry;
3202 	}
3203 	if (wbc->range_cyclic && !done)
3204 		done_index = 0;
3205 	if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0))
3206 		mapping->writeback_index = done_index;
3207 
3208 	if (nwritten)
3209 		f2fs_submit_merged_write_cond(F2FS_M_SB(mapping), mapping->host,
3210 								NULL, 0, DATA);
3211 	/* submit cached bio of IPU write */
3212 	if (bio)
3213 		f2fs_submit_merged_ipu_write(sbi, &bio, NULL);
3214 
3215 #ifdef CONFIG_F2FS_FS_COMPRESSION
3216 	if (pages != pages_local)
3217 		kfree(pages);
3218 #endif
3219 
3220 	return ret;
3221 }
3222 
__should_serialize_io(struct inode * inode,struct writeback_control * wbc)3223 static inline bool __should_serialize_io(struct inode *inode,
3224 					struct writeback_control *wbc)
3225 {
3226 	/* to avoid deadlock in path of data flush */
3227 	if (F2FS_I(inode)->wb_task)
3228 		return false;
3229 
3230 	if (!S_ISREG(inode->i_mode))
3231 		return false;
3232 	if (IS_NOQUOTA(inode))
3233 		return false;
3234 
3235 	if (f2fs_need_compress_data(inode))
3236 		return true;
3237 	if (wbc->sync_mode != WB_SYNC_ALL)
3238 		return true;
3239 	if (get_dirty_pages(inode) >= SM_I(F2FS_I_SB(inode))->min_seq_blocks)
3240 		return true;
3241 	return false;
3242 }
3243 
__f2fs_write_data_pages(struct address_space * mapping,struct writeback_control * wbc,enum iostat_type io_type)3244 static int __f2fs_write_data_pages(struct address_space *mapping,
3245 						struct writeback_control *wbc,
3246 						enum iostat_type io_type)
3247 {
3248 	struct inode *inode = mapping->host;
3249 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
3250 	struct blk_plug plug;
3251 	int ret;
3252 	bool locked = false;
3253 
3254 	/* skip writing if there is no dirty page in this inode */
3255 	if (!get_dirty_pages(inode) && wbc->sync_mode == WB_SYNC_NONE)
3256 		return 0;
3257 
3258 	/* during POR, we don't need to trigger writepage at all. */
3259 	if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
3260 		goto skip_write;
3261 
3262 	if ((S_ISDIR(inode->i_mode) || IS_NOQUOTA(inode)) &&
3263 			wbc->sync_mode == WB_SYNC_NONE &&
3264 			get_dirty_pages(inode) < nr_pages_to_skip(sbi, DATA) &&
3265 			f2fs_available_free_memory(sbi, DIRTY_DENTS))
3266 		goto skip_write;
3267 
3268 	/* skip writing in file defragment preparing stage */
3269 	if (is_inode_flag_set(inode, FI_SKIP_WRITES))
3270 		goto skip_write;
3271 
3272 	trace_f2fs_writepages(mapping->host, wbc, DATA);
3273 
3274 	/* to avoid spliting IOs due to mixed WB_SYNC_ALL and WB_SYNC_NONE */
3275 	if (wbc->sync_mode == WB_SYNC_ALL)
3276 		atomic_inc(&sbi->wb_sync_req[DATA]);
3277 	else if (atomic_read(&sbi->wb_sync_req[DATA])) {
3278 		/* to avoid potential deadlock */
3279 		if (current->plug)
3280 			blk_finish_plug(current->plug);
3281 		goto skip_write;
3282 	}
3283 
3284 	if (__should_serialize_io(inode, wbc)) {
3285 		mutex_lock(&sbi->writepages);
3286 		locked = true;
3287 	}
3288 
3289 	blk_start_plug(&plug);
3290 	ret = f2fs_write_cache_pages(mapping, wbc, io_type);
3291 	blk_finish_plug(&plug);
3292 
3293 	if (locked)
3294 		mutex_unlock(&sbi->writepages);
3295 
3296 	if (wbc->sync_mode == WB_SYNC_ALL)
3297 		atomic_dec(&sbi->wb_sync_req[DATA]);
3298 	/*
3299 	 * if some pages were truncated, we cannot guarantee its mapping->host
3300 	 * to detect pending bios.
3301 	 */
3302 
3303 	f2fs_remove_dirty_inode(inode);
3304 	return ret;
3305 
3306 skip_write:
3307 	wbc->pages_skipped += get_dirty_pages(inode);
3308 	trace_f2fs_writepages(mapping->host, wbc, DATA);
3309 	return 0;
3310 }
3311 
f2fs_write_data_pages(struct address_space * mapping,struct writeback_control * wbc)3312 static int f2fs_write_data_pages(struct address_space *mapping,
3313 			    struct writeback_control *wbc)
3314 {
3315 	struct inode *inode = mapping->host;
3316 
3317 	return __f2fs_write_data_pages(mapping, wbc,
3318 			F2FS_I(inode)->cp_task == current ?
3319 			FS_CP_DATA_IO : FS_DATA_IO);
3320 }
3321 
f2fs_write_failed(struct inode * inode,loff_t to)3322 void f2fs_write_failed(struct inode *inode, loff_t to)
3323 {
3324 	loff_t i_size = i_size_read(inode);
3325 
3326 	if (IS_NOQUOTA(inode))
3327 		return;
3328 
3329 	/* In the fs-verity case, f2fs_end_enable_verity() does the truncate */
3330 	if (to > i_size && !f2fs_verity_in_progress(inode)) {
3331 		f2fs_down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
3332 		filemap_invalidate_lock(inode->i_mapping);
3333 
3334 		truncate_pagecache(inode, i_size);
3335 		f2fs_truncate_blocks(inode, i_size, true);
3336 
3337 		filemap_invalidate_unlock(inode->i_mapping);
3338 		f2fs_up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
3339 	}
3340 }
3341 
prepare_write_begin(struct f2fs_sb_info * sbi,struct folio * folio,loff_t pos,unsigned int len,block_t * blk_addr,bool * node_changed)3342 static int prepare_write_begin(struct f2fs_sb_info *sbi,
3343 			struct folio *folio, loff_t pos, unsigned int len,
3344 			block_t *blk_addr, bool *node_changed)
3345 {
3346 	struct inode *inode = folio->mapping->host;
3347 	pgoff_t index = folio->index;
3348 	struct dnode_of_data dn;
3349 	struct folio *ifolio;
3350 	bool locked = false;
3351 	int flag = F2FS_GET_BLOCK_PRE_AIO;
3352 	int err = 0;
3353 
3354 	/*
3355 	 * If a whole page is being written and we already preallocated all the
3356 	 * blocks, then there is no need to get a block address now.
3357 	 */
3358 	if (len == PAGE_SIZE && is_inode_flag_set(inode, FI_PREALLOCATED_ALL))
3359 		return 0;
3360 
3361 	/* f2fs_lock_op avoids race between write CP and convert_inline_page */
3362 	if (f2fs_has_inline_data(inode)) {
3363 		if (pos + len > MAX_INLINE_DATA(inode))
3364 			flag = F2FS_GET_BLOCK_DEFAULT;
3365 		f2fs_map_lock(sbi, flag);
3366 		locked = true;
3367 	} else if ((pos & PAGE_MASK) >= i_size_read(inode)) {
3368 		f2fs_map_lock(sbi, flag);
3369 		locked = true;
3370 	}
3371 
3372 restart:
3373 	/* check inline_data */
3374 	ifolio = f2fs_get_inode_folio(sbi, inode->i_ino);
3375 	if (IS_ERR(ifolio)) {
3376 		err = PTR_ERR(ifolio);
3377 		goto unlock_out;
3378 	}
3379 
3380 	set_new_dnode(&dn, inode, ifolio, ifolio, 0);
3381 
3382 	if (f2fs_has_inline_data(inode)) {
3383 		if (pos + len <= MAX_INLINE_DATA(inode)) {
3384 			f2fs_do_read_inline_data(folio, ifolio);
3385 			set_inode_flag(inode, FI_DATA_EXIST);
3386 			if (inode->i_nlink)
3387 				folio_set_f2fs_inline(ifolio);
3388 			goto out;
3389 		}
3390 		err = f2fs_convert_inline_folio(&dn, folio);
3391 		if (err || dn.data_blkaddr != NULL_ADDR)
3392 			goto out;
3393 	}
3394 
3395 	if (!f2fs_lookup_read_extent_cache_block(inode, index,
3396 						 &dn.data_blkaddr)) {
3397 		if (IS_DEVICE_ALIASING(inode)) {
3398 			err = -ENODATA;
3399 			goto out;
3400 		}
3401 
3402 		if (locked) {
3403 			err = f2fs_reserve_block(&dn, index);
3404 			goto out;
3405 		}
3406 
3407 		/* hole case */
3408 		err = f2fs_get_dnode_of_data(&dn, index, LOOKUP_NODE);
3409 		if (!err && dn.data_blkaddr != NULL_ADDR)
3410 			goto out;
3411 		f2fs_put_dnode(&dn);
3412 		f2fs_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO);
3413 		WARN_ON(flag != F2FS_GET_BLOCK_PRE_AIO);
3414 		locked = true;
3415 		goto restart;
3416 	}
3417 out:
3418 	if (!err) {
3419 		/* convert_inline_page can make node_changed */
3420 		*blk_addr = dn.data_blkaddr;
3421 		*node_changed = dn.node_changed;
3422 	}
3423 	f2fs_put_dnode(&dn);
3424 unlock_out:
3425 	if (locked)
3426 		f2fs_map_unlock(sbi, flag);
3427 	return err;
3428 }
3429 
__find_data_block(struct inode * inode,pgoff_t index,block_t * blk_addr)3430 static int __find_data_block(struct inode *inode, pgoff_t index,
3431 				block_t *blk_addr)
3432 {
3433 	struct dnode_of_data dn;
3434 	struct folio *ifolio;
3435 	int err = 0;
3436 
3437 	ifolio = f2fs_get_inode_folio(F2FS_I_SB(inode), inode->i_ino);
3438 	if (IS_ERR(ifolio))
3439 		return PTR_ERR(ifolio);
3440 
3441 	set_new_dnode(&dn, inode, ifolio, ifolio, 0);
3442 
3443 	if (!f2fs_lookup_read_extent_cache_block(inode, index,
3444 						 &dn.data_blkaddr)) {
3445 		/* hole case */
3446 		err = f2fs_get_dnode_of_data(&dn, index, LOOKUP_NODE);
3447 		if (err) {
3448 			dn.data_blkaddr = NULL_ADDR;
3449 			err = 0;
3450 		}
3451 	}
3452 	*blk_addr = dn.data_blkaddr;
3453 	f2fs_put_dnode(&dn);
3454 	return err;
3455 }
3456 
__reserve_data_block(struct inode * inode,pgoff_t index,block_t * blk_addr,bool * node_changed)3457 static int __reserve_data_block(struct inode *inode, pgoff_t index,
3458 				block_t *blk_addr, bool *node_changed)
3459 {
3460 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
3461 	struct dnode_of_data dn;
3462 	struct folio *ifolio;
3463 	int err = 0;
3464 
3465 	f2fs_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO);
3466 
3467 	ifolio = f2fs_get_inode_folio(sbi, inode->i_ino);
3468 	if (IS_ERR(ifolio)) {
3469 		err = PTR_ERR(ifolio);
3470 		goto unlock_out;
3471 	}
3472 	set_new_dnode(&dn, inode, ifolio, ifolio, 0);
3473 
3474 	if (!f2fs_lookup_read_extent_cache_block(dn.inode, index,
3475 						&dn.data_blkaddr))
3476 		err = f2fs_reserve_block(&dn, index);
3477 
3478 	*blk_addr = dn.data_blkaddr;
3479 	*node_changed = dn.node_changed;
3480 	f2fs_put_dnode(&dn);
3481 
3482 unlock_out:
3483 	f2fs_map_unlock(sbi, F2FS_GET_BLOCK_PRE_AIO);
3484 	return err;
3485 }
3486 
prepare_atomic_write_begin(struct f2fs_sb_info * sbi,struct folio * folio,loff_t pos,unsigned int len,block_t * blk_addr,bool * node_changed,bool * use_cow)3487 static int prepare_atomic_write_begin(struct f2fs_sb_info *sbi,
3488 			struct folio *folio, loff_t pos, unsigned int len,
3489 			block_t *blk_addr, bool *node_changed, bool *use_cow)
3490 {
3491 	struct inode *inode = folio->mapping->host;
3492 	struct inode *cow_inode = F2FS_I(inode)->cow_inode;
3493 	pgoff_t index = folio->index;
3494 	int err = 0;
3495 	block_t ori_blk_addr = NULL_ADDR;
3496 
3497 	/* If pos is beyond the end of file, reserve a new block in COW inode */
3498 	if ((pos & PAGE_MASK) >= i_size_read(inode))
3499 		goto reserve_block;
3500 
3501 	/* Look for the block in COW inode first */
3502 	err = __find_data_block(cow_inode, index, blk_addr);
3503 	if (err) {
3504 		return err;
3505 	} else if (*blk_addr != NULL_ADDR) {
3506 		*use_cow = true;
3507 		return 0;
3508 	}
3509 
3510 	if (is_inode_flag_set(inode, FI_ATOMIC_REPLACE))
3511 		goto reserve_block;
3512 
3513 	/* Look for the block in the original inode */
3514 	err = __find_data_block(inode, index, &ori_blk_addr);
3515 	if (err)
3516 		return err;
3517 
3518 reserve_block:
3519 	/* Finally, we should reserve a new block in COW inode for the update */
3520 	err = __reserve_data_block(cow_inode, index, blk_addr, node_changed);
3521 	if (err)
3522 		return err;
3523 	inc_atomic_write_cnt(inode);
3524 
3525 	if (ori_blk_addr != NULL_ADDR)
3526 		*blk_addr = ori_blk_addr;
3527 	return 0;
3528 }
3529 
f2fs_write_begin(const struct kiocb * iocb,struct address_space * mapping,loff_t pos,unsigned len,struct folio ** foliop,void ** fsdata)3530 static int f2fs_write_begin(const struct kiocb *iocb,
3531 			    struct address_space *mapping,
3532 			    loff_t pos, unsigned len, struct folio **foliop,
3533 			    void **fsdata)
3534 {
3535 	struct inode *inode = mapping->host;
3536 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
3537 	struct folio *folio;
3538 	pgoff_t index = pos >> PAGE_SHIFT;
3539 	bool need_balance = false;
3540 	bool use_cow = false;
3541 	block_t blkaddr = NULL_ADDR;
3542 	int err = 0;
3543 
3544 	trace_f2fs_write_begin(inode, pos, len);
3545 
3546 	if (!f2fs_is_checkpoint_ready(sbi)) {
3547 		err = -ENOSPC;
3548 		goto fail;
3549 	}
3550 
3551 	/*
3552 	 * We should check this at this moment to avoid deadlock on inode page
3553 	 * and #0 page. The locking rule for inline_data conversion should be:
3554 	 * folio_lock(folio #0) -> folio_lock(inode_page)
3555 	 */
3556 	if (index != 0) {
3557 		err = f2fs_convert_inline_inode(inode);
3558 		if (err)
3559 			goto fail;
3560 	}
3561 
3562 #ifdef CONFIG_F2FS_FS_COMPRESSION
3563 	if (f2fs_compressed_file(inode)) {
3564 		int ret;
3565 		struct page *page;
3566 
3567 		*fsdata = NULL;
3568 
3569 		if (len == PAGE_SIZE && !(f2fs_is_atomic_file(inode)))
3570 			goto repeat;
3571 
3572 		ret = f2fs_prepare_compress_overwrite(inode, &page,
3573 							index, fsdata);
3574 		if (ret < 0) {
3575 			err = ret;
3576 			goto fail;
3577 		} else if (ret) {
3578 			*foliop = page_folio(page);
3579 			return 0;
3580 		}
3581 	}
3582 #endif
3583 
3584 repeat:
3585 	/*
3586 	 * Do not use FGP_STABLE to avoid deadlock.
3587 	 * Will wait that below with our IO control.
3588 	 */
3589 	folio = __filemap_get_folio(mapping, index,
3590 				FGP_LOCK | FGP_WRITE | FGP_CREAT, GFP_NOFS);
3591 	if (IS_ERR(folio)) {
3592 		err = PTR_ERR(folio);
3593 		goto fail;
3594 	}
3595 
3596 	/* TODO: cluster can be compressed due to race with .writepage */
3597 
3598 	*foliop = folio;
3599 
3600 	if (f2fs_is_atomic_file(inode))
3601 		err = prepare_atomic_write_begin(sbi, folio, pos, len,
3602 					&blkaddr, &need_balance, &use_cow);
3603 	else
3604 		err = prepare_write_begin(sbi, folio, pos, len,
3605 					&blkaddr, &need_balance);
3606 	if (err)
3607 		goto put_folio;
3608 
3609 	if (need_balance && !IS_NOQUOTA(inode) &&
3610 			has_not_enough_free_secs(sbi, 0, 0)) {
3611 		folio_unlock(folio);
3612 		f2fs_balance_fs(sbi, true);
3613 		folio_lock(folio);
3614 		if (folio->mapping != mapping) {
3615 			/* The folio got truncated from under us */
3616 			folio_unlock(folio);
3617 			folio_put(folio);
3618 			goto repeat;
3619 		}
3620 	}
3621 
3622 	f2fs_folio_wait_writeback(folio, DATA, false, true);
3623 
3624 	if (len == folio_size(folio) || folio_test_uptodate(folio))
3625 		return 0;
3626 
3627 	if (!(pos & (PAGE_SIZE - 1)) && (pos + len) >= i_size_read(inode) &&
3628 	    !f2fs_verity_in_progress(inode)) {
3629 		folio_zero_segment(folio, len, folio_size(folio));
3630 		return 0;
3631 	}
3632 
3633 	if (blkaddr == NEW_ADDR) {
3634 		folio_zero_segment(folio, 0, folio_size(folio));
3635 		folio_mark_uptodate(folio);
3636 	} else {
3637 		if (!f2fs_is_valid_blkaddr(sbi, blkaddr,
3638 				DATA_GENERIC_ENHANCE_READ)) {
3639 			err = -EFSCORRUPTED;
3640 			goto put_folio;
3641 		}
3642 		err = f2fs_submit_page_read(use_cow ?
3643 				F2FS_I(inode)->cow_inode : inode,
3644 				folio, blkaddr, 0, true);
3645 		if (err)
3646 			goto put_folio;
3647 
3648 		folio_lock(folio);
3649 		if (unlikely(folio->mapping != mapping)) {
3650 			folio_unlock(folio);
3651 			folio_put(folio);
3652 			goto repeat;
3653 		}
3654 		if (unlikely(!folio_test_uptodate(folio))) {
3655 			err = -EIO;
3656 			goto put_folio;
3657 		}
3658 	}
3659 	return 0;
3660 
3661 put_folio:
3662 	folio_unlock(folio);
3663 	folio_put(folio);
3664 fail:
3665 	f2fs_write_failed(inode, pos + len);
3666 	return err;
3667 }
3668 
f2fs_write_end(const struct kiocb * iocb,struct address_space * mapping,loff_t pos,unsigned len,unsigned copied,struct folio * folio,void * fsdata)3669 static int f2fs_write_end(const struct kiocb *iocb,
3670 			struct address_space *mapping,
3671 			loff_t pos, unsigned len, unsigned copied,
3672 			struct folio *folio, void *fsdata)
3673 {
3674 	struct inode *inode = folio->mapping->host;
3675 
3676 	trace_f2fs_write_end(inode, pos, len, copied);
3677 
3678 	/*
3679 	 * This should be come from len == PAGE_SIZE, and we expect copied
3680 	 * should be PAGE_SIZE. Otherwise, we treat it with zero copied and
3681 	 * let generic_perform_write() try to copy data again through copied=0.
3682 	 */
3683 	if (!folio_test_uptodate(folio)) {
3684 		if (unlikely(copied != len))
3685 			copied = 0;
3686 		else
3687 			folio_mark_uptodate(folio);
3688 	}
3689 
3690 #ifdef CONFIG_F2FS_FS_COMPRESSION
3691 	/* overwrite compressed file */
3692 	if (f2fs_compressed_file(inode) && fsdata) {
3693 		f2fs_compress_write_end(inode, fsdata, folio->index, copied);
3694 		f2fs_update_time(F2FS_I_SB(inode), REQ_TIME);
3695 
3696 		if (pos + copied > i_size_read(inode) &&
3697 				!f2fs_verity_in_progress(inode))
3698 			f2fs_i_size_write(inode, pos + copied);
3699 		return copied;
3700 	}
3701 #endif
3702 
3703 	if (!copied)
3704 		goto unlock_out;
3705 
3706 	folio_mark_dirty(folio);
3707 
3708 	if (f2fs_is_atomic_file(inode))
3709 		folio_set_f2fs_atomic(folio);
3710 
3711 	if (pos + copied > i_size_read(inode) &&
3712 	    !f2fs_verity_in_progress(inode)) {
3713 		f2fs_i_size_write(inode, pos + copied);
3714 		if (f2fs_is_atomic_file(inode))
3715 			f2fs_i_size_write(F2FS_I(inode)->cow_inode,
3716 					pos + copied);
3717 	}
3718 unlock_out:
3719 	folio_unlock(folio);
3720 	folio_put(folio);
3721 	f2fs_update_time(F2FS_I_SB(inode), REQ_TIME);
3722 	return copied;
3723 }
3724 
f2fs_invalidate_folio(struct folio * folio,size_t offset,size_t length)3725 void f2fs_invalidate_folio(struct folio *folio, size_t offset, size_t length)
3726 {
3727 	struct inode *inode = folio->mapping->host;
3728 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
3729 
3730 	if (inode->i_ino >= F2FS_ROOT_INO(sbi) &&
3731 				(offset || length != folio_size(folio)))
3732 		return;
3733 
3734 	if (folio_test_dirty(folio)) {
3735 		if (inode->i_ino == F2FS_META_INO(sbi)) {
3736 			dec_page_count(sbi, F2FS_DIRTY_META);
3737 		} else if (inode->i_ino == F2FS_NODE_INO(sbi)) {
3738 			dec_page_count(sbi, F2FS_DIRTY_NODES);
3739 		} else {
3740 			inode_dec_dirty_pages(inode);
3741 			f2fs_remove_dirty_inode(inode);
3742 		}
3743 	}
3744 	folio_detach_private(folio);
3745 }
3746 
f2fs_release_folio(struct folio * folio,gfp_t wait)3747 bool f2fs_release_folio(struct folio *folio, gfp_t wait)
3748 {
3749 	/* If this is dirty folio, keep private data */
3750 	if (folio_test_dirty(folio))
3751 		return false;
3752 
3753 	folio_detach_private(folio);
3754 	return true;
3755 }
3756 
f2fs_dirty_data_folio(struct address_space * mapping,struct folio * folio)3757 static bool f2fs_dirty_data_folio(struct address_space *mapping,
3758 		struct folio *folio)
3759 {
3760 	struct inode *inode = mapping->host;
3761 
3762 	trace_f2fs_set_page_dirty(folio, DATA);
3763 
3764 	if (!folio_test_uptodate(folio))
3765 		folio_mark_uptodate(folio);
3766 	BUG_ON(folio_test_swapcache(folio));
3767 
3768 	if (filemap_dirty_folio(mapping, folio)) {
3769 		f2fs_update_dirty_folio(inode, folio);
3770 		return true;
3771 	}
3772 	return false;
3773 }
3774 
3775 
f2fs_bmap_compress(struct inode * inode,sector_t block)3776 static sector_t f2fs_bmap_compress(struct inode *inode, sector_t block)
3777 {
3778 #ifdef CONFIG_F2FS_FS_COMPRESSION
3779 	struct dnode_of_data dn;
3780 	sector_t start_idx, blknr = 0;
3781 	int ret;
3782 
3783 	start_idx = round_down(block, F2FS_I(inode)->i_cluster_size);
3784 
3785 	set_new_dnode(&dn, inode, NULL, NULL, 0);
3786 	ret = f2fs_get_dnode_of_data(&dn, start_idx, LOOKUP_NODE);
3787 	if (ret)
3788 		return 0;
3789 
3790 	if (dn.data_blkaddr != COMPRESS_ADDR) {
3791 		dn.ofs_in_node += block - start_idx;
3792 		blknr = f2fs_data_blkaddr(&dn);
3793 		if (!__is_valid_data_blkaddr(blknr))
3794 			blknr = 0;
3795 	}
3796 
3797 	f2fs_put_dnode(&dn);
3798 	return blknr;
3799 #else
3800 	return 0;
3801 #endif
3802 }
3803 
3804 
f2fs_bmap(struct address_space * mapping,sector_t block)3805 static sector_t f2fs_bmap(struct address_space *mapping, sector_t block)
3806 {
3807 	struct inode *inode = mapping->host;
3808 	sector_t blknr = 0;
3809 
3810 	if (f2fs_has_inline_data(inode))
3811 		goto out;
3812 
3813 	/* make sure allocating whole blocks */
3814 	if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY))
3815 		filemap_write_and_wait(mapping);
3816 
3817 	/* Block number less than F2FS MAX BLOCKS */
3818 	if (unlikely(block >= max_file_blocks(inode)))
3819 		goto out;
3820 
3821 	if (f2fs_compressed_file(inode)) {
3822 		blknr = f2fs_bmap_compress(inode, block);
3823 	} else {
3824 		struct f2fs_map_blocks map;
3825 
3826 		memset(&map, 0, sizeof(map));
3827 		map.m_lblk = block;
3828 		map.m_len = 1;
3829 		map.m_next_pgofs = NULL;
3830 		map.m_seg_type = NO_CHECK_TYPE;
3831 
3832 		if (!f2fs_map_blocks(inode, &map, F2FS_GET_BLOCK_BMAP))
3833 			blknr = map.m_pblk;
3834 	}
3835 out:
3836 	trace_f2fs_bmap(inode, block, blknr);
3837 	return blknr;
3838 }
3839 
3840 #ifdef CONFIG_SWAP
f2fs_migrate_blocks(struct inode * inode,block_t start_blk,unsigned int blkcnt)3841 static int f2fs_migrate_blocks(struct inode *inode, block_t start_blk,
3842 							unsigned int blkcnt)
3843 {
3844 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
3845 	unsigned int blkofs;
3846 	unsigned int blk_per_sec = BLKS_PER_SEC(sbi);
3847 	unsigned int end_blk = start_blk + blkcnt - 1;
3848 	unsigned int secidx = start_blk / blk_per_sec;
3849 	unsigned int end_sec;
3850 	int ret = 0;
3851 
3852 	if (!blkcnt)
3853 		return 0;
3854 	end_sec = end_blk / blk_per_sec;
3855 
3856 	f2fs_down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
3857 	filemap_invalidate_lock(inode->i_mapping);
3858 
3859 	set_inode_flag(inode, FI_ALIGNED_WRITE);
3860 	set_inode_flag(inode, FI_OPU_WRITE);
3861 
3862 	for (; secidx <= end_sec; secidx++) {
3863 		unsigned int blkofs_end = secidx == end_sec ?
3864 				end_blk % blk_per_sec : blk_per_sec - 1;
3865 
3866 		f2fs_down_write(&sbi->pin_sem);
3867 
3868 		ret = f2fs_allocate_pinning_section(sbi);
3869 		if (ret) {
3870 			f2fs_up_write(&sbi->pin_sem);
3871 			break;
3872 		}
3873 
3874 		set_inode_flag(inode, FI_SKIP_WRITES);
3875 
3876 		for (blkofs = 0; blkofs <= blkofs_end; blkofs++) {
3877 			struct folio *folio;
3878 			unsigned int blkidx = secidx * blk_per_sec + blkofs;
3879 
3880 			folio = f2fs_get_lock_data_folio(inode, blkidx, true);
3881 			if (IS_ERR(folio)) {
3882 				f2fs_up_write(&sbi->pin_sem);
3883 				ret = PTR_ERR(folio);
3884 				goto done;
3885 			}
3886 
3887 			folio_mark_dirty(folio);
3888 			f2fs_folio_put(folio, true);
3889 		}
3890 
3891 		clear_inode_flag(inode, FI_SKIP_WRITES);
3892 
3893 		ret = filemap_fdatawrite(inode->i_mapping);
3894 
3895 		f2fs_up_write(&sbi->pin_sem);
3896 
3897 		if (ret)
3898 			break;
3899 	}
3900 
3901 done:
3902 	clear_inode_flag(inode, FI_SKIP_WRITES);
3903 	clear_inode_flag(inode, FI_OPU_WRITE);
3904 	clear_inode_flag(inode, FI_ALIGNED_WRITE);
3905 
3906 	filemap_invalidate_unlock(inode->i_mapping);
3907 	f2fs_up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
3908 
3909 	return ret;
3910 }
3911 
check_swap_activate(struct swap_info_struct * sis,struct file * swap_file,sector_t * span)3912 static int check_swap_activate(struct swap_info_struct *sis,
3913 				struct file *swap_file, sector_t *span)
3914 {
3915 	struct address_space *mapping = swap_file->f_mapping;
3916 	struct inode *inode = mapping->host;
3917 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
3918 	block_t cur_lblock;
3919 	block_t last_lblock;
3920 	block_t pblock;
3921 	block_t lowest_pblock = -1;
3922 	block_t highest_pblock = 0;
3923 	int nr_extents = 0;
3924 	unsigned int nr_pblocks;
3925 	unsigned int blks_per_sec = BLKS_PER_SEC(sbi);
3926 	unsigned int not_aligned = 0;
3927 	int ret = 0;
3928 
3929 	/*
3930 	 * Map all the blocks into the extent list.  This code doesn't try
3931 	 * to be very smart.
3932 	 */
3933 	cur_lblock = 0;
3934 	last_lblock = F2FS_BYTES_TO_BLK(i_size_read(inode));
3935 
3936 	while (cur_lblock < last_lblock && cur_lblock < sis->max) {
3937 		struct f2fs_map_blocks map;
3938 retry:
3939 		cond_resched();
3940 
3941 		memset(&map, 0, sizeof(map));
3942 		map.m_lblk = cur_lblock;
3943 		map.m_len = last_lblock - cur_lblock;
3944 		map.m_next_pgofs = NULL;
3945 		map.m_next_extent = NULL;
3946 		map.m_seg_type = NO_CHECK_TYPE;
3947 		map.m_may_create = false;
3948 
3949 		ret = f2fs_map_blocks(inode, &map, F2FS_GET_BLOCK_FIEMAP);
3950 		if (ret)
3951 			goto out;
3952 
3953 		/* hole */
3954 		if (!(map.m_flags & F2FS_MAP_FLAGS)) {
3955 			f2fs_err(sbi, "Swapfile has holes");
3956 			ret = -EINVAL;
3957 			goto out;
3958 		}
3959 
3960 		pblock = map.m_pblk;
3961 		nr_pblocks = map.m_len;
3962 
3963 		if ((pblock - SM_I(sbi)->main_blkaddr) % blks_per_sec ||
3964 				nr_pblocks % blks_per_sec ||
3965 				f2fs_is_sequential_zone_area(sbi, pblock)) {
3966 			bool last_extent = false;
3967 
3968 			not_aligned++;
3969 
3970 			nr_pblocks = roundup(nr_pblocks, blks_per_sec);
3971 			if (cur_lblock + nr_pblocks > sis->max)
3972 				nr_pblocks -= blks_per_sec;
3973 
3974 			/* this extent is last one */
3975 			if (!nr_pblocks) {
3976 				nr_pblocks = last_lblock - cur_lblock;
3977 				last_extent = true;
3978 			}
3979 
3980 			ret = f2fs_migrate_blocks(inode, cur_lblock,
3981 							nr_pblocks);
3982 			if (ret) {
3983 				if (ret == -ENOENT)
3984 					ret = -EINVAL;
3985 				goto out;
3986 			}
3987 
3988 			if (!last_extent)
3989 				goto retry;
3990 		}
3991 
3992 		if (cur_lblock + nr_pblocks >= sis->max)
3993 			nr_pblocks = sis->max - cur_lblock;
3994 
3995 		if (cur_lblock) {	/* exclude the header page */
3996 			if (pblock < lowest_pblock)
3997 				lowest_pblock = pblock;
3998 			if (pblock + nr_pblocks - 1 > highest_pblock)
3999 				highest_pblock = pblock + nr_pblocks - 1;
4000 		}
4001 
4002 		/*
4003 		 * We found a PAGE_SIZE-length, PAGE_SIZE-aligned run of blocks
4004 		 */
4005 		ret = add_swap_extent(sis, cur_lblock, nr_pblocks, pblock);
4006 		if (ret < 0)
4007 			goto out;
4008 		nr_extents += ret;
4009 		cur_lblock += nr_pblocks;
4010 	}
4011 	ret = nr_extents;
4012 	*span = 1 + highest_pblock - lowest_pblock;
4013 	if (cur_lblock == 0)
4014 		cur_lblock = 1;	/* force Empty message */
4015 	sis->max = cur_lblock;
4016 	sis->pages = cur_lblock - 1;
4017 out:
4018 	if (not_aligned)
4019 		f2fs_warn(sbi, "Swapfile (%u) is not align to section: 1) creat(), 2) ioctl(F2FS_IOC_SET_PIN_FILE), 3) fallocate(%lu * N)",
4020 			  not_aligned, blks_per_sec * F2FS_BLKSIZE);
4021 	return ret;
4022 }
4023 
f2fs_swap_activate(struct swap_info_struct * sis,struct file * file,sector_t * span)4024 static int f2fs_swap_activate(struct swap_info_struct *sis, struct file *file,
4025 				sector_t *span)
4026 {
4027 	struct inode *inode = file_inode(file);
4028 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
4029 	int ret;
4030 
4031 	if (!S_ISREG(inode->i_mode))
4032 		return -EINVAL;
4033 
4034 	if (f2fs_readonly(sbi->sb))
4035 		return -EROFS;
4036 
4037 	if (f2fs_lfs_mode(sbi) && !f2fs_sb_has_blkzoned(sbi)) {
4038 		f2fs_err(sbi, "Swapfile not supported in LFS mode");
4039 		return -EINVAL;
4040 	}
4041 
4042 	ret = f2fs_convert_inline_inode(inode);
4043 	if (ret)
4044 		return ret;
4045 
4046 	if (!f2fs_disable_compressed_file(inode))
4047 		return -EINVAL;
4048 
4049 	ret = filemap_fdatawrite(inode->i_mapping);
4050 	if (ret < 0)
4051 		return ret;
4052 
4053 	f2fs_precache_extents(inode);
4054 
4055 	ret = check_swap_activate(sis, file, span);
4056 	if (ret < 0)
4057 		return ret;
4058 
4059 	stat_inc_swapfile_inode(inode);
4060 	set_inode_flag(inode, FI_PIN_FILE);
4061 	f2fs_update_time(sbi, REQ_TIME);
4062 	return ret;
4063 }
4064 
f2fs_swap_deactivate(struct file * file)4065 static void f2fs_swap_deactivate(struct file *file)
4066 {
4067 	struct inode *inode = file_inode(file);
4068 
4069 	stat_dec_swapfile_inode(inode);
4070 	clear_inode_flag(inode, FI_PIN_FILE);
4071 }
4072 #else
f2fs_swap_activate(struct swap_info_struct * sis,struct file * file,sector_t * span)4073 static int f2fs_swap_activate(struct swap_info_struct *sis, struct file *file,
4074 				sector_t *span)
4075 {
4076 	return -EOPNOTSUPP;
4077 }
4078 
f2fs_swap_deactivate(struct file * file)4079 static void f2fs_swap_deactivate(struct file *file)
4080 {
4081 }
4082 #endif
4083 
4084 const struct address_space_operations f2fs_dblock_aops = {
4085 	.read_folio	= f2fs_read_data_folio,
4086 	.readahead	= f2fs_readahead,
4087 	.writepages	= f2fs_write_data_pages,
4088 	.write_begin	= f2fs_write_begin,
4089 	.write_end	= f2fs_write_end,
4090 	.dirty_folio	= f2fs_dirty_data_folio,
4091 	.migrate_folio	= filemap_migrate_folio,
4092 	.invalidate_folio = f2fs_invalidate_folio,
4093 	.release_folio	= f2fs_release_folio,
4094 	.bmap		= f2fs_bmap,
4095 	.swap_activate  = f2fs_swap_activate,
4096 	.swap_deactivate = f2fs_swap_deactivate,
4097 };
4098 
f2fs_clear_page_cache_dirty_tag(struct folio * folio)4099 void f2fs_clear_page_cache_dirty_tag(struct folio *folio)
4100 {
4101 	struct address_space *mapping = folio->mapping;
4102 	unsigned long flags;
4103 
4104 	xa_lock_irqsave(&mapping->i_pages, flags);
4105 	__xa_clear_mark(&mapping->i_pages, folio->index,
4106 						PAGECACHE_TAG_DIRTY);
4107 	xa_unlock_irqrestore(&mapping->i_pages, flags);
4108 }
4109 
f2fs_init_post_read_processing(void)4110 int __init f2fs_init_post_read_processing(void)
4111 {
4112 	bio_post_read_ctx_cache =
4113 		kmem_cache_create("f2fs_bio_post_read_ctx",
4114 				  sizeof(struct bio_post_read_ctx), 0, 0, NULL);
4115 	if (!bio_post_read_ctx_cache)
4116 		goto fail;
4117 	bio_post_read_ctx_pool =
4118 		mempool_create_slab_pool(NUM_PREALLOC_POST_READ_CTXS,
4119 					 bio_post_read_ctx_cache);
4120 	if (!bio_post_read_ctx_pool)
4121 		goto fail_free_cache;
4122 	return 0;
4123 
4124 fail_free_cache:
4125 	kmem_cache_destroy(bio_post_read_ctx_cache);
4126 fail:
4127 	return -ENOMEM;
4128 }
4129 
f2fs_destroy_post_read_processing(void)4130 void f2fs_destroy_post_read_processing(void)
4131 {
4132 	mempool_destroy(bio_post_read_ctx_pool);
4133 	kmem_cache_destroy(bio_post_read_ctx_cache);
4134 }
4135 
f2fs_init_post_read_wq(struct f2fs_sb_info * sbi)4136 int f2fs_init_post_read_wq(struct f2fs_sb_info *sbi)
4137 {
4138 	if (!f2fs_sb_has_encrypt(sbi) &&
4139 		!f2fs_sb_has_verity(sbi) &&
4140 		!f2fs_sb_has_compression(sbi))
4141 		return 0;
4142 
4143 	sbi->post_read_wq = alloc_workqueue("f2fs_post_read_wq",
4144 						 WQ_UNBOUND | WQ_HIGHPRI,
4145 						 num_online_cpus());
4146 	return sbi->post_read_wq ? 0 : -ENOMEM;
4147 }
4148 
f2fs_destroy_post_read_wq(struct f2fs_sb_info * sbi)4149 void f2fs_destroy_post_read_wq(struct f2fs_sb_info *sbi)
4150 {
4151 	if (sbi->post_read_wq)
4152 		destroy_workqueue(sbi->post_read_wq);
4153 }
4154 
f2fs_init_bio_entry_cache(void)4155 int __init f2fs_init_bio_entry_cache(void)
4156 {
4157 	bio_entry_slab = f2fs_kmem_cache_create("f2fs_bio_entry_slab",
4158 			sizeof(struct bio_entry));
4159 	return bio_entry_slab ? 0 : -ENOMEM;
4160 }
4161 
f2fs_destroy_bio_entry_cache(void)4162 void f2fs_destroy_bio_entry_cache(void)
4163 {
4164 	kmem_cache_destroy(bio_entry_slab);
4165 }
4166 
f2fs_iomap_begin(struct inode * inode,loff_t offset,loff_t length,unsigned int flags,struct iomap * iomap,struct iomap * srcmap)4167 static int f2fs_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
4168 			    unsigned int flags, struct iomap *iomap,
4169 			    struct iomap *srcmap)
4170 {
4171 	struct f2fs_map_blocks map = { NULL, };
4172 	pgoff_t next_pgofs = 0;
4173 	int err;
4174 
4175 	map.m_lblk = F2FS_BYTES_TO_BLK(offset);
4176 	map.m_len = F2FS_BYTES_TO_BLK(offset + length - 1) - map.m_lblk + 1;
4177 	map.m_next_pgofs = &next_pgofs;
4178 	map.m_seg_type = f2fs_rw_hint_to_seg_type(F2FS_I_SB(inode),
4179 						inode->i_write_hint);
4180 	if (flags & IOMAP_WRITE && iomap->private) {
4181 		map.m_last_pblk = (unsigned long)iomap->private;
4182 		iomap->private = NULL;
4183 	}
4184 
4185 	/*
4186 	 * If the blocks being overwritten are already allocated,
4187 	 * f2fs_map_lock and f2fs_balance_fs are not necessary.
4188 	 */
4189 	if ((flags & IOMAP_WRITE) &&
4190 		!f2fs_overwrite_io(inode, offset, length))
4191 		map.m_may_create = true;
4192 
4193 	err = f2fs_map_blocks(inode, &map, F2FS_GET_BLOCK_DIO);
4194 	if (err)
4195 		return err;
4196 
4197 	iomap->offset = F2FS_BLK_TO_BYTES(map.m_lblk);
4198 
4199 	/*
4200 	 * When inline encryption is enabled, sometimes I/O to an encrypted file
4201 	 * has to be broken up to guarantee DUN contiguity.  Handle this by
4202 	 * limiting the length of the mapping returned.
4203 	 */
4204 	map.m_len = fscrypt_limit_io_blocks(inode, map.m_lblk, map.m_len);
4205 
4206 	/*
4207 	 * We should never see delalloc or compressed extents here based on
4208 	 * prior flushing and checks.
4209 	 */
4210 	if (WARN_ON_ONCE(map.m_pblk == COMPRESS_ADDR))
4211 		return -EINVAL;
4212 
4213 	if (map.m_flags & F2FS_MAP_MAPPED) {
4214 		if (WARN_ON_ONCE(map.m_pblk == NEW_ADDR))
4215 			return -EINVAL;
4216 
4217 		iomap->length = F2FS_BLK_TO_BYTES(map.m_len);
4218 		iomap->type = IOMAP_MAPPED;
4219 		iomap->flags |= IOMAP_F_MERGED;
4220 		iomap->bdev = map.m_bdev;
4221 		iomap->addr = F2FS_BLK_TO_BYTES(map.m_pblk);
4222 
4223 		if (flags & IOMAP_WRITE && map.m_last_pblk)
4224 			iomap->private = (void *)map.m_last_pblk;
4225 	} else {
4226 		if (flags & IOMAP_WRITE)
4227 			return -ENOTBLK;
4228 
4229 		if (map.m_pblk == NULL_ADDR) {
4230 			iomap->length = F2FS_BLK_TO_BYTES(next_pgofs) -
4231 							iomap->offset;
4232 			iomap->type = IOMAP_HOLE;
4233 		} else if (map.m_pblk == NEW_ADDR) {
4234 			iomap->length = F2FS_BLK_TO_BYTES(map.m_len);
4235 			iomap->type = IOMAP_UNWRITTEN;
4236 		} else {
4237 			f2fs_bug_on(F2FS_I_SB(inode), 1);
4238 		}
4239 		iomap->addr = IOMAP_NULL_ADDR;
4240 	}
4241 
4242 	if (map.m_flags & F2FS_MAP_NEW)
4243 		iomap->flags |= IOMAP_F_NEW;
4244 	if ((inode->i_state & I_DIRTY_DATASYNC) ||
4245 	    offset + length > i_size_read(inode))
4246 		iomap->flags |= IOMAP_F_DIRTY;
4247 
4248 	return 0;
4249 }
4250 
4251 const struct iomap_ops f2fs_iomap_ops = {
4252 	.iomap_begin	= f2fs_iomap_begin,
4253 };
4254