xref: /linux/fs/bcachefs/fs-io-buffered.c (revision 3df692169e8486fc3dd91fcd5ea81c27a0bac033)
1 // SPDX-License-Identifier: GPL-2.0
2 #ifndef NO_BCACHEFS_FS
3 
4 #include "bcachefs.h"
5 #include "alloc_foreground.h"
6 #include "bkey_buf.h"
7 #include "fs-io.h"
8 #include "fs-io-buffered.h"
9 #include "fs-io-direct.h"
10 #include "fs-io-pagecache.h"
11 #include "io_read.h"
12 #include "io_write.h"
13 
14 #include <linux/backing-dev.h>
15 #include <linux/pagemap.h>
16 #include <linux/writeback.h>
17 
18 static inline bool bio_full(struct bio *bio, unsigned len)
19 {
20 	if (bio->bi_vcnt >= bio->bi_max_vecs)
21 		return true;
22 	if (bio->bi_iter.bi_size > UINT_MAX - len)
23 		return true;
24 	return false;
25 }
26 
27 /* readpage(s): */
28 
29 static void bch2_readpages_end_io(struct bio *bio)
30 {
31 	struct folio_iter fi;
32 
33 	bio_for_each_folio_all(fi, bio) {
34 		if (!bio->bi_status) {
35 			folio_mark_uptodate(fi.folio);
36 		} else {
37 			folio_clear_uptodate(fi.folio);
38 			folio_set_error(fi.folio);
39 		}
40 		folio_unlock(fi.folio);
41 	}
42 
43 	bio_put(bio);
44 }
45 
46 struct readpages_iter {
47 	struct address_space	*mapping;
48 	unsigned		idx;
49 	folios			folios;
50 };
51 
52 static int readpages_iter_init(struct readpages_iter *iter,
53 			       struct readahead_control *ractl)
54 {
55 	struct folio **fi;
56 	int ret;
57 
58 	memset(iter, 0, sizeof(*iter));
59 
60 	iter->mapping = ractl->mapping;
61 
62 	ret = bch2_filemap_get_contig_folios_d(iter->mapping,
63 				ractl->_index << PAGE_SHIFT,
64 				(ractl->_index + ractl->_nr_pages) << PAGE_SHIFT,
65 				0, mapping_gfp_mask(iter->mapping),
66 				&iter->folios);
67 	if (ret)
68 		return ret;
69 
70 	darray_for_each(iter->folios, fi) {
71 		ractl->_nr_pages -= 1U << folio_order(*fi);
72 		__bch2_folio_create(*fi, __GFP_NOFAIL|GFP_KERNEL);
73 		folio_put(*fi);
74 		folio_put(*fi);
75 	}
76 
77 	return 0;
78 }
79 
80 static inline struct folio *readpage_iter_peek(struct readpages_iter *iter)
81 {
82 	if (iter->idx >= iter->folios.nr)
83 		return NULL;
84 	return iter->folios.data[iter->idx];
85 }
86 
87 static inline void readpage_iter_advance(struct readpages_iter *iter)
88 {
89 	iter->idx++;
90 }
91 
92 static bool extent_partial_reads_expensive(struct bkey_s_c k)
93 {
94 	struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k);
95 	struct bch_extent_crc_unpacked crc;
96 	const union bch_extent_entry *i;
97 
98 	bkey_for_each_crc(k.k, ptrs, crc, i)
99 		if (crc.csum_type || crc.compression_type)
100 			return true;
101 	return false;
102 }
103 
104 static int readpage_bio_extend(struct btree_trans *trans,
105 			       struct readpages_iter *iter,
106 			       struct bio *bio,
107 			       unsigned sectors_this_extent,
108 			       bool get_more)
109 {
110 	/* Don't hold btree locks while allocating memory: */
111 	bch2_trans_unlock(trans);
112 
113 	while (bio_sectors(bio) < sectors_this_extent &&
114 	       bio->bi_vcnt < bio->bi_max_vecs) {
115 		struct folio *folio = readpage_iter_peek(iter);
116 		int ret;
117 
118 		if (folio) {
119 			readpage_iter_advance(iter);
120 		} else {
121 			pgoff_t folio_offset = bio_end_sector(bio) >> PAGE_SECTORS_SHIFT;
122 
123 			if (!get_more)
124 				break;
125 
126 			folio = xa_load(&iter->mapping->i_pages, folio_offset);
127 			if (folio && !xa_is_value(folio))
128 				break;
129 
130 			folio = filemap_alloc_folio(readahead_gfp_mask(iter->mapping), 0);
131 			if (!folio)
132 				break;
133 
134 			if (!__bch2_folio_create(folio, GFP_KERNEL)) {
135 				folio_put(folio);
136 				break;
137 			}
138 
139 			ret = filemap_add_folio(iter->mapping, folio, folio_offset, GFP_KERNEL);
140 			if (ret) {
141 				__bch2_folio_release(folio);
142 				folio_put(folio);
143 				break;
144 			}
145 
146 			folio_put(folio);
147 		}
148 
149 		BUG_ON(folio_sector(folio) != bio_end_sector(bio));
150 
151 		BUG_ON(!bio_add_folio(bio, folio, folio_size(folio), 0));
152 	}
153 
154 	return bch2_trans_relock(trans);
155 }
156 
157 static void bchfs_read(struct btree_trans *trans,
158 		       struct bch_read_bio *rbio,
159 		       subvol_inum inum,
160 		       struct readpages_iter *readpages_iter)
161 {
162 	struct bch_fs *c = trans->c;
163 	struct btree_iter iter;
164 	struct bkey_buf sk;
165 	int flags = BCH_READ_RETRY_IF_STALE|
166 		BCH_READ_MAY_PROMOTE;
167 	u32 snapshot;
168 	int ret = 0;
169 
170 	rbio->c = c;
171 	rbio->start_time = local_clock();
172 	rbio->subvol = inum.subvol;
173 
174 	bch2_bkey_buf_init(&sk);
175 retry:
176 	bch2_trans_begin(trans);
177 	iter = (struct btree_iter) { NULL };
178 
179 	ret = bch2_subvolume_get_snapshot(trans, inum.subvol, &snapshot);
180 	if (ret)
181 		goto err;
182 
183 	bch2_trans_iter_init(trans, &iter, BTREE_ID_extents,
184 			     SPOS(inum.inum, rbio->bio.bi_iter.bi_sector, snapshot),
185 			     BTREE_ITER_SLOTS);
186 	while (1) {
187 		struct bkey_s_c k;
188 		unsigned bytes, sectors, offset_into_extent;
189 		enum btree_id data_btree = BTREE_ID_extents;
190 
191 		/*
192 		 * read_extent -> io_time_reset may cause a transaction restart
193 		 * without returning an error, we need to check for that here:
194 		 */
195 		ret = bch2_trans_relock(trans);
196 		if (ret)
197 			break;
198 
199 		bch2_btree_iter_set_pos(&iter,
200 				POS(inum.inum, rbio->bio.bi_iter.bi_sector));
201 
202 		k = bch2_btree_iter_peek_slot(&iter);
203 		ret = bkey_err(k);
204 		if (ret)
205 			break;
206 
207 		offset_into_extent = iter.pos.offset -
208 			bkey_start_offset(k.k);
209 		sectors = k.k->size - offset_into_extent;
210 
211 		bch2_bkey_buf_reassemble(&sk, c, k);
212 
213 		ret = bch2_read_indirect_extent(trans, &data_btree,
214 					&offset_into_extent, &sk);
215 		if (ret)
216 			break;
217 
218 		k = bkey_i_to_s_c(sk.k);
219 
220 		sectors = min(sectors, k.k->size - offset_into_extent);
221 
222 		if (readpages_iter) {
223 			ret = readpage_bio_extend(trans, readpages_iter, &rbio->bio, sectors,
224 						  extent_partial_reads_expensive(k));
225 			if (ret)
226 				break;
227 		}
228 
229 		bytes = min(sectors, bio_sectors(&rbio->bio)) << 9;
230 		swap(rbio->bio.bi_iter.bi_size, bytes);
231 
232 		if (rbio->bio.bi_iter.bi_size == bytes)
233 			flags |= BCH_READ_LAST_FRAGMENT;
234 
235 		bch2_bio_page_state_set(&rbio->bio, k);
236 
237 		bch2_read_extent(trans, rbio, iter.pos,
238 				 data_btree, k, offset_into_extent, flags);
239 
240 		if (flags & BCH_READ_LAST_FRAGMENT)
241 			break;
242 
243 		swap(rbio->bio.bi_iter.bi_size, bytes);
244 		bio_advance(&rbio->bio, bytes);
245 
246 		ret = btree_trans_too_many_iters(trans);
247 		if (ret)
248 			break;
249 	}
250 err:
251 	bch2_trans_iter_exit(trans, &iter);
252 
253 	if (bch2_err_matches(ret, BCH_ERR_transaction_restart))
254 		goto retry;
255 
256 	if (ret) {
257 		bch_err_inum_offset_ratelimited(c,
258 				iter.pos.inode,
259 				iter.pos.offset << 9,
260 				"read error %i from btree lookup", ret);
261 		rbio->bio.bi_status = BLK_STS_IOERR;
262 		bio_endio(&rbio->bio);
263 	}
264 
265 	bch2_bkey_buf_exit(&sk, c);
266 }
267 
268 void bch2_readahead(struct readahead_control *ractl)
269 {
270 	struct bch_inode_info *inode = to_bch_ei(ractl->mapping->host);
271 	struct bch_fs *c = inode->v.i_sb->s_fs_info;
272 	struct bch_io_opts opts;
273 	struct btree_trans *trans = bch2_trans_get(c);
274 	struct folio *folio;
275 	struct readpages_iter readpages_iter;
276 	int ret;
277 
278 	bch2_inode_opts_get(&opts, c, &inode->ei_inode);
279 
280 	ret = readpages_iter_init(&readpages_iter, ractl);
281 	BUG_ON(ret);
282 
283 	bch2_pagecache_add_get(inode);
284 
285 	while ((folio = readpage_iter_peek(&readpages_iter))) {
286 		unsigned n = min_t(unsigned,
287 				   readpages_iter.folios.nr -
288 				   readpages_iter.idx,
289 				   BIO_MAX_VECS);
290 		struct bch_read_bio *rbio =
291 			rbio_init(bio_alloc_bioset(NULL, n, REQ_OP_READ,
292 						   GFP_KERNEL, &c->bio_read),
293 				  opts);
294 
295 		readpage_iter_advance(&readpages_iter);
296 
297 		rbio->bio.bi_iter.bi_sector = folio_sector(folio);
298 		rbio->bio.bi_end_io = bch2_readpages_end_io;
299 		BUG_ON(!bio_add_folio(&rbio->bio, folio, folio_size(folio), 0));
300 
301 		bchfs_read(trans, rbio, inode_inum(inode),
302 			   &readpages_iter);
303 		bch2_trans_unlock(trans);
304 	}
305 
306 	bch2_pagecache_add_put(inode);
307 
308 	bch2_trans_put(trans);
309 	darray_exit(&readpages_iter.folios);
310 }
311 
312 static void __bchfs_readfolio(struct bch_fs *c, struct bch_read_bio *rbio,
313 			     subvol_inum inum, struct folio *folio)
314 {
315 	bch2_folio_create(folio, __GFP_NOFAIL);
316 
317 	rbio->bio.bi_opf = REQ_OP_READ|REQ_SYNC;
318 	rbio->bio.bi_iter.bi_sector = folio_sector(folio);
319 	BUG_ON(!bio_add_folio(&rbio->bio, folio, folio_size(folio), 0));
320 
321 	bch2_trans_run(c, (bchfs_read(trans, rbio, inum, NULL), 0));
322 }
323 
324 static void bch2_read_single_folio_end_io(struct bio *bio)
325 {
326 	complete(bio->bi_private);
327 }
328 
329 int bch2_read_single_folio(struct folio *folio, struct address_space *mapping)
330 {
331 	struct bch_inode_info *inode = to_bch_ei(mapping->host);
332 	struct bch_fs *c = inode->v.i_sb->s_fs_info;
333 	struct bch_read_bio *rbio;
334 	struct bch_io_opts opts;
335 	int ret;
336 	DECLARE_COMPLETION_ONSTACK(done);
337 
338 	bch2_inode_opts_get(&opts, c, &inode->ei_inode);
339 
340 	rbio = rbio_init(bio_alloc_bioset(NULL, 1, REQ_OP_READ, GFP_KERNEL, &c->bio_read),
341 			 opts);
342 	rbio->bio.bi_private = &done;
343 	rbio->bio.bi_end_io = bch2_read_single_folio_end_io;
344 
345 	__bchfs_readfolio(c, rbio, inode_inum(inode), folio);
346 	wait_for_completion(&done);
347 
348 	ret = blk_status_to_errno(rbio->bio.bi_status);
349 	bio_put(&rbio->bio);
350 
351 	if (ret < 0)
352 		return ret;
353 
354 	folio_mark_uptodate(folio);
355 	return 0;
356 }
357 
358 int bch2_read_folio(struct file *file, struct folio *folio)
359 {
360 	int ret;
361 
362 	ret = bch2_read_single_folio(folio, folio->mapping);
363 	folio_unlock(folio);
364 	return bch2_err_class(ret);
365 }
366 
367 /* writepages: */
368 
369 struct bch_writepage_io {
370 	struct bch_inode_info		*inode;
371 
372 	/* must be last: */
373 	struct bch_write_op		op;
374 };
375 
376 struct bch_writepage_state {
377 	struct bch_writepage_io	*io;
378 	struct bch_io_opts	opts;
379 	struct bch_folio_sector	*tmp;
380 	unsigned		tmp_sectors;
381 };
382 
383 static inline struct bch_writepage_state bch_writepage_state_init(struct bch_fs *c,
384 								  struct bch_inode_info *inode)
385 {
386 	struct bch_writepage_state ret = { 0 };
387 
388 	bch2_inode_opts_get(&ret.opts, c, &inode->ei_inode);
389 	return ret;
390 }
391 
392 /*
393  * Determine when a writepage io is full. We have to limit writepage bios to a
394  * single page per bvec (i.e. 1MB with 4k pages) because that is the limit to
395  * what the bounce path in bch2_write_extent() can handle. In theory we could
396  * loosen this restriction for non-bounce I/O, but we don't have that context
397  * here. Ideally, we can up this limit and make it configurable in the future
398  * when the bounce path can be enhanced to accommodate larger source bios.
399  */
400 static inline bool bch_io_full(struct bch_writepage_io *io, unsigned len)
401 {
402 	struct bio *bio = &io->op.wbio.bio;
403 	return bio_full(bio, len) ||
404 		(bio->bi_iter.bi_size + len > BIO_MAX_VECS * PAGE_SIZE);
405 }
406 
407 static void bch2_writepage_io_done(struct bch_write_op *op)
408 {
409 	struct bch_writepage_io *io =
410 		container_of(op, struct bch_writepage_io, op);
411 	struct bch_fs *c = io->op.c;
412 	struct bio *bio = &io->op.wbio.bio;
413 	struct folio_iter fi;
414 	unsigned i;
415 
416 	if (io->op.error) {
417 		set_bit(EI_INODE_ERROR, &io->inode->ei_flags);
418 
419 		bio_for_each_folio_all(fi, bio) {
420 			struct bch_folio *s;
421 
422 			folio_set_error(fi.folio);
423 			mapping_set_error(fi.folio->mapping, -EIO);
424 
425 			s = __bch2_folio(fi.folio);
426 			spin_lock(&s->lock);
427 			for (i = 0; i < folio_sectors(fi.folio); i++)
428 				s->s[i].nr_replicas = 0;
429 			spin_unlock(&s->lock);
430 		}
431 	}
432 
433 	if (io->op.flags & BCH_WRITE_WROTE_DATA_INLINE) {
434 		bio_for_each_folio_all(fi, bio) {
435 			struct bch_folio *s;
436 
437 			s = __bch2_folio(fi.folio);
438 			spin_lock(&s->lock);
439 			for (i = 0; i < folio_sectors(fi.folio); i++)
440 				s->s[i].nr_replicas = 0;
441 			spin_unlock(&s->lock);
442 		}
443 	}
444 
445 	/*
446 	 * racing with fallocate can cause us to add fewer sectors than
447 	 * expected - but we shouldn't add more sectors than expected:
448 	 */
449 	WARN_ON_ONCE(io->op.i_sectors_delta > 0);
450 
451 	/*
452 	 * (error (due to going RO) halfway through a page can screw that up
453 	 * slightly)
454 	 * XXX wtf?
455 	   BUG_ON(io->op.op.i_sectors_delta >= PAGE_SECTORS);
456 	 */
457 
458 	/*
459 	 * PageWriteback is effectively our ref on the inode - fixup i_blocks
460 	 * before calling end_page_writeback:
461 	 */
462 	bch2_i_sectors_acct(c, io->inode, NULL, io->op.i_sectors_delta);
463 
464 	bio_for_each_folio_all(fi, bio) {
465 		struct bch_folio *s = __bch2_folio(fi.folio);
466 
467 		if (atomic_dec_and_test(&s->write_count))
468 			folio_end_writeback(fi.folio);
469 	}
470 
471 	bio_put(&io->op.wbio.bio);
472 }
473 
474 static void bch2_writepage_do_io(struct bch_writepage_state *w)
475 {
476 	struct bch_writepage_io *io = w->io;
477 
478 	w->io = NULL;
479 	closure_call(&io->op.cl, bch2_write, NULL, NULL);
480 }
481 
482 /*
483  * Get a bch_writepage_io and add @page to it - appending to an existing one if
484  * possible, else allocating a new one:
485  */
486 static void bch2_writepage_io_alloc(struct bch_fs *c,
487 				    struct writeback_control *wbc,
488 				    struct bch_writepage_state *w,
489 				    struct bch_inode_info *inode,
490 				    u64 sector,
491 				    unsigned nr_replicas)
492 {
493 	struct bch_write_op *op;
494 
495 	w->io = container_of(bio_alloc_bioset(NULL, BIO_MAX_VECS,
496 					      REQ_OP_WRITE,
497 					      GFP_KERNEL,
498 					      &c->writepage_bioset),
499 			     struct bch_writepage_io, op.wbio.bio);
500 
501 	w->io->inode		= inode;
502 	op			= &w->io->op;
503 	bch2_write_op_init(op, c, w->opts);
504 	op->target		= w->opts.foreground_target;
505 	op->nr_replicas		= nr_replicas;
506 	op->res.nr_replicas	= nr_replicas;
507 	op->write_point		= writepoint_hashed(inode->ei_last_dirtied);
508 	op->subvol		= inode->ei_subvol;
509 	op->pos			= POS(inode->v.i_ino, sector);
510 	op->end_io		= bch2_writepage_io_done;
511 	op->devs_need_flush	= &inode->ei_devs_need_flush;
512 	op->wbio.bio.bi_iter.bi_sector = sector;
513 	op->wbio.bio.bi_opf	= wbc_to_write_flags(wbc);
514 }
515 
516 static int __bch2_writepage(struct folio *folio,
517 			    struct writeback_control *wbc,
518 			    void *data)
519 {
520 	struct bch_inode_info *inode = to_bch_ei(folio->mapping->host);
521 	struct bch_fs *c = inode->v.i_sb->s_fs_info;
522 	struct bch_writepage_state *w = data;
523 	struct bch_folio *s;
524 	unsigned i, offset, f_sectors, nr_replicas_this_write = U32_MAX;
525 	loff_t i_size = i_size_read(&inode->v);
526 	int ret;
527 
528 	EBUG_ON(!folio_test_uptodate(folio));
529 
530 	/* Is the folio fully inside i_size? */
531 	if (folio_end_pos(folio) <= i_size)
532 		goto do_io;
533 
534 	/* Is the folio fully outside i_size? (truncate in progress) */
535 	if (folio_pos(folio) >= i_size) {
536 		folio_unlock(folio);
537 		return 0;
538 	}
539 
540 	/*
541 	 * The folio straddles i_size.  It must be zeroed out on each and every
542 	 * writepage invocation because it may be mmapped.  "A file is mapped
543 	 * in multiples of the folio size.  For a file that is not a multiple of
544 	 * the  folio size, the remaining memory is zeroed when mapped, and
545 	 * writes to that region are not written out to the file."
546 	 */
547 	folio_zero_segment(folio,
548 			   i_size - folio_pos(folio),
549 			   folio_size(folio));
550 do_io:
551 	f_sectors = folio_sectors(folio);
552 	s = bch2_folio(folio);
553 
554 	if (f_sectors > w->tmp_sectors) {
555 		kfree(w->tmp);
556 		w->tmp = kcalloc(f_sectors, sizeof(struct bch_folio_sector), __GFP_NOFAIL);
557 		w->tmp_sectors = f_sectors;
558 	}
559 
560 	/*
561 	 * Things get really hairy with errors during writeback:
562 	 */
563 	ret = bch2_get_folio_disk_reservation(c, inode, folio, false);
564 	BUG_ON(ret);
565 
566 	/* Before unlocking the page, get copy of reservations: */
567 	spin_lock(&s->lock);
568 	memcpy(w->tmp, s->s, sizeof(struct bch_folio_sector) * f_sectors);
569 
570 	for (i = 0; i < f_sectors; i++) {
571 		if (s->s[i].state < SECTOR_dirty)
572 			continue;
573 
574 		nr_replicas_this_write =
575 			min_t(unsigned, nr_replicas_this_write,
576 			      s->s[i].nr_replicas +
577 			      s->s[i].replicas_reserved);
578 	}
579 
580 	for (i = 0; i < f_sectors; i++) {
581 		if (s->s[i].state < SECTOR_dirty)
582 			continue;
583 
584 		s->s[i].nr_replicas = w->opts.compression
585 			? 0 : nr_replicas_this_write;
586 
587 		s->s[i].replicas_reserved = 0;
588 		bch2_folio_sector_set(folio, s, i, SECTOR_allocated);
589 	}
590 	spin_unlock(&s->lock);
591 
592 	BUG_ON(atomic_read(&s->write_count));
593 	atomic_set(&s->write_count, 1);
594 
595 	BUG_ON(folio_test_writeback(folio));
596 	folio_start_writeback(folio);
597 
598 	folio_unlock(folio);
599 
600 	offset = 0;
601 	while (1) {
602 		unsigned sectors = 0, dirty_sectors = 0, reserved_sectors = 0;
603 		u64 sector;
604 
605 		while (offset < f_sectors &&
606 		       w->tmp[offset].state < SECTOR_dirty)
607 			offset++;
608 
609 		if (offset == f_sectors)
610 			break;
611 
612 		while (offset + sectors < f_sectors &&
613 		       w->tmp[offset + sectors].state >= SECTOR_dirty) {
614 			reserved_sectors += w->tmp[offset + sectors].replicas_reserved;
615 			dirty_sectors += w->tmp[offset + sectors].state == SECTOR_dirty;
616 			sectors++;
617 		}
618 		BUG_ON(!sectors);
619 
620 		sector = folio_sector(folio) + offset;
621 
622 		if (w->io &&
623 		    (w->io->op.res.nr_replicas != nr_replicas_this_write ||
624 		     bch_io_full(w->io, sectors << 9) ||
625 		     bio_end_sector(&w->io->op.wbio.bio) != sector))
626 			bch2_writepage_do_io(w);
627 
628 		if (!w->io)
629 			bch2_writepage_io_alloc(c, wbc, w, inode, sector,
630 						nr_replicas_this_write);
631 
632 		atomic_inc(&s->write_count);
633 
634 		BUG_ON(inode != w->io->inode);
635 		BUG_ON(!bio_add_folio(&w->io->op.wbio.bio, folio,
636 				     sectors << 9, offset << 9));
637 
638 		/* Check for writing past i_size: */
639 		WARN_ONCE((bio_end_sector(&w->io->op.wbio.bio) << 9) >
640 			  round_up(i_size, block_bytes(c)) &&
641 			  !test_bit(BCH_FS_EMERGENCY_RO, &c->flags),
642 			  "writing past i_size: %llu > %llu (unrounded %llu)\n",
643 			  bio_end_sector(&w->io->op.wbio.bio) << 9,
644 			  round_up(i_size, block_bytes(c)),
645 			  i_size);
646 
647 		w->io->op.res.sectors += reserved_sectors;
648 		w->io->op.i_sectors_delta -= dirty_sectors;
649 		w->io->op.new_i_size = i_size;
650 
651 		offset += sectors;
652 	}
653 
654 	if (atomic_dec_and_test(&s->write_count))
655 		folio_end_writeback(folio);
656 
657 	return 0;
658 }
659 
660 int bch2_writepages(struct address_space *mapping, struct writeback_control *wbc)
661 {
662 	struct bch_fs *c = mapping->host->i_sb->s_fs_info;
663 	struct bch_writepage_state w =
664 		bch_writepage_state_init(c, to_bch_ei(mapping->host));
665 	struct blk_plug plug;
666 	int ret;
667 
668 	blk_start_plug(&plug);
669 	ret = write_cache_pages(mapping, wbc, __bch2_writepage, &w);
670 	if (w.io)
671 		bch2_writepage_do_io(&w);
672 	blk_finish_plug(&plug);
673 	kfree(w.tmp);
674 	return bch2_err_class(ret);
675 }
676 
677 /* buffered writes: */
678 
679 int bch2_write_begin(struct file *file, struct address_space *mapping,
680 		     loff_t pos, unsigned len,
681 		     struct page **pagep, void **fsdata)
682 {
683 	struct bch_inode_info *inode = to_bch_ei(mapping->host);
684 	struct bch_fs *c = inode->v.i_sb->s_fs_info;
685 	struct bch2_folio_reservation *res;
686 	struct folio *folio;
687 	unsigned offset;
688 	int ret = -ENOMEM;
689 
690 	res = kmalloc(sizeof(*res), GFP_KERNEL);
691 	if (!res)
692 		return -ENOMEM;
693 
694 	bch2_folio_reservation_init(c, inode, res);
695 	*fsdata = res;
696 
697 	bch2_pagecache_add_get(inode);
698 
699 	folio = __filemap_get_folio(mapping, pos >> PAGE_SHIFT,
700 				FGP_LOCK|FGP_WRITE|FGP_CREAT|FGP_STABLE,
701 				mapping_gfp_mask(mapping));
702 	if (IS_ERR_OR_NULL(folio))
703 		goto err_unlock;
704 
705 	offset = pos - folio_pos(folio);
706 	len = min_t(size_t, len, folio_end_pos(folio) - pos);
707 
708 	if (folio_test_uptodate(folio))
709 		goto out;
710 
711 	/* If we're writing entire folio, don't need to read it in first: */
712 	if (!offset && len == folio_size(folio))
713 		goto out;
714 
715 	if (!offset && pos + len >= inode->v.i_size) {
716 		folio_zero_segment(folio, len, folio_size(folio));
717 		flush_dcache_folio(folio);
718 		goto out;
719 	}
720 
721 	if (folio_pos(folio) >= inode->v.i_size) {
722 		folio_zero_segments(folio, 0, offset, offset + len, folio_size(folio));
723 		flush_dcache_folio(folio);
724 		goto out;
725 	}
726 readpage:
727 	ret = bch2_read_single_folio(folio, mapping);
728 	if (ret)
729 		goto err;
730 out:
731 	ret = bch2_folio_set(c, inode_inum(inode), &folio, 1);
732 	if (ret)
733 		goto err;
734 
735 	ret = bch2_folio_reservation_get(c, inode, folio, res, offset, len);
736 	if (ret) {
737 		if (!folio_test_uptodate(folio)) {
738 			/*
739 			 * If the folio hasn't been read in, we won't know if we
740 			 * actually need a reservation - we don't actually need
741 			 * to read here, we just need to check if the folio is
742 			 * fully backed by uncompressed data:
743 			 */
744 			goto readpage;
745 		}
746 
747 		goto err;
748 	}
749 
750 	*pagep = &folio->page;
751 	return 0;
752 err:
753 	folio_unlock(folio);
754 	folio_put(folio);
755 	*pagep = NULL;
756 err_unlock:
757 	bch2_pagecache_add_put(inode);
758 	kfree(res);
759 	*fsdata = NULL;
760 	return bch2_err_class(ret);
761 }
762 
763 int bch2_write_end(struct file *file, struct address_space *mapping,
764 		   loff_t pos, unsigned len, unsigned copied,
765 		   struct page *page, void *fsdata)
766 {
767 	struct bch_inode_info *inode = to_bch_ei(mapping->host);
768 	struct bch_fs *c = inode->v.i_sb->s_fs_info;
769 	struct bch2_folio_reservation *res = fsdata;
770 	struct folio *folio = page_folio(page);
771 	unsigned offset = pos - folio_pos(folio);
772 
773 	lockdep_assert_held(&inode->v.i_rwsem);
774 	BUG_ON(offset + copied > folio_size(folio));
775 
776 	if (unlikely(copied < len && !folio_test_uptodate(folio))) {
777 		/*
778 		 * The folio needs to be read in, but that would destroy
779 		 * our partial write - simplest thing is to just force
780 		 * userspace to redo the write:
781 		 */
782 		folio_zero_range(folio, 0, folio_size(folio));
783 		flush_dcache_folio(folio);
784 		copied = 0;
785 	}
786 
787 	spin_lock(&inode->v.i_lock);
788 	if (pos + copied > inode->v.i_size)
789 		i_size_write(&inode->v, pos + copied);
790 	spin_unlock(&inode->v.i_lock);
791 
792 	if (copied) {
793 		if (!folio_test_uptodate(folio))
794 			folio_mark_uptodate(folio);
795 
796 		bch2_set_folio_dirty(c, inode, folio, res, offset, copied);
797 
798 		inode->ei_last_dirtied = (unsigned long) current;
799 	}
800 
801 	folio_unlock(folio);
802 	folio_put(folio);
803 	bch2_pagecache_add_put(inode);
804 
805 	bch2_folio_reservation_put(c, inode, res);
806 	kfree(res);
807 
808 	return copied;
809 }
810 
811 static noinline void folios_trunc(folios *fs, struct folio **fi)
812 {
813 	while (fs->data + fs->nr > fi) {
814 		struct folio *f = darray_pop(fs);
815 
816 		folio_unlock(f);
817 		folio_put(f);
818 	}
819 }
820 
821 static int __bch2_buffered_write(struct bch_inode_info *inode,
822 				 struct address_space *mapping,
823 				 struct iov_iter *iter,
824 				 loff_t pos, unsigned len)
825 {
826 	struct bch_fs *c = inode->v.i_sb->s_fs_info;
827 	struct bch2_folio_reservation res;
828 	folios fs;
829 	struct folio **fi, *f;
830 	unsigned copied = 0, f_offset, f_copied;
831 	u64 end = pos + len, f_pos, f_len;
832 	loff_t last_folio_pos = inode->v.i_size;
833 	int ret = 0;
834 
835 	BUG_ON(!len);
836 
837 	bch2_folio_reservation_init(c, inode, &res);
838 	darray_init(&fs);
839 
840 	ret = bch2_filemap_get_contig_folios_d(mapping, pos, end,
841 				   FGP_LOCK|FGP_WRITE|FGP_STABLE|FGP_CREAT,
842 				   mapping_gfp_mask(mapping),
843 				   &fs);
844 	if (ret)
845 		goto out;
846 
847 	BUG_ON(!fs.nr);
848 
849 	f = darray_first(fs);
850 	if (pos != folio_pos(f) && !folio_test_uptodate(f)) {
851 		ret = bch2_read_single_folio(f, mapping);
852 		if (ret)
853 			goto out;
854 	}
855 
856 	f = darray_last(fs);
857 	end = min(end, folio_end_pos(f));
858 	last_folio_pos = folio_pos(f);
859 	if (end != folio_end_pos(f) && !folio_test_uptodate(f)) {
860 		if (end >= inode->v.i_size) {
861 			folio_zero_range(f, 0, folio_size(f));
862 		} else {
863 			ret = bch2_read_single_folio(f, mapping);
864 			if (ret)
865 				goto out;
866 		}
867 	}
868 
869 	ret = bch2_folio_set(c, inode_inum(inode), fs.data, fs.nr);
870 	if (ret)
871 		goto out;
872 
873 	f_pos = pos;
874 	f_offset = pos - folio_pos(darray_first(fs));
875 	darray_for_each(fs, fi) {
876 		f = *fi;
877 		f_len = min(end, folio_end_pos(f)) - f_pos;
878 
879 		/*
880 		 * XXX: per POSIX and fstests generic/275, on -ENOSPC we're
881 		 * supposed to write as much as we have disk space for.
882 		 *
883 		 * On failure here we should still write out a partial page if
884 		 * we aren't completely out of disk space - we don't do that
885 		 * yet:
886 		 */
887 		ret = bch2_folio_reservation_get(c, inode, f, &res, f_offset, f_len);
888 		if (unlikely(ret)) {
889 			folios_trunc(&fs, fi);
890 			if (!fs.nr)
891 				goto out;
892 
893 			end = min(end, folio_end_pos(darray_last(fs)));
894 			break;
895 		}
896 
897 		f_pos = folio_end_pos(f);
898 		f_offset = 0;
899 	}
900 
901 	if (mapping_writably_mapped(mapping))
902 		darray_for_each(fs, fi)
903 			flush_dcache_folio(*fi);
904 
905 	f_pos = pos;
906 	f_offset = pos - folio_pos(darray_first(fs));
907 	darray_for_each(fs, fi) {
908 		f = *fi;
909 		f_len = min(end, folio_end_pos(f)) - f_pos;
910 		f_copied = copy_page_from_iter_atomic(&f->page, f_offset, f_len, iter);
911 		if (!f_copied) {
912 			folios_trunc(&fs, fi);
913 			break;
914 		}
915 
916 		if (!folio_test_uptodate(f) &&
917 		    f_copied != folio_size(f) &&
918 		    pos + copied + f_copied < inode->v.i_size) {
919 			iov_iter_revert(iter, f_copied);
920 			folio_zero_range(f, 0, folio_size(f));
921 			folios_trunc(&fs, fi);
922 			break;
923 		}
924 
925 		flush_dcache_folio(f);
926 		copied += f_copied;
927 
928 		if (f_copied != f_len) {
929 			folios_trunc(&fs, fi + 1);
930 			break;
931 		}
932 
933 		f_pos = folio_end_pos(f);
934 		f_offset = 0;
935 	}
936 
937 	if (!copied)
938 		goto out;
939 
940 	end = pos + copied;
941 
942 	spin_lock(&inode->v.i_lock);
943 	if (end > inode->v.i_size)
944 		i_size_write(&inode->v, end);
945 	spin_unlock(&inode->v.i_lock);
946 
947 	f_pos = pos;
948 	f_offset = pos - folio_pos(darray_first(fs));
949 	darray_for_each(fs, fi) {
950 		f = *fi;
951 		f_len = min(end, folio_end_pos(f)) - f_pos;
952 
953 		if (!folio_test_uptodate(f))
954 			folio_mark_uptodate(f);
955 
956 		bch2_set_folio_dirty(c, inode, f, &res, f_offset, f_len);
957 
958 		f_pos = folio_end_pos(f);
959 		f_offset = 0;
960 	}
961 
962 	inode->ei_last_dirtied = (unsigned long) current;
963 out:
964 	darray_for_each(fs, fi) {
965 		folio_unlock(*fi);
966 		folio_put(*fi);
967 	}
968 
969 	/*
970 	 * If the last folio added to the mapping starts beyond current EOF, we
971 	 * performed a short write but left around at least one post-EOF folio.
972 	 * Clean up the mapping before we return.
973 	 */
974 	if (last_folio_pos >= inode->v.i_size)
975 		truncate_pagecache(&inode->v, inode->v.i_size);
976 
977 	darray_exit(&fs);
978 	bch2_folio_reservation_put(c, inode, &res);
979 
980 	return copied ?: ret;
981 }
982 
983 static ssize_t bch2_buffered_write(struct kiocb *iocb, struct iov_iter *iter)
984 {
985 	struct file *file = iocb->ki_filp;
986 	struct address_space *mapping = file->f_mapping;
987 	struct bch_inode_info *inode = file_bch_inode(file);
988 	loff_t pos = iocb->ki_pos;
989 	ssize_t written = 0;
990 	int ret = 0;
991 
992 	bch2_pagecache_add_get(inode);
993 
994 	do {
995 		unsigned offset = pos & (PAGE_SIZE - 1);
996 		unsigned bytes = iov_iter_count(iter);
997 again:
998 		/*
999 		 * Bring in the user page that we will copy from _first_.
1000 		 * Otherwise there's a nasty deadlock on copying from the
1001 		 * same page as we're writing to, without it being marked
1002 		 * up-to-date.
1003 		 *
1004 		 * Not only is this an optimisation, but it is also required
1005 		 * to check that the address is actually valid, when atomic
1006 		 * usercopies are used, below.
1007 		 */
1008 		if (unlikely(fault_in_iov_iter_readable(iter, bytes))) {
1009 			bytes = min_t(unsigned long, iov_iter_count(iter),
1010 				      PAGE_SIZE - offset);
1011 
1012 			if (unlikely(fault_in_iov_iter_readable(iter, bytes))) {
1013 				ret = -EFAULT;
1014 				break;
1015 			}
1016 		}
1017 
1018 		if (unlikely(fatal_signal_pending(current))) {
1019 			ret = -EINTR;
1020 			break;
1021 		}
1022 
1023 		ret = __bch2_buffered_write(inode, mapping, iter, pos, bytes);
1024 		if (unlikely(ret < 0))
1025 			break;
1026 
1027 		cond_resched();
1028 
1029 		if (unlikely(ret == 0)) {
1030 			/*
1031 			 * If we were unable to copy any data at all, we must
1032 			 * fall back to a single segment length write.
1033 			 *
1034 			 * If we didn't fallback here, we could livelock
1035 			 * because not all segments in the iov can be copied at
1036 			 * once without a pagefault.
1037 			 */
1038 			bytes = min_t(unsigned long, PAGE_SIZE - offset,
1039 				      iov_iter_single_seg_count(iter));
1040 			goto again;
1041 		}
1042 		pos += ret;
1043 		written += ret;
1044 		ret = 0;
1045 
1046 		balance_dirty_pages_ratelimited(mapping);
1047 	} while (iov_iter_count(iter));
1048 
1049 	bch2_pagecache_add_put(inode);
1050 
1051 	return written ? written : ret;
1052 }
1053 
1054 ssize_t bch2_write_iter(struct kiocb *iocb, struct iov_iter *from)
1055 {
1056 	struct file *file = iocb->ki_filp;
1057 	struct bch_inode_info *inode = file_bch_inode(file);
1058 	ssize_t ret;
1059 
1060 	if (iocb->ki_flags & IOCB_DIRECT) {
1061 		ret = bch2_direct_write(iocb, from);
1062 		goto out;
1063 	}
1064 
1065 	inode_lock(&inode->v);
1066 
1067 	ret = generic_write_checks(iocb, from);
1068 	if (ret <= 0)
1069 		goto unlock;
1070 
1071 	ret = file_remove_privs(file);
1072 	if (ret)
1073 		goto unlock;
1074 
1075 	ret = file_update_time(file);
1076 	if (ret)
1077 		goto unlock;
1078 
1079 	ret = bch2_buffered_write(iocb, from);
1080 	if (likely(ret > 0))
1081 		iocb->ki_pos += ret;
1082 unlock:
1083 	inode_unlock(&inode->v);
1084 
1085 	if (ret > 0)
1086 		ret = generic_write_sync(iocb, ret);
1087 out:
1088 	return bch2_err_class(ret);
1089 }
1090 
1091 void bch2_fs_fs_io_buffered_exit(struct bch_fs *c)
1092 {
1093 	bioset_exit(&c->writepage_bioset);
1094 }
1095 
1096 int bch2_fs_fs_io_buffered_init(struct bch_fs *c)
1097 {
1098 	if (bioset_init(&c->writepage_bioset,
1099 			4, offsetof(struct bch_writepage_io, op.wbio.bio),
1100 			BIOSET_NEED_BVECS))
1101 		return -BCH_ERR_ENOMEM_writepage_bioset_init;
1102 
1103 	return 0;
1104 }
1105 
1106 #endif /* NO_BCACHEFS_FS */
1107