xref: /linux/fs/btrfs/extent_io.c (revision 5ca7fe213ba3113dde19c4cd46347c16d9e69f81)
1 // SPDX-License-Identifier: GPL-2.0
2 
3 #include <linux/bitops.h>
4 #include <linux/slab.h>
5 #include <linux/bio.h>
6 #include <linux/mm.h>
7 #include <linux/pagemap.h>
8 #include <linux/page-flags.h>
9 #include <linux/sched/mm.h>
10 #include <linux/spinlock.h>
11 #include <linux/blkdev.h>
12 #include <linux/swap.h>
13 #include <linux/writeback.h>
14 #include <linux/pagevec.h>
15 #include <linux/prefetch.h>
16 #include <linux/fsverity.h>
17 #include "extent_io.h"
18 #include "extent-io-tree.h"
19 #include "extent_map.h"
20 #include "ctree.h"
21 #include "btrfs_inode.h"
22 #include "bio.h"
23 #include "locking.h"
24 #include "backref.h"
25 #include "disk-io.h"
26 #include "subpage.h"
27 #include "zoned.h"
28 #include "block-group.h"
29 #include "compression.h"
30 #include "fs.h"
31 #include "accessors.h"
32 #include "file-item.h"
33 #include "file.h"
34 #include "dev-replace.h"
35 #include "super.h"
36 #include "transaction.h"
37 
38 static struct kmem_cache *extent_buffer_cache;
39 
40 #ifdef CONFIG_BTRFS_DEBUG
btrfs_leak_debug_add_eb(struct extent_buffer * eb)41 static inline void btrfs_leak_debug_add_eb(struct extent_buffer *eb)
42 {
43 	struct btrfs_fs_info *fs_info = eb->fs_info;
44 	unsigned long flags;
45 
46 	spin_lock_irqsave(&fs_info->eb_leak_lock, flags);
47 	list_add(&eb->leak_list, &fs_info->allocated_ebs);
48 	spin_unlock_irqrestore(&fs_info->eb_leak_lock, flags);
49 }
50 
btrfs_leak_debug_del_eb(struct extent_buffer * eb)51 static inline void btrfs_leak_debug_del_eb(struct extent_buffer *eb)
52 {
53 	struct btrfs_fs_info *fs_info = eb->fs_info;
54 	unsigned long flags;
55 
56 	spin_lock_irqsave(&fs_info->eb_leak_lock, flags);
57 	list_del(&eb->leak_list);
58 	spin_unlock_irqrestore(&fs_info->eb_leak_lock, flags);
59 }
60 
btrfs_extent_buffer_leak_debug_check(struct btrfs_fs_info * fs_info)61 void btrfs_extent_buffer_leak_debug_check(struct btrfs_fs_info *fs_info)
62 {
63 	struct extent_buffer *eb;
64 	unsigned long flags;
65 
66 	/*
67 	 * If we didn't get into open_ctree our allocated_ebs will not be
68 	 * initialized, so just skip this.
69 	 */
70 	if (!fs_info->allocated_ebs.next)
71 		return;
72 
73 	WARN_ON(!list_empty(&fs_info->allocated_ebs));
74 	spin_lock_irqsave(&fs_info->eb_leak_lock, flags);
75 	while (!list_empty(&fs_info->allocated_ebs)) {
76 		eb = list_first_entry(&fs_info->allocated_ebs,
77 				      struct extent_buffer, leak_list);
78 		pr_err(
79 	"BTRFS: buffer leak start %llu len %u refs %d bflags %lu owner %llu\n",
80 		       eb->start, eb->len, atomic_read(&eb->refs), eb->bflags,
81 		       btrfs_header_owner(eb));
82 		list_del(&eb->leak_list);
83 		WARN_ON_ONCE(1);
84 		kmem_cache_free(extent_buffer_cache, eb);
85 	}
86 	spin_unlock_irqrestore(&fs_info->eb_leak_lock, flags);
87 }
88 #else
89 #define btrfs_leak_debug_add_eb(eb)			do {} while (0)
90 #define btrfs_leak_debug_del_eb(eb)			do {} while (0)
91 #endif
92 
93 /*
94  * Structure to record info about the bio being assembled, and other info like
95  * how many bytes are there before stripe/ordered extent boundary.
96  */
97 struct btrfs_bio_ctrl {
98 	struct btrfs_bio *bbio;
99 	/* Last byte contained in bbio + 1 . */
100 	loff_t next_file_offset;
101 	enum btrfs_compression_type compress_type;
102 	u32 len_to_oe_boundary;
103 	blk_opf_t opf;
104 	btrfs_bio_end_io_t end_io_func;
105 	struct writeback_control *wbc;
106 
107 	/*
108 	 * The sectors of the page which are going to be submitted by
109 	 * extent_writepage_io().
110 	 * This is to avoid touching ranges covered by compression/inline.
111 	 */
112 	unsigned long submit_bitmap;
113 };
114 
submit_one_bio(struct btrfs_bio_ctrl * bio_ctrl)115 static void submit_one_bio(struct btrfs_bio_ctrl *bio_ctrl)
116 {
117 	struct btrfs_bio *bbio = bio_ctrl->bbio;
118 
119 	if (!bbio)
120 		return;
121 
122 	/* Caller should ensure the bio has at least some range added */
123 	ASSERT(bbio->bio.bi_iter.bi_size);
124 
125 	if (btrfs_op(&bbio->bio) == BTRFS_MAP_READ &&
126 	    bio_ctrl->compress_type != BTRFS_COMPRESS_NONE)
127 		btrfs_submit_compressed_read(bbio);
128 	else
129 		btrfs_submit_bbio(bbio, 0);
130 
131 	/* The bbio is owned by the end_io handler now */
132 	bio_ctrl->bbio = NULL;
133 }
134 
135 /*
136  * Submit or fail the current bio in the bio_ctrl structure.
137  */
submit_write_bio(struct btrfs_bio_ctrl * bio_ctrl,int ret)138 static void submit_write_bio(struct btrfs_bio_ctrl *bio_ctrl, int ret)
139 {
140 	struct btrfs_bio *bbio = bio_ctrl->bbio;
141 
142 	if (!bbio)
143 		return;
144 
145 	if (ret) {
146 		ASSERT(ret < 0);
147 		btrfs_bio_end_io(bbio, errno_to_blk_status(ret));
148 		/* The bio is owned by the end_io handler now */
149 		bio_ctrl->bbio = NULL;
150 	} else {
151 		submit_one_bio(bio_ctrl);
152 	}
153 }
154 
extent_buffer_init_cachep(void)155 int __init extent_buffer_init_cachep(void)
156 {
157 	extent_buffer_cache = kmem_cache_create("btrfs_extent_buffer",
158 						sizeof(struct extent_buffer), 0, 0,
159 						NULL);
160 	if (!extent_buffer_cache)
161 		return -ENOMEM;
162 
163 	return 0;
164 }
165 
extent_buffer_free_cachep(void)166 void __cold extent_buffer_free_cachep(void)
167 {
168 	/*
169 	 * Make sure all delayed rcu free are flushed before we
170 	 * destroy caches.
171 	 */
172 	rcu_barrier();
173 	kmem_cache_destroy(extent_buffer_cache);
174 }
175 
process_one_folio(struct btrfs_fs_info * fs_info,struct folio * folio,const struct folio * locked_folio,unsigned long page_ops,u64 start,u64 end)176 static void process_one_folio(struct btrfs_fs_info *fs_info,
177 			      struct folio *folio, const struct folio *locked_folio,
178 			      unsigned long page_ops, u64 start, u64 end)
179 {
180 	u32 len;
181 
182 	ASSERT(end + 1 - start != 0 && end + 1 - start < U32_MAX);
183 	len = end + 1 - start;
184 
185 	if (page_ops & PAGE_SET_ORDERED)
186 		btrfs_folio_clamp_set_ordered(fs_info, folio, start, len);
187 	if (page_ops & PAGE_START_WRITEBACK) {
188 		btrfs_folio_clamp_clear_dirty(fs_info, folio, start, len);
189 		btrfs_folio_clamp_set_writeback(fs_info, folio, start, len);
190 	}
191 	if (page_ops & PAGE_END_WRITEBACK)
192 		btrfs_folio_clamp_clear_writeback(fs_info, folio, start, len);
193 
194 	if (folio != locked_folio && (page_ops & PAGE_UNLOCK))
195 		btrfs_folio_end_lock(fs_info, folio, start, len);
196 }
197 
__process_folios_contig(struct address_space * mapping,const struct folio * locked_folio,u64 start,u64 end,unsigned long page_ops)198 static void __process_folios_contig(struct address_space *mapping,
199 				    const struct folio *locked_folio, u64 start,
200 				    u64 end, unsigned long page_ops)
201 {
202 	struct btrfs_fs_info *fs_info = inode_to_fs_info(mapping->host);
203 	pgoff_t index = start >> PAGE_SHIFT;
204 	pgoff_t end_index = end >> PAGE_SHIFT;
205 	struct folio_batch fbatch;
206 	int i;
207 
208 	folio_batch_init(&fbatch);
209 	while (index <= end_index) {
210 		int found_folios;
211 
212 		found_folios = filemap_get_folios_contig(mapping, &index,
213 				end_index, &fbatch);
214 		for (i = 0; i < found_folios; i++) {
215 			struct folio *folio = fbatch.folios[i];
216 
217 			process_one_folio(fs_info, folio, locked_folio,
218 					  page_ops, start, end);
219 		}
220 		folio_batch_release(&fbatch);
221 		cond_resched();
222 	}
223 }
224 
unlock_delalloc_folio(const struct inode * inode,struct folio * locked_folio,u64 start,u64 end)225 static noinline void unlock_delalloc_folio(const struct inode *inode,
226 					   struct folio *locked_folio,
227 					   u64 start, u64 end)
228 {
229 	ASSERT(locked_folio);
230 
231 	__process_folios_contig(inode->i_mapping, locked_folio, start, end,
232 				PAGE_UNLOCK);
233 }
234 
lock_delalloc_folios(struct inode * inode,struct folio * locked_folio,u64 start,u64 end)235 static noinline int lock_delalloc_folios(struct inode *inode,
236 					 struct folio *locked_folio,
237 					 u64 start, u64 end)
238 {
239 	struct btrfs_fs_info *fs_info = inode_to_fs_info(inode);
240 	struct address_space *mapping = inode->i_mapping;
241 	pgoff_t index = start >> PAGE_SHIFT;
242 	pgoff_t end_index = end >> PAGE_SHIFT;
243 	u64 processed_end = start;
244 	struct folio_batch fbatch;
245 
246 	folio_batch_init(&fbatch);
247 	while (index <= end_index) {
248 		unsigned int found_folios, i;
249 
250 		found_folios = filemap_get_folios_contig(mapping, &index,
251 				end_index, &fbatch);
252 		if (found_folios == 0)
253 			goto out;
254 
255 		for (i = 0; i < found_folios; i++) {
256 			struct folio *folio = fbatch.folios[i];
257 			u64 range_start;
258 			u32 range_len;
259 
260 			if (folio == locked_folio)
261 				continue;
262 
263 			folio_lock(folio);
264 			if (!folio_test_dirty(folio) || folio->mapping != mapping) {
265 				folio_unlock(folio);
266 				goto out;
267 			}
268 			range_start = max_t(u64, folio_pos(folio), start);
269 			range_len = min_t(u64, folio_pos(folio) + folio_size(folio),
270 					  end + 1) - range_start;
271 			btrfs_folio_set_lock(fs_info, folio, range_start, range_len);
272 
273 			processed_end = range_start + range_len - 1;
274 		}
275 		folio_batch_release(&fbatch);
276 		cond_resched();
277 	}
278 
279 	return 0;
280 out:
281 	folio_batch_release(&fbatch);
282 	if (processed_end > start)
283 		unlock_delalloc_folio(inode, locked_folio, start, processed_end);
284 	return -EAGAIN;
285 }
286 
287 /*
288  * Find and lock a contiguous range of bytes in the file marked as delalloc, no
289  * more than @max_bytes.
290  *
291  * @start:	The original start bytenr to search.
292  *		Will store the extent range start bytenr.
293  * @end:	The original end bytenr of the search range
294  *		Will store the extent range end bytenr.
295  *
296  * Return true if we find a delalloc range which starts inside the original
297  * range, and @start/@end will store the delalloc range start/end.
298  *
299  * Return false if we can't find any delalloc range which starts inside the
300  * original range, and @start/@end will be the non-delalloc range start/end.
301  */
302 EXPORT_FOR_TESTS
find_lock_delalloc_range(struct inode * inode,struct folio * locked_folio,u64 * start,u64 * end)303 noinline_for_stack bool find_lock_delalloc_range(struct inode *inode,
304 						 struct folio *locked_folio,
305 						 u64 *start, u64 *end)
306 {
307 	struct btrfs_fs_info *fs_info = inode_to_fs_info(inode);
308 	struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
309 	const u64 orig_start = *start;
310 	const u64 orig_end = *end;
311 	/* The sanity tests may not set a valid fs_info. */
312 	u64 max_bytes = fs_info ? fs_info->max_extent_size : BTRFS_MAX_EXTENT_SIZE;
313 	u64 delalloc_start;
314 	u64 delalloc_end;
315 	bool found;
316 	struct extent_state *cached_state = NULL;
317 	int ret;
318 	int loops = 0;
319 
320 	/* Caller should pass a valid @end to indicate the search range end */
321 	ASSERT(orig_end > orig_start);
322 
323 	/* The range should at least cover part of the folio */
324 	ASSERT(!(orig_start >= folio_pos(locked_folio) + folio_size(locked_folio) ||
325 		 orig_end <= folio_pos(locked_folio)));
326 again:
327 	/* step one, find a bunch of delalloc bytes starting at start */
328 	delalloc_start = *start;
329 	delalloc_end = 0;
330 	found = btrfs_find_delalloc_range(tree, &delalloc_start, &delalloc_end,
331 					  max_bytes, &cached_state);
332 	if (!found || delalloc_end <= *start || delalloc_start > orig_end) {
333 		*start = delalloc_start;
334 
335 		/* @delalloc_end can be -1, never go beyond @orig_end */
336 		*end = min(delalloc_end, orig_end);
337 		btrfs_free_extent_state(cached_state);
338 		return false;
339 	}
340 
341 	/*
342 	 * start comes from the offset of locked_folio.  We have to lock
343 	 * folios in order, so we can't process delalloc bytes before
344 	 * locked_folio
345 	 */
346 	if (delalloc_start < *start)
347 		delalloc_start = *start;
348 
349 	/*
350 	 * make sure to limit the number of folios we try to lock down
351 	 */
352 	if (delalloc_end + 1 - delalloc_start > max_bytes)
353 		delalloc_end = delalloc_start + max_bytes - 1;
354 
355 	/* step two, lock all the folioss after the folios that has start */
356 	ret = lock_delalloc_folios(inode, locked_folio, delalloc_start,
357 				   delalloc_end);
358 	ASSERT(!ret || ret == -EAGAIN);
359 	if (ret == -EAGAIN) {
360 		/* some of the folios are gone, lets avoid looping by
361 		 * shortening the size of the delalloc range we're searching
362 		 */
363 		btrfs_free_extent_state(cached_state);
364 		cached_state = NULL;
365 		if (!loops) {
366 			max_bytes = PAGE_SIZE;
367 			loops = 1;
368 			goto again;
369 		} else {
370 			found = false;
371 			goto out_failed;
372 		}
373 	}
374 
375 	/* step three, lock the state bits for the whole range */
376 	btrfs_lock_extent(tree, delalloc_start, delalloc_end, &cached_state);
377 
378 	/* then test to make sure it is all still delalloc */
379 	ret = btrfs_test_range_bit(tree, delalloc_start, delalloc_end,
380 				   EXTENT_DELALLOC, cached_state);
381 
382 	btrfs_unlock_extent(tree, delalloc_start, delalloc_end, &cached_state);
383 	if (!ret) {
384 		unlock_delalloc_folio(inode, locked_folio, delalloc_start,
385 				      delalloc_end);
386 		cond_resched();
387 		goto again;
388 	}
389 	*start = delalloc_start;
390 	*end = delalloc_end;
391 out_failed:
392 	return found;
393 }
394 
extent_clear_unlock_delalloc(struct btrfs_inode * inode,u64 start,u64 end,const struct folio * locked_folio,struct extent_state ** cached,u32 clear_bits,unsigned long page_ops)395 void extent_clear_unlock_delalloc(struct btrfs_inode *inode, u64 start, u64 end,
396 				  const struct folio *locked_folio,
397 				  struct extent_state **cached,
398 				  u32 clear_bits, unsigned long page_ops)
399 {
400 	btrfs_clear_extent_bit(&inode->io_tree, start, end, clear_bits, cached);
401 
402 	__process_folios_contig(inode->vfs_inode.i_mapping, locked_folio, start,
403 				end, page_ops);
404 }
405 
btrfs_verify_folio(struct folio * folio,u64 start,u32 len)406 static bool btrfs_verify_folio(struct folio *folio, u64 start, u32 len)
407 {
408 	struct btrfs_fs_info *fs_info = folio_to_fs_info(folio);
409 
410 	if (!fsverity_active(folio->mapping->host) ||
411 	    btrfs_folio_test_uptodate(fs_info, folio, start, len) ||
412 	    start >= i_size_read(folio->mapping->host))
413 		return true;
414 	return fsverity_verify_folio(folio);
415 }
416 
end_folio_read(struct folio * folio,bool uptodate,u64 start,u32 len)417 static void end_folio_read(struct folio *folio, bool uptodate, u64 start, u32 len)
418 {
419 	struct btrfs_fs_info *fs_info = folio_to_fs_info(folio);
420 
421 	ASSERT(folio_pos(folio) <= start &&
422 	       start + len <= folio_pos(folio) + folio_size(folio));
423 
424 	if (uptodate && btrfs_verify_folio(folio, start, len))
425 		btrfs_folio_set_uptodate(fs_info, folio, start, len);
426 	else
427 		btrfs_folio_clear_uptodate(fs_info, folio, start, len);
428 
429 	if (!btrfs_is_subpage(fs_info, folio))
430 		folio_unlock(folio);
431 	else
432 		btrfs_folio_end_lock(fs_info, folio, start, len);
433 }
434 
435 /*
436  * After a write IO is done, we need to:
437  *
438  * - clear the uptodate bits on error
439  * - clear the writeback bits in the extent tree for the range
440  * - filio_end_writeback()  if there is no more pending io for the folio
441  *
442  * Scheduling is not allowed, so the extent state tree is expected
443  * to have one and only one object corresponding to this IO.
444  */
end_bbio_data_write(struct btrfs_bio * bbio)445 static void end_bbio_data_write(struct btrfs_bio *bbio)
446 {
447 	struct btrfs_fs_info *fs_info = bbio->fs_info;
448 	struct bio *bio = &bbio->bio;
449 	int error = blk_status_to_errno(bio->bi_status);
450 	struct folio_iter fi;
451 	const u32 sectorsize = fs_info->sectorsize;
452 
453 	ASSERT(!bio_flagged(bio, BIO_CLONED));
454 	bio_for_each_folio_all(fi, bio) {
455 		struct folio *folio = fi.folio;
456 		u64 start = folio_pos(folio) + fi.offset;
457 		u32 len = fi.length;
458 
459 		/* Our read/write should always be sector aligned. */
460 		if (!IS_ALIGNED(fi.offset, sectorsize))
461 			btrfs_err(fs_info,
462 		"partial page write in btrfs with offset %zu and length %zu",
463 				  fi.offset, fi.length);
464 		else if (!IS_ALIGNED(fi.length, sectorsize))
465 			btrfs_info(fs_info,
466 		"incomplete page write with offset %zu and length %zu",
467 				   fi.offset, fi.length);
468 
469 		btrfs_finish_ordered_extent(bbio->ordered, folio, start, len,
470 					    !error);
471 		if (error)
472 			mapping_set_error(folio->mapping, error);
473 		btrfs_folio_clear_writeback(fs_info, folio, start, len);
474 	}
475 
476 	bio_put(bio);
477 }
478 
begin_folio_read(struct btrfs_fs_info * fs_info,struct folio * folio)479 static void begin_folio_read(struct btrfs_fs_info *fs_info, struct folio *folio)
480 {
481 	ASSERT(folio_test_locked(folio));
482 	if (!btrfs_is_subpage(fs_info, folio))
483 		return;
484 
485 	ASSERT(folio_test_private(folio));
486 	btrfs_folio_set_lock(fs_info, folio, folio_pos(folio), folio_size(folio));
487 }
488 
489 /*
490  * After a data read IO is done, we need to:
491  *
492  * - clear the uptodate bits on error
493  * - set the uptodate bits if things worked
494  * - set the folio up to date if all extents in the tree are uptodate
495  * - clear the lock bit in the extent tree
496  * - unlock the folio if there are no other extents locked for it
497  *
498  * Scheduling is not allowed, so the extent state tree is expected
499  * to have one and only one object corresponding to this IO.
500  */
end_bbio_data_read(struct btrfs_bio * bbio)501 static void end_bbio_data_read(struct btrfs_bio *bbio)
502 {
503 	struct btrfs_fs_info *fs_info = bbio->fs_info;
504 	struct bio *bio = &bbio->bio;
505 	struct folio_iter fi;
506 
507 	ASSERT(!bio_flagged(bio, BIO_CLONED));
508 	bio_for_each_folio_all(fi, &bbio->bio) {
509 		bool uptodate = !bio->bi_status;
510 		struct folio *folio = fi.folio;
511 		struct inode *inode = folio->mapping->host;
512 		u64 start = folio_pos(folio) + fi.offset;
513 
514 		btrfs_debug(fs_info,
515 			"%s: bi_sector=%llu, err=%d, mirror=%u",
516 			__func__, bio->bi_iter.bi_sector, bio->bi_status,
517 			bbio->mirror_num);
518 
519 
520 		if (likely(uptodate)) {
521 			u64 end = start + fi.length - 1;
522 			loff_t i_size = i_size_read(inode);
523 
524 			/*
525 			 * Zero out the remaining part if this range straddles
526 			 * i_size.
527 			 *
528 			 * Here we should only zero the range inside the folio,
529 			 * not touch anything else.
530 			 *
531 			 * NOTE: i_size is exclusive while end is inclusive and
532 			 * folio_contains() takes PAGE_SIZE units.
533 			 */
534 			if (folio_contains(folio, i_size >> PAGE_SHIFT) &&
535 			    i_size <= end) {
536 				u32 zero_start = max(offset_in_folio(folio, i_size),
537 						     offset_in_folio(folio, start));
538 				u32 zero_len = offset_in_folio(folio, end) + 1 -
539 					       zero_start;
540 
541 				folio_zero_range(folio, zero_start, zero_len);
542 			}
543 		}
544 
545 		/* Update page status and unlock. */
546 		end_folio_read(folio, uptodate, start, fi.length);
547 	}
548 	bio_put(bio);
549 }
550 
551 /*
552  * Populate every free slot in a provided array with folios using GFP_NOFS.
553  *
554  * @nr_folios:   number of folios to allocate
555  * @folio_array: the array to fill with folios; any existing non-NULL entries in
556  *		 the array will be skipped
557  *
558  * Return: 0        if all folios were able to be allocated;
559  *         -ENOMEM  otherwise, the partially allocated folios would be freed and
560  *                  the array slots zeroed
561  */
btrfs_alloc_folio_array(unsigned int nr_folios,struct folio ** folio_array)562 int btrfs_alloc_folio_array(unsigned int nr_folios, struct folio **folio_array)
563 {
564 	for (int i = 0; i < nr_folios; i++) {
565 		if (folio_array[i])
566 			continue;
567 		folio_array[i] = folio_alloc(GFP_NOFS, 0);
568 		if (!folio_array[i])
569 			goto error;
570 	}
571 	return 0;
572 error:
573 	for (int i = 0; i < nr_folios; i++) {
574 		if (folio_array[i])
575 			folio_put(folio_array[i]);
576 	}
577 	return -ENOMEM;
578 }
579 
580 /*
581  * Populate every free slot in a provided array with pages, using GFP_NOFS.
582  *
583  * @nr_pages:   number of pages to allocate
584  * @page_array: the array to fill with pages; any existing non-null entries in
585  *		the array will be skipped
586  * @nofail:	whether using __GFP_NOFAIL flag
587  *
588  * Return: 0        if all pages were able to be allocated;
589  *         -ENOMEM  otherwise, the partially allocated pages would be freed and
590  *                  the array slots zeroed
591  */
btrfs_alloc_page_array(unsigned int nr_pages,struct page ** page_array,bool nofail)592 int btrfs_alloc_page_array(unsigned int nr_pages, struct page **page_array,
593 			   bool nofail)
594 {
595 	const gfp_t gfp = nofail ? (GFP_NOFS | __GFP_NOFAIL) : GFP_NOFS;
596 	unsigned int allocated;
597 
598 	for (allocated = 0; allocated < nr_pages;) {
599 		unsigned int last = allocated;
600 
601 		allocated = alloc_pages_bulk(gfp, nr_pages, page_array);
602 		if (unlikely(allocated == last)) {
603 			/* No progress, fail and do cleanup. */
604 			for (int i = 0; i < allocated; i++) {
605 				__free_page(page_array[i]);
606 				page_array[i] = NULL;
607 			}
608 			return -ENOMEM;
609 		}
610 	}
611 	return 0;
612 }
613 
614 /*
615  * Populate needed folios for the extent buffer.
616  *
617  * For now, the folios populated are always in order 0 (aka, single page).
618  */
alloc_eb_folio_array(struct extent_buffer * eb,bool nofail)619 static int alloc_eb_folio_array(struct extent_buffer *eb, bool nofail)
620 {
621 	struct page *page_array[INLINE_EXTENT_BUFFER_PAGES] = { 0 };
622 	int num_pages = num_extent_pages(eb);
623 	int ret;
624 
625 	ret = btrfs_alloc_page_array(num_pages, page_array, nofail);
626 	if (ret < 0)
627 		return ret;
628 
629 	for (int i = 0; i < num_pages; i++)
630 		eb->folios[i] = page_folio(page_array[i]);
631 	eb->folio_size = PAGE_SIZE;
632 	eb->folio_shift = PAGE_SHIFT;
633 	return 0;
634 }
635 
btrfs_bio_is_contig(struct btrfs_bio_ctrl * bio_ctrl,u64 disk_bytenr,loff_t file_offset)636 static bool btrfs_bio_is_contig(struct btrfs_bio_ctrl *bio_ctrl,
637 				u64 disk_bytenr, loff_t file_offset)
638 {
639 	struct bio *bio = &bio_ctrl->bbio->bio;
640 	const sector_t sector = disk_bytenr >> SECTOR_SHIFT;
641 
642 	if (bio_ctrl->compress_type != BTRFS_COMPRESS_NONE) {
643 		/*
644 		 * For compression, all IO should have its logical bytenr set
645 		 * to the starting bytenr of the compressed extent.
646 		 */
647 		return bio->bi_iter.bi_sector == sector;
648 	}
649 
650 	/*
651 	 * To merge into a bio both the disk sector and the logical offset in
652 	 * the file need to be contiguous.
653 	 */
654 	return bio_ctrl->next_file_offset == file_offset &&
655 		bio_end_sector(bio) == sector;
656 }
657 
alloc_new_bio(struct btrfs_inode * inode,struct btrfs_bio_ctrl * bio_ctrl,u64 disk_bytenr,u64 file_offset)658 static void alloc_new_bio(struct btrfs_inode *inode,
659 			  struct btrfs_bio_ctrl *bio_ctrl,
660 			  u64 disk_bytenr, u64 file_offset)
661 {
662 	struct btrfs_fs_info *fs_info = inode->root->fs_info;
663 	struct btrfs_bio *bbio;
664 
665 	bbio = btrfs_bio_alloc(BIO_MAX_VECS, bio_ctrl->opf, fs_info,
666 			       bio_ctrl->end_io_func, NULL);
667 	bbio->bio.bi_iter.bi_sector = disk_bytenr >> SECTOR_SHIFT;
668 	bbio->bio.bi_write_hint = inode->vfs_inode.i_write_hint;
669 	bbio->inode = inode;
670 	bbio->file_offset = file_offset;
671 	bio_ctrl->bbio = bbio;
672 	bio_ctrl->len_to_oe_boundary = U32_MAX;
673 	bio_ctrl->next_file_offset = file_offset;
674 
675 	/* Limit data write bios to the ordered boundary. */
676 	if (bio_ctrl->wbc) {
677 		struct btrfs_ordered_extent *ordered;
678 
679 		ordered = btrfs_lookup_ordered_extent(inode, file_offset);
680 		if (ordered) {
681 			bio_ctrl->len_to_oe_boundary = min_t(u32, U32_MAX,
682 					ordered->file_offset +
683 					ordered->disk_num_bytes - file_offset);
684 			bbio->ordered = ordered;
685 		}
686 
687 		/*
688 		 * Pick the last added device to support cgroup writeback.  For
689 		 * multi-device file systems this means blk-cgroup policies have
690 		 * to always be set on the last added/replaced device.
691 		 * This is a bit odd but has been like that for a long time.
692 		 */
693 		bio_set_dev(&bbio->bio, fs_info->fs_devices->latest_dev->bdev);
694 		wbc_init_bio(bio_ctrl->wbc, &bbio->bio);
695 	}
696 }
697 
698 /*
699  * @disk_bytenr: logical bytenr where the write will be
700  * @page:	page to add to the bio
701  * @size:	portion of page that we want to write to
702  * @pg_offset:	offset of the new bio or to check whether we are adding
703  *              a contiguous page to the previous one
704  *
705  * The will either add the page into the existing @bio_ctrl->bbio, or allocate a
706  * new one in @bio_ctrl->bbio.
707  * The mirror number for this IO should already be initizlied in
708  * @bio_ctrl->mirror_num.
709  */
submit_extent_folio(struct btrfs_bio_ctrl * bio_ctrl,u64 disk_bytenr,struct folio * folio,size_t size,unsigned long pg_offset)710 static void submit_extent_folio(struct btrfs_bio_ctrl *bio_ctrl,
711 			       u64 disk_bytenr, struct folio *folio,
712 			       size_t size, unsigned long pg_offset)
713 {
714 	struct btrfs_inode *inode = folio_to_inode(folio);
715 	loff_t file_offset = folio_pos(folio) + pg_offset;
716 
717 	ASSERT(pg_offset + size <= folio_size(folio));
718 	ASSERT(bio_ctrl->end_io_func);
719 
720 	if (bio_ctrl->bbio &&
721 	    !btrfs_bio_is_contig(bio_ctrl, disk_bytenr, file_offset))
722 		submit_one_bio(bio_ctrl);
723 
724 	do {
725 		u32 len = size;
726 
727 		/* Allocate new bio if needed */
728 		if (!bio_ctrl->bbio)
729 			alloc_new_bio(inode, bio_ctrl, disk_bytenr, file_offset);
730 
731 		/* Cap to the current ordered extent boundary if there is one. */
732 		if (len > bio_ctrl->len_to_oe_boundary) {
733 			ASSERT(bio_ctrl->compress_type == BTRFS_COMPRESS_NONE);
734 			ASSERT(is_data_inode(inode));
735 			len = bio_ctrl->len_to_oe_boundary;
736 		}
737 
738 		if (!bio_add_folio(&bio_ctrl->bbio->bio, folio, len, pg_offset)) {
739 			/* bio full: move on to a new one */
740 			submit_one_bio(bio_ctrl);
741 			continue;
742 		}
743 		bio_ctrl->next_file_offset += len;
744 
745 		if (bio_ctrl->wbc)
746 			wbc_account_cgroup_owner(bio_ctrl->wbc, folio, len);
747 
748 		size -= len;
749 		pg_offset += len;
750 		disk_bytenr += len;
751 		file_offset += len;
752 
753 		/*
754 		 * len_to_oe_boundary defaults to U32_MAX, which isn't folio or
755 		 * sector aligned.  alloc_new_bio() then sets it to the end of
756 		 * our ordered extent for writes into zoned devices.
757 		 *
758 		 * When len_to_oe_boundary is tracking an ordered extent, we
759 		 * trust the ordered extent code to align things properly, and
760 		 * the check above to cap our write to the ordered extent
761 		 * boundary is correct.
762 		 *
763 		 * When len_to_oe_boundary is U32_MAX, the cap above would
764 		 * result in a 4095 byte IO for the last folio right before
765 		 * we hit the bio limit of UINT_MAX.  bio_add_folio() has all
766 		 * the checks required to make sure we don't overflow the bio,
767 		 * and we should just ignore len_to_oe_boundary completely
768 		 * unless we're using it to track an ordered extent.
769 		 *
770 		 * It's pretty hard to make a bio sized U32_MAX, but it can
771 		 * happen when the page cache is able to feed us contiguous
772 		 * folios for large extents.
773 		 */
774 		if (bio_ctrl->len_to_oe_boundary != U32_MAX)
775 			bio_ctrl->len_to_oe_boundary -= len;
776 
777 		/* Ordered extent boundary: move on to a new bio. */
778 		if (bio_ctrl->len_to_oe_boundary == 0)
779 			submit_one_bio(bio_ctrl);
780 	} while (size);
781 }
782 
attach_extent_buffer_folio(struct extent_buffer * eb,struct folio * folio,struct btrfs_subpage * prealloc)783 static int attach_extent_buffer_folio(struct extent_buffer *eb,
784 				      struct folio *folio,
785 				      struct btrfs_subpage *prealloc)
786 {
787 	struct btrfs_fs_info *fs_info = eb->fs_info;
788 	int ret = 0;
789 
790 	/*
791 	 * If the page is mapped to btree inode, we should hold the private
792 	 * lock to prevent race.
793 	 * For cloned or dummy extent buffers, their pages are not mapped and
794 	 * will not race with any other ebs.
795 	 */
796 	if (folio->mapping)
797 		lockdep_assert_held(&folio->mapping->i_private_lock);
798 
799 	if (!btrfs_meta_is_subpage(fs_info)) {
800 		if (!folio_test_private(folio))
801 			folio_attach_private(folio, eb);
802 		else
803 			WARN_ON(folio_get_private(folio) != eb);
804 		return 0;
805 	}
806 
807 	/* Already mapped, just free prealloc */
808 	if (folio_test_private(folio)) {
809 		btrfs_free_subpage(prealloc);
810 		return 0;
811 	}
812 
813 	if (prealloc)
814 		/* Has preallocated memory for subpage */
815 		folio_attach_private(folio, prealloc);
816 	else
817 		/* Do new allocation to attach subpage */
818 		ret = btrfs_attach_subpage(fs_info, folio, BTRFS_SUBPAGE_METADATA);
819 	return ret;
820 }
821 
set_folio_extent_mapped(struct folio * folio)822 int set_folio_extent_mapped(struct folio *folio)
823 {
824 	struct btrfs_fs_info *fs_info;
825 
826 	ASSERT(folio->mapping);
827 
828 	if (folio_test_private(folio))
829 		return 0;
830 
831 	fs_info = folio_to_fs_info(folio);
832 
833 	if (btrfs_is_subpage(fs_info, folio))
834 		return btrfs_attach_subpage(fs_info, folio, BTRFS_SUBPAGE_DATA);
835 
836 	folio_attach_private(folio, (void *)EXTENT_FOLIO_PRIVATE);
837 	return 0;
838 }
839 
clear_folio_extent_mapped(struct folio * folio)840 void clear_folio_extent_mapped(struct folio *folio)
841 {
842 	struct btrfs_fs_info *fs_info;
843 
844 	ASSERT(folio->mapping);
845 
846 	if (!folio_test_private(folio))
847 		return;
848 
849 	fs_info = folio_to_fs_info(folio);
850 	if (btrfs_is_subpage(fs_info, folio))
851 		return btrfs_detach_subpage(fs_info, folio, BTRFS_SUBPAGE_DATA);
852 
853 	folio_detach_private(folio);
854 }
855 
get_extent_map(struct btrfs_inode * inode,struct folio * folio,u64 start,u64 len,struct extent_map ** em_cached)856 static struct extent_map *get_extent_map(struct btrfs_inode *inode,
857 					 struct folio *folio, u64 start,
858 					 u64 len, struct extent_map **em_cached)
859 {
860 	struct extent_map *em;
861 
862 	ASSERT(em_cached);
863 
864 	if (*em_cached) {
865 		em = *em_cached;
866 		if (btrfs_extent_map_in_tree(em) && start >= em->start &&
867 		    start < btrfs_extent_map_end(em)) {
868 			refcount_inc(&em->refs);
869 			return em;
870 		}
871 
872 		btrfs_free_extent_map(em);
873 		*em_cached = NULL;
874 	}
875 
876 	em = btrfs_get_extent(inode, folio, start, len);
877 	if (!IS_ERR(em)) {
878 		BUG_ON(*em_cached);
879 		refcount_inc(&em->refs);
880 		*em_cached = em;
881 	}
882 
883 	return em;
884 }
885 /*
886  * basic readpage implementation.  Locked extent state structs are inserted
887  * into the tree that are removed when the IO is done (by the end_io
888  * handlers)
889  * XXX JDM: This needs looking at to ensure proper page locking
890  * return 0 on success, otherwise return error
891  */
btrfs_do_readpage(struct folio * folio,struct extent_map ** em_cached,struct btrfs_bio_ctrl * bio_ctrl,u64 * prev_em_start)892 static int btrfs_do_readpage(struct folio *folio, struct extent_map **em_cached,
893 		      struct btrfs_bio_ctrl *bio_ctrl, u64 *prev_em_start)
894 {
895 	struct inode *inode = folio->mapping->host;
896 	struct btrfs_fs_info *fs_info = inode_to_fs_info(inode);
897 	u64 start = folio_pos(folio);
898 	const u64 end = start + folio_size(folio) - 1;
899 	u64 extent_offset;
900 	u64 last_byte = i_size_read(inode);
901 	struct extent_map *em;
902 	int ret = 0;
903 	const size_t blocksize = fs_info->sectorsize;
904 
905 	ret = set_folio_extent_mapped(folio);
906 	if (ret < 0) {
907 		folio_unlock(folio);
908 		return ret;
909 	}
910 
911 	if (folio_contains(folio, last_byte >> PAGE_SHIFT)) {
912 		size_t zero_offset = offset_in_folio(folio, last_byte);
913 
914 		if (zero_offset)
915 			folio_zero_range(folio, zero_offset,
916 					 folio_size(folio) - zero_offset);
917 	}
918 	bio_ctrl->end_io_func = end_bbio_data_read;
919 	begin_folio_read(fs_info, folio);
920 	for (u64 cur = start; cur <= end; cur += blocksize) {
921 		enum btrfs_compression_type compress_type = BTRFS_COMPRESS_NONE;
922 		unsigned long pg_offset = offset_in_folio(folio, cur);
923 		bool force_bio_submit = false;
924 		u64 disk_bytenr;
925 		u64 block_start;
926 
927 		ASSERT(IS_ALIGNED(cur, fs_info->sectorsize));
928 		if (cur >= last_byte) {
929 			folio_zero_range(folio, pg_offset, end - cur + 1);
930 			end_folio_read(folio, true, cur, end - cur + 1);
931 			break;
932 		}
933 		if (btrfs_folio_test_uptodate(fs_info, folio, cur, blocksize)) {
934 			end_folio_read(folio, true, cur, blocksize);
935 			continue;
936 		}
937 		em = get_extent_map(BTRFS_I(inode), folio, cur, end - cur + 1, em_cached);
938 		if (IS_ERR(em)) {
939 			end_folio_read(folio, false, cur, end + 1 - cur);
940 			return PTR_ERR(em);
941 		}
942 		extent_offset = cur - em->start;
943 		BUG_ON(btrfs_extent_map_end(em) <= cur);
944 		BUG_ON(end < cur);
945 
946 		compress_type = btrfs_extent_map_compression(em);
947 
948 		if (compress_type != BTRFS_COMPRESS_NONE)
949 			disk_bytenr = em->disk_bytenr;
950 		else
951 			disk_bytenr = btrfs_extent_map_block_start(em) + extent_offset;
952 
953 		if (em->flags & EXTENT_FLAG_PREALLOC)
954 			block_start = EXTENT_MAP_HOLE;
955 		else
956 			block_start = btrfs_extent_map_block_start(em);
957 
958 		/*
959 		 * If we have a file range that points to a compressed extent
960 		 * and it's followed by a consecutive file range that points
961 		 * to the same compressed extent (possibly with a different
962 		 * offset and/or length, so it either points to the whole extent
963 		 * or only part of it), we must make sure we do not submit a
964 		 * single bio to populate the folios for the 2 ranges because
965 		 * this makes the compressed extent read zero out the folios
966 		 * belonging to the 2nd range. Imagine the following scenario:
967 		 *
968 		 *  File layout
969 		 *  [0 - 8K]                     [8K - 24K]
970 		 *    |                               |
971 		 *    |                               |
972 		 * points to extent X,         points to extent X,
973 		 * offset 4K, length of 8K     offset 0, length 16K
974 		 *
975 		 * [extent X, compressed length = 4K uncompressed length = 16K]
976 		 *
977 		 * If the bio to read the compressed extent covers both ranges,
978 		 * it will decompress extent X into the folios belonging to the
979 		 * first range and then it will stop, zeroing out the remaining
980 		 * folios that belong to the other range that points to extent X.
981 		 * So here we make sure we submit 2 bios, one for the first
982 		 * range and another one for the third range. Both will target
983 		 * the same physical extent from disk, but we can't currently
984 		 * make the compressed bio endio callback populate the folios
985 		 * for both ranges because each compressed bio is tightly
986 		 * coupled with a single extent map, and each range can have
987 		 * an extent map with a different offset value relative to the
988 		 * uncompressed data of our extent and different lengths. This
989 		 * is a corner case so we prioritize correctness over
990 		 * non-optimal behavior (submitting 2 bios for the same extent).
991 		 */
992 		if (compress_type != BTRFS_COMPRESS_NONE &&
993 		    prev_em_start && *prev_em_start != (u64)-1 &&
994 		    *prev_em_start != em->start)
995 			force_bio_submit = true;
996 
997 		if (prev_em_start)
998 			*prev_em_start = em->start;
999 
1000 		btrfs_free_extent_map(em);
1001 		em = NULL;
1002 
1003 		/* we've found a hole, just zero and go on */
1004 		if (block_start == EXTENT_MAP_HOLE) {
1005 			folio_zero_range(folio, pg_offset, blocksize);
1006 			end_folio_read(folio, true, cur, blocksize);
1007 			continue;
1008 		}
1009 		/* the get_extent function already copied into the folio */
1010 		if (block_start == EXTENT_MAP_INLINE) {
1011 			end_folio_read(folio, true, cur, blocksize);
1012 			continue;
1013 		}
1014 
1015 		if (bio_ctrl->compress_type != compress_type) {
1016 			submit_one_bio(bio_ctrl);
1017 			bio_ctrl->compress_type = compress_type;
1018 		}
1019 
1020 		if (force_bio_submit)
1021 			submit_one_bio(bio_ctrl);
1022 		submit_extent_folio(bio_ctrl, disk_bytenr, folio, blocksize,
1023 				    pg_offset);
1024 	}
1025 	return 0;
1026 }
1027 
1028 /*
1029  * Check if we can skip waiting the @ordered extent covering the block at @fileoff.
1030  *
1031  * @fileoff:	Both input and output.
1032  *		Input as the file offset where the check should start at.
1033  *		Output as where the next check should start at,
1034  *		if the function returns true.
1035  *
1036  * Return true if we can skip to @fileoff. The caller needs to check the new
1037  * @fileoff value to make sure it covers the full range, before skipping the
1038  * full OE.
1039  *
1040  * Return false if we must wait for the ordered extent.
1041  */
can_skip_one_ordered_range(struct btrfs_inode * inode,struct btrfs_ordered_extent * ordered,u64 * fileoff)1042 static bool can_skip_one_ordered_range(struct btrfs_inode *inode,
1043 				       struct btrfs_ordered_extent *ordered,
1044 				       u64 *fileoff)
1045 {
1046 	const struct btrfs_fs_info *fs_info = inode->root->fs_info;
1047 	struct folio *folio;
1048 	const u32 blocksize = fs_info->sectorsize;
1049 	u64 cur = *fileoff;
1050 	bool ret;
1051 
1052 	folio = filemap_get_folio(inode->vfs_inode.i_mapping, cur >> PAGE_SHIFT);
1053 
1054 	/*
1055 	 * We should have locked the folio(s) for range [start, end], thus
1056 	 * there must be a folio and it must be locked.
1057 	 */
1058 	ASSERT(!IS_ERR(folio));
1059 	ASSERT(folio_test_locked(folio));
1060 
1061 	/*
1062 	 * There are several cases for the folio and OE combination:
1063 	 *
1064 	 * 1) Folio has no private flag
1065 	 *    The OE has all its IO done but not yet finished, and folio got
1066 	 *    invalidated.
1067 	 *
1068 	 * Have we have to wait for the OE to finish, as it may contain the
1069 	 * to-be-inserted data checksum.
1070 	 * Without the data checksum inserted into the csum tree, read will
1071 	 * just fail with missing csum.
1072 	 */
1073 	if (!folio_test_private(folio)) {
1074 		ret = false;
1075 		goto out;
1076 	}
1077 
1078 	/*
1079 	 * 2) The first block is DIRTY.
1080 	 *
1081 	 * This means the OE is created by some other folios whose file pos is
1082 	 * before this one. And since we are holding the folio lock, the writeback
1083 	 * of this folio cannot start.
1084 	 *
1085 	 * We must skip the whole OE, because it will never start until we
1086 	 * finished our folio read and unlocked the folio.
1087 	 */
1088 	if (btrfs_folio_test_dirty(fs_info, folio, cur, blocksize)) {
1089 		u64 range_len = min(folio_pos(folio) + folio_size(folio),
1090 				    ordered->file_offset + ordered->num_bytes) - cur;
1091 
1092 		ret = true;
1093 		/*
1094 		 * At least inside the folio, all the remaining blocks should
1095 		 * also be dirty.
1096 		 */
1097 		ASSERT(btrfs_folio_test_dirty(fs_info, folio, cur, range_len));
1098 		*fileoff = ordered->file_offset + ordered->num_bytes;
1099 		goto out;
1100 	}
1101 
1102 	/*
1103 	 * 3) The first block is uptodate.
1104 	 *
1105 	 * At least the first block can be skipped, but we are still not fully
1106 	 * sure. E.g. if the OE has some other folios in the range that cannot
1107 	 * be skipped.
1108 	 * So we return true and update @next_ret to the OE/folio boundary.
1109 	 */
1110 	if (btrfs_folio_test_uptodate(fs_info, folio, cur, blocksize)) {
1111 		u64 range_len = min(folio_pos(folio) + folio_size(folio),
1112 				    ordered->file_offset + ordered->num_bytes) - cur;
1113 
1114 		/*
1115 		 * The whole range to the OE end or folio boundary should also
1116 		 * be uptodate.
1117 		 */
1118 		ASSERT(btrfs_folio_test_uptodate(fs_info, folio, cur, range_len));
1119 		ret = true;
1120 		*fileoff = cur + range_len;
1121 		goto out;
1122 	}
1123 
1124 	/*
1125 	 * 4) The first block is not uptodate.
1126 	 *
1127 	 * This means the folio is invalidated after the writeback was finished,
1128 	 * but by some other operations (e.g. block aligned buffered write) the
1129 	 * folio is inserted into filemap.
1130 	 * Very much the same as case 1).
1131 	 */
1132 	ret = false;
1133 out:
1134 	folio_put(folio);
1135 	return ret;
1136 }
1137 
can_skip_ordered_extent(struct btrfs_inode * inode,struct btrfs_ordered_extent * ordered,u64 start,u64 end)1138 static bool can_skip_ordered_extent(struct btrfs_inode *inode,
1139 				    struct btrfs_ordered_extent *ordered,
1140 				    u64 start, u64 end)
1141 {
1142 	const u64 range_end = min(end, ordered->file_offset + ordered->num_bytes - 1);
1143 	u64 cur = max(start, ordered->file_offset);
1144 
1145 	while (cur < range_end) {
1146 		bool can_skip;
1147 
1148 		can_skip = can_skip_one_ordered_range(inode, ordered, &cur);
1149 		if (!can_skip)
1150 			return false;
1151 	}
1152 	return true;
1153 }
1154 
1155 /*
1156  * Locking helper to make sure we get a stable view of extent maps for the
1157  * involved range.
1158  *
1159  * This is for folio read paths (read and readahead), thus the involved range
1160  * should have all the folios locked.
1161  */
lock_extents_for_read(struct btrfs_inode * inode,u64 start,u64 end,struct extent_state ** cached_state)1162 static void lock_extents_for_read(struct btrfs_inode *inode, u64 start, u64 end,
1163 				  struct extent_state **cached_state)
1164 {
1165 	u64 cur_pos;
1166 
1167 	/* Caller must provide a valid @cached_state. */
1168 	ASSERT(cached_state);
1169 
1170 	/* The range must at least be page aligned, as all read paths are folio based. */
1171 	ASSERT(IS_ALIGNED(start, PAGE_SIZE));
1172 	ASSERT(IS_ALIGNED(end + 1, PAGE_SIZE));
1173 
1174 again:
1175 	btrfs_lock_extent(&inode->io_tree, start, end, cached_state);
1176 	cur_pos = start;
1177 	while (cur_pos < end) {
1178 		struct btrfs_ordered_extent *ordered;
1179 
1180 		ordered = btrfs_lookup_ordered_range(inode, cur_pos,
1181 						     end - cur_pos + 1);
1182 		/*
1183 		 * No ordered extents in the range, and we hold the extent lock,
1184 		 * no one can modify the extent maps in the range, we're safe to return.
1185 		 */
1186 		if (!ordered)
1187 			break;
1188 
1189 		/* Check if we can skip waiting for the whole OE. */
1190 		if (can_skip_ordered_extent(inode, ordered, start, end)) {
1191 			cur_pos = min(ordered->file_offset + ordered->num_bytes,
1192 				      end + 1);
1193 			btrfs_put_ordered_extent(ordered);
1194 			continue;
1195 		}
1196 
1197 		/* Now wait for the OE to finish. */
1198 		btrfs_unlock_extent(&inode->io_tree, start, end, cached_state);
1199 		btrfs_start_ordered_extent_nowriteback(ordered, start, end + 1 - start);
1200 		btrfs_put_ordered_extent(ordered);
1201 		/* We have unlocked the whole range, restart from the beginning. */
1202 		goto again;
1203 	}
1204 }
1205 
btrfs_read_folio(struct file * file,struct folio * folio)1206 int btrfs_read_folio(struct file *file, struct folio *folio)
1207 {
1208 	struct btrfs_inode *inode = folio_to_inode(folio);
1209 	const u64 start = folio_pos(folio);
1210 	const u64 end = start + folio_size(folio) - 1;
1211 	struct extent_state *cached_state = NULL;
1212 	struct btrfs_bio_ctrl bio_ctrl = { .opf = REQ_OP_READ };
1213 	struct extent_map *em_cached = NULL;
1214 	int ret;
1215 
1216 	lock_extents_for_read(inode, start, end, &cached_state);
1217 	ret = btrfs_do_readpage(folio, &em_cached, &bio_ctrl, NULL);
1218 	btrfs_unlock_extent(&inode->io_tree, start, end, &cached_state);
1219 
1220 	btrfs_free_extent_map(em_cached);
1221 
1222 	/*
1223 	 * If btrfs_do_readpage() failed we will want to submit the assembled
1224 	 * bio to do the cleanup.
1225 	 */
1226 	submit_one_bio(&bio_ctrl);
1227 	return ret;
1228 }
1229 
set_delalloc_bitmap(struct folio * folio,unsigned long * delalloc_bitmap,u64 start,u32 len)1230 static void set_delalloc_bitmap(struct folio *folio, unsigned long *delalloc_bitmap,
1231 				u64 start, u32 len)
1232 {
1233 	struct btrfs_fs_info *fs_info = folio_to_fs_info(folio);
1234 	const u64 folio_start = folio_pos(folio);
1235 	unsigned int start_bit;
1236 	unsigned int nbits;
1237 
1238 	ASSERT(start >= folio_start && start + len <= folio_start + folio_size(folio));
1239 	start_bit = (start - folio_start) >> fs_info->sectorsize_bits;
1240 	nbits = len >> fs_info->sectorsize_bits;
1241 	ASSERT(bitmap_test_range_all_zero(delalloc_bitmap, start_bit, nbits));
1242 	bitmap_set(delalloc_bitmap, start_bit, nbits);
1243 }
1244 
find_next_delalloc_bitmap(struct folio * folio,unsigned long * delalloc_bitmap,u64 start,u64 * found_start,u32 * found_len)1245 static bool find_next_delalloc_bitmap(struct folio *folio,
1246 				      unsigned long *delalloc_bitmap, u64 start,
1247 				      u64 *found_start, u32 *found_len)
1248 {
1249 	struct btrfs_fs_info *fs_info = folio_to_fs_info(folio);
1250 	const u64 folio_start = folio_pos(folio);
1251 	const unsigned int bitmap_size = btrfs_blocks_per_folio(fs_info, folio);
1252 	unsigned int start_bit;
1253 	unsigned int first_zero;
1254 	unsigned int first_set;
1255 
1256 	ASSERT(start >= folio_start && start < folio_start + folio_size(folio));
1257 
1258 	start_bit = (start - folio_start) >> fs_info->sectorsize_bits;
1259 	first_set = find_next_bit(delalloc_bitmap, bitmap_size, start_bit);
1260 	if (first_set >= bitmap_size)
1261 		return false;
1262 
1263 	*found_start = folio_start + (first_set << fs_info->sectorsize_bits);
1264 	first_zero = find_next_zero_bit(delalloc_bitmap, bitmap_size, first_set);
1265 	*found_len = (first_zero - first_set) << fs_info->sectorsize_bits;
1266 	return true;
1267 }
1268 
1269 /*
1270  * Do all of the delayed allocation setup.
1271  *
1272  * Return >0 if all the dirty blocks are submitted async (compression) or inlined.
1273  * The @folio should no longer be touched (treat it as already unlocked).
1274  *
1275  * Return 0 if there is still dirty block that needs to be submitted through
1276  * extent_writepage_io().
1277  * bio_ctrl->submit_bitmap will indicate which blocks of the folio should be
1278  * submitted, and @folio is still kept locked.
1279  *
1280  * Return <0 if there is any error hit.
1281  * Any allocated ordered extent range covering this folio will be marked
1282  * finished (IOERR), and @folio is still kept locked.
1283  */
writepage_delalloc(struct btrfs_inode * inode,struct folio * folio,struct btrfs_bio_ctrl * bio_ctrl)1284 static noinline_for_stack int writepage_delalloc(struct btrfs_inode *inode,
1285 						 struct folio *folio,
1286 						 struct btrfs_bio_ctrl *bio_ctrl)
1287 {
1288 	struct btrfs_fs_info *fs_info = inode_to_fs_info(&inode->vfs_inode);
1289 	struct writeback_control *wbc = bio_ctrl->wbc;
1290 	const bool is_subpage = btrfs_is_subpage(fs_info, folio);
1291 	const u64 page_start = folio_pos(folio);
1292 	const u64 page_end = page_start + folio_size(folio) - 1;
1293 	const unsigned int blocks_per_folio = btrfs_blocks_per_folio(fs_info, folio);
1294 	unsigned long delalloc_bitmap = 0;
1295 	/*
1296 	 * Save the last found delalloc end. As the delalloc end can go beyond
1297 	 * page boundary, thus we cannot rely on subpage bitmap to locate the
1298 	 * last delalloc end.
1299 	 */
1300 	u64 last_delalloc_end = 0;
1301 	/*
1302 	 * The range end (exclusive) of the last successfully finished delalloc
1303 	 * range.
1304 	 * Any range covered by ordered extent must either be manually marked
1305 	 * finished (error handling), or has IO submitted (and finish the
1306 	 * ordered extent normally).
1307 	 *
1308 	 * This records the end of ordered extent cleanup if we hit an error.
1309 	 */
1310 	u64 last_finished_delalloc_end = page_start;
1311 	u64 delalloc_start = page_start;
1312 	u64 delalloc_end = page_end;
1313 	u64 delalloc_to_write = 0;
1314 	int ret = 0;
1315 	int bit;
1316 
1317 	/* Save the dirty bitmap as our submission bitmap will be a subset of it. */
1318 	if (btrfs_is_subpage(fs_info, folio)) {
1319 		ASSERT(blocks_per_folio > 1);
1320 		btrfs_get_subpage_dirty_bitmap(fs_info, folio, &bio_ctrl->submit_bitmap);
1321 	} else {
1322 		bio_ctrl->submit_bitmap = 1;
1323 	}
1324 
1325 	for_each_set_bit(bit, &bio_ctrl->submit_bitmap, blocks_per_folio) {
1326 		u64 start = page_start + (bit << fs_info->sectorsize_bits);
1327 
1328 		btrfs_folio_set_lock(fs_info, folio, start, fs_info->sectorsize);
1329 	}
1330 
1331 	/* Lock all (subpage) delalloc ranges inside the folio first. */
1332 	while (delalloc_start < page_end) {
1333 		delalloc_end = page_end;
1334 		if (!find_lock_delalloc_range(&inode->vfs_inode, folio,
1335 					      &delalloc_start, &delalloc_end)) {
1336 			delalloc_start = delalloc_end + 1;
1337 			continue;
1338 		}
1339 		set_delalloc_bitmap(folio, &delalloc_bitmap, delalloc_start,
1340 				    min(delalloc_end, page_end) + 1 - delalloc_start);
1341 		last_delalloc_end = delalloc_end;
1342 		delalloc_start = delalloc_end + 1;
1343 	}
1344 	delalloc_start = page_start;
1345 
1346 	if (!last_delalloc_end)
1347 		goto out;
1348 
1349 	/* Run the delalloc ranges for the above locked ranges. */
1350 	while (delalloc_start < page_end) {
1351 		u64 found_start;
1352 		u32 found_len;
1353 		bool found;
1354 
1355 		if (!is_subpage) {
1356 			/*
1357 			 * For non-subpage case, the found delalloc range must
1358 			 * cover this folio and there must be only one locked
1359 			 * delalloc range.
1360 			 */
1361 			found_start = page_start;
1362 			found_len = last_delalloc_end + 1 - found_start;
1363 			found = true;
1364 		} else {
1365 			found = find_next_delalloc_bitmap(folio, &delalloc_bitmap,
1366 					delalloc_start, &found_start, &found_len);
1367 		}
1368 		if (!found)
1369 			break;
1370 		/*
1371 		 * The subpage range covers the last sector, the delalloc range may
1372 		 * end beyond the folio boundary, use the saved delalloc_end
1373 		 * instead.
1374 		 */
1375 		if (found_start + found_len >= page_end)
1376 			found_len = last_delalloc_end + 1 - found_start;
1377 
1378 		if (ret >= 0) {
1379 			/*
1380 			 * Some delalloc range may be created by previous folios.
1381 			 * Thus we still need to clean up this range during error
1382 			 * handling.
1383 			 */
1384 			last_finished_delalloc_end = found_start;
1385 			/* No errors hit so far, run the current delalloc range. */
1386 			ret = btrfs_run_delalloc_range(inode, folio,
1387 						       found_start,
1388 						       found_start + found_len - 1,
1389 						       wbc);
1390 			if (ret >= 0)
1391 				last_finished_delalloc_end = found_start + found_len;
1392 			if (unlikely(ret < 0))
1393 				btrfs_err_rl(fs_info,
1394 "failed to run delalloc range, root=%lld ino=%llu folio=%llu submit_bitmap=%*pbl start=%llu len=%u: %d",
1395 					     btrfs_root_id(inode->root),
1396 					     btrfs_ino(inode),
1397 					     folio_pos(folio),
1398 					     blocks_per_folio,
1399 					     &bio_ctrl->submit_bitmap,
1400 					     found_start, found_len, ret);
1401 		} else {
1402 			/*
1403 			 * We've hit an error during previous delalloc range,
1404 			 * have to cleanup the remaining locked ranges.
1405 			 */
1406 			btrfs_unlock_extent(&inode->io_tree, found_start,
1407 					    found_start + found_len - 1, NULL);
1408 			unlock_delalloc_folio(&inode->vfs_inode, folio,
1409 					      found_start,
1410 					      found_start + found_len - 1);
1411 		}
1412 
1413 		/*
1414 		 * We have some ranges that's going to be submitted asynchronously
1415 		 * (compression or inline).  These range have their own control
1416 		 * on when to unlock the pages.  We should not touch them
1417 		 * anymore, so clear the range from the submission bitmap.
1418 		 */
1419 		if (ret > 0) {
1420 			unsigned int start_bit = (found_start - page_start) >>
1421 						 fs_info->sectorsize_bits;
1422 			unsigned int end_bit = (min(page_end + 1, found_start + found_len) -
1423 						page_start) >> fs_info->sectorsize_bits;
1424 			bitmap_clear(&bio_ctrl->submit_bitmap, start_bit, end_bit - start_bit);
1425 		}
1426 		/*
1427 		 * Above btrfs_run_delalloc_range() may have unlocked the folio,
1428 		 * thus for the last range, we cannot touch the folio anymore.
1429 		 */
1430 		if (found_start + found_len >= last_delalloc_end + 1)
1431 			break;
1432 
1433 		delalloc_start = found_start + found_len;
1434 	}
1435 	/*
1436 	 * It's possible we had some ordered extents created before we hit
1437 	 * an error, cleanup non-async successfully created delalloc ranges.
1438 	 */
1439 	if (unlikely(ret < 0)) {
1440 		unsigned int bitmap_size = min(
1441 				(last_finished_delalloc_end - page_start) >>
1442 				fs_info->sectorsize_bits,
1443 				blocks_per_folio);
1444 
1445 		for_each_set_bit(bit, &bio_ctrl->submit_bitmap, bitmap_size)
1446 			btrfs_mark_ordered_io_finished(inode, folio,
1447 				page_start + (bit << fs_info->sectorsize_bits),
1448 				fs_info->sectorsize, false);
1449 		return ret;
1450 	}
1451 out:
1452 	if (last_delalloc_end)
1453 		delalloc_end = last_delalloc_end;
1454 	else
1455 		delalloc_end = page_end;
1456 	/*
1457 	 * delalloc_end is already one less than the total length, so
1458 	 * we don't subtract one from PAGE_SIZE.
1459 	 */
1460 	delalloc_to_write +=
1461 		DIV_ROUND_UP(delalloc_end + 1 - page_start, PAGE_SIZE);
1462 
1463 	/*
1464 	 * If all ranges are submitted asynchronously, we just need to account
1465 	 * for them here.
1466 	 */
1467 	if (bitmap_empty(&bio_ctrl->submit_bitmap, blocks_per_folio)) {
1468 		wbc->nr_to_write -= delalloc_to_write;
1469 		return 1;
1470 	}
1471 
1472 	if (wbc->nr_to_write < delalloc_to_write) {
1473 		int thresh = 8192;
1474 
1475 		if (delalloc_to_write < thresh * 2)
1476 			thresh = delalloc_to_write;
1477 		wbc->nr_to_write = min_t(u64, delalloc_to_write,
1478 					 thresh);
1479 	}
1480 
1481 	return 0;
1482 }
1483 
1484 /*
1485  * Return 0 if we have submitted or queued the sector for submission.
1486  * Return <0 for critical errors.
1487  *
1488  * Caller should make sure filepos < i_size and handle filepos >= i_size case.
1489  */
submit_one_sector(struct btrfs_inode * inode,struct folio * folio,u64 filepos,struct btrfs_bio_ctrl * bio_ctrl,loff_t i_size)1490 static int submit_one_sector(struct btrfs_inode *inode,
1491 			     struct folio *folio,
1492 			     u64 filepos, struct btrfs_bio_ctrl *bio_ctrl,
1493 			     loff_t i_size)
1494 {
1495 	struct btrfs_fs_info *fs_info = inode->root->fs_info;
1496 	struct extent_map *em;
1497 	u64 block_start;
1498 	u64 disk_bytenr;
1499 	u64 extent_offset;
1500 	u64 em_end;
1501 	const u32 sectorsize = fs_info->sectorsize;
1502 
1503 	ASSERT(IS_ALIGNED(filepos, sectorsize));
1504 
1505 	/* @filepos >= i_size case should be handled by the caller. */
1506 	ASSERT(filepos < i_size);
1507 
1508 	em = btrfs_get_extent(inode, NULL, filepos, sectorsize);
1509 	if (IS_ERR(em))
1510 		return PTR_ERR(em);
1511 
1512 	extent_offset = filepos - em->start;
1513 	em_end = btrfs_extent_map_end(em);
1514 	ASSERT(filepos <= em_end);
1515 	ASSERT(IS_ALIGNED(em->start, sectorsize));
1516 	ASSERT(IS_ALIGNED(em->len, sectorsize));
1517 
1518 	block_start = btrfs_extent_map_block_start(em);
1519 	disk_bytenr = btrfs_extent_map_block_start(em) + extent_offset;
1520 
1521 	ASSERT(!btrfs_extent_map_is_compressed(em));
1522 	ASSERT(block_start != EXTENT_MAP_HOLE);
1523 	ASSERT(block_start != EXTENT_MAP_INLINE);
1524 
1525 	btrfs_free_extent_map(em);
1526 	em = NULL;
1527 
1528 	/*
1529 	 * Although the PageDirty bit is cleared before entering this
1530 	 * function, subpage dirty bit is not cleared.
1531 	 * So clear subpage dirty bit here so next time we won't submit
1532 	 * a folio for a range already written to disk.
1533 	 */
1534 	btrfs_folio_clear_dirty(fs_info, folio, filepos, sectorsize);
1535 	btrfs_folio_set_writeback(fs_info, folio, filepos, sectorsize);
1536 	/*
1537 	 * Above call should set the whole folio with writeback flag, even
1538 	 * just for a single subpage sector.
1539 	 * As long as the folio is properly locked and the range is correct,
1540 	 * we should always get the folio with writeback flag.
1541 	 */
1542 	ASSERT(folio_test_writeback(folio));
1543 
1544 	submit_extent_folio(bio_ctrl, disk_bytenr, folio,
1545 			    sectorsize, filepos - folio_pos(folio));
1546 	return 0;
1547 }
1548 
1549 /*
1550  * Helper for extent_writepage().  This calls the writepage start hooks,
1551  * and does the loop to map the page into extents and bios.
1552  *
1553  * We return 1 if the IO is started and the page is unlocked,
1554  * 0 if all went well (page still locked)
1555  * < 0 if there were errors (page still locked)
1556  */
extent_writepage_io(struct btrfs_inode * inode,struct folio * folio,u64 start,u32 len,struct btrfs_bio_ctrl * bio_ctrl,loff_t i_size)1557 static noinline_for_stack int extent_writepage_io(struct btrfs_inode *inode,
1558 						  struct folio *folio,
1559 						  u64 start, u32 len,
1560 						  struct btrfs_bio_ctrl *bio_ctrl,
1561 						  loff_t i_size)
1562 {
1563 	struct btrfs_fs_info *fs_info = inode->root->fs_info;
1564 	unsigned long range_bitmap = 0;
1565 	bool submitted_io = false;
1566 	bool error = false;
1567 	const u64 folio_start = folio_pos(folio);
1568 	const unsigned int blocks_per_folio = btrfs_blocks_per_folio(fs_info, folio);
1569 	u64 cur;
1570 	int bit;
1571 	int ret = 0;
1572 
1573 	ASSERT(start >= folio_start &&
1574 	       start + len <= folio_start + folio_size(folio));
1575 
1576 	ret = btrfs_writepage_cow_fixup(folio);
1577 	if (ret == -EAGAIN) {
1578 		/* Fixup worker will requeue */
1579 		folio_redirty_for_writepage(bio_ctrl->wbc, folio);
1580 		folio_unlock(folio);
1581 		return 1;
1582 	}
1583 	if (ret < 0)
1584 		return ret;
1585 
1586 	for (cur = start; cur < start + len; cur += fs_info->sectorsize)
1587 		set_bit((cur - folio_start) >> fs_info->sectorsize_bits, &range_bitmap);
1588 	bitmap_and(&bio_ctrl->submit_bitmap, &bio_ctrl->submit_bitmap, &range_bitmap,
1589 		   blocks_per_folio);
1590 
1591 	bio_ctrl->end_io_func = end_bbio_data_write;
1592 
1593 	for_each_set_bit(bit, &bio_ctrl->submit_bitmap, blocks_per_folio) {
1594 		cur = folio_pos(folio) + (bit << fs_info->sectorsize_bits);
1595 
1596 		if (cur >= i_size) {
1597 			btrfs_mark_ordered_io_finished(inode, folio, cur,
1598 						       start + len - cur, true);
1599 			/*
1600 			 * This range is beyond i_size, thus we don't need to
1601 			 * bother writing back.
1602 			 * But we still need to clear the dirty subpage bit, or
1603 			 * the next time the folio gets dirtied, we will try to
1604 			 * writeback the sectors with subpage dirty bits,
1605 			 * causing writeback without ordered extent.
1606 			 */
1607 			btrfs_folio_clear_dirty(fs_info, folio, cur,
1608 						start + len - cur);
1609 			break;
1610 		}
1611 		ret = submit_one_sector(inode, folio, cur, bio_ctrl, i_size);
1612 		if (unlikely(ret < 0)) {
1613 			/*
1614 			 * bio_ctrl may contain a bio crossing several folios.
1615 			 * Submit it immediately so that the bio has a chance
1616 			 * to finish normally, other than marked as error.
1617 			 */
1618 			submit_one_bio(bio_ctrl);
1619 			/*
1620 			 * Failed to grab the extent map which should be very rare.
1621 			 * Since there is no bio submitted to finish the ordered
1622 			 * extent, we have to manually finish this sector.
1623 			 */
1624 			btrfs_mark_ordered_io_finished(inode, folio, cur,
1625 						       fs_info->sectorsize, false);
1626 			error = true;
1627 			continue;
1628 		}
1629 		submitted_io = true;
1630 	}
1631 
1632 	/*
1633 	 * If we didn't submitted any sector (>= i_size), folio dirty get
1634 	 * cleared but PAGECACHE_TAG_DIRTY is not cleared (only cleared
1635 	 * by folio_start_writeback() if the folio is not dirty).
1636 	 *
1637 	 * Here we set writeback and clear for the range. If the full folio
1638 	 * is no longer dirty then we clear the PAGECACHE_TAG_DIRTY tag.
1639 	 *
1640 	 * If we hit any error, the corresponding sector will still be dirty
1641 	 * thus no need to clear PAGECACHE_TAG_DIRTY.
1642 	 */
1643 	if (!submitted_io && !error) {
1644 		btrfs_folio_set_writeback(fs_info, folio, start, len);
1645 		btrfs_folio_clear_writeback(fs_info, folio, start, len);
1646 	}
1647 	return ret;
1648 }
1649 
1650 /*
1651  * the writepage semantics are similar to regular writepage.  extent
1652  * records are inserted to lock ranges in the tree, and as dirty areas
1653  * are found, they are marked writeback.  Then the lock bits are removed
1654  * and the end_io handler clears the writeback ranges
1655  *
1656  * Return 0 if everything goes well.
1657  * Return <0 for error.
1658  */
extent_writepage(struct folio * folio,struct btrfs_bio_ctrl * bio_ctrl)1659 static int extent_writepage(struct folio *folio, struct btrfs_bio_ctrl *bio_ctrl)
1660 {
1661 	struct btrfs_inode *inode = BTRFS_I(folio->mapping->host);
1662 	struct btrfs_fs_info *fs_info = inode->root->fs_info;
1663 	int ret;
1664 	size_t pg_offset;
1665 	loff_t i_size = i_size_read(&inode->vfs_inode);
1666 	unsigned long end_index = i_size >> PAGE_SHIFT;
1667 	const unsigned int blocks_per_folio = btrfs_blocks_per_folio(fs_info, folio);
1668 
1669 	trace_extent_writepage(folio, &inode->vfs_inode, bio_ctrl->wbc);
1670 
1671 	WARN_ON(!folio_test_locked(folio));
1672 
1673 	pg_offset = offset_in_folio(folio, i_size);
1674 	if (folio->index > end_index ||
1675 	   (folio->index == end_index && !pg_offset)) {
1676 		folio_invalidate(folio, 0, folio_size(folio));
1677 		folio_unlock(folio);
1678 		return 0;
1679 	}
1680 
1681 	if (folio_contains(folio, end_index))
1682 		folio_zero_range(folio, pg_offset, folio_size(folio) - pg_offset);
1683 
1684 	/*
1685 	 * Default to unlock the whole folio.
1686 	 * The proper bitmap can only be initialized until writepage_delalloc().
1687 	 */
1688 	bio_ctrl->submit_bitmap = (unsigned long)-1;
1689 
1690 	/*
1691 	 * If the page is dirty but without private set, it's marked dirty
1692 	 * without informing the fs.
1693 	 * Nowadays that is a bug, since the introduction of
1694 	 * pin_user_pages*().
1695 	 *
1696 	 * So here we check if the page has private set to rule out such
1697 	 * case.
1698 	 * But we also have a long history of relying on the COW fixup,
1699 	 * so here we only enable this check for experimental builds until
1700 	 * we're sure it's safe.
1701 	 */
1702 	if (IS_ENABLED(CONFIG_BTRFS_EXPERIMENTAL) &&
1703 	    unlikely(!folio_test_private(folio))) {
1704 		WARN_ON(IS_ENABLED(CONFIG_BTRFS_DEBUG));
1705 		btrfs_err_rl(fs_info,
1706 	"root %lld ino %llu folio %llu is marked dirty without notifying the fs",
1707 			     inode->root->root_key.objectid,
1708 			     btrfs_ino(inode), folio_pos(folio));
1709 		ret = -EUCLEAN;
1710 		goto done;
1711 	}
1712 
1713 	ret = set_folio_extent_mapped(folio);
1714 	if (ret < 0)
1715 		goto done;
1716 
1717 	ret = writepage_delalloc(inode, folio, bio_ctrl);
1718 	if (ret == 1)
1719 		return 0;
1720 	if (ret)
1721 		goto done;
1722 
1723 	ret = extent_writepage_io(inode, folio, folio_pos(folio),
1724 				  folio_size(folio), bio_ctrl, i_size);
1725 	if (ret == 1)
1726 		return 0;
1727 	if (ret < 0)
1728 		btrfs_err_rl(fs_info,
1729 "failed to submit blocks, root=%lld inode=%llu folio=%llu submit_bitmap=%*pbl: %d",
1730 			     btrfs_root_id(inode->root), btrfs_ino(inode),
1731 			     folio_pos(folio), blocks_per_folio,
1732 			     &bio_ctrl->submit_bitmap, ret);
1733 
1734 	bio_ctrl->wbc->nr_to_write--;
1735 
1736 done:
1737 	if (ret < 0)
1738 		mapping_set_error(folio->mapping, ret);
1739 	/*
1740 	 * Only unlock ranges that are submitted. As there can be some async
1741 	 * submitted ranges inside the folio.
1742 	 */
1743 	btrfs_folio_end_lock_bitmap(fs_info, folio, bio_ctrl->submit_bitmap);
1744 	ASSERT(ret <= 0);
1745 	return ret;
1746 }
1747 
1748 /*
1749  * Lock extent buffer status and pages for writeback.
1750  *
1751  * Return %false if the extent buffer doesn't need to be submitted (e.g. the
1752  * extent buffer is not dirty)
1753  * Return %true is the extent buffer is submitted to bio.
1754  */
lock_extent_buffer_for_io(struct extent_buffer * eb,struct writeback_control * wbc)1755 static noinline_for_stack bool lock_extent_buffer_for_io(struct extent_buffer *eb,
1756 			  struct writeback_control *wbc)
1757 {
1758 	struct btrfs_fs_info *fs_info = eb->fs_info;
1759 	bool ret = false;
1760 
1761 	btrfs_tree_lock(eb);
1762 	while (test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags)) {
1763 		btrfs_tree_unlock(eb);
1764 		if (wbc->sync_mode != WB_SYNC_ALL)
1765 			return false;
1766 		wait_on_extent_buffer_writeback(eb);
1767 		btrfs_tree_lock(eb);
1768 	}
1769 
1770 	/*
1771 	 * We need to do this to prevent races in people who check if the eb is
1772 	 * under IO since we can end up having no IO bits set for a short period
1773 	 * of time.
1774 	 */
1775 	spin_lock(&eb->refs_lock);
1776 	if (test_and_clear_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)) {
1777 		XA_STATE(xas, &fs_info->buffer_tree, eb->start >> fs_info->sectorsize_bits);
1778 		unsigned long flags;
1779 
1780 		set_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags);
1781 		spin_unlock(&eb->refs_lock);
1782 
1783 		xas_lock_irqsave(&xas, flags);
1784 		xas_load(&xas);
1785 		xas_set_mark(&xas, PAGECACHE_TAG_WRITEBACK);
1786 		xas_clear_mark(&xas, PAGECACHE_TAG_DIRTY);
1787 		xas_unlock_irqrestore(&xas, flags);
1788 
1789 		btrfs_set_header_flag(eb, BTRFS_HEADER_FLAG_WRITTEN);
1790 		percpu_counter_add_batch(&fs_info->dirty_metadata_bytes,
1791 					 -eb->len,
1792 					 fs_info->dirty_metadata_batch);
1793 		ret = true;
1794 	} else {
1795 		spin_unlock(&eb->refs_lock);
1796 	}
1797 	btrfs_tree_unlock(eb);
1798 	return ret;
1799 }
1800 
set_btree_ioerr(struct extent_buffer * eb)1801 static void set_btree_ioerr(struct extent_buffer *eb)
1802 {
1803 	struct btrfs_fs_info *fs_info = eb->fs_info;
1804 
1805 	set_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags);
1806 
1807 	/*
1808 	 * A read may stumble upon this buffer later, make sure that it gets an
1809 	 * error and knows there was an error.
1810 	 */
1811 	clear_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
1812 
1813 	/*
1814 	 * We need to set the mapping with the io error as well because a write
1815 	 * error will flip the file system readonly, and then syncfs() will
1816 	 * return a 0 because we are readonly if we don't modify the err seq for
1817 	 * the superblock.
1818 	 */
1819 	mapping_set_error(eb->fs_info->btree_inode->i_mapping, -EIO);
1820 
1821 	/*
1822 	 * If writeback for a btree extent that doesn't belong to a log tree
1823 	 * failed, increment the counter transaction->eb_write_errors.
1824 	 * We do this because while the transaction is running and before it's
1825 	 * committing (when we call filemap_fdata[write|wait]_range against
1826 	 * the btree inode), we might have
1827 	 * btree_inode->i_mapping->a_ops->writepages() called by the VM - if it
1828 	 * returns an error or an error happens during writeback, when we're
1829 	 * committing the transaction we wouldn't know about it, since the pages
1830 	 * can be no longer dirty nor marked anymore for writeback (if a
1831 	 * subsequent modification to the extent buffer didn't happen before the
1832 	 * transaction commit), which makes filemap_fdata[write|wait]_range not
1833 	 * able to find the pages which contain errors at transaction
1834 	 * commit time. So if this happens we must abort the transaction,
1835 	 * otherwise we commit a super block with btree roots that point to
1836 	 * btree nodes/leafs whose content on disk is invalid - either garbage
1837 	 * or the content of some node/leaf from a past generation that got
1838 	 * cowed or deleted and is no longer valid.
1839 	 *
1840 	 * Note: setting AS_EIO/AS_ENOSPC in the btree inode's i_mapping would
1841 	 * not be enough - we need to distinguish between log tree extents vs
1842 	 * non-log tree extents, and the next filemap_fdatawait_range() call
1843 	 * will catch and clear such errors in the mapping - and that call might
1844 	 * be from a log sync and not from a transaction commit. Also, checking
1845 	 * for the eb flag EXTENT_BUFFER_WRITE_ERR at transaction commit time is
1846 	 * not done and would not be reliable - the eb might have been released
1847 	 * from memory and reading it back again means that flag would not be
1848 	 * set (since it's a runtime flag, not persisted on disk).
1849 	 *
1850 	 * Using the flags below in the btree inode also makes us achieve the
1851 	 * goal of AS_EIO/AS_ENOSPC when writepages() returns success, started
1852 	 * writeback for all dirty pages and before filemap_fdatawait_range()
1853 	 * is called, the writeback for all dirty pages had already finished
1854 	 * with errors - because we were not using AS_EIO/AS_ENOSPC,
1855 	 * filemap_fdatawait_range() would return success, as it could not know
1856 	 * that writeback errors happened (the pages were no longer tagged for
1857 	 * writeback).
1858 	 */
1859 	switch (eb->log_index) {
1860 	case -1:
1861 		set_bit(BTRFS_FS_BTREE_ERR, &fs_info->flags);
1862 		break;
1863 	case 0:
1864 		set_bit(BTRFS_FS_LOG1_ERR, &fs_info->flags);
1865 		break;
1866 	case 1:
1867 		set_bit(BTRFS_FS_LOG2_ERR, &fs_info->flags);
1868 		break;
1869 	default:
1870 		BUG(); /* unexpected, logic error */
1871 	}
1872 }
1873 
buffer_tree_set_mark(const struct extent_buffer * eb,xa_mark_t mark)1874 static void buffer_tree_set_mark(const struct extent_buffer *eb, xa_mark_t mark)
1875 {
1876 	struct btrfs_fs_info *fs_info = eb->fs_info;
1877 	XA_STATE(xas, &fs_info->buffer_tree, eb->start >> fs_info->sectorsize_bits);
1878 	unsigned long flags;
1879 
1880 	xas_lock_irqsave(&xas, flags);
1881 	xas_load(&xas);
1882 	xas_set_mark(&xas, mark);
1883 	xas_unlock_irqrestore(&xas, flags);
1884 }
1885 
buffer_tree_clear_mark(const struct extent_buffer * eb,xa_mark_t mark)1886 static void buffer_tree_clear_mark(const struct extent_buffer *eb, xa_mark_t mark)
1887 {
1888 	struct btrfs_fs_info *fs_info = eb->fs_info;
1889 	XA_STATE(xas, &fs_info->buffer_tree, eb->start >> fs_info->sectorsize_bits);
1890 	unsigned long flags;
1891 
1892 	xas_lock_irqsave(&xas, flags);
1893 	xas_load(&xas);
1894 	xas_clear_mark(&xas, mark);
1895 	xas_unlock_irqrestore(&xas, flags);
1896 }
1897 
buffer_tree_tag_for_writeback(struct btrfs_fs_info * fs_info,unsigned long start,unsigned long end)1898 static void buffer_tree_tag_for_writeback(struct btrfs_fs_info *fs_info,
1899 					  unsigned long start, unsigned long end)
1900 {
1901 	XA_STATE(xas, &fs_info->buffer_tree, start);
1902 	unsigned int tagged = 0;
1903 	void *eb;
1904 
1905 	xas_lock_irq(&xas);
1906 	xas_for_each_marked(&xas, eb, end, PAGECACHE_TAG_DIRTY) {
1907 		xas_set_mark(&xas, PAGECACHE_TAG_TOWRITE);
1908 		if (++tagged % XA_CHECK_SCHED)
1909 			continue;
1910 		xas_pause(&xas);
1911 		xas_unlock_irq(&xas);
1912 		cond_resched();
1913 		xas_lock_irq(&xas);
1914 	}
1915 	xas_unlock_irq(&xas);
1916 }
1917 
1918 struct eb_batch {
1919 	unsigned int nr;
1920 	unsigned int cur;
1921 	struct extent_buffer *ebs[PAGEVEC_SIZE];
1922 };
1923 
eb_batch_add(struct eb_batch * batch,struct extent_buffer * eb)1924 static inline bool eb_batch_add(struct eb_batch *batch, struct extent_buffer *eb)
1925 {
1926 	batch->ebs[batch->nr++] = eb;
1927 	return (batch->nr < PAGEVEC_SIZE);
1928 }
1929 
eb_batch_init(struct eb_batch * batch)1930 static inline void eb_batch_init(struct eb_batch *batch)
1931 {
1932 	batch->nr = 0;
1933 	batch->cur = 0;
1934 }
1935 
eb_batch_next(struct eb_batch * batch)1936 static inline struct extent_buffer *eb_batch_next(struct eb_batch *batch)
1937 {
1938 	if (batch->cur >= batch->nr)
1939 		return NULL;
1940 	return batch->ebs[batch->cur++];
1941 }
1942 
eb_batch_release(struct eb_batch * batch)1943 static inline void eb_batch_release(struct eb_batch *batch)
1944 {
1945 	for (unsigned int i = 0; i < batch->nr; i++)
1946 		free_extent_buffer(batch->ebs[i]);
1947 	eb_batch_init(batch);
1948 }
1949 
find_get_eb(struct xa_state * xas,unsigned long max,xa_mark_t mark)1950 static inline struct extent_buffer *find_get_eb(struct xa_state *xas, unsigned long max,
1951 						xa_mark_t mark)
1952 {
1953 	struct extent_buffer *eb;
1954 
1955 retry:
1956 	eb = xas_find_marked(xas, max, mark);
1957 
1958 	if (xas_retry(xas, eb))
1959 		goto retry;
1960 
1961 	if (!eb)
1962 		return NULL;
1963 
1964 	if (!atomic_inc_not_zero(&eb->refs)) {
1965 		xas_reset(xas);
1966 		goto retry;
1967 	}
1968 
1969 	if (unlikely(eb != xas_reload(xas))) {
1970 		free_extent_buffer(eb);
1971 		xas_reset(xas);
1972 		goto retry;
1973 	}
1974 
1975 	return eb;
1976 }
1977 
buffer_tree_get_ebs_tag(struct btrfs_fs_info * fs_info,unsigned long * start,unsigned long end,xa_mark_t tag,struct eb_batch * batch)1978 static unsigned int buffer_tree_get_ebs_tag(struct btrfs_fs_info *fs_info,
1979 					    unsigned long *start,
1980 					    unsigned long end, xa_mark_t tag,
1981 					    struct eb_batch *batch)
1982 {
1983 	XA_STATE(xas, &fs_info->buffer_tree, *start);
1984 	struct extent_buffer *eb;
1985 
1986 	rcu_read_lock();
1987 	while ((eb = find_get_eb(&xas, end, tag)) != NULL) {
1988 		if (!eb_batch_add(batch, eb)) {
1989 			*start = ((eb->start + eb->len) >> fs_info->sectorsize_bits);
1990 			goto out;
1991 		}
1992 	}
1993 	if (end == ULONG_MAX)
1994 		*start = ULONG_MAX;
1995 	else
1996 		*start = end + 1;
1997 out:
1998 	rcu_read_unlock();
1999 
2000 	return batch->nr;
2001 }
2002 
2003 /*
2004  * The endio specific version which won't touch any unsafe spinlock in endio
2005  * context.
2006  */
find_extent_buffer_nolock(struct btrfs_fs_info * fs_info,u64 start)2007 static struct extent_buffer *find_extent_buffer_nolock(
2008 		struct btrfs_fs_info *fs_info, u64 start)
2009 {
2010 	struct extent_buffer *eb;
2011 	unsigned long index = (start >> fs_info->sectorsize_bits);
2012 
2013 	rcu_read_lock();
2014 	eb = xa_load(&fs_info->buffer_tree, index);
2015 	if (eb && !atomic_inc_not_zero(&eb->refs))
2016 		eb = NULL;
2017 	rcu_read_unlock();
2018 	return eb;
2019 }
2020 
end_bbio_meta_write(struct btrfs_bio * bbio)2021 static void end_bbio_meta_write(struct btrfs_bio *bbio)
2022 {
2023 	struct extent_buffer *eb = bbio->private;
2024 	struct folio_iter fi;
2025 
2026 	if (bbio->bio.bi_status != BLK_STS_OK)
2027 		set_btree_ioerr(eb);
2028 
2029 	bio_for_each_folio_all(fi, &bbio->bio) {
2030 		btrfs_meta_folio_clear_writeback(fi.folio, eb);
2031 	}
2032 
2033 	buffer_tree_clear_mark(eb, PAGECACHE_TAG_WRITEBACK);
2034 	clear_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags);
2035 	smp_mb__after_atomic();
2036 	wake_up_bit(&eb->bflags, EXTENT_BUFFER_WRITEBACK);
2037 
2038 	bio_put(&bbio->bio);
2039 }
2040 
prepare_eb_write(struct extent_buffer * eb)2041 static void prepare_eb_write(struct extent_buffer *eb)
2042 {
2043 	u32 nritems;
2044 	unsigned long start;
2045 	unsigned long end;
2046 
2047 	clear_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags);
2048 
2049 	/* Set btree blocks beyond nritems with 0 to avoid stale content */
2050 	nritems = btrfs_header_nritems(eb);
2051 	if (btrfs_header_level(eb) > 0) {
2052 		end = btrfs_node_key_ptr_offset(eb, nritems);
2053 		memzero_extent_buffer(eb, end, eb->len - end);
2054 	} else {
2055 		/*
2056 		 * Leaf:
2057 		 * header 0 1 2 .. N ... data_N .. data_2 data_1 data_0
2058 		 */
2059 		start = btrfs_item_nr_offset(eb, nritems);
2060 		end = btrfs_item_nr_offset(eb, 0);
2061 		if (nritems == 0)
2062 			end += BTRFS_LEAF_DATA_SIZE(eb->fs_info);
2063 		else
2064 			end += btrfs_item_offset(eb, nritems - 1);
2065 		memzero_extent_buffer(eb, start, end - start);
2066 	}
2067 }
2068 
write_one_eb(struct extent_buffer * eb,struct writeback_control * wbc)2069 static noinline_for_stack void write_one_eb(struct extent_buffer *eb,
2070 					    struct writeback_control *wbc)
2071 {
2072 	struct btrfs_fs_info *fs_info = eb->fs_info;
2073 	struct btrfs_bio *bbio;
2074 
2075 	prepare_eb_write(eb);
2076 
2077 	bbio = btrfs_bio_alloc(INLINE_EXTENT_BUFFER_PAGES,
2078 			       REQ_OP_WRITE | REQ_META | wbc_to_write_flags(wbc),
2079 			       eb->fs_info, end_bbio_meta_write, eb);
2080 	bbio->bio.bi_iter.bi_sector = eb->start >> SECTOR_SHIFT;
2081 	bio_set_dev(&bbio->bio, fs_info->fs_devices->latest_dev->bdev);
2082 	wbc_init_bio(wbc, &bbio->bio);
2083 	bbio->inode = BTRFS_I(eb->fs_info->btree_inode);
2084 	bbio->file_offset = eb->start;
2085 	for (int i = 0; i < num_extent_folios(eb); i++) {
2086 		struct folio *folio = eb->folios[i];
2087 		u64 range_start = max_t(u64, eb->start, folio_pos(folio));
2088 		u32 range_len = min_t(u64, folio_pos(folio) + folio_size(folio),
2089 				      eb->start + eb->len) - range_start;
2090 
2091 		folio_lock(folio);
2092 		btrfs_meta_folio_clear_dirty(folio, eb);
2093 		btrfs_meta_folio_set_writeback(folio, eb);
2094 		if (!folio_test_dirty(folio))
2095 			wbc->nr_to_write -= folio_nr_pages(folio);
2096 		bio_add_folio_nofail(&bbio->bio, folio, range_len,
2097 				     offset_in_folio(folio, range_start));
2098 		wbc_account_cgroup_owner(wbc, folio, range_len);
2099 		folio_unlock(folio);
2100 	}
2101 	btrfs_submit_bbio(bbio, 0);
2102 }
2103 
2104 /*
2105  * Wait for all eb writeback in the given range to finish.
2106  *
2107  * @fs_info:	The fs_info for this file system.
2108  * @start:	The offset of the range to start waiting on writeback.
2109  * @end:	The end of the range, inclusive. This is meant to be used in
2110  *		conjuction with wait_marked_extents, so this will usually be
2111  *		the_next_eb->start - 1.
2112  */
btrfs_btree_wait_writeback_range(struct btrfs_fs_info * fs_info,u64 start,u64 end)2113 void btrfs_btree_wait_writeback_range(struct btrfs_fs_info *fs_info, u64 start,
2114 				      u64 end)
2115 {
2116 	struct eb_batch batch;
2117 	unsigned long start_index = (start >> fs_info->sectorsize_bits);
2118 	unsigned long end_index = (end >> fs_info->sectorsize_bits);
2119 
2120 	eb_batch_init(&batch);
2121 	while (start_index <= end_index) {
2122 		struct extent_buffer *eb;
2123 		unsigned int nr_ebs;
2124 
2125 		nr_ebs = buffer_tree_get_ebs_tag(fs_info, &start_index, end_index,
2126 						 PAGECACHE_TAG_WRITEBACK, &batch);
2127 		if (!nr_ebs)
2128 			break;
2129 
2130 		while ((eb = eb_batch_next(&batch)) != NULL)
2131 			wait_on_extent_buffer_writeback(eb);
2132 		eb_batch_release(&batch);
2133 		cond_resched();
2134 	}
2135 }
2136 
btree_write_cache_pages(struct address_space * mapping,struct writeback_control * wbc)2137 int btree_write_cache_pages(struct address_space *mapping,
2138 				   struct writeback_control *wbc)
2139 {
2140 	struct btrfs_eb_write_context ctx = { .wbc = wbc };
2141 	struct btrfs_fs_info *fs_info = inode_to_fs_info(mapping->host);
2142 	int ret = 0;
2143 	int done = 0;
2144 	int nr_to_write_done = 0;
2145 	struct eb_batch batch;
2146 	unsigned int nr_ebs;
2147 	unsigned long index;
2148 	unsigned long end;
2149 	int scanned = 0;
2150 	xa_mark_t tag;
2151 
2152 	eb_batch_init(&batch);
2153 	if (wbc->range_cyclic) {
2154 		index = ((mapping->writeback_index << PAGE_SHIFT) >> fs_info->sectorsize_bits);
2155 		end = -1;
2156 
2157 		/*
2158 		 * Start from the beginning does not need to cycle over the
2159 		 * range, mark it as scanned.
2160 		 */
2161 		scanned = (index == 0);
2162 	} else {
2163 		index = (wbc->range_start >> fs_info->sectorsize_bits);
2164 		end = (wbc->range_end >> fs_info->sectorsize_bits);
2165 
2166 		scanned = 1;
2167 	}
2168 	if (wbc->sync_mode == WB_SYNC_ALL)
2169 		tag = PAGECACHE_TAG_TOWRITE;
2170 	else
2171 		tag = PAGECACHE_TAG_DIRTY;
2172 	btrfs_zoned_meta_io_lock(fs_info);
2173 retry:
2174 	if (wbc->sync_mode == WB_SYNC_ALL)
2175 		buffer_tree_tag_for_writeback(fs_info, index, end);
2176 	while (!done && !nr_to_write_done && (index <= end) &&
2177 	       (nr_ebs = buffer_tree_get_ebs_tag(fs_info, &index, end, tag, &batch))) {
2178 		struct extent_buffer *eb;
2179 
2180 		while ((eb = eb_batch_next(&batch)) != NULL) {
2181 			ctx.eb = eb;
2182 
2183 			ret = btrfs_check_meta_write_pointer(eb->fs_info, &ctx);
2184 			if (ret) {
2185 				if (ret == -EBUSY)
2186 					ret = 0;
2187 
2188 				if (ret) {
2189 					done = 1;
2190 					break;
2191 				}
2192 				continue;
2193 			}
2194 
2195 			if (!lock_extent_buffer_for_io(eb, wbc))
2196 				continue;
2197 
2198 			/* Implies write in zoned mode. */
2199 			if (ctx.zoned_bg) {
2200 				/* Mark the last eb in the block group. */
2201 				btrfs_schedule_zone_finish_bg(ctx.zoned_bg, eb);
2202 				ctx.zoned_bg->meta_write_pointer += eb->len;
2203 			}
2204 			write_one_eb(eb, wbc);
2205 		}
2206 		nr_to_write_done = (wbc->nr_to_write <= 0);
2207 		eb_batch_release(&batch);
2208 		cond_resched();
2209 	}
2210 	if (!scanned && !done) {
2211 		/*
2212 		 * We hit the last page and there is more work to be done: wrap
2213 		 * back to the start of the file
2214 		 */
2215 		scanned = 1;
2216 		index = 0;
2217 		goto retry;
2218 	}
2219 	/*
2220 	 * If something went wrong, don't allow any metadata write bio to be
2221 	 * submitted.
2222 	 *
2223 	 * This would prevent use-after-free if we had dirty pages not
2224 	 * cleaned up, which can still happen by fuzzed images.
2225 	 *
2226 	 * - Bad extent tree
2227 	 *   Allowing existing tree block to be allocated for other trees.
2228 	 *
2229 	 * - Log tree operations
2230 	 *   Exiting tree blocks get allocated to log tree, bumps its
2231 	 *   generation, then get cleaned in tree re-balance.
2232 	 *   Such tree block will not be written back, since it's clean,
2233 	 *   thus no WRITTEN flag set.
2234 	 *   And after log writes back, this tree block is not traced by
2235 	 *   any dirty extent_io_tree.
2236 	 *
2237 	 * - Offending tree block gets re-dirtied from its original owner
2238 	 *   Since it has bumped generation, no WRITTEN flag, it can be
2239 	 *   reused without COWing. This tree block will not be traced
2240 	 *   by btrfs_transaction::dirty_pages.
2241 	 *
2242 	 *   Now such dirty tree block will not be cleaned by any dirty
2243 	 *   extent io tree. Thus we don't want to submit such wild eb
2244 	 *   if the fs already has error.
2245 	 *
2246 	 * We can get ret > 0 from submit_extent_folio() indicating how many ebs
2247 	 * were submitted. Reset it to 0 to avoid false alerts for the caller.
2248 	 */
2249 	if (ret > 0)
2250 		ret = 0;
2251 	if (!ret && BTRFS_FS_ERROR(fs_info))
2252 		ret = -EROFS;
2253 
2254 	if (ctx.zoned_bg)
2255 		btrfs_put_block_group(ctx.zoned_bg);
2256 	btrfs_zoned_meta_io_unlock(fs_info);
2257 	return ret;
2258 }
2259 
2260 /*
2261  * Walk the list of dirty pages of the given address space and write all of them.
2262  *
2263  * @mapping:   address space structure to write
2264  * @wbc:       subtract the number of written pages from *@wbc->nr_to_write
2265  * @bio_ctrl:  holds context for the write, namely the bio
2266  *
2267  * If a page is already under I/O, write_cache_pages() skips it, even
2268  * if it's dirty.  This is desirable behaviour for memory-cleaning writeback,
2269  * but it is INCORRECT for data-integrity system calls such as fsync().  fsync()
2270  * and msync() need to guarantee that all the data which was dirty at the time
2271  * the call was made get new I/O started against them.  If wbc->sync_mode is
2272  * WB_SYNC_ALL then we were called for data integrity and we must wait for
2273  * existing IO to complete.
2274  */
extent_write_cache_pages(struct address_space * mapping,struct btrfs_bio_ctrl * bio_ctrl)2275 static int extent_write_cache_pages(struct address_space *mapping,
2276 			     struct btrfs_bio_ctrl *bio_ctrl)
2277 {
2278 	struct writeback_control *wbc = bio_ctrl->wbc;
2279 	struct inode *inode = mapping->host;
2280 	int ret = 0;
2281 	int done = 0;
2282 	int nr_to_write_done = 0;
2283 	struct folio_batch fbatch;
2284 	unsigned int nr_folios;
2285 	pgoff_t index;
2286 	pgoff_t end;		/* Inclusive */
2287 	pgoff_t done_index;
2288 	int range_whole = 0;
2289 	int scanned = 0;
2290 	xa_mark_t tag;
2291 
2292 	/*
2293 	 * We have to hold onto the inode so that ordered extents can do their
2294 	 * work when the IO finishes.  The alternative to this is failing to add
2295 	 * an ordered extent if the igrab() fails there and that is a huge pain
2296 	 * to deal with, so instead just hold onto the inode throughout the
2297 	 * writepages operation.  If it fails here we are freeing up the inode
2298 	 * anyway and we'd rather not waste our time writing out stuff that is
2299 	 * going to be truncated anyway.
2300 	 */
2301 	if (!igrab(inode))
2302 		return 0;
2303 
2304 	folio_batch_init(&fbatch);
2305 	if (wbc->range_cyclic) {
2306 		index = mapping->writeback_index; /* Start from prev offset */
2307 		end = -1;
2308 		/*
2309 		 * Start from the beginning does not need to cycle over the
2310 		 * range, mark it as scanned.
2311 		 */
2312 		scanned = (index == 0);
2313 	} else {
2314 		index = wbc->range_start >> PAGE_SHIFT;
2315 		end = wbc->range_end >> PAGE_SHIFT;
2316 		if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
2317 			range_whole = 1;
2318 		scanned = 1;
2319 	}
2320 
2321 	/*
2322 	 * We do the tagged writepage as long as the snapshot flush bit is set
2323 	 * and we are the first one who do the filemap_flush() on this inode.
2324 	 *
2325 	 * The nr_to_write == LONG_MAX is needed to make sure other flushers do
2326 	 * not race in and drop the bit.
2327 	 */
2328 	if (range_whole && wbc->nr_to_write == LONG_MAX &&
2329 	    test_and_clear_bit(BTRFS_INODE_SNAPSHOT_FLUSH,
2330 			       &BTRFS_I(inode)->runtime_flags))
2331 		wbc->tagged_writepages = 1;
2332 
2333 	if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
2334 		tag = PAGECACHE_TAG_TOWRITE;
2335 	else
2336 		tag = PAGECACHE_TAG_DIRTY;
2337 retry:
2338 	if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
2339 		tag_pages_for_writeback(mapping, index, end);
2340 	done_index = index;
2341 	while (!done && !nr_to_write_done && (index <= end) &&
2342 			(nr_folios = filemap_get_folios_tag(mapping, &index,
2343 							end, tag, &fbatch))) {
2344 		unsigned i;
2345 
2346 		for (i = 0; i < nr_folios; i++) {
2347 			struct folio *folio = fbatch.folios[i];
2348 
2349 			done_index = folio_next_index(folio);
2350 			/*
2351 			 * At this point we hold neither the i_pages lock nor
2352 			 * the folio lock: the folio may be truncated or
2353 			 * invalidated (changing folio->mapping to NULL).
2354 			 */
2355 			if (!folio_trylock(folio)) {
2356 				submit_write_bio(bio_ctrl, 0);
2357 				folio_lock(folio);
2358 			}
2359 
2360 			if (unlikely(folio->mapping != mapping)) {
2361 				folio_unlock(folio);
2362 				continue;
2363 			}
2364 
2365 			if (!folio_test_dirty(folio)) {
2366 				/* Someone wrote it for us. */
2367 				folio_unlock(folio);
2368 				continue;
2369 			}
2370 
2371 			/*
2372 			 * For subpage case, compression can lead to mixed
2373 			 * writeback and dirty flags, e.g:
2374 			 * 0     32K    64K    96K    128K
2375 			 * |     |//////||/////|   |//|
2376 			 *
2377 			 * In above case, [32K, 96K) is asynchronously submitted
2378 			 * for compression, and [124K, 128K) needs to be written back.
2379 			 *
2380 			 * If we didn't wait wrtiteback for page 64K, [128K, 128K)
2381 			 * won't be submitted as the page still has writeback flag
2382 			 * and will be skipped in the next check.
2383 			 *
2384 			 * This mixed writeback and dirty case is only possible for
2385 			 * subpage case.
2386 			 *
2387 			 * TODO: Remove this check after migrating compression to
2388 			 * regular submission.
2389 			 */
2390 			if (wbc->sync_mode != WB_SYNC_NONE ||
2391 			    btrfs_is_subpage(inode_to_fs_info(inode), folio)) {
2392 				if (folio_test_writeback(folio))
2393 					submit_write_bio(bio_ctrl, 0);
2394 				folio_wait_writeback(folio);
2395 			}
2396 
2397 			if (folio_test_writeback(folio) ||
2398 			    !folio_clear_dirty_for_io(folio)) {
2399 				folio_unlock(folio);
2400 				continue;
2401 			}
2402 
2403 			ret = extent_writepage(folio, bio_ctrl);
2404 			if (ret < 0) {
2405 				done = 1;
2406 				break;
2407 			}
2408 
2409 			/*
2410 			 * The filesystem may choose to bump up nr_to_write.
2411 			 * We have to make sure to honor the new nr_to_write
2412 			 * at any time.
2413 			 */
2414 			nr_to_write_done = (wbc->sync_mode == WB_SYNC_NONE &&
2415 					    wbc->nr_to_write <= 0);
2416 		}
2417 		folio_batch_release(&fbatch);
2418 		cond_resched();
2419 	}
2420 	if (!scanned && !done) {
2421 		/*
2422 		 * We hit the last page and there is more work to be done: wrap
2423 		 * back to the start of the file
2424 		 */
2425 		scanned = 1;
2426 		index = 0;
2427 
2428 		/*
2429 		 * If we're looping we could run into a page that is locked by a
2430 		 * writer and that writer could be waiting on writeback for a
2431 		 * page in our current bio, and thus deadlock, so flush the
2432 		 * write bio here.
2433 		 */
2434 		submit_write_bio(bio_ctrl, 0);
2435 		goto retry;
2436 	}
2437 
2438 	if (wbc->range_cyclic || (wbc->nr_to_write > 0 && range_whole))
2439 		mapping->writeback_index = done_index;
2440 
2441 	btrfs_add_delayed_iput(BTRFS_I(inode));
2442 	return ret;
2443 }
2444 
2445 /*
2446  * Submit the pages in the range to bio for call sites which delalloc range has
2447  * already been ran (aka, ordered extent inserted) and all pages are still
2448  * locked.
2449  */
extent_write_locked_range(struct inode * inode,const struct folio * locked_folio,u64 start,u64 end,struct writeback_control * wbc,bool pages_dirty)2450 void extent_write_locked_range(struct inode *inode, const struct folio *locked_folio,
2451 			       u64 start, u64 end, struct writeback_control *wbc,
2452 			       bool pages_dirty)
2453 {
2454 	bool found_error = false;
2455 	int ret = 0;
2456 	struct address_space *mapping = inode->i_mapping;
2457 	struct btrfs_fs_info *fs_info = inode_to_fs_info(inode);
2458 	const u32 sectorsize = fs_info->sectorsize;
2459 	loff_t i_size = i_size_read(inode);
2460 	u64 cur = start;
2461 	struct btrfs_bio_ctrl bio_ctrl = {
2462 		.wbc = wbc,
2463 		.opf = REQ_OP_WRITE | wbc_to_write_flags(wbc),
2464 	};
2465 
2466 	if (wbc->no_cgroup_owner)
2467 		bio_ctrl.opf |= REQ_BTRFS_CGROUP_PUNT;
2468 
2469 	ASSERT(IS_ALIGNED(start, sectorsize) && IS_ALIGNED(end + 1, sectorsize));
2470 
2471 	while (cur <= end) {
2472 		u64 cur_end;
2473 		u32 cur_len;
2474 		struct folio *folio;
2475 
2476 		folio = filemap_get_folio(mapping, cur >> PAGE_SHIFT);
2477 
2478 		/*
2479 		 * This shouldn't happen, the pages are pinned and locked, this
2480 		 * code is just in case, but shouldn't actually be run.
2481 		 */
2482 		if (IS_ERR(folio)) {
2483 			cur_end = min(round_down(cur, PAGE_SIZE) + PAGE_SIZE - 1, end);
2484 			cur_len = cur_end + 1 - cur;
2485 			btrfs_mark_ordered_io_finished(BTRFS_I(inode), NULL,
2486 						       cur, cur_len, false);
2487 			mapping_set_error(mapping, PTR_ERR(folio));
2488 			cur = cur_end;
2489 			continue;
2490 		}
2491 
2492 		cur_end = min_t(u64, folio_pos(folio) + folio_size(folio) - 1, end);
2493 		cur_len = cur_end + 1 - cur;
2494 
2495 		ASSERT(folio_test_locked(folio));
2496 		if (pages_dirty && folio != locked_folio)
2497 			ASSERT(folio_test_dirty(folio));
2498 
2499 		/*
2500 		 * Set the submission bitmap to submit all sectors.
2501 		 * extent_writepage_io() will do the truncation correctly.
2502 		 */
2503 		bio_ctrl.submit_bitmap = (unsigned long)-1;
2504 		ret = extent_writepage_io(BTRFS_I(inode), folio, cur, cur_len,
2505 					  &bio_ctrl, i_size);
2506 		if (ret == 1)
2507 			goto next_page;
2508 
2509 		if (ret)
2510 			mapping_set_error(mapping, ret);
2511 		btrfs_folio_end_lock(fs_info, folio, cur, cur_len);
2512 		if (ret < 0)
2513 			found_error = true;
2514 next_page:
2515 		folio_put(folio);
2516 		cur = cur_end + 1;
2517 	}
2518 
2519 	submit_write_bio(&bio_ctrl, found_error ? ret : 0);
2520 }
2521 
btrfs_writepages(struct address_space * mapping,struct writeback_control * wbc)2522 int btrfs_writepages(struct address_space *mapping, struct writeback_control *wbc)
2523 {
2524 	struct inode *inode = mapping->host;
2525 	int ret = 0;
2526 	struct btrfs_bio_ctrl bio_ctrl = {
2527 		.wbc = wbc,
2528 		.opf = REQ_OP_WRITE | wbc_to_write_flags(wbc),
2529 	};
2530 
2531 	/*
2532 	 * Allow only a single thread to do the reloc work in zoned mode to
2533 	 * protect the write pointer updates.
2534 	 */
2535 	btrfs_zoned_data_reloc_lock(BTRFS_I(inode));
2536 	ret = extent_write_cache_pages(mapping, &bio_ctrl);
2537 	submit_write_bio(&bio_ctrl, ret);
2538 	btrfs_zoned_data_reloc_unlock(BTRFS_I(inode));
2539 	return ret;
2540 }
2541 
btrfs_readahead(struct readahead_control * rac)2542 void btrfs_readahead(struct readahead_control *rac)
2543 {
2544 	struct btrfs_bio_ctrl bio_ctrl = { .opf = REQ_OP_READ | REQ_RAHEAD };
2545 	struct folio *folio;
2546 	struct btrfs_inode *inode = BTRFS_I(rac->mapping->host);
2547 	const u64 start = readahead_pos(rac);
2548 	const u64 end = start + readahead_length(rac) - 1;
2549 	struct extent_state *cached_state = NULL;
2550 	struct extent_map *em_cached = NULL;
2551 	u64 prev_em_start = (u64)-1;
2552 
2553 	lock_extents_for_read(inode, start, end, &cached_state);
2554 
2555 	while ((folio = readahead_folio(rac)) != NULL)
2556 		btrfs_do_readpage(folio, &em_cached, &bio_ctrl, &prev_em_start);
2557 
2558 	btrfs_unlock_extent(&inode->io_tree, start, end, &cached_state);
2559 
2560 	if (em_cached)
2561 		btrfs_free_extent_map(em_cached);
2562 	submit_one_bio(&bio_ctrl);
2563 }
2564 
2565 /*
2566  * basic invalidate_folio code, this waits on any locked or writeback
2567  * ranges corresponding to the folio, and then deletes any extent state
2568  * records from the tree
2569  */
extent_invalidate_folio(struct extent_io_tree * tree,struct folio * folio,size_t offset)2570 int extent_invalidate_folio(struct extent_io_tree *tree,
2571 			  struct folio *folio, size_t offset)
2572 {
2573 	struct extent_state *cached_state = NULL;
2574 	u64 start = folio_pos(folio);
2575 	u64 end = start + folio_size(folio) - 1;
2576 	size_t blocksize = folio_to_fs_info(folio)->sectorsize;
2577 
2578 	/* This function is only called for the btree inode */
2579 	ASSERT(tree->owner == IO_TREE_BTREE_INODE_IO);
2580 
2581 	start += ALIGN(offset, blocksize);
2582 	if (start > end)
2583 		return 0;
2584 
2585 	btrfs_lock_extent(tree, start, end, &cached_state);
2586 	folio_wait_writeback(folio);
2587 
2588 	/*
2589 	 * Currently for btree io tree, only EXTENT_LOCKED is utilized,
2590 	 * so here we only need to unlock the extent range to free any
2591 	 * existing extent state.
2592 	 */
2593 	btrfs_unlock_extent(tree, start, end, &cached_state);
2594 	return 0;
2595 }
2596 
2597 /*
2598  * A helper for struct address_space_operations::release_folio, this tests for
2599  * areas of the folio that are locked or under IO and drops the related state
2600  * bits if it is safe to drop the folio.
2601  */
try_release_extent_state(struct extent_io_tree * tree,struct folio * folio)2602 static bool try_release_extent_state(struct extent_io_tree *tree,
2603 				     struct folio *folio)
2604 {
2605 	struct extent_state *cached_state = NULL;
2606 	u64 start = folio_pos(folio);
2607 	u64 end = start + folio_size(folio) - 1;
2608 	u32 range_bits;
2609 	u32 clear_bits;
2610 	bool ret = false;
2611 	int ret2;
2612 
2613 	btrfs_get_range_bits(tree, start, end, &range_bits, &cached_state);
2614 
2615 	/*
2616 	 * We can release the folio if it's locked only for ordered extent
2617 	 * completion, since that doesn't require using the folio.
2618 	 */
2619 	if ((range_bits & EXTENT_LOCKED) &&
2620 	    !(range_bits & EXTENT_FINISHING_ORDERED))
2621 		goto out;
2622 
2623 	clear_bits = ~(EXTENT_LOCKED | EXTENT_NODATASUM | EXTENT_DELALLOC_NEW |
2624 		       EXTENT_CTLBITS | EXTENT_QGROUP_RESERVED |
2625 		       EXTENT_FINISHING_ORDERED);
2626 	/*
2627 	 * At this point we can safely clear everything except the locked,
2628 	 * nodatasum, delalloc new and finishing ordered bits. The delalloc new
2629 	 * bit will be cleared by ordered extent completion.
2630 	 */
2631 	ret2 = btrfs_clear_extent_bit(tree, start, end, clear_bits, &cached_state);
2632 	/*
2633 	 * If clear_extent_bit failed for enomem reasons, we can't allow the
2634 	 * release to continue.
2635 	 */
2636 	if (ret2 == 0)
2637 		ret = true;
2638 out:
2639 	btrfs_free_extent_state(cached_state);
2640 
2641 	return ret;
2642 }
2643 
2644 /*
2645  * a helper for release_folio.  As long as there are no locked extents
2646  * in the range corresponding to the page, both state records and extent
2647  * map records are removed
2648  */
try_release_extent_mapping(struct folio * folio,gfp_t mask)2649 bool try_release_extent_mapping(struct folio *folio, gfp_t mask)
2650 {
2651 	u64 start = folio_pos(folio);
2652 	u64 end = start + folio_size(folio) - 1;
2653 	struct btrfs_inode *inode = folio_to_inode(folio);
2654 	struct extent_io_tree *io_tree = &inode->io_tree;
2655 
2656 	while (start <= end) {
2657 		const u64 cur_gen = btrfs_get_fs_generation(inode->root->fs_info);
2658 		const u64 len = end - start + 1;
2659 		struct extent_map_tree *extent_tree = &inode->extent_tree;
2660 		struct extent_map *em;
2661 
2662 		write_lock(&extent_tree->lock);
2663 		em = btrfs_lookup_extent_mapping(extent_tree, start, len);
2664 		if (!em) {
2665 			write_unlock(&extent_tree->lock);
2666 			break;
2667 		}
2668 		if ((em->flags & EXTENT_FLAG_PINNED) || em->start != start) {
2669 			write_unlock(&extent_tree->lock);
2670 			btrfs_free_extent_map(em);
2671 			break;
2672 		}
2673 		if (btrfs_test_range_bit_exists(io_tree, em->start,
2674 						btrfs_extent_map_end(em) - 1,
2675 						EXTENT_LOCKED))
2676 			goto next;
2677 		/*
2678 		 * If it's not in the list of modified extents, used by a fast
2679 		 * fsync, we can remove it. If it's being logged we can safely
2680 		 * remove it since fsync took an extra reference on the em.
2681 		 */
2682 		if (list_empty(&em->list) || (em->flags & EXTENT_FLAG_LOGGING))
2683 			goto remove_em;
2684 		/*
2685 		 * If it's in the list of modified extents, remove it only if
2686 		 * its generation is older then the current one, in which case
2687 		 * we don't need it for a fast fsync. Otherwise don't remove it,
2688 		 * we could be racing with an ongoing fast fsync that could miss
2689 		 * the new extent.
2690 		 */
2691 		if (em->generation >= cur_gen)
2692 			goto next;
2693 remove_em:
2694 		/*
2695 		 * We only remove extent maps that are not in the list of
2696 		 * modified extents or that are in the list but with a
2697 		 * generation lower then the current generation, so there is no
2698 		 * need to set the full fsync flag on the inode (it hurts the
2699 		 * fsync performance for workloads with a data size that exceeds
2700 		 * or is close to the system's memory).
2701 		 */
2702 		btrfs_remove_extent_mapping(inode, em);
2703 		/* Once for the inode's extent map tree. */
2704 		btrfs_free_extent_map(em);
2705 next:
2706 		start = btrfs_extent_map_end(em);
2707 		write_unlock(&extent_tree->lock);
2708 
2709 		/* Once for us, for the lookup_extent_mapping() reference. */
2710 		btrfs_free_extent_map(em);
2711 
2712 		if (need_resched()) {
2713 			/*
2714 			 * If we need to resched but we can't block just exit
2715 			 * and leave any remaining extent maps.
2716 			 */
2717 			if (!gfpflags_allow_blocking(mask))
2718 				break;
2719 
2720 			cond_resched();
2721 		}
2722 	}
2723 	return try_release_extent_state(io_tree, folio);
2724 }
2725 
extent_buffer_under_io(const struct extent_buffer * eb)2726 static int extent_buffer_under_io(const struct extent_buffer *eb)
2727 {
2728 	return (test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags) ||
2729 		test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
2730 }
2731 
folio_range_has_eb(struct folio * folio)2732 static bool folio_range_has_eb(struct folio *folio)
2733 {
2734 	struct btrfs_subpage *subpage;
2735 
2736 	lockdep_assert_held(&folio->mapping->i_private_lock);
2737 
2738 	if (folio_test_private(folio)) {
2739 		subpage = folio_get_private(folio);
2740 		if (atomic_read(&subpage->eb_refs))
2741 			return true;
2742 	}
2743 	return false;
2744 }
2745 
detach_extent_buffer_folio(const struct extent_buffer * eb,struct folio * folio)2746 static void detach_extent_buffer_folio(const struct extent_buffer *eb, struct folio *folio)
2747 {
2748 	struct btrfs_fs_info *fs_info = eb->fs_info;
2749 	struct address_space *mapping = folio->mapping;
2750 	const bool mapped = !test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags);
2751 
2752 	/*
2753 	 * For mapped eb, we're going to change the folio private, which should
2754 	 * be done under the i_private_lock.
2755 	 */
2756 	if (mapped)
2757 		spin_lock(&mapping->i_private_lock);
2758 
2759 	if (!folio_test_private(folio)) {
2760 		if (mapped)
2761 			spin_unlock(&mapping->i_private_lock);
2762 		return;
2763 	}
2764 
2765 	if (!btrfs_meta_is_subpage(fs_info)) {
2766 		/*
2767 		 * We do this since we'll remove the pages after we've removed
2768 		 * the eb from the xarray, so we could race and have this page
2769 		 * now attached to the new eb.  So only clear folio if it's
2770 		 * still connected to this eb.
2771 		 */
2772 		if (folio_test_private(folio) && folio_get_private(folio) == eb) {
2773 			BUG_ON(test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
2774 			BUG_ON(folio_test_dirty(folio));
2775 			BUG_ON(folio_test_writeback(folio));
2776 			/* We need to make sure we haven't be attached to a new eb. */
2777 			folio_detach_private(folio);
2778 		}
2779 		if (mapped)
2780 			spin_unlock(&mapping->i_private_lock);
2781 		return;
2782 	}
2783 
2784 	/*
2785 	 * For subpage, we can have dummy eb with folio private attached.  In
2786 	 * this case, we can directly detach the private as such folio is only
2787 	 * attached to one dummy eb, no sharing.
2788 	 */
2789 	if (!mapped) {
2790 		btrfs_detach_subpage(fs_info, folio, BTRFS_SUBPAGE_METADATA);
2791 		return;
2792 	}
2793 
2794 	btrfs_folio_dec_eb_refs(fs_info, folio);
2795 
2796 	/*
2797 	 * We can only detach the folio private if there are no other ebs in the
2798 	 * page range and no unfinished IO.
2799 	 */
2800 	if (!folio_range_has_eb(folio))
2801 		btrfs_detach_subpage(fs_info, folio, BTRFS_SUBPAGE_METADATA);
2802 
2803 	spin_unlock(&mapping->i_private_lock);
2804 }
2805 
2806 /* Release all folios attached to the extent buffer */
btrfs_release_extent_buffer_folios(const struct extent_buffer * eb)2807 static void btrfs_release_extent_buffer_folios(const struct extent_buffer *eb)
2808 {
2809 	ASSERT(!extent_buffer_under_io(eb));
2810 
2811 	for (int i = 0; i < INLINE_EXTENT_BUFFER_PAGES; i++) {
2812 		struct folio *folio = eb->folios[i];
2813 
2814 		if (!folio)
2815 			continue;
2816 
2817 		detach_extent_buffer_folio(eb, folio);
2818 	}
2819 }
2820 
2821 /*
2822  * Helper for releasing the extent buffer.
2823  */
btrfs_release_extent_buffer(struct extent_buffer * eb)2824 static inline void btrfs_release_extent_buffer(struct extent_buffer *eb)
2825 {
2826 	btrfs_release_extent_buffer_folios(eb);
2827 	btrfs_leak_debug_del_eb(eb);
2828 	kmem_cache_free(extent_buffer_cache, eb);
2829 }
2830 
__alloc_extent_buffer(struct btrfs_fs_info * fs_info,u64 start)2831 static struct extent_buffer *__alloc_extent_buffer(struct btrfs_fs_info *fs_info,
2832 						   u64 start)
2833 {
2834 	struct extent_buffer *eb = NULL;
2835 
2836 	eb = kmem_cache_zalloc(extent_buffer_cache, GFP_NOFS|__GFP_NOFAIL);
2837 	eb->start = start;
2838 	eb->len = fs_info->nodesize;
2839 	eb->fs_info = fs_info;
2840 	init_rwsem(&eb->lock);
2841 
2842 	btrfs_leak_debug_add_eb(eb);
2843 
2844 	spin_lock_init(&eb->refs_lock);
2845 	atomic_set(&eb->refs, 1);
2846 
2847 	ASSERT(eb->len <= BTRFS_MAX_METADATA_BLOCKSIZE);
2848 
2849 	return eb;
2850 }
2851 
2852 /*
2853  * For use in eb allocation error cleanup paths, as btrfs_release_extent_buffer()
2854  * does not call folio_put(), and we need to set the folios to NULL so that
2855  * btrfs_release_extent_buffer() will not detach them a second time.
2856  */
cleanup_extent_buffer_folios(struct extent_buffer * eb)2857 static void cleanup_extent_buffer_folios(struct extent_buffer *eb)
2858 {
2859 	const int num_folios = num_extent_folios(eb);
2860 
2861 	/* We canont use num_extent_folios() as loop bound as eb->folios changes. */
2862 	for (int i = 0; i < num_folios; i++) {
2863 		ASSERT(eb->folios[i]);
2864 		detach_extent_buffer_folio(eb, eb->folios[i]);
2865 		folio_put(eb->folios[i]);
2866 		eb->folios[i] = NULL;
2867 	}
2868 }
2869 
btrfs_clone_extent_buffer(const struct extent_buffer * src)2870 struct extent_buffer *btrfs_clone_extent_buffer(const struct extent_buffer *src)
2871 {
2872 	struct extent_buffer *new;
2873 	int num_folios;
2874 	int ret;
2875 
2876 	new = __alloc_extent_buffer(src->fs_info, src->start);
2877 	if (new == NULL)
2878 		return NULL;
2879 
2880 	/*
2881 	 * Set UNMAPPED before calling btrfs_release_extent_buffer(), as
2882 	 * btrfs_release_extent_buffer() have different behavior for
2883 	 * UNMAPPED subpage extent buffer.
2884 	 */
2885 	set_bit(EXTENT_BUFFER_UNMAPPED, &new->bflags);
2886 
2887 	ret = alloc_eb_folio_array(new, false);
2888 	if (ret)
2889 		goto release_eb;
2890 
2891 	ASSERT(num_extent_folios(src) == num_extent_folios(new),
2892 	       "%d != %d", num_extent_folios(src), num_extent_folios(new));
2893 	/* Explicitly use the cached num_extent value from now on. */
2894 	num_folios = num_extent_folios(src);
2895 	for (int i = 0; i < num_folios; i++) {
2896 		struct folio *folio = new->folios[i];
2897 
2898 		ret = attach_extent_buffer_folio(new, folio, NULL);
2899 		if (ret < 0)
2900 			goto cleanup_folios;
2901 		WARN_ON(folio_test_dirty(folio));
2902 	}
2903 	for (int i = 0; i < num_folios; i++)
2904 		folio_put(new->folios[i]);
2905 
2906 	copy_extent_buffer_full(new, src);
2907 	set_extent_buffer_uptodate(new);
2908 
2909 	return new;
2910 
2911 cleanup_folios:
2912 	cleanup_extent_buffer_folios(new);
2913 release_eb:
2914 	btrfs_release_extent_buffer(new);
2915 	return NULL;
2916 }
2917 
alloc_dummy_extent_buffer(struct btrfs_fs_info * fs_info,u64 start)2918 struct extent_buffer *alloc_dummy_extent_buffer(struct btrfs_fs_info *fs_info,
2919 						u64 start)
2920 {
2921 	struct extent_buffer *eb;
2922 	int ret;
2923 
2924 	eb = __alloc_extent_buffer(fs_info, start);
2925 	if (!eb)
2926 		return NULL;
2927 
2928 	ret = alloc_eb_folio_array(eb, false);
2929 	if (ret)
2930 		goto release_eb;
2931 
2932 	for (int i = 0; i < num_extent_folios(eb); i++) {
2933 		ret = attach_extent_buffer_folio(eb, eb->folios[i], NULL);
2934 		if (ret < 0)
2935 			goto cleanup_folios;
2936 	}
2937 	for (int i = 0; i < num_extent_folios(eb); i++)
2938 		folio_put(eb->folios[i]);
2939 
2940 	set_extent_buffer_uptodate(eb);
2941 	btrfs_set_header_nritems(eb, 0);
2942 	set_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags);
2943 
2944 	return eb;
2945 
2946 cleanup_folios:
2947 	cleanup_extent_buffer_folios(eb);
2948 release_eb:
2949 	btrfs_release_extent_buffer(eb);
2950 	return NULL;
2951 }
2952 
check_buffer_tree_ref(struct extent_buffer * eb)2953 static void check_buffer_tree_ref(struct extent_buffer *eb)
2954 {
2955 	int refs;
2956 	/*
2957 	 * The TREE_REF bit is first set when the extent_buffer is added to the
2958 	 * xarray. It is also reset, if unset, when a new reference is created
2959 	 * by find_extent_buffer.
2960 	 *
2961 	 * It is only cleared in two cases: freeing the last non-tree
2962 	 * reference to the extent_buffer when its STALE bit is set or
2963 	 * calling release_folio when the tree reference is the only reference.
2964 	 *
2965 	 * In both cases, care is taken to ensure that the extent_buffer's
2966 	 * pages are not under io. However, release_folio can be concurrently
2967 	 * called with creating new references, which is prone to race
2968 	 * conditions between the calls to check_buffer_tree_ref in those
2969 	 * codepaths and clearing TREE_REF in try_release_extent_buffer.
2970 	 *
2971 	 * The actual lifetime of the extent_buffer in the xarray is adequately
2972 	 * protected by the refcount, but the TREE_REF bit and its corresponding
2973 	 * reference are not. To protect against this class of races, we call
2974 	 * check_buffer_tree_ref() from the code paths which trigger io. Note that
2975 	 * once io is initiated, TREE_REF can no longer be cleared, so that is
2976 	 * the moment at which any such race is best fixed.
2977 	 */
2978 	refs = atomic_read(&eb->refs);
2979 	if (refs >= 2 && test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
2980 		return;
2981 
2982 	spin_lock(&eb->refs_lock);
2983 	if (!test_and_set_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
2984 		atomic_inc(&eb->refs);
2985 	spin_unlock(&eb->refs_lock);
2986 }
2987 
mark_extent_buffer_accessed(struct extent_buffer * eb)2988 static void mark_extent_buffer_accessed(struct extent_buffer *eb)
2989 {
2990 	check_buffer_tree_ref(eb);
2991 
2992 	for (int i = 0; i < num_extent_folios(eb); i++)
2993 		folio_mark_accessed(eb->folios[i]);
2994 }
2995 
find_extent_buffer(struct btrfs_fs_info * fs_info,u64 start)2996 struct extent_buffer *find_extent_buffer(struct btrfs_fs_info *fs_info,
2997 					 u64 start)
2998 {
2999 	struct extent_buffer *eb;
3000 
3001 	eb = find_extent_buffer_nolock(fs_info, start);
3002 	if (!eb)
3003 		return NULL;
3004 	/*
3005 	 * Lock our eb's refs_lock to avoid races with free_extent_buffer().
3006 	 * When we get our eb it might be flagged with EXTENT_BUFFER_STALE and
3007 	 * another task running free_extent_buffer() might have seen that flag
3008 	 * set, eb->refs == 2, that the buffer isn't under IO (dirty and
3009 	 * writeback flags not set) and it's still in the tree (flag
3010 	 * EXTENT_BUFFER_TREE_REF set), therefore being in the process of
3011 	 * decrementing the extent buffer's reference count twice.  So here we
3012 	 * could race and increment the eb's reference count, clear its stale
3013 	 * flag, mark it as dirty and drop our reference before the other task
3014 	 * finishes executing free_extent_buffer, which would later result in
3015 	 * an attempt to free an extent buffer that is dirty.
3016 	 */
3017 	if (test_bit(EXTENT_BUFFER_STALE, &eb->bflags)) {
3018 		spin_lock(&eb->refs_lock);
3019 		spin_unlock(&eb->refs_lock);
3020 	}
3021 	mark_extent_buffer_accessed(eb);
3022 	return eb;
3023 }
3024 
alloc_test_extent_buffer(struct btrfs_fs_info * fs_info,u64 start)3025 struct extent_buffer *alloc_test_extent_buffer(struct btrfs_fs_info *fs_info,
3026 					u64 start)
3027 {
3028 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
3029 	struct extent_buffer *eb, *exists = NULL;
3030 	int ret;
3031 
3032 	eb = find_extent_buffer(fs_info, start);
3033 	if (eb)
3034 		return eb;
3035 	eb = alloc_dummy_extent_buffer(fs_info, start);
3036 	if (!eb)
3037 		return ERR_PTR(-ENOMEM);
3038 	eb->fs_info = fs_info;
3039 again:
3040 	xa_lock_irq(&fs_info->buffer_tree);
3041 	exists = __xa_cmpxchg(&fs_info->buffer_tree, start >> fs_info->sectorsize_bits,
3042 			      NULL, eb, GFP_NOFS);
3043 	if (xa_is_err(exists)) {
3044 		ret = xa_err(exists);
3045 		xa_unlock_irq(&fs_info->buffer_tree);
3046 		btrfs_release_extent_buffer(eb);
3047 		return ERR_PTR(ret);
3048 	}
3049 	if (exists) {
3050 		if (!atomic_inc_not_zero(&exists->refs)) {
3051 			/* The extent buffer is being freed, retry. */
3052 			xa_unlock_irq(&fs_info->buffer_tree);
3053 			goto again;
3054 		}
3055 		xa_unlock_irq(&fs_info->buffer_tree);
3056 		btrfs_release_extent_buffer(eb);
3057 		return exists;
3058 	}
3059 	xa_unlock_irq(&fs_info->buffer_tree);
3060 	check_buffer_tree_ref(eb);
3061 
3062 	return eb;
3063 #else
3064 	/* Stub to avoid linker error when compiled with optimizations turned off. */
3065 	return NULL;
3066 #endif
3067 }
3068 
grab_extent_buffer(struct btrfs_fs_info * fs_info,struct folio * folio)3069 static struct extent_buffer *grab_extent_buffer(struct btrfs_fs_info *fs_info,
3070 						struct folio *folio)
3071 {
3072 	struct extent_buffer *exists;
3073 
3074 	lockdep_assert_held(&folio->mapping->i_private_lock);
3075 
3076 	/*
3077 	 * For subpage case, we completely rely on xarray to ensure we don't try
3078 	 * to insert two ebs for the same bytenr.  So here we always return NULL
3079 	 * and just continue.
3080 	 */
3081 	if (btrfs_meta_is_subpage(fs_info))
3082 		return NULL;
3083 
3084 	/* Page not yet attached to an extent buffer */
3085 	if (!folio_test_private(folio))
3086 		return NULL;
3087 
3088 	/*
3089 	 * We could have already allocated an eb for this folio and attached one
3090 	 * so lets see if we can get a ref on the existing eb, and if we can we
3091 	 * know it's good and we can just return that one, else we know we can
3092 	 * just overwrite folio private.
3093 	 */
3094 	exists = folio_get_private(folio);
3095 	if (atomic_inc_not_zero(&exists->refs))
3096 		return exists;
3097 
3098 	WARN_ON(folio_test_dirty(folio));
3099 	folio_detach_private(folio);
3100 	return NULL;
3101 }
3102 
3103 /*
3104  * Validate alignment constraints of eb at logical address @start.
3105  */
check_eb_alignment(struct btrfs_fs_info * fs_info,u64 start)3106 static bool check_eb_alignment(struct btrfs_fs_info *fs_info, u64 start)
3107 {
3108 	if (!IS_ALIGNED(start, fs_info->sectorsize)) {
3109 		btrfs_err(fs_info, "bad tree block start %llu", start);
3110 		return true;
3111 	}
3112 
3113 	if (fs_info->nodesize < PAGE_SIZE && !IS_ALIGNED(start, fs_info->nodesize)) {
3114 		btrfs_err(fs_info,
3115 		"tree block is not nodesize aligned, start %llu nodesize %u",
3116 			  start, fs_info->nodesize);
3117 		return true;
3118 	}
3119 	if (fs_info->nodesize >= PAGE_SIZE &&
3120 	    !PAGE_ALIGNED(start)) {
3121 		btrfs_err(fs_info,
3122 		"tree block is not page aligned, start %llu nodesize %u",
3123 			  start, fs_info->nodesize);
3124 		return true;
3125 	}
3126 	if (!IS_ALIGNED(start, fs_info->nodesize) &&
3127 	    !test_and_set_bit(BTRFS_FS_UNALIGNED_TREE_BLOCK, &fs_info->flags)) {
3128 		btrfs_warn(fs_info,
3129 "tree block not nodesize aligned, start %llu nodesize %u, can be resolved by a full metadata balance",
3130 			      start, fs_info->nodesize);
3131 	}
3132 	return false;
3133 }
3134 
3135 /*
3136  * Return 0 if eb->folios[i] is attached to btree inode successfully.
3137  * Return >0 if there is already another extent buffer for the range,
3138  * and @found_eb_ret would be updated.
3139  * Return -EAGAIN if the filemap has an existing folio but with different size
3140  * than @eb.
3141  * The caller needs to free the existing folios and retry using the same order.
3142  */
attach_eb_folio_to_filemap(struct extent_buffer * eb,int i,struct btrfs_subpage * prealloc,struct extent_buffer ** found_eb_ret)3143 static int attach_eb_folio_to_filemap(struct extent_buffer *eb, int i,
3144 				      struct btrfs_subpage *prealloc,
3145 				      struct extent_buffer **found_eb_ret)
3146 {
3147 
3148 	struct btrfs_fs_info *fs_info = eb->fs_info;
3149 	struct address_space *mapping = fs_info->btree_inode->i_mapping;
3150 	const unsigned long index = eb->start >> PAGE_SHIFT;
3151 	struct folio *existing_folio;
3152 	int ret;
3153 
3154 	ASSERT(found_eb_ret);
3155 
3156 	/* Caller should ensure the folio exists. */
3157 	ASSERT(eb->folios[i]);
3158 
3159 retry:
3160 	existing_folio = NULL;
3161 	ret = filemap_add_folio(mapping, eb->folios[i], index + i,
3162 				GFP_NOFS | __GFP_NOFAIL);
3163 	if (!ret)
3164 		goto finish;
3165 
3166 	existing_folio = filemap_lock_folio(mapping, index + i);
3167 	/* The page cache only exists for a very short time, just retry. */
3168 	if (IS_ERR(existing_folio))
3169 		goto retry;
3170 
3171 	/* For now, we should only have single-page folios for btree inode. */
3172 	ASSERT(folio_nr_pages(existing_folio) == 1);
3173 
3174 	if (folio_size(existing_folio) != eb->folio_size) {
3175 		folio_unlock(existing_folio);
3176 		folio_put(existing_folio);
3177 		return -EAGAIN;
3178 	}
3179 
3180 finish:
3181 	spin_lock(&mapping->i_private_lock);
3182 	if (existing_folio && btrfs_meta_is_subpage(fs_info)) {
3183 		/* We're going to reuse the existing page, can drop our folio now. */
3184 		__free_page(folio_page(eb->folios[i], 0));
3185 		eb->folios[i] = existing_folio;
3186 	} else if (existing_folio) {
3187 		struct extent_buffer *existing_eb;
3188 
3189 		existing_eb = grab_extent_buffer(fs_info, existing_folio);
3190 		if (existing_eb) {
3191 			/* The extent buffer still exists, we can use it directly. */
3192 			*found_eb_ret = existing_eb;
3193 			spin_unlock(&mapping->i_private_lock);
3194 			folio_unlock(existing_folio);
3195 			folio_put(existing_folio);
3196 			return 1;
3197 		}
3198 		/* The extent buffer no longer exists, we can reuse the folio. */
3199 		__free_page(folio_page(eb->folios[i], 0));
3200 		eb->folios[i] = existing_folio;
3201 	}
3202 	eb->folio_size = folio_size(eb->folios[i]);
3203 	eb->folio_shift = folio_shift(eb->folios[i]);
3204 	/* Should not fail, as we have preallocated the memory. */
3205 	ret = attach_extent_buffer_folio(eb, eb->folios[i], prealloc);
3206 	ASSERT(!ret);
3207 	/*
3208 	 * To inform we have an extra eb under allocation, so that
3209 	 * detach_extent_buffer_page() won't release the folio private when the
3210 	 * eb hasn't been inserted into the xarray yet.
3211 	 *
3212 	 * The ref will be decreased when the eb releases the page, in
3213 	 * detach_extent_buffer_page().  Thus needs no special handling in the
3214 	 * error path.
3215 	 */
3216 	btrfs_folio_inc_eb_refs(fs_info, eb->folios[i]);
3217 	spin_unlock(&mapping->i_private_lock);
3218 	return 0;
3219 }
3220 
alloc_extent_buffer(struct btrfs_fs_info * fs_info,u64 start,u64 owner_root,int level)3221 struct extent_buffer *alloc_extent_buffer(struct btrfs_fs_info *fs_info,
3222 					  u64 start, u64 owner_root, int level)
3223 {
3224 	int attached = 0;
3225 	struct extent_buffer *eb;
3226 	struct extent_buffer *existing_eb = NULL;
3227 	struct btrfs_subpage *prealloc = NULL;
3228 	u64 lockdep_owner = owner_root;
3229 	bool page_contig = true;
3230 	int uptodate = 1;
3231 	int ret;
3232 
3233 	if (check_eb_alignment(fs_info, start))
3234 		return ERR_PTR(-EINVAL);
3235 
3236 #if BITS_PER_LONG == 32
3237 	if (start >= MAX_LFS_FILESIZE) {
3238 		btrfs_err_rl(fs_info,
3239 		"extent buffer %llu is beyond 32bit page cache limit", start);
3240 		btrfs_err_32bit_limit(fs_info);
3241 		return ERR_PTR(-EOVERFLOW);
3242 	}
3243 	if (start >= BTRFS_32BIT_EARLY_WARN_THRESHOLD)
3244 		btrfs_warn_32bit_limit(fs_info);
3245 #endif
3246 
3247 	eb = find_extent_buffer(fs_info, start);
3248 	if (eb)
3249 		return eb;
3250 
3251 	eb = __alloc_extent_buffer(fs_info, start);
3252 	if (!eb)
3253 		return ERR_PTR(-ENOMEM);
3254 
3255 	/*
3256 	 * The reloc trees are just snapshots, so we need them to appear to be
3257 	 * just like any other fs tree WRT lockdep.
3258 	 */
3259 	if (lockdep_owner == BTRFS_TREE_RELOC_OBJECTID)
3260 		lockdep_owner = BTRFS_FS_TREE_OBJECTID;
3261 
3262 	btrfs_set_buffer_lockdep_class(lockdep_owner, eb, level);
3263 
3264 	/*
3265 	 * Preallocate folio private for subpage case, so that we won't
3266 	 * allocate memory with i_private_lock nor page lock hold.
3267 	 *
3268 	 * The memory will be freed by attach_extent_buffer_page() or freed
3269 	 * manually if we exit earlier.
3270 	 */
3271 	if (btrfs_meta_is_subpage(fs_info)) {
3272 		prealloc = btrfs_alloc_subpage(fs_info, PAGE_SIZE, BTRFS_SUBPAGE_METADATA);
3273 		if (IS_ERR(prealloc)) {
3274 			ret = PTR_ERR(prealloc);
3275 			goto out;
3276 		}
3277 	}
3278 
3279 reallocate:
3280 	/* Allocate all pages first. */
3281 	ret = alloc_eb_folio_array(eb, true);
3282 	if (ret < 0) {
3283 		btrfs_free_subpage(prealloc);
3284 		goto out;
3285 	}
3286 
3287 	/* Attach all pages to the filemap. */
3288 	for (int i = 0; i < num_extent_folios(eb); i++) {
3289 		struct folio *folio;
3290 
3291 		ret = attach_eb_folio_to_filemap(eb, i, prealloc, &existing_eb);
3292 		if (ret > 0) {
3293 			ASSERT(existing_eb);
3294 			goto out;
3295 		}
3296 
3297 		/*
3298 		 * TODO: Special handling for a corner case where the order of
3299 		 * folios mismatch between the new eb and filemap.
3300 		 *
3301 		 * This happens when:
3302 		 *
3303 		 * - the new eb is using higher order folio
3304 		 *
3305 		 * - the filemap is still using 0-order folios for the range
3306 		 *   This can happen at the previous eb allocation, and we don't
3307 		 *   have higher order folio for the call.
3308 		 *
3309 		 * - the existing eb has already been freed
3310 		 *
3311 		 * In this case, we have to free the existing folios first, and
3312 		 * re-allocate using the same order.
3313 		 * Thankfully this is not going to happen yet, as we're still
3314 		 * using 0-order folios.
3315 		 */
3316 		if (unlikely(ret == -EAGAIN)) {
3317 			DEBUG_WARN("folio order mismatch between new eb and filemap");
3318 			goto reallocate;
3319 		}
3320 		attached++;
3321 
3322 		/*
3323 		 * Only after attach_eb_folio_to_filemap(), eb->folios[] is
3324 		 * reliable, as we may choose to reuse the existing page cache
3325 		 * and free the allocated page.
3326 		 */
3327 		folio = eb->folios[i];
3328 		WARN_ON(btrfs_meta_folio_test_dirty(folio, eb));
3329 
3330 		/*
3331 		 * Check if the current page is physically contiguous with previous eb
3332 		 * page.
3333 		 * At this stage, either we allocated a large folio, thus @i
3334 		 * would only be 0, or we fall back to per-page allocation.
3335 		 */
3336 		if (i && folio_page(eb->folios[i - 1], 0) + 1 != folio_page(folio, 0))
3337 			page_contig = false;
3338 
3339 		if (!btrfs_meta_folio_test_uptodate(folio, eb))
3340 			uptodate = 0;
3341 
3342 		/*
3343 		 * We can't unlock the pages just yet since the extent buffer
3344 		 * hasn't been properly inserted into the xarray, this opens a
3345 		 * race with btree_release_folio() which can free a page while we
3346 		 * are still filling in all pages for the buffer and we could crash.
3347 		 */
3348 	}
3349 	if (uptodate)
3350 		set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
3351 	/* All pages are physically contiguous, can skip cross page handling. */
3352 	if (page_contig)
3353 		eb->addr = folio_address(eb->folios[0]) + offset_in_page(eb->start);
3354 again:
3355 	xa_lock_irq(&fs_info->buffer_tree);
3356 	existing_eb = __xa_cmpxchg(&fs_info->buffer_tree,
3357 				   start >> fs_info->sectorsize_bits, NULL, eb,
3358 				   GFP_NOFS);
3359 	if (xa_is_err(existing_eb)) {
3360 		ret = xa_err(existing_eb);
3361 		xa_unlock_irq(&fs_info->buffer_tree);
3362 		goto out;
3363 	}
3364 	if (existing_eb) {
3365 		if (!atomic_inc_not_zero(&existing_eb->refs)) {
3366 			xa_unlock_irq(&fs_info->buffer_tree);
3367 			goto again;
3368 		}
3369 		xa_unlock_irq(&fs_info->buffer_tree);
3370 		goto out;
3371 	}
3372 	xa_unlock_irq(&fs_info->buffer_tree);
3373 
3374 	/* add one reference for the tree */
3375 	check_buffer_tree_ref(eb);
3376 
3377 	/*
3378 	 * Now it's safe to unlock the pages because any calls to
3379 	 * btree_release_folio will correctly detect that a page belongs to a
3380 	 * live buffer and won't free them prematurely.
3381 	 */
3382 	for (int i = 0; i < num_extent_folios(eb); i++) {
3383 		folio_unlock(eb->folios[i]);
3384 		/*
3385 		 * A folio that has been added to an address_space mapping
3386 		 * should not continue holding the refcount from its original
3387 		 * allocation indefinitely.
3388 		 */
3389 		folio_put(eb->folios[i]);
3390 	}
3391 	return eb;
3392 
3393 out:
3394 	WARN_ON(!atomic_dec_and_test(&eb->refs));
3395 
3396 	/*
3397 	 * Any attached folios need to be detached before we unlock them.  This
3398 	 * is because when we're inserting our new folios into the mapping, and
3399 	 * then attaching our eb to that folio.  If we fail to insert our folio
3400 	 * we'll lookup the folio for that index, and grab that EB.  We do not
3401 	 * want that to grab this eb, as we're getting ready to free it.  So we
3402 	 * have to detach it first and then unlock it.
3403 	 *
3404 	 * Note: the bounds is num_extent_pages() as we need to go through all slots.
3405 	 */
3406 	for (int i = 0; i < num_extent_pages(eb); i++) {
3407 		struct folio *folio = eb->folios[i];
3408 
3409 		if (i < attached) {
3410 			ASSERT(folio);
3411 			detach_extent_buffer_folio(eb, folio);
3412 			folio_unlock(folio);
3413 		} else if (!folio) {
3414 			continue;
3415 		}
3416 
3417 		folio_put(folio);
3418 		eb->folios[i] = NULL;
3419 	}
3420 	btrfs_release_extent_buffer(eb);
3421 	if (ret < 0)
3422 		return ERR_PTR(ret);
3423 	ASSERT(existing_eb);
3424 	return existing_eb;
3425 }
3426 
btrfs_release_extent_buffer_rcu(struct rcu_head * head)3427 static inline void btrfs_release_extent_buffer_rcu(struct rcu_head *head)
3428 {
3429 	struct extent_buffer *eb =
3430 			container_of(head, struct extent_buffer, rcu_head);
3431 
3432 	kmem_cache_free(extent_buffer_cache, eb);
3433 }
3434 
release_extent_buffer(struct extent_buffer * eb)3435 static int release_extent_buffer(struct extent_buffer *eb)
3436 	__releases(&eb->refs_lock)
3437 {
3438 	lockdep_assert_held(&eb->refs_lock);
3439 
3440 	WARN_ON(atomic_read(&eb->refs) == 0);
3441 	if (atomic_dec_and_test(&eb->refs)) {
3442 		struct btrfs_fs_info *fs_info = eb->fs_info;
3443 
3444 		spin_unlock(&eb->refs_lock);
3445 
3446 		/*
3447 		 * We're erasing, theoretically there will be no allocations, so
3448 		 * just use GFP_ATOMIC.
3449 		 *
3450 		 * We use cmpxchg instead of erase because we do not know if
3451 		 * this eb is actually in the tree or not, we could be cleaning
3452 		 * up an eb that we allocated but never inserted into the tree.
3453 		 * Thus use cmpxchg to remove it from the tree if it is there,
3454 		 * or leave the other entry if this isn't in the tree.
3455 		 *
3456 		 * The documentation says that putting a NULL value is the same
3457 		 * as erase as long as XA_FLAGS_ALLOC is not set, which it isn't
3458 		 * in this case.
3459 		 */
3460 		xa_cmpxchg_irq(&fs_info->buffer_tree,
3461 			       eb->start >> fs_info->sectorsize_bits, eb, NULL,
3462 			       GFP_ATOMIC);
3463 
3464 		btrfs_leak_debug_del_eb(eb);
3465 		/* Should be safe to release folios at this point. */
3466 		btrfs_release_extent_buffer_folios(eb);
3467 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
3468 		if (unlikely(test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags))) {
3469 			kmem_cache_free(extent_buffer_cache, eb);
3470 			return 1;
3471 		}
3472 #endif
3473 		call_rcu(&eb->rcu_head, btrfs_release_extent_buffer_rcu);
3474 		return 1;
3475 	}
3476 	spin_unlock(&eb->refs_lock);
3477 
3478 	return 0;
3479 }
3480 
free_extent_buffer(struct extent_buffer * eb)3481 void free_extent_buffer(struct extent_buffer *eb)
3482 {
3483 	int refs;
3484 	if (!eb)
3485 		return;
3486 
3487 	refs = atomic_read(&eb->refs);
3488 	while (1) {
3489 		if ((!test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags) && refs <= 3)
3490 		    || (test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags) &&
3491 			refs == 1))
3492 			break;
3493 		if (atomic_try_cmpxchg(&eb->refs, &refs, refs - 1))
3494 			return;
3495 	}
3496 
3497 	spin_lock(&eb->refs_lock);
3498 	if (atomic_read(&eb->refs) == 2 &&
3499 	    test_bit(EXTENT_BUFFER_STALE, &eb->bflags) &&
3500 	    !extent_buffer_under_io(eb) &&
3501 	    test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
3502 		atomic_dec(&eb->refs);
3503 
3504 	/*
3505 	 * I know this is terrible, but it's temporary until we stop tracking
3506 	 * the uptodate bits and such for the extent buffers.
3507 	 */
3508 	release_extent_buffer(eb);
3509 }
3510 
free_extent_buffer_stale(struct extent_buffer * eb)3511 void free_extent_buffer_stale(struct extent_buffer *eb)
3512 {
3513 	if (!eb)
3514 		return;
3515 
3516 	spin_lock(&eb->refs_lock);
3517 	set_bit(EXTENT_BUFFER_STALE, &eb->bflags);
3518 
3519 	if (atomic_read(&eb->refs) == 2 && !extent_buffer_under_io(eb) &&
3520 	    test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
3521 		atomic_dec(&eb->refs);
3522 	release_extent_buffer(eb);
3523 }
3524 
btree_clear_folio_dirty_tag(struct folio * folio)3525 static void btree_clear_folio_dirty_tag(struct folio *folio)
3526 {
3527 	ASSERT(!folio_test_dirty(folio));
3528 	ASSERT(folio_test_locked(folio));
3529 	xa_lock_irq(&folio->mapping->i_pages);
3530 	if (!folio_test_dirty(folio))
3531 		__xa_clear_mark(&folio->mapping->i_pages, folio->index,
3532 				PAGECACHE_TAG_DIRTY);
3533 	xa_unlock_irq(&folio->mapping->i_pages);
3534 }
3535 
btrfs_clear_buffer_dirty(struct btrfs_trans_handle * trans,struct extent_buffer * eb)3536 void btrfs_clear_buffer_dirty(struct btrfs_trans_handle *trans,
3537 			      struct extent_buffer *eb)
3538 {
3539 	struct btrfs_fs_info *fs_info = eb->fs_info;
3540 
3541 	btrfs_assert_tree_write_locked(eb);
3542 
3543 	if (trans && btrfs_header_generation(eb) != trans->transid)
3544 		return;
3545 
3546 	/*
3547 	 * Instead of clearing the dirty flag off of the buffer, mark it as
3548 	 * EXTENT_BUFFER_ZONED_ZEROOUT. This allows us to preserve
3549 	 * write-ordering in zoned mode, without the need to later re-dirty
3550 	 * the extent_buffer.
3551 	 *
3552 	 * The actual zeroout of the buffer will happen later in
3553 	 * btree_csum_one_bio.
3554 	 */
3555 	if (btrfs_is_zoned(fs_info) && test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)) {
3556 		set_bit(EXTENT_BUFFER_ZONED_ZEROOUT, &eb->bflags);
3557 		return;
3558 	}
3559 
3560 	if (!test_and_clear_bit(EXTENT_BUFFER_DIRTY, &eb->bflags))
3561 		return;
3562 
3563 	buffer_tree_clear_mark(eb, PAGECACHE_TAG_DIRTY);
3564 	percpu_counter_add_batch(&fs_info->dirty_metadata_bytes, -eb->len,
3565 				 fs_info->dirty_metadata_batch);
3566 
3567 	for (int i = 0; i < num_extent_folios(eb); i++) {
3568 		struct folio *folio = eb->folios[i];
3569 		bool last;
3570 
3571 		if (!folio_test_dirty(folio))
3572 			continue;
3573 		folio_lock(folio);
3574 		last = btrfs_meta_folio_clear_and_test_dirty(folio, eb);
3575 		if (last)
3576 			btree_clear_folio_dirty_tag(folio);
3577 		folio_unlock(folio);
3578 	}
3579 	WARN_ON(atomic_read(&eb->refs) == 0);
3580 }
3581 
set_extent_buffer_dirty(struct extent_buffer * eb)3582 void set_extent_buffer_dirty(struct extent_buffer *eb)
3583 {
3584 	bool was_dirty;
3585 
3586 	check_buffer_tree_ref(eb);
3587 
3588 	was_dirty = test_and_set_bit(EXTENT_BUFFER_DIRTY, &eb->bflags);
3589 
3590 	WARN_ON(atomic_read(&eb->refs) == 0);
3591 	WARN_ON(!test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags));
3592 	WARN_ON(test_bit(EXTENT_BUFFER_ZONED_ZEROOUT, &eb->bflags));
3593 
3594 	if (!was_dirty) {
3595 		bool subpage = btrfs_meta_is_subpage(eb->fs_info);
3596 
3597 		/*
3598 		 * For subpage case, we can have other extent buffers in the
3599 		 * same page, and in clear_extent_buffer_dirty() we
3600 		 * have to clear page dirty without subpage lock held.
3601 		 * This can cause race where our page gets dirty cleared after
3602 		 * we just set it.
3603 		 *
3604 		 * Thankfully, clear_extent_buffer_dirty() has locked
3605 		 * its page for other reasons, we can use page lock to prevent
3606 		 * the above race.
3607 		 */
3608 		if (subpage)
3609 			folio_lock(eb->folios[0]);
3610 		for (int i = 0; i < num_extent_folios(eb); i++)
3611 			btrfs_meta_folio_set_dirty(eb->folios[i], eb);
3612 		buffer_tree_set_mark(eb, PAGECACHE_TAG_DIRTY);
3613 		if (subpage)
3614 			folio_unlock(eb->folios[0]);
3615 		percpu_counter_add_batch(&eb->fs_info->dirty_metadata_bytes,
3616 					 eb->len,
3617 					 eb->fs_info->dirty_metadata_batch);
3618 	}
3619 #ifdef CONFIG_BTRFS_DEBUG
3620 	for (int i = 0; i < num_extent_folios(eb); i++)
3621 		ASSERT(folio_test_dirty(eb->folios[i]));
3622 #endif
3623 }
3624 
clear_extent_buffer_uptodate(struct extent_buffer * eb)3625 void clear_extent_buffer_uptodate(struct extent_buffer *eb)
3626 {
3627 
3628 	clear_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
3629 	for (int i = 0; i < num_extent_folios(eb); i++) {
3630 		struct folio *folio = eb->folios[i];
3631 
3632 		if (!folio)
3633 			continue;
3634 
3635 		btrfs_meta_folio_clear_uptodate(folio, eb);
3636 	}
3637 }
3638 
set_extent_buffer_uptodate(struct extent_buffer * eb)3639 void set_extent_buffer_uptodate(struct extent_buffer *eb)
3640 {
3641 
3642 	set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
3643 	for (int i = 0; i < num_extent_folios(eb); i++)
3644 		btrfs_meta_folio_set_uptodate(eb->folios[i], eb);
3645 }
3646 
clear_extent_buffer_reading(struct extent_buffer * eb)3647 static void clear_extent_buffer_reading(struct extent_buffer *eb)
3648 {
3649 	clear_bit(EXTENT_BUFFER_READING, &eb->bflags);
3650 	smp_mb__after_atomic();
3651 	wake_up_bit(&eb->bflags, EXTENT_BUFFER_READING);
3652 }
3653 
end_bbio_meta_read(struct btrfs_bio * bbio)3654 static void end_bbio_meta_read(struct btrfs_bio *bbio)
3655 {
3656 	struct extent_buffer *eb = bbio->private;
3657 	bool uptodate = !bbio->bio.bi_status;
3658 
3659 	/*
3660 	 * If the extent buffer is marked UPTODATE before the read operation
3661 	 * completes, other calls to read_extent_buffer_pages() will return
3662 	 * early without waiting for the read to finish, causing data races.
3663 	 */
3664 	WARN_ON(test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags));
3665 
3666 	eb->read_mirror = bbio->mirror_num;
3667 
3668 	if (uptodate &&
3669 	    btrfs_validate_extent_buffer(eb, &bbio->parent_check) < 0)
3670 		uptodate = false;
3671 
3672 	if (uptodate)
3673 		set_extent_buffer_uptodate(eb);
3674 	else
3675 		clear_extent_buffer_uptodate(eb);
3676 
3677 	clear_extent_buffer_reading(eb);
3678 	free_extent_buffer(eb);
3679 
3680 	bio_put(&bbio->bio);
3681 }
3682 
read_extent_buffer_pages_nowait(struct extent_buffer * eb,int mirror_num,const struct btrfs_tree_parent_check * check)3683 int read_extent_buffer_pages_nowait(struct extent_buffer *eb, int mirror_num,
3684 				    const struct btrfs_tree_parent_check *check)
3685 {
3686 	struct btrfs_bio *bbio;
3687 
3688 	if (test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags))
3689 		return 0;
3690 
3691 	/*
3692 	 * We could have had EXTENT_BUFFER_UPTODATE cleared by the write
3693 	 * operation, which could potentially still be in flight.  In this case
3694 	 * we simply want to return an error.
3695 	 */
3696 	if (unlikely(test_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags)))
3697 		return -EIO;
3698 
3699 	/* Someone else is already reading the buffer, just wait for it. */
3700 	if (test_and_set_bit(EXTENT_BUFFER_READING, &eb->bflags))
3701 		return 0;
3702 
3703 	/*
3704 	 * Between the initial test_bit(EXTENT_BUFFER_UPTODATE) and the above
3705 	 * test_and_set_bit(EXTENT_BUFFER_READING), someone else could have
3706 	 * started and finished reading the same eb.  In this case, UPTODATE
3707 	 * will now be set, and we shouldn't read it in again.
3708 	 */
3709 	if (unlikely(test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags))) {
3710 		clear_extent_buffer_reading(eb);
3711 		return 0;
3712 	}
3713 
3714 	eb->read_mirror = 0;
3715 	check_buffer_tree_ref(eb);
3716 	atomic_inc(&eb->refs);
3717 
3718 	bbio = btrfs_bio_alloc(INLINE_EXTENT_BUFFER_PAGES,
3719 			       REQ_OP_READ | REQ_META, eb->fs_info,
3720 			       end_bbio_meta_read, eb);
3721 	bbio->bio.bi_iter.bi_sector = eb->start >> SECTOR_SHIFT;
3722 	bbio->inode = BTRFS_I(eb->fs_info->btree_inode);
3723 	bbio->file_offset = eb->start;
3724 	memcpy(&bbio->parent_check, check, sizeof(*check));
3725 	for (int i = 0; i < num_extent_folios(eb); i++) {
3726 		struct folio *folio = eb->folios[i];
3727 		u64 range_start = max_t(u64, eb->start, folio_pos(folio));
3728 		u32 range_len = min_t(u64, folio_pos(folio) + folio_size(folio),
3729 				      eb->start + eb->len) - range_start;
3730 
3731 		bio_add_folio_nofail(&bbio->bio, folio, range_len,
3732 				     offset_in_folio(folio, range_start));
3733 	}
3734 	btrfs_submit_bbio(bbio, mirror_num);
3735 	return 0;
3736 }
3737 
read_extent_buffer_pages(struct extent_buffer * eb,int mirror_num,const struct btrfs_tree_parent_check * check)3738 int read_extent_buffer_pages(struct extent_buffer *eb, int mirror_num,
3739 			     const struct btrfs_tree_parent_check *check)
3740 {
3741 	int ret;
3742 
3743 	ret = read_extent_buffer_pages_nowait(eb, mirror_num, check);
3744 	if (ret < 0)
3745 		return ret;
3746 
3747 	wait_on_bit_io(&eb->bflags, EXTENT_BUFFER_READING, TASK_UNINTERRUPTIBLE);
3748 	if (!test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags))
3749 		return -EIO;
3750 	return 0;
3751 }
3752 
report_eb_range(const struct extent_buffer * eb,unsigned long start,unsigned long len)3753 static bool report_eb_range(const struct extent_buffer *eb, unsigned long start,
3754 			    unsigned long len)
3755 {
3756 	btrfs_warn(eb->fs_info,
3757 		"access to eb bytenr %llu len %u out of range start %lu len %lu",
3758 		eb->start, eb->len, start, len);
3759 	DEBUG_WARN();
3760 
3761 	return true;
3762 }
3763 
3764 /*
3765  * Check if the [start, start + len) range is valid before reading/writing
3766  * the eb.
3767  * NOTE: @start and @len are offset inside the eb, not logical address.
3768  *
3769  * Caller should not touch the dst/src memory if this function returns error.
3770  */
check_eb_range(const struct extent_buffer * eb,unsigned long start,unsigned long len)3771 static inline int check_eb_range(const struct extent_buffer *eb,
3772 				 unsigned long start, unsigned long len)
3773 {
3774 	unsigned long offset;
3775 
3776 	/* start, start + len should not go beyond eb->len nor overflow */
3777 	if (unlikely(check_add_overflow(start, len, &offset) || offset > eb->len))
3778 		return report_eb_range(eb, start, len);
3779 
3780 	return false;
3781 }
3782 
read_extent_buffer(const struct extent_buffer * eb,void * dstv,unsigned long start,unsigned long len)3783 void read_extent_buffer(const struct extent_buffer *eb, void *dstv,
3784 			unsigned long start, unsigned long len)
3785 {
3786 	const int unit_size = eb->folio_size;
3787 	size_t cur;
3788 	size_t offset;
3789 	char *dst = (char *)dstv;
3790 	unsigned long i = get_eb_folio_index(eb, start);
3791 
3792 	if (check_eb_range(eb, start, len)) {
3793 		/*
3794 		 * Invalid range hit, reset the memory, so callers won't get
3795 		 * some random garbage for their uninitialized memory.
3796 		 */
3797 		memset(dstv, 0, len);
3798 		return;
3799 	}
3800 
3801 	if (eb->addr) {
3802 		memcpy(dstv, eb->addr + start, len);
3803 		return;
3804 	}
3805 
3806 	offset = get_eb_offset_in_folio(eb, start);
3807 
3808 	while (len > 0) {
3809 		char *kaddr;
3810 
3811 		cur = min(len, unit_size - offset);
3812 		kaddr = folio_address(eb->folios[i]);
3813 		memcpy(dst, kaddr + offset, cur);
3814 
3815 		dst += cur;
3816 		len -= cur;
3817 		offset = 0;
3818 		i++;
3819 	}
3820 }
3821 
read_extent_buffer_to_user_nofault(const struct extent_buffer * eb,void __user * dstv,unsigned long start,unsigned long len)3822 int read_extent_buffer_to_user_nofault(const struct extent_buffer *eb,
3823 				       void __user *dstv,
3824 				       unsigned long start, unsigned long len)
3825 {
3826 	const int unit_size = eb->folio_size;
3827 	size_t cur;
3828 	size_t offset;
3829 	char __user *dst = (char __user *)dstv;
3830 	unsigned long i = get_eb_folio_index(eb, start);
3831 	int ret = 0;
3832 
3833 	WARN_ON(start > eb->len);
3834 	WARN_ON(start + len > eb->start + eb->len);
3835 
3836 	if (eb->addr) {
3837 		if (copy_to_user_nofault(dstv, eb->addr + start, len))
3838 			ret = -EFAULT;
3839 		return ret;
3840 	}
3841 
3842 	offset = get_eb_offset_in_folio(eb, start);
3843 
3844 	while (len > 0) {
3845 		char *kaddr;
3846 
3847 		cur = min(len, unit_size - offset);
3848 		kaddr = folio_address(eb->folios[i]);
3849 		if (copy_to_user_nofault(dst, kaddr + offset, cur)) {
3850 			ret = -EFAULT;
3851 			break;
3852 		}
3853 
3854 		dst += cur;
3855 		len -= cur;
3856 		offset = 0;
3857 		i++;
3858 	}
3859 
3860 	return ret;
3861 }
3862 
memcmp_extent_buffer(const struct extent_buffer * eb,const void * ptrv,unsigned long start,unsigned long len)3863 int memcmp_extent_buffer(const struct extent_buffer *eb, const void *ptrv,
3864 			 unsigned long start, unsigned long len)
3865 {
3866 	const int unit_size = eb->folio_size;
3867 	size_t cur;
3868 	size_t offset;
3869 	char *kaddr;
3870 	char *ptr = (char *)ptrv;
3871 	unsigned long i = get_eb_folio_index(eb, start);
3872 	int ret = 0;
3873 
3874 	if (check_eb_range(eb, start, len))
3875 		return -EINVAL;
3876 
3877 	if (eb->addr)
3878 		return memcmp(ptrv, eb->addr + start, len);
3879 
3880 	offset = get_eb_offset_in_folio(eb, start);
3881 
3882 	while (len > 0) {
3883 		cur = min(len, unit_size - offset);
3884 		kaddr = folio_address(eb->folios[i]);
3885 		ret = memcmp(ptr, kaddr + offset, cur);
3886 		if (ret)
3887 			break;
3888 
3889 		ptr += cur;
3890 		len -= cur;
3891 		offset = 0;
3892 		i++;
3893 	}
3894 	return ret;
3895 }
3896 
3897 /*
3898  * Check that the extent buffer is uptodate.
3899  *
3900  * For regular sector size == PAGE_SIZE case, check if @page is uptodate.
3901  * For subpage case, check if the range covered by the eb has EXTENT_UPTODATE.
3902  */
assert_eb_folio_uptodate(const struct extent_buffer * eb,int i)3903 static void assert_eb_folio_uptodate(const struct extent_buffer *eb, int i)
3904 {
3905 	struct btrfs_fs_info *fs_info = eb->fs_info;
3906 	struct folio *folio = eb->folios[i];
3907 
3908 	ASSERT(folio);
3909 
3910 	/*
3911 	 * If we are using the commit root we could potentially clear a page
3912 	 * Uptodate while we're using the extent buffer that we've previously
3913 	 * looked up.  We don't want to complain in this case, as the page was
3914 	 * valid before, we just didn't write it out.  Instead we want to catch
3915 	 * the case where we didn't actually read the block properly, which
3916 	 * would have !PageUptodate and !EXTENT_BUFFER_WRITE_ERR.
3917 	 */
3918 	if (test_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags))
3919 		return;
3920 
3921 	if (btrfs_meta_is_subpage(fs_info)) {
3922 		folio = eb->folios[0];
3923 		ASSERT(i == 0);
3924 		if (WARN_ON(!btrfs_subpage_test_uptodate(fs_info, folio,
3925 							 eb->start, eb->len)))
3926 			btrfs_subpage_dump_bitmap(fs_info, folio, eb->start, eb->len);
3927 	} else {
3928 		WARN_ON(!folio_test_uptodate(folio));
3929 	}
3930 }
3931 
__write_extent_buffer(const struct extent_buffer * eb,const void * srcv,unsigned long start,unsigned long len,bool use_memmove)3932 static void __write_extent_buffer(const struct extent_buffer *eb,
3933 				  const void *srcv, unsigned long start,
3934 				  unsigned long len, bool use_memmove)
3935 {
3936 	const int unit_size = eb->folio_size;
3937 	size_t cur;
3938 	size_t offset;
3939 	char *kaddr;
3940 	const char *src = (const char *)srcv;
3941 	unsigned long i = get_eb_folio_index(eb, start);
3942 	/* For unmapped (dummy) ebs, no need to check their uptodate status. */
3943 	const bool check_uptodate = !test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags);
3944 
3945 	if (check_eb_range(eb, start, len))
3946 		return;
3947 
3948 	if (eb->addr) {
3949 		if (use_memmove)
3950 			memmove(eb->addr + start, srcv, len);
3951 		else
3952 			memcpy(eb->addr + start, srcv, len);
3953 		return;
3954 	}
3955 
3956 	offset = get_eb_offset_in_folio(eb, start);
3957 
3958 	while (len > 0) {
3959 		if (check_uptodate)
3960 			assert_eb_folio_uptodate(eb, i);
3961 
3962 		cur = min(len, unit_size - offset);
3963 		kaddr = folio_address(eb->folios[i]);
3964 		if (use_memmove)
3965 			memmove(kaddr + offset, src, cur);
3966 		else
3967 			memcpy(kaddr + offset, src, cur);
3968 
3969 		src += cur;
3970 		len -= cur;
3971 		offset = 0;
3972 		i++;
3973 	}
3974 }
3975 
write_extent_buffer(const struct extent_buffer * eb,const void * srcv,unsigned long start,unsigned long len)3976 void write_extent_buffer(const struct extent_buffer *eb, const void *srcv,
3977 			 unsigned long start, unsigned long len)
3978 {
3979 	return __write_extent_buffer(eb, srcv, start, len, false);
3980 }
3981 
memset_extent_buffer(const struct extent_buffer * eb,int c,unsigned long start,unsigned long len)3982 static void memset_extent_buffer(const struct extent_buffer *eb, int c,
3983 				 unsigned long start, unsigned long len)
3984 {
3985 	const int unit_size = eb->folio_size;
3986 	unsigned long cur = start;
3987 
3988 	if (eb->addr) {
3989 		memset(eb->addr + start, c, len);
3990 		return;
3991 	}
3992 
3993 	while (cur < start + len) {
3994 		unsigned long index = get_eb_folio_index(eb, cur);
3995 		unsigned int offset = get_eb_offset_in_folio(eb, cur);
3996 		unsigned int cur_len = min(start + len - cur, unit_size - offset);
3997 
3998 		assert_eb_folio_uptodate(eb, index);
3999 		memset(folio_address(eb->folios[index]) + offset, c, cur_len);
4000 
4001 		cur += cur_len;
4002 	}
4003 }
4004 
memzero_extent_buffer(const struct extent_buffer * eb,unsigned long start,unsigned long len)4005 void memzero_extent_buffer(const struct extent_buffer *eb, unsigned long start,
4006 			   unsigned long len)
4007 {
4008 	if (check_eb_range(eb, start, len))
4009 		return;
4010 	return memset_extent_buffer(eb, 0, start, len);
4011 }
4012 
copy_extent_buffer_full(const struct extent_buffer * dst,const struct extent_buffer * src)4013 void copy_extent_buffer_full(const struct extent_buffer *dst,
4014 			     const struct extent_buffer *src)
4015 {
4016 	const int unit_size = src->folio_size;
4017 	unsigned long cur = 0;
4018 
4019 	ASSERT(dst->len == src->len);
4020 
4021 	while (cur < src->len) {
4022 		unsigned long index = get_eb_folio_index(src, cur);
4023 		unsigned long offset = get_eb_offset_in_folio(src, cur);
4024 		unsigned long cur_len = min(src->len, unit_size - offset);
4025 		void *addr = folio_address(src->folios[index]) + offset;
4026 
4027 		write_extent_buffer(dst, addr, cur, cur_len);
4028 
4029 		cur += cur_len;
4030 	}
4031 }
4032 
copy_extent_buffer(const struct extent_buffer * dst,const struct extent_buffer * src,unsigned long dst_offset,unsigned long src_offset,unsigned long len)4033 void copy_extent_buffer(const struct extent_buffer *dst,
4034 			const struct extent_buffer *src,
4035 			unsigned long dst_offset, unsigned long src_offset,
4036 			unsigned long len)
4037 {
4038 	const int unit_size = dst->folio_size;
4039 	u64 dst_len = dst->len;
4040 	size_t cur;
4041 	size_t offset;
4042 	char *kaddr;
4043 	unsigned long i = get_eb_folio_index(dst, dst_offset);
4044 
4045 	if (check_eb_range(dst, dst_offset, len) ||
4046 	    check_eb_range(src, src_offset, len))
4047 		return;
4048 
4049 	WARN_ON(src->len != dst_len);
4050 
4051 	offset = get_eb_offset_in_folio(dst, dst_offset);
4052 
4053 	while (len > 0) {
4054 		assert_eb_folio_uptodate(dst, i);
4055 
4056 		cur = min(len, (unsigned long)(unit_size - offset));
4057 
4058 		kaddr = folio_address(dst->folios[i]);
4059 		read_extent_buffer(src, kaddr + offset, src_offset, cur);
4060 
4061 		src_offset += cur;
4062 		len -= cur;
4063 		offset = 0;
4064 		i++;
4065 	}
4066 }
4067 
4068 /*
4069  * Calculate the folio and offset of the byte containing the given bit number.
4070  *
4071  * @eb:           the extent buffer
4072  * @start:        offset of the bitmap item in the extent buffer
4073  * @nr:           bit number
4074  * @folio_index:  return index of the folio in the extent buffer that contains
4075  *                the given bit number
4076  * @folio_offset: return offset into the folio given by folio_index
4077  *
4078  * This helper hides the ugliness of finding the byte in an extent buffer which
4079  * contains a given bit.
4080  */
eb_bitmap_offset(const struct extent_buffer * eb,unsigned long start,unsigned long nr,unsigned long * folio_index,size_t * folio_offset)4081 static inline void eb_bitmap_offset(const struct extent_buffer *eb,
4082 				    unsigned long start, unsigned long nr,
4083 				    unsigned long *folio_index,
4084 				    size_t *folio_offset)
4085 {
4086 	size_t byte_offset = BIT_BYTE(nr);
4087 	size_t offset;
4088 
4089 	/*
4090 	 * The byte we want is the offset of the extent buffer + the offset of
4091 	 * the bitmap item in the extent buffer + the offset of the byte in the
4092 	 * bitmap item.
4093 	 */
4094 	offset = start + offset_in_eb_folio(eb, eb->start) + byte_offset;
4095 
4096 	*folio_index = offset >> eb->folio_shift;
4097 	*folio_offset = offset_in_eb_folio(eb, offset);
4098 }
4099 
4100 /*
4101  * Determine whether a bit in a bitmap item is set.
4102  *
4103  * @eb:     the extent buffer
4104  * @start:  offset of the bitmap item in the extent buffer
4105  * @nr:     bit number to test
4106  */
extent_buffer_test_bit(const struct extent_buffer * eb,unsigned long start,unsigned long nr)4107 int extent_buffer_test_bit(const struct extent_buffer *eb, unsigned long start,
4108 			   unsigned long nr)
4109 {
4110 	unsigned long i;
4111 	size_t offset;
4112 	u8 *kaddr;
4113 
4114 	eb_bitmap_offset(eb, start, nr, &i, &offset);
4115 	assert_eb_folio_uptodate(eb, i);
4116 	kaddr = folio_address(eb->folios[i]);
4117 	return 1U & (kaddr[offset] >> (nr & (BITS_PER_BYTE - 1)));
4118 }
4119 
extent_buffer_get_byte(const struct extent_buffer * eb,unsigned long bytenr)4120 static u8 *extent_buffer_get_byte(const struct extent_buffer *eb, unsigned long bytenr)
4121 {
4122 	unsigned long index = get_eb_folio_index(eb, bytenr);
4123 
4124 	if (check_eb_range(eb, bytenr, 1))
4125 		return NULL;
4126 	return folio_address(eb->folios[index]) + get_eb_offset_in_folio(eb, bytenr);
4127 }
4128 
4129 /*
4130  * Set an area of a bitmap to 1.
4131  *
4132  * @eb:     the extent buffer
4133  * @start:  offset of the bitmap item in the extent buffer
4134  * @pos:    bit number of the first bit
4135  * @len:    number of bits to set
4136  */
extent_buffer_bitmap_set(const struct extent_buffer * eb,unsigned long start,unsigned long pos,unsigned long len)4137 void extent_buffer_bitmap_set(const struct extent_buffer *eb, unsigned long start,
4138 			      unsigned long pos, unsigned long len)
4139 {
4140 	unsigned int first_byte = start + BIT_BYTE(pos);
4141 	unsigned int last_byte = start + BIT_BYTE(pos + len - 1);
4142 	const bool same_byte = (first_byte == last_byte);
4143 	u8 mask = BITMAP_FIRST_BYTE_MASK(pos);
4144 	u8 *kaddr;
4145 
4146 	if (same_byte)
4147 		mask &= BITMAP_LAST_BYTE_MASK(pos + len);
4148 
4149 	/* Handle the first byte. */
4150 	kaddr = extent_buffer_get_byte(eb, first_byte);
4151 	*kaddr |= mask;
4152 	if (same_byte)
4153 		return;
4154 
4155 	/* Handle the byte aligned part. */
4156 	ASSERT(first_byte + 1 <= last_byte);
4157 	memset_extent_buffer(eb, 0xff, first_byte + 1, last_byte - first_byte - 1);
4158 
4159 	/* Handle the last byte. */
4160 	kaddr = extent_buffer_get_byte(eb, last_byte);
4161 	*kaddr |= BITMAP_LAST_BYTE_MASK(pos + len);
4162 }
4163 
4164 
4165 /*
4166  * Clear an area of a bitmap.
4167  *
4168  * @eb:     the extent buffer
4169  * @start:  offset of the bitmap item in the extent buffer
4170  * @pos:    bit number of the first bit
4171  * @len:    number of bits to clear
4172  */
extent_buffer_bitmap_clear(const struct extent_buffer * eb,unsigned long start,unsigned long pos,unsigned long len)4173 void extent_buffer_bitmap_clear(const struct extent_buffer *eb,
4174 				unsigned long start, unsigned long pos,
4175 				unsigned long len)
4176 {
4177 	unsigned int first_byte = start + BIT_BYTE(pos);
4178 	unsigned int last_byte = start + BIT_BYTE(pos + len - 1);
4179 	const bool same_byte = (first_byte == last_byte);
4180 	u8 mask = BITMAP_FIRST_BYTE_MASK(pos);
4181 	u8 *kaddr;
4182 
4183 	if (same_byte)
4184 		mask &= BITMAP_LAST_BYTE_MASK(pos + len);
4185 
4186 	/* Handle the first byte. */
4187 	kaddr = extent_buffer_get_byte(eb, first_byte);
4188 	*kaddr &= ~mask;
4189 	if (same_byte)
4190 		return;
4191 
4192 	/* Handle the byte aligned part. */
4193 	ASSERT(first_byte + 1 <= last_byte);
4194 	memset_extent_buffer(eb, 0, first_byte + 1, last_byte - first_byte - 1);
4195 
4196 	/* Handle the last byte. */
4197 	kaddr = extent_buffer_get_byte(eb, last_byte);
4198 	*kaddr &= ~BITMAP_LAST_BYTE_MASK(pos + len);
4199 }
4200 
areas_overlap(unsigned long src,unsigned long dst,unsigned long len)4201 static inline bool areas_overlap(unsigned long src, unsigned long dst, unsigned long len)
4202 {
4203 	unsigned long distance = (src > dst) ? src - dst : dst - src;
4204 	return distance < len;
4205 }
4206 
memcpy_extent_buffer(const struct extent_buffer * dst,unsigned long dst_offset,unsigned long src_offset,unsigned long len)4207 void memcpy_extent_buffer(const struct extent_buffer *dst,
4208 			  unsigned long dst_offset, unsigned long src_offset,
4209 			  unsigned long len)
4210 {
4211 	const int unit_size = dst->folio_size;
4212 	unsigned long cur_off = 0;
4213 
4214 	if (check_eb_range(dst, dst_offset, len) ||
4215 	    check_eb_range(dst, src_offset, len))
4216 		return;
4217 
4218 	if (dst->addr) {
4219 		const bool use_memmove = areas_overlap(src_offset, dst_offset, len);
4220 
4221 		if (use_memmove)
4222 			memmove(dst->addr + dst_offset, dst->addr + src_offset, len);
4223 		else
4224 			memcpy(dst->addr + dst_offset, dst->addr + src_offset, len);
4225 		return;
4226 	}
4227 
4228 	while (cur_off < len) {
4229 		unsigned long cur_src = cur_off + src_offset;
4230 		unsigned long folio_index = get_eb_folio_index(dst, cur_src);
4231 		unsigned long folio_off = get_eb_offset_in_folio(dst, cur_src);
4232 		unsigned long cur_len = min(src_offset + len - cur_src,
4233 					    unit_size - folio_off);
4234 		void *src_addr = folio_address(dst->folios[folio_index]) + folio_off;
4235 		const bool use_memmove = areas_overlap(src_offset + cur_off,
4236 						       dst_offset + cur_off, cur_len);
4237 
4238 		__write_extent_buffer(dst, src_addr, dst_offset + cur_off, cur_len,
4239 				      use_memmove);
4240 		cur_off += cur_len;
4241 	}
4242 }
4243 
memmove_extent_buffer(const struct extent_buffer * dst,unsigned long dst_offset,unsigned long src_offset,unsigned long len)4244 void memmove_extent_buffer(const struct extent_buffer *dst,
4245 			   unsigned long dst_offset, unsigned long src_offset,
4246 			   unsigned long len)
4247 {
4248 	unsigned long dst_end = dst_offset + len - 1;
4249 	unsigned long src_end = src_offset + len - 1;
4250 
4251 	if (check_eb_range(dst, dst_offset, len) ||
4252 	    check_eb_range(dst, src_offset, len))
4253 		return;
4254 
4255 	if (dst_offset < src_offset) {
4256 		memcpy_extent_buffer(dst, dst_offset, src_offset, len);
4257 		return;
4258 	}
4259 
4260 	if (dst->addr) {
4261 		memmove(dst->addr + dst_offset, dst->addr + src_offset, len);
4262 		return;
4263 	}
4264 
4265 	while (len > 0) {
4266 		unsigned long src_i;
4267 		size_t cur;
4268 		size_t dst_off_in_folio;
4269 		size_t src_off_in_folio;
4270 		void *src_addr;
4271 		bool use_memmove;
4272 
4273 		src_i = get_eb_folio_index(dst, src_end);
4274 
4275 		dst_off_in_folio = get_eb_offset_in_folio(dst, dst_end);
4276 		src_off_in_folio = get_eb_offset_in_folio(dst, src_end);
4277 
4278 		cur = min_t(unsigned long, len, src_off_in_folio + 1);
4279 		cur = min(cur, dst_off_in_folio + 1);
4280 
4281 		src_addr = folio_address(dst->folios[src_i]) + src_off_in_folio -
4282 					 cur + 1;
4283 		use_memmove = areas_overlap(src_end - cur + 1, dst_end - cur + 1,
4284 					    cur);
4285 
4286 		__write_extent_buffer(dst, src_addr, dst_end - cur + 1, cur,
4287 				      use_memmove);
4288 
4289 		dst_end -= cur;
4290 		src_end -= cur;
4291 		len -= cur;
4292 	}
4293 }
4294 
try_release_subpage_extent_buffer(struct folio * folio)4295 static int try_release_subpage_extent_buffer(struct folio *folio)
4296 {
4297 	struct btrfs_fs_info *fs_info = folio_to_fs_info(folio);
4298 	struct extent_buffer *eb;
4299 	unsigned long start = (folio_pos(folio) >> fs_info->sectorsize_bits);
4300 	unsigned long index = start;
4301 	unsigned long end = index + (PAGE_SIZE >> fs_info->sectorsize_bits) - 1;
4302 	int ret;
4303 
4304 	xa_lock_irq(&fs_info->buffer_tree);
4305 	xa_for_each_range(&fs_info->buffer_tree, index, eb, start, end) {
4306 		/*
4307 		 * The same as try_release_extent_buffer(), to ensure the eb
4308 		 * won't disappear out from under us.
4309 		 */
4310 		spin_lock(&eb->refs_lock);
4311 		if (atomic_read(&eb->refs) != 1 || extent_buffer_under_io(eb)) {
4312 			spin_unlock(&eb->refs_lock);
4313 			continue;
4314 		}
4315 
4316 		/*
4317 		 * If tree ref isn't set then we know the ref on this eb is a
4318 		 * real ref, so just return, this eb will likely be freed soon
4319 		 * anyway.
4320 		 */
4321 		if (!test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) {
4322 			spin_unlock(&eb->refs_lock);
4323 			break;
4324 		}
4325 
4326 		/*
4327 		 * Here we don't care about the return value, we will always
4328 		 * check the folio private at the end.  And
4329 		 * release_extent_buffer() will release the refs_lock.
4330 		 */
4331 		xa_unlock_irq(&fs_info->buffer_tree);
4332 		release_extent_buffer(eb);
4333 		xa_lock_irq(&fs_info->buffer_tree);
4334 	}
4335 	xa_unlock_irq(&fs_info->buffer_tree);
4336 
4337 	/*
4338 	 * Finally to check if we have cleared folio private, as if we have
4339 	 * released all ebs in the page, the folio private should be cleared now.
4340 	 */
4341 	spin_lock(&folio->mapping->i_private_lock);
4342 	if (!folio_test_private(folio))
4343 		ret = 1;
4344 	else
4345 		ret = 0;
4346 	spin_unlock(&folio->mapping->i_private_lock);
4347 	return ret;
4348 
4349 }
4350 
try_release_extent_buffer(struct folio * folio)4351 int try_release_extent_buffer(struct folio *folio)
4352 {
4353 	struct extent_buffer *eb;
4354 
4355 	if (btrfs_meta_is_subpage(folio_to_fs_info(folio)))
4356 		return try_release_subpage_extent_buffer(folio);
4357 
4358 	/*
4359 	 * We need to make sure nobody is changing folio private, as we rely on
4360 	 * folio private as the pointer to extent buffer.
4361 	 */
4362 	spin_lock(&folio->mapping->i_private_lock);
4363 	if (!folio_test_private(folio)) {
4364 		spin_unlock(&folio->mapping->i_private_lock);
4365 		return 1;
4366 	}
4367 
4368 	eb = folio_get_private(folio);
4369 	BUG_ON(!eb);
4370 
4371 	/*
4372 	 * This is a little awful but should be ok, we need to make sure that
4373 	 * the eb doesn't disappear out from under us while we're looking at
4374 	 * this page.
4375 	 */
4376 	spin_lock(&eb->refs_lock);
4377 	if (atomic_read(&eb->refs) != 1 || extent_buffer_under_io(eb)) {
4378 		spin_unlock(&eb->refs_lock);
4379 		spin_unlock(&folio->mapping->i_private_lock);
4380 		return 0;
4381 	}
4382 	spin_unlock(&folio->mapping->i_private_lock);
4383 
4384 	/*
4385 	 * If tree ref isn't set then we know the ref on this eb is a real ref,
4386 	 * so just return, this page will likely be freed soon anyway.
4387 	 */
4388 	if (!test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) {
4389 		spin_unlock(&eb->refs_lock);
4390 		return 0;
4391 	}
4392 
4393 	return release_extent_buffer(eb);
4394 }
4395 
4396 /*
4397  * Attempt to readahead a child block.
4398  *
4399  * @fs_info:	the fs_info
4400  * @bytenr:	bytenr to read
4401  * @owner_root: objectid of the root that owns this eb
4402  * @gen:	generation for the uptodate check, can be 0
4403  * @level:	level for the eb
4404  *
4405  * Attempt to readahead a tree block at @bytenr.  If @gen is 0 then we do a
4406  * normal uptodate check of the eb, without checking the generation.  If we have
4407  * to read the block we will not block on anything.
4408  */
btrfs_readahead_tree_block(struct btrfs_fs_info * fs_info,u64 bytenr,u64 owner_root,u64 gen,int level)4409 void btrfs_readahead_tree_block(struct btrfs_fs_info *fs_info,
4410 				u64 bytenr, u64 owner_root, u64 gen, int level)
4411 {
4412 	struct btrfs_tree_parent_check check = {
4413 		.level = level,
4414 		.transid = gen
4415 	};
4416 	struct extent_buffer *eb;
4417 	int ret;
4418 
4419 	eb = btrfs_find_create_tree_block(fs_info, bytenr, owner_root, level);
4420 	if (IS_ERR(eb))
4421 		return;
4422 
4423 	if (btrfs_buffer_uptodate(eb, gen, 1)) {
4424 		free_extent_buffer(eb);
4425 		return;
4426 	}
4427 
4428 	ret = read_extent_buffer_pages_nowait(eb, 0, &check);
4429 	if (ret < 0)
4430 		free_extent_buffer_stale(eb);
4431 	else
4432 		free_extent_buffer(eb);
4433 }
4434 
4435 /*
4436  * Readahead a node's child block.
4437  *
4438  * @node:	parent node we're reading from
4439  * @slot:	slot in the parent node for the child we want to read
4440  *
4441  * A helper for btrfs_readahead_tree_block, we simply read the bytenr pointed at
4442  * the slot in the node provided.
4443  */
btrfs_readahead_node_child(struct extent_buffer * node,int slot)4444 void btrfs_readahead_node_child(struct extent_buffer *node, int slot)
4445 {
4446 	btrfs_readahead_tree_block(node->fs_info,
4447 				   btrfs_node_blockptr(node, slot),
4448 				   btrfs_header_owner(node),
4449 				   btrfs_node_ptr_generation(node, slot),
4450 				   btrfs_header_level(node) - 1);
4451 }
4452