xref: /linux/fs/btrfs/scrub.c (revision f3827213abae9291b7525b05e6fd29b1f0536ce6)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) 2011, 2012 STRATO.  All rights reserved.
4  */
5 
6 #include <linux/blkdev.h>
7 #include <linux/ratelimit.h>
8 #include <linux/sched/mm.h>
9 #include <crypto/hash.h>
10 #include "ctree.h"
11 #include "discard.h"
12 #include "volumes.h"
13 #include "disk-io.h"
14 #include "ordered-data.h"
15 #include "transaction.h"
16 #include "backref.h"
17 #include "extent_io.h"
18 #include "dev-replace.h"
19 #include "raid56.h"
20 #include "block-group.h"
21 #include "zoned.h"
22 #include "fs.h"
23 #include "accessors.h"
24 #include "file-item.h"
25 #include "scrub.h"
26 #include "raid-stripe-tree.h"
27 
28 /*
29  * This is only the first step towards a full-features scrub. It reads all
30  * extent and super block and verifies the checksums. In case a bad checksum
31  * is found or the extent cannot be read, good data will be written back if
32  * any can be found.
33  *
34  * Future enhancements:
35  *  - In case an unrepairable extent is encountered, track which files are
36  *    affected and report them
37  *  - track and record media errors, throw out bad devices
38  *  - add a mode to also read unallocated space
39  */
40 
41 struct scrub_ctx;
42 
43 /*
44  * The following value only influences the performance.
45  *
46  * This determines how many stripes would be submitted in one go,
47  * which is 512KiB (BTRFS_STRIPE_LEN * SCRUB_STRIPES_PER_GROUP).
48  */
49 #define SCRUB_STRIPES_PER_GROUP		8
50 
51 /*
52  * How many groups we have for each sctx.
53  *
54  * This would be 8M per device, the same value as the old scrub in-flight bios
55  * size limit.
56  */
57 #define SCRUB_GROUPS_PER_SCTX		16
58 
59 #define SCRUB_TOTAL_STRIPES		(SCRUB_GROUPS_PER_SCTX * SCRUB_STRIPES_PER_GROUP)
60 
61 /*
62  * The following value times PAGE_SIZE needs to be large enough to match the
63  * largest node/leaf/sector size that shall be supported.
64  */
65 #define SCRUB_MAX_SECTORS_PER_BLOCK	(BTRFS_MAX_METADATA_BLOCKSIZE / SZ_4K)
66 
67 /* Represent one sector and its needed info to verify the content. */
68 struct scrub_sector_verification {
69 	union {
70 		/*
71 		 * Csum pointer for data csum verification.  Should point to a
72 		 * sector csum inside scrub_stripe::csums.
73 		 *
74 		 * NULL if this data sector has no csum.
75 		 */
76 		u8 *csum;
77 
78 		/*
79 		 * Extra info for metadata verification.  All sectors inside a
80 		 * tree block share the same generation.
81 		 */
82 		u64 generation;
83 	};
84 };
85 
86 enum scrub_stripe_flags {
87 	/* Set when @mirror_num, @dev, @physical and @logical are set. */
88 	SCRUB_STRIPE_FLAG_INITIALIZED,
89 
90 	/* Set when the read-repair is finished. */
91 	SCRUB_STRIPE_FLAG_REPAIR_DONE,
92 
93 	/*
94 	 * Set for data stripes if it's triggered from P/Q stripe.
95 	 * During such scrub, we should not report errors in data stripes, nor
96 	 * update the accounting.
97 	 */
98 	SCRUB_STRIPE_FLAG_NO_REPORT,
99 };
100 
101 /*
102  * We have multiple bitmaps for one scrub_stripe.
103  * However each bitmap has at most (BTRFS_STRIPE_LEN / blocksize) bits,
104  * which is normally 16, and much smaller than BITS_PER_LONG (32 or 64).
105  *
106  * So to reduce memory usage for each scrub_stripe, we pack those bitmaps
107  * into a larger one.
108  *
109  * These enum records where the sub-bitmap are inside the larger one.
110  * Each subbitmap starts at scrub_bitmap_nr_##name * nr_sectors bit.
111  */
112 enum {
113 	/* Which blocks are covered by extent items. */
114 	scrub_bitmap_nr_has_extent = 0,
115 
116 	/* Which blocks are metadata. */
117 	scrub_bitmap_nr_is_metadata,
118 
119 	/*
120 	 * Which blocks have errors, including IO, csum, and metadata
121 	 * errors.
122 	 * This sub-bitmap is the OR results of the next few error related
123 	 * sub-bitmaps.
124 	 */
125 	scrub_bitmap_nr_error,
126 	scrub_bitmap_nr_io_error,
127 	scrub_bitmap_nr_csum_error,
128 	scrub_bitmap_nr_meta_error,
129 	scrub_bitmap_nr_meta_gen_error,
130 	scrub_bitmap_nr_last,
131 };
132 
133 #define SCRUB_STRIPE_MAX_FOLIOS		(BTRFS_STRIPE_LEN / PAGE_SIZE)
134 
135 /*
136  * Represent one contiguous range with a length of BTRFS_STRIPE_LEN.
137  */
138 struct scrub_stripe {
139 	struct scrub_ctx *sctx;
140 	struct btrfs_block_group *bg;
141 
142 	struct folio *folios[SCRUB_STRIPE_MAX_FOLIOS];
143 	struct scrub_sector_verification *sectors;
144 
145 	struct btrfs_device *dev;
146 	u64 logical;
147 	u64 physical;
148 
149 	u16 mirror_num;
150 
151 	/* Should be BTRFS_STRIPE_LEN / sectorsize. */
152 	u16 nr_sectors;
153 
154 	/*
155 	 * How many data/meta extents are in this stripe.  Only for scrub status
156 	 * reporting purposes.
157 	 */
158 	u16 nr_data_extents;
159 	u16 nr_meta_extents;
160 
161 	atomic_t pending_io;
162 	wait_queue_head_t io_wait;
163 	wait_queue_head_t repair_wait;
164 
165 	/*
166 	 * Indicate the states of the stripe.  Bits are defined in
167 	 * scrub_stripe_flags enum.
168 	 */
169 	unsigned long state;
170 
171 	/* The large bitmap contains all the sub-bitmaps. */
172 	unsigned long bitmaps[BITS_TO_LONGS(scrub_bitmap_nr_last *
173 					    (BTRFS_STRIPE_LEN / BTRFS_MIN_BLOCKSIZE))];
174 
175 	/*
176 	 * For writeback (repair or replace) error reporting.
177 	 * This one is protected by a spinlock, thus can not be packed into
178 	 * the larger bitmap.
179 	 */
180 	unsigned long write_error_bitmap;
181 
182 	/* Writeback can be concurrent, thus we need to protect the bitmap. */
183 	spinlock_t write_error_lock;
184 
185 	/*
186 	 * Checksum for the whole stripe if this stripe is inside a data block
187 	 * group.
188 	 */
189 	u8 *csums;
190 
191 	struct work_struct work;
192 };
193 
194 struct scrub_ctx {
195 	struct scrub_stripe	stripes[SCRUB_TOTAL_STRIPES];
196 	struct scrub_stripe	*raid56_data_stripes;
197 	struct btrfs_fs_info	*fs_info;
198 	struct btrfs_path	extent_path;
199 	struct btrfs_path	csum_path;
200 	int			first_free;
201 	int			cur_stripe;
202 	atomic_t		cancel_req;
203 	int			readonly;
204 
205 	/* State of IO submission throttling affecting the associated device */
206 	ktime_t			throttle_deadline;
207 	u64			throttle_sent;
208 
209 	bool			is_dev_replace;
210 	u64			write_pointer;
211 
212 	struct mutex            wr_lock;
213 	struct btrfs_device     *wr_tgtdev;
214 
215 	/*
216 	 * statistics
217 	 */
218 	struct btrfs_scrub_progress stat;
219 	spinlock_t		stat_lock;
220 
221 	/*
222 	 * Use a ref counter to avoid use-after-free issues. Scrub workers
223 	 * decrement bios_in_flight and workers_pending and then do a wakeup
224 	 * on the list_wait wait queue. We must ensure the main scrub task
225 	 * doesn't free the scrub context before or while the workers are
226 	 * doing the wakeup() call.
227 	 */
228 	refcount_t              refs;
229 };
230 
231 #define scrub_calc_start_bit(stripe, name, block_nr)			\
232 ({									\
233 	unsigned int __start_bit;					\
234 									\
235 	ASSERT(block_nr < stripe->nr_sectors,				\
236 		"nr_sectors=%u block_nr=%u", stripe->nr_sectors, block_nr); \
237 	__start_bit = scrub_bitmap_nr_##name * stripe->nr_sectors + block_nr; \
238 	__start_bit;							\
239 })
240 
241 #define IMPLEMENT_SCRUB_BITMAP_OPS(name)				\
242 static inline void scrub_bitmap_set_##name(struct scrub_stripe *stripe,	\
243 				    unsigned int block_nr,		\
244 				    unsigned int nr_blocks)		\
245 {									\
246 	const unsigned int start_bit = scrub_calc_start_bit(stripe,	\
247 							    name, block_nr); \
248 									\
249 	bitmap_set(stripe->bitmaps, start_bit, nr_blocks);		\
250 }									\
251 static inline void scrub_bitmap_clear_##name(struct scrub_stripe *stripe, \
252 				      unsigned int block_nr,		\
253 				      unsigned int nr_blocks)		\
254 {									\
255 	const unsigned int start_bit = scrub_calc_start_bit(stripe, name, \
256 							    block_nr);	\
257 									\
258 	bitmap_clear(stripe->bitmaps, start_bit, nr_blocks);		\
259 }									\
260 static inline bool scrub_bitmap_test_bit_##name(struct scrub_stripe *stripe, \
261 				     unsigned int block_nr)		\
262 {									\
263 	const unsigned int start_bit = scrub_calc_start_bit(stripe, name, \
264 							    block_nr);	\
265 									\
266 	return test_bit(start_bit, stripe->bitmaps);			\
267 }									\
268 static inline void scrub_bitmap_set_bit_##name(struct scrub_stripe *stripe, \
269 				     unsigned int block_nr)		\
270 {									\
271 	const unsigned int start_bit = scrub_calc_start_bit(stripe, name, \
272 							    block_nr);	\
273 									\
274 	set_bit(start_bit, stripe->bitmaps);				\
275 }									\
276 static inline void scrub_bitmap_clear_bit_##name(struct scrub_stripe *stripe, \
277 				     unsigned int block_nr)		\
278 {									\
279 	const unsigned int start_bit = scrub_calc_start_bit(stripe, name, \
280 							    block_nr);	\
281 									\
282 	clear_bit(start_bit, stripe->bitmaps);				\
283 }									\
284 static inline unsigned long scrub_bitmap_read_##name(struct scrub_stripe *stripe) \
285 {									\
286 	const unsigned int nr_blocks = stripe->nr_sectors;		\
287 									\
288 	ASSERT(nr_blocks > 0 && nr_blocks <= BITS_PER_LONG,		\
289 	       "nr_blocks=%u BITS_PER_LONG=%u",				\
290 	       nr_blocks, BITS_PER_LONG);				\
291 									\
292 	return bitmap_read(stripe->bitmaps, nr_blocks * scrub_bitmap_nr_##name, \
293 			   stripe->nr_sectors);				\
294 }									\
295 static inline bool scrub_bitmap_empty_##name(struct scrub_stripe *stripe) \
296 {									\
297 	unsigned long bitmap = scrub_bitmap_read_##name(stripe);	\
298 									\
299 	return bitmap_empty(&bitmap, stripe->nr_sectors);		\
300 }									\
301 static inline unsigned int scrub_bitmap_weight_##name(struct scrub_stripe *stripe) \
302 {									\
303 	unsigned long bitmap = scrub_bitmap_read_##name(stripe);	\
304 									\
305 	return bitmap_weight(&bitmap, stripe->nr_sectors);		\
306 }
307 IMPLEMENT_SCRUB_BITMAP_OPS(has_extent);
308 IMPLEMENT_SCRUB_BITMAP_OPS(is_metadata);
309 IMPLEMENT_SCRUB_BITMAP_OPS(error);
310 IMPLEMENT_SCRUB_BITMAP_OPS(io_error);
311 IMPLEMENT_SCRUB_BITMAP_OPS(csum_error);
312 IMPLEMENT_SCRUB_BITMAP_OPS(meta_error);
313 IMPLEMENT_SCRUB_BITMAP_OPS(meta_gen_error);
314 
315 struct scrub_warning {
316 	struct btrfs_path	*path;
317 	u64			extent_item_size;
318 	const char		*errstr;
319 	u64			physical;
320 	u64			logical;
321 	struct btrfs_device	*dev;
322 };
323 
324 struct scrub_error_records {
325 	/*
326 	 * Bitmap recording which blocks hit errors (IO/csum/...) during the
327 	 * initial read.
328 	 */
329 	unsigned long init_error_bitmap;
330 
331 	unsigned int nr_io_errors;
332 	unsigned int nr_csum_errors;
333 	unsigned int nr_meta_errors;
334 	unsigned int nr_meta_gen_errors;
335 };
336 
release_scrub_stripe(struct scrub_stripe * stripe)337 static void release_scrub_stripe(struct scrub_stripe *stripe)
338 {
339 	if (!stripe)
340 		return;
341 
342 	for (int i = 0; i < SCRUB_STRIPE_MAX_FOLIOS; i++) {
343 		if (stripe->folios[i])
344 			folio_put(stripe->folios[i]);
345 		stripe->folios[i] = NULL;
346 	}
347 	kfree(stripe->sectors);
348 	kfree(stripe->csums);
349 	stripe->sectors = NULL;
350 	stripe->csums = NULL;
351 	stripe->sctx = NULL;
352 	stripe->state = 0;
353 }
354 
init_scrub_stripe(struct btrfs_fs_info * fs_info,struct scrub_stripe * stripe)355 static int init_scrub_stripe(struct btrfs_fs_info *fs_info,
356 			     struct scrub_stripe *stripe)
357 {
358 	const u32 min_folio_shift = PAGE_SHIFT + fs_info->block_min_order;
359 	int ret;
360 
361 	memset(stripe, 0, sizeof(*stripe));
362 
363 	stripe->nr_sectors = BTRFS_STRIPE_LEN >> fs_info->sectorsize_bits;
364 	stripe->state = 0;
365 
366 	init_waitqueue_head(&stripe->io_wait);
367 	init_waitqueue_head(&stripe->repair_wait);
368 	atomic_set(&stripe->pending_io, 0);
369 	spin_lock_init(&stripe->write_error_lock);
370 
371 	ASSERT(BTRFS_STRIPE_LEN >> min_folio_shift <= SCRUB_STRIPE_MAX_FOLIOS);
372 	ret = btrfs_alloc_folio_array(BTRFS_STRIPE_LEN >> min_folio_shift,
373 				      fs_info->block_min_order, stripe->folios);
374 	if (ret < 0)
375 		goto error;
376 
377 	stripe->sectors = kcalloc(stripe->nr_sectors,
378 				  sizeof(struct scrub_sector_verification),
379 				  GFP_KERNEL);
380 	if (!stripe->sectors)
381 		goto error;
382 
383 	stripe->csums = kcalloc(BTRFS_STRIPE_LEN >> fs_info->sectorsize_bits,
384 				fs_info->csum_size, GFP_KERNEL);
385 	if (!stripe->csums)
386 		goto error;
387 	return 0;
388 error:
389 	release_scrub_stripe(stripe);
390 	return -ENOMEM;
391 }
392 
wait_scrub_stripe_io(struct scrub_stripe * stripe)393 static void wait_scrub_stripe_io(struct scrub_stripe *stripe)
394 {
395 	wait_event(stripe->io_wait, atomic_read(&stripe->pending_io) == 0);
396 }
397 
398 static void scrub_put_ctx(struct scrub_ctx *sctx);
399 
__scrub_blocked_if_needed(struct btrfs_fs_info * fs_info)400 static void __scrub_blocked_if_needed(struct btrfs_fs_info *fs_info)
401 {
402 	while (atomic_read(&fs_info->scrub_pause_req)) {
403 		mutex_unlock(&fs_info->scrub_lock);
404 		wait_event(fs_info->scrub_pause_wait,
405 		   atomic_read(&fs_info->scrub_pause_req) == 0);
406 		mutex_lock(&fs_info->scrub_lock);
407 	}
408 }
409 
scrub_pause_on(struct btrfs_fs_info * fs_info)410 static void scrub_pause_on(struct btrfs_fs_info *fs_info)
411 {
412 	atomic_inc(&fs_info->scrubs_paused);
413 	wake_up(&fs_info->scrub_pause_wait);
414 }
415 
scrub_pause_off(struct btrfs_fs_info * fs_info)416 static void scrub_pause_off(struct btrfs_fs_info *fs_info)
417 {
418 	mutex_lock(&fs_info->scrub_lock);
419 	__scrub_blocked_if_needed(fs_info);
420 	atomic_dec(&fs_info->scrubs_paused);
421 	mutex_unlock(&fs_info->scrub_lock);
422 
423 	wake_up(&fs_info->scrub_pause_wait);
424 }
425 
scrub_blocked_if_needed(struct btrfs_fs_info * fs_info)426 static void scrub_blocked_if_needed(struct btrfs_fs_info *fs_info)
427 {
428 	scrub_pause_on(fs_info);
429 	scrub_pause_off(fs_info);
430 }
431 
scrub_free_ctx(struct scrub_ctx * sctx)432 static noinline_for_stack void scrub_free_ctx(struct scrub_ctx *sctx)
433 {
434 	int i;
435 
436 	if (!sctx)
437 		return;
438 
439 	for (i = 0; i < SCRUB_TOTAL_STRIPES; i++)
440 		release_scrub_stripe(&sctx->stripes[i]);
441 
442 	kvfree(sctx);
443 }
444 
scrub_put_ctx(struct scrub_ctx * sctx)445 static void scrub_put_ctx(struct scrub_ctx *sctx)
446 {
447 	if (refcount_dec_and_test(&sctx->refs))
448 		scrub_free_ctx(sctx);
449 }
450 
scrub_setup_ctx(struct btrfs_fs_info * fs_info,bool is_dev_replace)451 static noinline_for_stack struct scrub_ctx *scrub_setup_ctx(
452 		struct btrfs_fs_info *fs_info, bool is_dev_replace)
453 {
454 	struct scrub_ctx *sctx;
455 	int		i;
456 
457 	/* Since sctx has inline 128 stripes, it can go beyond 64K easily.  Use
458 	 * kvzalloc().
459 	 */
460 	sctx = kvzalloc(sizeof(*sctx), GFP_KERNEL);
461 	if (!sctx)
462 		goto nomem;
463 	refcount_set(&sctx->refs, 1);
464 	sctx->is_dev_replace = is_dev_replace;
465 	sctx->fs_info = fs_info;
466 	sctx->extent_path.search_commit_root = 1;
467 	sctx->extent_path.skip_locking = 1;
468 	sctx->csum_path.search_commit_root = 1;
469 	sctx->csum_path.skip_locking = 1;
470 	for (i = 0; i < SCRUB_TOTAL_STRIPES; i++) {
471 		int ret;
472 
473 		ret = init_scrub_stripe(fs_info, &sctx->stripes[i]);
474 		if (ret < 0)
475 			goto nomem;
476 		sctx->stripes[i].sctx = sctx;
477 	}
478 	sctx->first_free = 0;
479 	atomic_set(&sctx->cancel_req, 0);
480 
481 	spin_lock_init(&sctx->stat_lock);
482 	sctx->throttle_deadline = 0;
483 
484 	mutex_init(&sctx->wr_lock);
485 	if (is_dev_replace) {
486 		WARN_ON(!fs_info->dev_replace.tgtdev);
487 		sctx->wr_tgtdev = fs_info->dev_replace.tgtdev;
488 	}
489 
490 	return sctx;
491 
492 nomem:
493 	scrub_free_ctx(sctx);
494 	return ERR_PTR(-ENOMEM);
495 }
496 
scrub_print_warning_inode(u64 inum,u64 offset,u64 num_bytes,u64 root,void * warn_ctx)497 static int scrub_print_warning_inode(u64 inum, u64 offset, u64 num_bytes,
498 				     u64 root, void *warn_ctx)
499 {
500 	u32 nlink;
501 	int ret;
502 	int i;
503 	unsigned nofs_flag;
504 	struct extent_buffer *eb;
505 	struct btrfs_inode_item *inode_item;
506 	struct scrub_warning *swarn = warn_ctx;
507 	struct btrfs_fs_info *fs_info = swarn->dev->fs_info;
508 	struct inode_fs_paths *ipath = NULL;
509 	struct btrfs_root *local_root;
510 	struct btrfs_key key;
511 
512 	local_root = btrfs_get_fs_root(fs_info, root, true);
513 	if (IS_ERR(local_root)) {
514 		ret = PTR_ERR(local_root);
515 		goto err;
516 	}
517 
518 	/*
519 	 * this makes the path point to (inum INODE_ITEM ioff)
520 	 */
521 	key.objectid = inum;
522 	key.type = BTRFS_INODE_ITEM_KEY;
523 	key.offset = 0;
524 
525 	ret = btrfs_search_slot(NULL, local_root, &key, swarn->path, 0, 0);
526 	if (ret) {
527 		btrfs_put_root(local_root);
528 		btrfs_release_path(swarn->path);
529 		goto err;
530 	}
531 
532 	eb = swarn->path->nodes[0];
533 	inode_item = btrfs_item_ptr(eb, swarn->path->slots[0],
534 					struct btrfs_inode_item);
535 	nlink = btrfs_inode_nlink(eb, inode_item);
536 	btrfs_release_path(swarn->path);
537 
538 	/*
539 	 * init_path might indirectly call vmalloc, or use GFP_KERNEL. Scrub
540 	 * uses GFP_NOFS in this context, so we keep it consistent but it does
541 	 * not seem to be strictly necessary.
542 	 */
543 	nofs_flag = memalloc_nofs_save();
544 	ipath = init_ipath(4096, local_root, swarn->path);
545 	memalloc_nofs_restore(nofs_flag);
546 	if (IS_ERR(ipath)) {
547 		btrfs_put_root(local_root);
548 		ret = PTR_ERR(ipath);
549 		ipath = NULL;
550 		goto err;
551 	}
552 	ret = paths_from_inode(inum, ipath);
553 
554 	if (ret < 0)
555 		goto err;
556 
557 	/*
558 	 * we deliberately ignore the bit ipath might have been too small to
559 	 * hold all of the paths here
560 	 */
561 	for (i = 0; i < ipath->fspath->elem_cnt; ++i)
562 		btrfs_warn(fs_info,
563 "scrub: %s at logical %llu on dev %s, physical %llu root %llu inode %llu offset %llu length %u links %u (path: %s)",
564 				  swarn->errstr, swarn->logical,
565 				  btrfs_dev_name(swarn->dev),
566 				  swarn->physical,
567 				  root, inum, offset,
568 				  fs_info->sectorsize, nlink,
569 				  (char *)(unsigned long)ipath->fspath->val[i]);
570 
571 	btrfs_put_root(local_root);
572 	free_ipath(ipath);
573 	return 0;
574 
575 err:
576 	btrfs_warn(fs_info,
577 			  "scrub: %s at logical %llu on dev %s, physical %llu root %llu inode %llu offset %llu: path resolving failed with ret=%d",
578 			  swarn->errstr, swarn->logical,
579 			  btrfs_dev_name(swarn->dev),
580 			  swarn->physical,
581 			  root, inum, offset, ret);
582 
583 	free_ipath(ipath);
584 	return 0;
585 }
586 
scrub_print_common_warning(const char * errstr,struct btrfs_device * dev,bool is_super,u64 logical,u64 physical)587 static void scrub_print_common_warning(const char *errstr, struct btrfs_device *dev,
588 				       bool is_super, u64 logical, u64 physical)
589 {
590 	struct btrfs_fs_info *fs_info = dev->fs_info;
591 	BTRFS_PATH_AUTO_FREE(path);
592 	struct btrfs_key found_key;
593 	struct extent_buffer *eb;
594 	struct btrfs_extent_item *ei;
595 	struct scrub_warning swarn;
596 	u64 flags = 0;
597 	u32 item_size;
598 	int ret;
599 
600 	/* Super block error, no need to search extent tree. */
601 	if (is_super) {
602 		btrfs_warn(fs_info, "scrub: %s on device %s, physical %llu",
603 				  errstr, btrfs_dev_name(dev), physical);
604 		return;
605 	}
606 	path = btrfs_alloc_path();
607 	if (!path)
608 		return;
609 
610 	swarn.physical = physical;
611 	swarn.logical = logical;
612 	swarn.errstr = errstr;
613 	swarn.dev = NULL;
614 
615 	ret = extent_from_logical(fs_info, swarn.logical, path, &found_key,
616 				  &flags);
617 	if (ret < 0)
618 		return;
619 
620 	swarn.extent_item_size = found_key.offset;
621 
622 	eb = path->nodes[0];
623 	ei = btrfs_item_ptr(eb, path->slots[0], struct btrfs_extent_item);
624 	item_size = btrfs_item_size(eb, path->slots[0]);
625 
626 	if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
627 		unsigned long ptr = 0;
628 		u8 ref_level;
629 		u64 ref_root;
630 
631 		while (true) {
632 			ret = tree_backref_for_extent(&ptr, eb, &found_key, ei,
633 						      item_size, &ref_root,
634 						      &ref_level);
635 			if (ret < 0) {
636 				btrfs_warn(fs_info,
637 		   "scrub: failed to resolve tree backref for logical %llu: %d",
638 					   swarn.logical, ret);
639 				break;
640 			}
641 			if (ret > 0)
642 				break;
643 			btrfs_warn(fs_info,
644 "scrub: %s at logical %llu on dev %s, physical %llu: metadata %s (level %d) in tree %llu",
645 				errstr, swarn.logical, btrfs_dev_name(dev),
646 				swarn.physical, (ref_level ? "node" : "leaf"),
647 				ref_level, ref_root);
648 		}
649 		btrfs_release_path(path);
650 	} else {
651 		struct btrfs_backref_walk_ctx ctx = { 0 };
652 
653 		btrfs_release_path(path);
654 
655 		ctx.bytenr = found_key.objectid;
656 		ctx.extent_item_pos = swarn.logical - found_key.objectid;
657 		ctx.fs_info = fs_info;
658 
659 		swarn.path = path;
660 		swarn.dev = dev;
661 
662 		iterate_extent_inodes(&ctx, true, scrub_print_warning_inode, &swarn);
663 	}
664 }
665 
fill_writer_pointer_gap(struct scrub_ctx * sctx,u64 physical)666 static int fill_writer_pointer_gap(struct scrub_ctx *sctx, u64 physical)
667 {
668 	int ret = 0;
669 	u64 length;
670 
671 	if (!btrfs_is_zoned(sctx->fs_info))
672 		return 0;
673 
674 	if (!btrfs_dev_is_sequential(sctx->wr_tgtdev, physical))
675 		return 0;
676 
677 	if (sctx->write_pointer < physical) {
678 		length = physical - sctx->write_pointer;
679 
680 		ret = btrfs_zoned_issue_zeroout(sctx->wr_tgtdev,
681 						sctx->write_pointer, length);
682 		if (!ret)
683 			sctx->write_pointer = physical;
684 	}
685 	return ret;
686 }
687 
scrub_stripe_get_kaddr(struct scrub_stripe * stripe,int sector_nr)688 static void *scrub_stripe_get_kaddr(struct scrub_stripe *stripe, int sector_nr)
689 {
690 	struct btrfs_fs_info *fs_info = stripe->bg->fs_info;
691 	const u32 min_folio_shift = PAGE_SHIFT + fs_info->block_min_order;
692 	u32 offset = (sector_nr << fs_info->sectorsize_bits);
693 	const struct folio *folio = stripe->folios[offset >> min_folio_shift];
694 
695 	/* stripe->folios[] is allocated by us and no highmem is allowed. */
696 	ASSERT(folio);
697 	ASSERT(!folio_test_partial_kmap(folio));
698 	return folio_address(folio) + offset_in_folio(folio, offset);
699 }
700 
scrub_stripe_get_paddr(struct scrub_stripe * stripe,int sector_nr)701 static phys_addr_t scrub_stripe_get_paddr(struct scrub_stripe *stripe, int sector_nr)
702 {
703 	struct btrfs_fs_info *fs_info = stripe->bg->fs_info;
704 	const u32 min_folio_shift = PAGE_SHIFT + fs_info->block_min_order;
705 	u32 offset = (sector_nr << fs_info->sectorsize_bits);
706 	const struct folio *folio = stripe->folios[offset >> min_folio_shift];
707 
708 	/* stripe->folios[] is allocated by us and no highmem is allowed. */
709 	ASSERT(folio);
710 	ASSERT(!folio_test_partial_kmap(folio));
711 	/* And the range must be contained inside the folio. */
712 	ASSERT(offset_in_folio(folio, offset) + fs_info->sectorsize <= folio_size(folio));
713 	return page_to_phys(folio_page(folio, 0)) + offset_in_folio(folio, offset);
714 }
715 
scrub_verify_one_metadata(struct scrub_stripe * stripe,int sector_nr)716 static void scrub_verify_one_metadata(struct scrub_stripe *stripe, int sector_nr)
717 {
718 	struct btrfs_fs_info *fs_info = stripe->bg->fs_info;
719 	const u32 sectors_per_tree = fs_info->nodesize >> fs_info->sectorsize_bits;
720 	const u64 logical = stripe->logical + (sector_nr << fs_info->sectorsize_bits);
721 	void *first_kaddr = scrub_stripe_get_kaddr(stripe, sector_nr);
722 	struct btrfs_header *header = first_kaddr;
723 	SHASH_DESC_ON_STACK(shash, fs_info->csum_shash);
724 	u8 on_disk_csum[BTRFS_CSUM_SIZE];
725 	u8 calculated_csum[BTRFS_CSUM_SIZE];
726 
727 	/*
728 	 * Here we don't have a good way to attach the pages (and subpages)
729 	 * to a dummy extent buffer, thus we have to directly grab the members
730 	 * from pages.
731 	 */
732 	memcpy(on_disk_csum, header->csum, fs_info->csum_size);
733 
734 	if (logical != btrfs_stack_header_bytenr(header)) {
735 		scrub_bitmap_set_meta_error(stripe, sector_nr, sectors_per_tree);
736 		scrub_bitmap_set_error(stripe, sector_nr, sectors_per_tree);
737 		btrfs_warn_rl(fs_info,
738 	  "scrub: tree block %llu mirror %u has bad bytenr, has %llu want %llu",
739 			      logical, stripe->mirror_num,
740 			      btrfs_stack_header_bytenr(header), logical);
741 		return;
742 	}
743 	if (memcmp(header->fsid, fs_info->fs_devices->metadata_uuid,
744 		   BTRFS_FSID_SIZE) != 0) {
745 		scrub_bitmap_set_meta_error(stripe, sector_nr, sectors_per_tree);
746 		scrub_bitmap_set_error(stripe, sector_nr, sectors_per_tree);
747 		btrfs_warn_rl(fs_info,
748 	      "scrub: tree block %llu mirror %u has bad fsid, has %pU want %pU",
749 			      logical, stripe->mirror_num,
750 			      header->fsid, fs_info->fs_devices->fsid);
751 		return;
752 	}
753 	if (memcmp(header->chunk_tree_uuid, fs_info->chunk_tree_uuid,
754 		   BTRFS_UUID_SIZE) != 0) {
755 		scrub_bitmap_set_meta_error(stripe, sector_nr, sectors_per_tree);
756 		scrub_bitmap_set_error(stripe, sector_nr, sectors_per_tree);
757 		btrfs_warn_rl(fs_info,
758    "scrub: tree block %llu mirror %u has bad chunk tree uuid, has %pU want %pU",
759 			      logical, stripe->mirror_num,
760 			      header->chunk_tree_uuid, fs_info->chunk_tree_uuid);
761 		return;
762 	}
763 
764 	/* Now check tree block csum. */
765 	shash->tfm = fs_info->csum_shash;
766 	crypto_shash_init(shash);
767 	crypto_shash_update(shash, first_kaddr + BTRFS_CSUM_SIZE,
768 			    fs_info->sectorsize - BTRFS_CSUM_SIZE);
769 
770 	for (int i = sector_nr + 1; i < sector_nr + sectors_per_tree; i++) {
771 		crypto_shash_update(shash, scrub_stripe_get_kaddr(stripe, i),
772 				    fs_info->sectorsize);
773 	}
774 
775 	crypto_shash_final(shash, calculated_csum);
776 	if (memcmp(calculated_csum, on_disk_csum, fs_info->csum_size) != 0) {
777 		scrub_bitmap_set_meta_error(stripe, sector_nr, sectors_per_tree);
778 		scrub_bitmap_set_error(stripe, sector_nr, sectors_per_tree);
779 		btrfs_warn_rl(fs_info,
780 "scrub: tree block %llu mirror %u has bad csum, has " CSUM_FMT " want " CSUM_FMT,
781 			      logical, stripe->mirror_num,
782 			      CSUM_FMT_VALUE(fs_info->csum_size, on_disk_csum),
783 			      CSUM_FMT_VALUE(fs_info->csum_size, calculated_csum));
784 		return;
785 	}
786 	if (stripe->sectors[sector_nr].generation !=
787 	    btrfs_stack_header_generation(header)) {
788 		scrub_bitmap_set_meta_gen_error(stripe, sector_nr, sectors_per_tree);
789 		scrub_bitmap_set_error(stripe, sector_nr, sectors_per_tree);
790 		btrfs_warn_rl(fs_info,
791       "scrub: tree block %llu mirror %u has bad generation, has %llu want %llu",
792 			      logical, stripe->mirror_num,
793 			      btrfs_stack_header_generation(header),
794 			      stripe->sectors[sector_nr].generation);
795 		return;
796 	}
797 	scrub_bitmap_clear_error(stripe, sector_nr, sectors_per_tree);
798 	scrub_bitmap_clear_csum_error(stripe, sector_nr, sectors_per_tree);
799 	scrub_bitmap_clear_meta_error(stripe, sector_nr, sectors_per_tree);
800 	scrub_bitmap_clear_meta_gen_error(stripe, sector_nr, sectors_per_tree);
801 }
802 
scrub_verify_one_sector(struct scrub_stripe * stripe,int sector_nr)803 static void scrub_verify_one_sector(struct scrub_stripe *stripe, int sector_nr)
804 {
805 	struct btrfs_fs_info *fs_info = stripe->bg->fs_info;
806 	struct scrub_sector_verification *sector = &stripe->sectors[sector_nr];
807 	const u32 sectors_per_tree = fs_info->nodesize >> fs_info->sectorsize_bits;
808 	phys_addr_t paddr = scrub_stripe_get_paddr(stripe, sector_nr);
809 	u8 csum_buf[BTRFS_CSUM_SIZE];
810 	int ret;
811 
812 	ASSERT(sector_nr >= 0 && sector_nr < stripe->nr_sectors);
813 
814 	/* Sector not utilized, skip it. */
815 	if (!scrub_bitmap_test_bit_has_extent(stripe, sector_nr))
816 		return;
817 
818 	/* IO error, no need to check. */
819 	if (scrub_bitmap_test_bit_io_error(stripe, sector_nr))
820 		return;
821 
822 	/* Metadata, verify the full tree block. */
823 	if (scrub_bitmap_test_bit_is_metadata(stripe, sector_nr)) {
824 		/*
825 		 * Check if the tree block crosses the stripe boundary.  If
826 		 * crossed the boundary, we cannot verify it but only give a
827 		 * warning.
828 		 *
829 		 * This can only happen on a very old filesystem where chunks
830 		 * are not ensured to be stripe aligned.
831 		 */
832 		if (unlikely(sector_nr + sectors_per_tree > stripe->nr_sectors)) {
833 			btrfs_warn_rl(fs_info,
834 			"scrub: tree block at %llu crosses stripe boundary %llu",
835 				      stripe->logical +
836 				      (sector_nr << fs_info->sectorsize_bits),
837 				      stripe->logical);
838 			return;
839 		}
840 		scrub_verify_one_metadata(stripe, sector_nr);
841 		return;
842 	}
843 
844 	/*
845 	 * Data is easier, we just verify the data csum (if we have it).  For
846 	 * cases without csum, we have no other choice but to trust it.
847 	 */
848 	if (!sector->csum) {
849 		scrub_bitmap_clear_bit_error(stripe, sector_nr);
850 		return;
851 	}
852 
853 	ret = btrfs_check_block_csum(fs_info, paddr, csum_buf, sector->csum);
854 	if (ret < 0) {
855 		scrub_bitmap_set_bit_csum_error(stripe, sector_nr);
856 		scrub_bitmap_set_bit_error(stripe, sector_nr);
857 	} else {
858 		scrub_bitmap_clear_bit_csum_error(stripe, sector_nr);
859 		scrub_bitmap_clear_bit_error(stripe, sector_nr);
860 	}
861 }
862 
863 /* Verify specified sectors of a stripe. */
scrub_verify_one_stripe(struct scrub_stripe * stripe,unsigned long bitmap)864 static void scrub_verify_one_stripe(struct scrub_stripe *stripe, unsigned long bitmap)
865 {
866 	struct btrfs_fs_info *fs_info = stripe->bg->fs_info;
867 	const u32 sectors_per_tree = fs_info->nodesize >> fs_info->sectorsize_bits;
868 	int sector_nr;
869 
870 	for_each_set_bit(sector_nr, &bitmap, stripe->nr_sectors) {
871 		scrub_verify_one_sector(stripe, sector_nr);
872 		if (scrub_bitmap_test_bit_is_metadata(stripe, sector_nr))
873 			sector_nr += sectors_per_tree - 1;
874 	}
875 }
876 
calc_sector_number(struct scrub_stripe * stripe,struct bio_vec * first_bvec)877 static int calc_sector_number(struct scrub_stripe *stripe, struct bio_vec *first_bvec)
878 {
879 	int i;
880 
881 	for (i = 0; i < stripe->nr_sectors; i++) {
882 		if (scrub_stripe_get_kaddr(stripe, i) == bvec_virt(first_bvec))
883 			break;
884 	}
885 	ASSERT(i < stripe->nr_sectors);
886 	return i;
887 }
888 
889 /*
890  * Repair read is different to the regular read:
891  *
892  * - Only reads the failed sectors
893  * - May have extra blocksize limits
894  */
scrub_repair_read_endio(struct btrfs_bio * bbio)895 static void scrub_repair_read_endio(struct btrfs_bio *bbio)
896 {
897 	struct scrub_stripe *stripe = bbio->private;
898 	struct btrfs_fs_info *fs_info = stripe->bg->fs_info;
899 	struct bio_vec *bvec;
900 	int sector_nr = calc_sector_number(stripe, bio_first_bvec_all(&bbio->bio));
901 	u32 bio_size = 0;
902 	int i;
903 
904 	ASSERT(sector_nr < stripe->nr_sectors);
905 
906 	bio_for_each_bvec_all(bvec, &bbio->bio, i)
907 		bio_size += bvec->bv_len;
908 
909 	if (bbio->bio.bi_status) {
910 		scrub_bitmap_set_io_error(stripe, sector_nr,
911 					  bio_size >> fs_info->sectorsize_bits);
912 		scrub_bitmap_set_error(stripe, sector_nr,
913 				       bio_size >> fs_info->sectorsize_bits);
914 	} else {
915 		scrub_bitmap_clear_io_error(stripe, sector_nr,
916 					  bio_size >> fs_info->sectorsize_bits);
917 	}
918 	bio_put(&bbio->bio);
919 	if (atomic_dec_and_test(&stripe->pending_io))
920 		wake_up(&stripe->io_wait);
921 }
922 
calc_next_mirror(int mirror,int num_copies)923 static int calc_next_mirror(int mirror, int num_copies)
924 {
925 	ASSERT(mirror <= num_copies);
926 	return (mirror + 1 > num_copies) ? 1 : mirror + 1;
927 }
928 
scrub_bio_add_sector(struct btrfs_bio * bbio,struct scrub_stripe * stripe,int sector_nr)929 static void scrub_bio_add_sector(struct btrfs_bio *bbio, struct scrub_stripe *stripe,
930 				 int sector_nr)
931 {
932 	void *kaddr = scrub_stripe_get_kaddr(stripe, sector_nr);
933 	int ret;
934 
935 	ret = bio_add_page(&bbio->bio, virt_to_page(kaddr), bbio->fs_info->sectorsize,
936 			   offset_in_page(kaddr));
937 	/*
938 	 * Caller should ensure the bbio has enough size.
939 	 * And we cannot use __bio_add_page(), which doesn't do any merge.
940 	 *
941 	 * Meanwhile for scrub_submit_initial_read() we fully rely on the merge
942 	 * to create the minimal amount of bio vectors, for fs block size < page
943 	 * size cases.
944 	 */
945 	ASSERT(ret == bbio->fs_info->sectorsize);
946 }
947 
scrub_stripe_submit_repair_read(struct scrub_stripe * stripe,int mirror,int blocksize,bool wait)948 static void scrub_stripe_submit_repair_read(struct scrub_stripe *stripe,
949 					    int mirror, int blocksize, bool wait)
950 {
951 	struct btrfs_fs_info *fs_info = stripe->bg->fs_info;
952 	struct btrfs_bio *bbio = NULL;
953 	const unsigned long old_error_bitmap = scrub_bitmap_read_error(stripe);
954 	int i;
955 
956 	ASSERT(stripe->mirror_num >= 1);
957 	ASSERT(atomic_read(&stripe->pending_io) == 0);
958 
959 	for_each_set_bit(i, &old_error_bitmap, stripe->nr_sectors) {
960 		/* The current sector cannot be merged, submit the bio. */
961 		if (bbio && ((i > 0 && !test_bit(i - 1, &old_error_bitmap)) ||
962 			     bbio->bio.bi_iter.bi_size >= blocksize)) {
963 			ASSERT(bbio->bio.bi_iter.bi_size);
964 			atomic_inc(&stripe->pending_io);
965 			btrfs_submit_bbio(bbio, mirror);
966 			if (wait)
967 				wait_scrub_stripe_io(stripe);
968 			bbio = NULL;
969 		}
970 
971 		if (!bbio) {
972 			bbio = btrfs_bio_alloc(stripe->nr_sectors, REQ_OP_READ,
973 				fs_info, scrub_repair_read_endio, stripe);
974 			bbio->bio.bi_iter.bi_sector = (stripe->logical +
975 				(i << fs_info->sectorsize_bits)) >> SECTOR_SHIFT;
976 		}
977 
978 		scrub_bio_add_sector(bbio, stripe, i);
979 	}
980 	if (bbio) {
981 		ASSERT(bbio->bio.bi_iter.bi_size);
982 		atomic_inc(&stripe->pending_io);
983 		btrfs_submit_bbio(bbio, mirror);
984 		if (wait)
985 			wait_scrub_stripe_io(stripe);
986 	}
987 }
988 
scrub_stripe_report_errors(struct scrub_ctx * sctx,struct scrub_stripe * stripe,const struct scrub_error_records * errors)989 static void scrub_stripe_report_errors(struct scrub_ctx *sctx,
990 				       struct scrub_stripe *stripe,
991 				       const struct scrub_error_records *errors)
992 {
993 	static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
994 				      DEFAULT_RATELIMIT_BURST);
995 	struct btrfs_fs_info *fs_info = sctx->fs_info;
996 	struct btrfs_device *dev = NULL;
997 	const unsigned long extent_bitmap = scrub_bitmap_read_has_extent(stripe);
998 	const unsigned long error_bitmap = scrub_bitmap_read_error(stripe);
999 	u64 physical = 0;
1000 	int nr_data_sectors = 0;
1001 	int nr_meta_sectors = 0;
1002 	int nr_nodatacsum_sectors = 0;
1003 	int nr_repaired_sectors = 0;
1004 	int sector_nr;
1005 
1006 	if (test_bit(SCRUB_STRIPE_FLAG_NO_REPORT, &stripe->state))
1007 		return;
1008 
1009 	/*
1010 	 * Init needed infos for error reporting.
1011 	 *
1012 	 * Although our scrub_stripe infrastructure is mostly based on btrfs_submit_bio()
1013 	 * thus no need for dev/physical, error reporting still needs dev and physical.
1014 	 */
1015 	if (!bitmap_empty(&errors->init_error_bitmap, stripe->nr_sectors)) {
1016 		u64 mapped_len = fs_info->sectorsize;
1017 		struct btrfs_io_context *bioc = NULL;
1018 		int stripe_index = stripe->mirror_num - 1;
1019 		int ret;
1020 
1021 		/* For scrub, our mirror_num should always start at 1. */
1022 		ASSERT(stripe->mirror_num >= 1);
1023 		ret = btrfs_map_block(fs_info, BTRFS_MAP_GET_READ_MIRRORS,
1024 				      stripe->logical, &mapped_len, &bioc,
1025 				      NULL, NULL);
1026 		/*
1027 		 * If we failed, dev will be NULL, and later detailed reports
1028 		 * will just be skipped.
1029 		 */
1030 		if (ret < 0)
1031 			goto skip;
1032 		physical = bioc->stripes[stripe_index].physical;
1033 		dev = bioc->stripes[stripe_index].dev;
1034 		btrfs_put_bioc(bioc);
1035 	}
1036 
1037 skip:
1038 	for_each_set_bit(sector_nr, &extent_bitmap, stripe->nr_sectors) {
1039 		bool repaired = false;
1040 
1041 		if (scrub_bitmap_test_bit_is_metadata(stripe, sector_nr)) {
1042 			nr_meta_sectors++;
1043 		} else {
1044 			nr_data_sectors++;
1045 			if (!stripe->sectors[sector_nr].csum)
1046 				nr_nodatacsum_sectors++;
1047 		}
1048 
1049 		if (test_bit(sector_nr, &errors->init_error_bitmap) &&
1050 		    !test_bit(sector_nr, &error_bitmap)) {
1051 			nr_repaired_sectors++;
1052 			repaired = true;
1053 		}
1054 
1055 		/* Good sector from the beginning, nothing need to be done. */
1056 		if (!test_bit(sector_nr, &errors->init_error_bitmap))
1057 			continue;
1058 
1059 		/*
1060 		 * Report error for the corrupted sectors.  If repaired, just
1061 		 * output the message of repaired message.
1062 		 */
1063 		if (repaired) {
1064 			if (dev) {
1065 				btrfs_err_rl(fs_info,
1066 		"scrub: fixed up error at logical %llu on dev %s physical %llu",
1067 					    stripe->logical, btrfs_dev_name(dev),
1068 					    physical);
1069 			} else {
1070 				btrfs_err_rl(fs_info,
1071 			   "scrub: fixed up error at logical %llu on mirror %u",
1072 					    stripe->logical, stripe->mirror_num);
1073 			}
1074 			continue;
1075 		}
1076 
1077 		/* The remaining are all for unrepaired. */
1078 		if (dev) {
1079 			btrfs_err_rl(fs_info,
1080 "scrub: unable to fixup (regular) error at logical %llu on dev %s physical %llu",
1081 					    stripe->logical, btrfs_dev_name(dev),
1082 					    physical);
1083 		} else {
1084 			btrfs_err_rl(fs_info,
1085 	  "scrub: unable to fixup (regular) error at logical %llu on mirror %u",
1086 					    stripe->logical, stripe->mirror_num);
1087 		}
1088 
1089 		if (scrub_bitmap_test_bit_io_error(stripe, sector_nr))
1090 			if (__ratelimit(&rs) && dev)
1091 				scrub_print_common_warning("i/o error", dev, false,
1092 						     stripe->logical, physical);
1093 		if (scrub_bitmap_test_bit_csum_error(stripe, sector_nr))
1094 			if (__ratelimit(&rs) && dev)
1095 				scrub_print_common_warning("checksum error", dev, false,
1096 						     stripe->logical, physical);
1097 		if (scrub_bitmap_test_bit_meta_error(stripe, sector_nr))
1098 			if (__ratelimit(&rs) && dev)
1099 				scrub_print_common_warning("header error", dev, false,
1100 						     stripe->logical, physical);
1101 		if (scrub_bitmap_test_bit_meta_gen_error(stripe, sector_nr))
1102 			if (__ratelimit(&rs) && dev)
1103 				scrub_print_common_warning("generation error", dev, false,
1104 						     stripe->logical, physical);
1105 	}
1106 
1107 	/* Update the device stats. */
1108 	for (int i = 0; i < errors->nr_io_errors; i++)
1109 		btrfs_dev_stat_inc_and_print(stripe->dev, BTRFS_DEV_STAT_READ_ERRS);
1110 	for (int i = 0; i < errors->nr_csum_errors; i++)
1111 		btrfs_dev_stat_inc_and_print(stripe->dev, BTRFS_DEV_STAT_CORRUPTION_ERRS);
1112 	/* Generation mismatch error is based on each metadata, not each block. */
1113 	for (int i = 0; i < errors->nr_meta_gen_errors;
1114 	     i += (fs_info->nodesize >> fs_info->sectorsize_bits))
1115 		btrfs_dev_stat_inc_and_print(stripe->dev, BTRFS_DEV_STAT_GENERATION_ERRS);
1116 
1117 	spin_lock(&sctx->stat_lock);
1118 	sctx->stat.data_extents_scrubbed += stripe->nr_data_extents;
1119 	sctx->stat.tree_extents_scrubbed += stripe->nr_meta_extents;
1120 	sctx->stat.data_bytes_scrubbed += nr_data_sectors << fs_info->sectorsize_bits;
1121 	sctx->stat.tree_bytes_scrubbed += nr_meta_sectors << fs_info->sectorsize_bits;
1122 	sctx->stat.no_csum += nr_nodatacsum_sectors;
1123 	sctx->stat.read_errors += errors->nr_io_errors;
1124 	sctx->stat.csum_errors += errors->nr_csum_errors;
1125 	sctx->stat.verify_errors += errors->nr_meta_errors +
1126 				    errors->nr_meta_gen_errors;
1127 	sctx->stat.uncorrectable_errors +=
1128 		bitmap_weight(&error_bitmap, stripe->nr_sectors);
1129 	sctx->stat.corrected_errors += nr_repaired_sectors;
1130 	spin_unlock(&sctx->stat_lock);
1131 }
1132 
1133 static void scrub_write_sectors(struct scrub_ctx *sctx, struct scrub_stripe *stripe,
1134 				unsigned long write_bitmap, bool dev_replace);
1135 
1136 /*
1137  * The main entrance for all read related scrub work, including:
1138  *
1139  * - Wait for the initial read to finish
1140  * - Verify and locate any bad sectors
1141  * - Go through the remaining mirrors and try to read as large blocksize as
1142  *   possible
1143  * - Go through all mirrors (including the failed mirror) sector-by-sector
1144  * - Submit writeback for repaired sectors
1145  *
1146  * Writeback for dev-replace does not happen here, it needs extra
1147  * synchronization for zoned devices.
1148  */
scrub_stripe_read_repair_worker(struct work_struct * work)1149 static void scrub_stripe_read_repair_worker(struct work_struct *work)
1150 {
1151 	struct scrub_stripe *stripe = container_of(work, struct scrub_stripe, work);
1152 	struct scrub_ctx *sctx = stripe->sctx;
1153 	struct btrfs_fs_info *fs_info = sctx->fs_info;
1154 	struct scrub_error_records errors = { 0 };
1155 	int num_copies = btrfs_num_copies(fs_info, stripe->bg->start,
1156 					  stripe->bg->length);
1157 	unsigned long repaired;
1158 	unsigned long error;
1159 	int mirror;
1160 	int i;
1161 
1162 	ASSERT(stripe->mirror_num > 0);
1163 
1164 	wait_scrub_stripe_io(stripe);
1165 	scrub_verify_one_stripe(stripe, scrub_bitmap_read_has_extent(stripe));
1166 	/* Save the initial failed bitmap for later repair and report usage. */
1167 	errors.init_error_bitmap = scrub_bitmap_read_error(stripe);
1168 	errors.nr_io_errors = scrub_bitmap_weight_io_error(stripe);
1169 	errors.nr_csum_errors = scrub_bitmap_weight_csum_error(stripe);
1170 	errors.nr_meta_errors = scrub_bitmap_weight_meta_error(stripe);
1171 	errors.nr_meta_gen_errors = scrub_bitmap_weight_meta_gen_error(stripe);
1172 
1173 	if (bitmap_empty(&errors.init_error_bitmap, stripe->nr_sectors))
1174 		goto out;
1175 
1176 	/*
1177 	 * Try all remaining mirrors.
1178 	 *
1179 	 * Here we still try to read as large block as possible, as this is
1180 	 * faster and we have extra safety nets to rely on.
1181 	 */
1182 	for (mirror = calc_next_mirror(stripe->mirror_num, num_copies);
1183 	     mirror != stripe->mirror_num;
1184 	     mirror = calc_next_mirror(mirror, num_copies)) {
1185 		const unsigned long old_error_bitmap = scrub_bitmap_read_error(stripe);
1186 
1187 		scrub_stripe_submit_repair_read(stripe, mirror,
1188 						BTRFS_STRIPE_LEN, false);
1189 		wait_scrub_stripe_io(stripe);
1190 		scrub_verify_one_stripe(stripe, old_error_bitmap);
1191 		if (scrub_bitmap_empty_error(stripe))
1192 			goto out;
1193 	}
1194 
1195 	/*
1196 	 * Last safety net, try re-checking all mirrors, including the failed
1197 	 * one, sector-by-sector.
1198 	 *
1199 	 * As if one sector failed the drive's internal csum, the whole read
1200 	 * containing the offending sector would be marked as error.
1201 	 * Thus here we do sector-by-sector read.
1202 	 *
1203 	 * This can be slow, thus we only try it as the last resort.
1204 	 */
1205 
1206 	for (i = 0, mirror = stripe->mirror_num;
1207 	     i < num_copies;
1208 	     i++, mirror = calc_next_mirror(mirror, num_copies)) {
1209 		const unsigned long old_error_bitmap = scrub_bitmap_read_error(stripe);
1210 
1211 		scrub_stripe_submit_repair_read(stripe, mirror,
1212 						fs_info->sectorsize, true);
1213 		wait_scrub_stripe_io(stripe);
1214 		scrub_verify_one_stripe(stripe, old_error_bitmap);
1215 		if (scrub_bitmap_empty_error(stripe))
1216 			goto out;
1217 	}
1218 out:
1219 	error = scrub_bitmap_read_error(stripe);
1220 	/*
1221 	 * Submit the repaired sectors.  For zoned case, we cannot do repair
1222 	 * in-place, but queue the bg to be relocated.
1223 	 */
1224 	bitmap_andnot(&repaired, &errors.init_error_bitmap, &error,
1225 		      stripe->nr_sectors);
1226 	if (!sctx->readonly && !bitmap_empty(&repaired, stripe->nr_sectors)) {
1227 		if (btrfs_is_zoned(fs_info)) {
1228 			btrfs_repair_one_zone(fs_info, sctx->stripes[0].bg->start);
1229 		} else {
1230 			scrub_write_sectors(sctx, stripe, repaired, false);
1231 			wait_scrub_stripe_io(stripe);
1232 		}
1233 	}
1234 
1235 	scrub_stripe_report_errors(sctx, stripe, &errors);
1236 	set_bit(SCRUB_STRIPE_FLAG_REPAIR_DONE, &stripe->state);
1237 	wake_up(&stripe->repair_wait);
1238 }
1239 
scrub_read_endio(struct btrfs_bio * bbio)1240 static void scrub_read_endio(struct btrfs_bio *bbio)
1241 {
1242 	struct scrub_stripe *stripe = bbio->private;
1243 	struct bio_vec *bvec;
1244 	int sector_nr = calc_sector_number(stripe, bio_first_bvec_all(&bbio->bio));
1245 	int num_sectors;
1246 	u32 bio_size = 0;
1247 	int i;
1248 
1249 	ASSERT(sector_nr < stripe->nr_sectors);
1250 	bio_for_each_bvec_all(bvec, &bbio->bio, i)
1251 		bio_size += bvec->bv_len;
1252 	num_sectors = bio_size >> stripe->bg->fs_info->sectorsize_bits;
1253 
1254 	if (bbio->bio.bi_status) {
1255 		scrub_bitmap_set_io_error(stripe, sector_nr, num_sectors);
1256 		scrub_bitmap_set_error(stripe, sector_nr, num_sectors);
1257 	} else {
1258 		scrub_bitmap_clear_io_error(stripe, sector_nr, num_sectors);
1259 	}
1260 	bio_put(&bbio->bio);
1261 	if (atomic_dec_and_test(&stripe->pending_io)) {
1262 		wake_up(&stripe->io_wait);
1263 		INIT_WORK(&stripe->work, scrub_stripe_read_repair_worker);
1264 		queue_work(stripe->bg->fs_info->scrub_workers, &stripe->work);
1265 	}
1266 }
1267 
scrub_write_endio(struct btrfs_bio * bbio)1268 static void scrub_write_endio(struct btrfs_bio *bbio)
1269 {
1270 	struct scrub_stripe *stripe = bbio->private;
1271 	struct btrfs_fs_info *fs_info = stripe->bg->fs_info;
1272 	struct bio_vec *bvec;
1273 	int sector_nr = calc_sector_number(stripe, bio_first_bvec_all(&bbio->bio));
1274 	u32 bio_size = 0;
1275 	int i;
1276 
1277 	bio_for_each_bvec_all(bvec, &bbio->bio, i)
1278 		bio_size += bvec->bv_len;
1279 
1280 	if (bbio->bio.bi_status) {
1281 		unsigned long flags;
1282 
1283 		spin_lock_irqsave(&stripe->write_error_lock, flags);
1284 		bitmap_set(&stripe->write_error_bitmap, sector_nr,
1285 			   bio_size >> fs_info->sectorsize_bits);
1286 		spin_unlock_irqrestore(&stripe->write_error_lock, flags);
1287 		for (int i = 0; i < (bio_size >> fs_info->sectorsize_bits); i++)
1288 			btrfs_dev_stat_inc_and_print(stripe->dev,
1289 						     BTRFS_DEV_STAT_WRITE_ERRS);
1290 	}
1291 	bio_put(&bbio->bio);
1292 
1293 	if (atomic_dec_and_test(&stripe->pending_io))
1294 		wake_up(&stripe->io_wait);
1295 }
1296 
scrub_submit_write_bio(struct scrub_ctx * sctx,struct scrub_stripe * stripe,struct btrfs_bio * bbio,bool dev_replace)1297 static void scrub_submit_write_bio(struct scrub_ctx *sctx,
1298 				   struct scrub_stripe *stripe,
1299 				   struct btrfs_bio *bbio, bool dev_replace)
1300 {
1301 	struct btrfs_fs_info *fs_info = sctx->fs_info;
1302 	u32 bio_len = bbio->bio.bi_iter.bi_size;
1303 	u32 bio_off = (bbio->bio.bi_iter.bi_sector << SECTOR_SHIFT) -
1304 		      stripe->logical;
1305 
1306 	fill_writer_pointer_gap(sctx, stripe->physical + bio_off);
1307 	atomic_inc(&stripe->pending_io);
1308 	btrfs_submit_repair_write(bbio, stripe->mirror_num, dev_replace);
1309 	if (!btrfs_is_zoned(fs_info))
1310 		return;
1311 	/*
1312 	 * For zoned writeback, queue depth must be 1, thus we must wait for
1313 	 * the write to finish before the next write.
1314 	 */
1315 	wait_scrub_stripe_io(stripe);
1316 
1317 	/*
1318 	 * And also need to update the write pointer if write finished
1319 	 * successfully.
1320 	 */
1321 	if (!test_bit(bio_off >> fs_info->sectorsize_bits,
1322 		      &stripe->write_error_bitmap))
1323 		sctx->write_pointer += bio_len;
1324 }
1325 
1326 /*
1327  * Submit the write bio(s) for the sectors specified by @write_bitmap.
1328  *
1329  * Here we utilize btrfs_submit_repair_write(), which has some extra benefits:
1330  *
1331  * - Only needs logical bytenr and mirror_num
1332  *   Just like the scrub read path
1333  *
1334  * - Would only result in writes to the specified mirror
1335  *   Unlike the regular writeback path, which would write back to all stripes
1336  *
1337  * - Handle dev-replace and read-repair writeback differently
1338  */
scrub_write_sectors(struct scrub_ctx * sctx,struct scrub_stripe * stripe,unsigned long write_bitmap,bool dev_replace)1339 static void scrub_write_sectors(struct scrub_ctx *sctx, struct scrub_stripe *stripe,
1340 				unsigned long write_bitmap, bool dev_replace)
1341 {
1342 	struct btrfs_fs_info *fs_info = stripe->bg->fs_info;
1343 	struct btrfs_bio *bbio = NULL;
1344 	int sector_nr;
1345 
1346 	for_each_set_bit(sector_nr, &write_bitmap, stripe->nr_sectors) {
1347 		/* We should only writeback sectors covered by an extent. */
1348 		ASSERT(scrub_bitmap_test_bit_has_extent(stripe, sector_nr));
1349 
1350 		/* Cannot merge with previous sector, submit the current one. */
1351 		if (bbio && sector_nr && !test_bit(sector_nr - 1, &write_bitmap)) {
1352 			scrub_submit_write_bio(sctx, stripe, bbio, dev_replace);
1353 			bbio = NULL;
1354 		}
1355 		if (!bbio) {
1356 			bbio = btrfs_bio_alloc(stripe->nr_sectors, REQ_OP_WRITE,
1357 					       fs_info, scrub_write_endio, stripe);
1358 			bbio->bio.bi_iter.bi_sector = (stripe->logical +
1359 				(sector_nr << fs_info->sectorsize_bits)) >>
1360 				SECTOR_SHIFT;
1361 		}
1362 		scrub_bio_add_sector(bbio, stripe, sector_nr);
1363 	}
1364 	if (bbio)
1365 		scrub_submit_write_bio(sctx, stripe, bbio, dev_replace);
1366 }
1367 
1368 /*
1369  * Throttling of IO submission, bandwidth-limit based, the timeslice is 1
1370  * second.  Limit can be set via /sys/fs/UUID/devinfo/devid/scrub_speed_max.
1371  */
scrub_throttle_dev_io(struct scrub_ctx * sctx,struct btrfs_device * device,unsigned int bio_size)1372 static void scrub_throttle_dev_io(struct scrub_ctx *sctx, struct btrfs_device *device,
1373 				  unsigned int bio_size)
1374 {
1375 	const int time_slice = 1000;
1376 	s64 delta;
1377 	ktime_t now;
1378 	u32 div;
1379 	u64 bwlimit;
1380 
1381 	bwlimit = READ_ONCE(device->scrub_speed_max);
1382 	if (bwlimit == 0)
1383 		return;
1384 
1385 	/*
1386 	 * Slice is divided into intervals when the IO is submitted, adjust by
1387 	 * bwlimit and maximum of 64 intervals.
1388 	 */
1389 	div = clamp(bwlimit / (16 * 1024 * 1024), 1, 64);
1390 
1391 	/* Start new epoch, set deadline */
1392 	now = ktime_get();
1393 	if (sctx->throttle_deadline == 0) {
1394 		sctx->throttle_deadline = ktime_add_ms(now, time_slice / div);
1395 		sctx->throttle_sent = 0;
1396 	}
1397 
1398 	/* Still in the time to send? */
1399 	if (ktime_before(now, sctx->throttle_deadline)) {
1400 		/* If current bio is within the limit, send it */
1401 		sctx->throttle_sent += bio_size;
1402 		if (sctx->throttle_sent <= div_u64(bwlimit, div))
1403 			return;
1404 
1405 		/* We're over the limit, sleep until the rest of the slice */
1406 		delta = ktime_ms_delta(sctx->throttle_deadline, now);
1407 	} else {
1408 		/* New request after deadline, start new epoch */
1409 		delta = 0;
1410 	}
1411 
1412 	if (delta) {
1413 		long timeout;
1414 
1415 		timeout = div_u64(delta * HZ, 1000);
1416 		schedule_timeout_interruptible(timeout);
1417 	}
1418 
1419 	/* Next call will start the deadline period */
1420 	sctx->throttle_deadline = 0;
1421 }
1422 
1423 /*
1424  * Given a physical address, this will calculate it's
1425  * logical offset. if this is a parity stripe, it will return
1426  * the most left data stripe's logical offset.
1427  *
1428  * return 0 if it is a data stripe, 1 means parity stripe.
1429  */
get_raid56_logic_offset(u64 physical,int num,struct btrfs_chunk_map * map,u64 * offset,u64 * stripe_start)1430 static int get_raid56_logic_offset(u64 physical, int num,
1431 				   struct btrfs_chunk_map *map, u64 *offset,
1432 				   u64 *stripe_start)
1433 {
1434 	int i;
1435 	int j = 0;
1436 	u64 last_offset;
1437 	const int data_stripes = nr_data_stripes(map);
1438 
1439 	last_offset = (physical - map->stripes[num].physical) * data_stripes;
1440 	if (stripe_start)
1441 		*stripe_start = last_offset;
1442 
1443 	*offset = last_offset;
1444 	for (i = 0; i < data_stripes; i++) {
1445 		u32 stripe_nr;
1446 		u32 stripe_index;
1447 		u32 rot;
1448 
1449 		*offset = last_offset + btrfs_stripe_nr_to_offset(i);
1450 
1451 		stripe_nr = (u32)(*offset >> BTRFS_STRIPE_LEN_SHIFT) / data_stripes;
1452 
1453 		/* Work out the disk rotation on this stripe-set */
1454 		rot = stripe_nr % map->num_stripes;
1455 		/* calculate which stripe this data locates */
1456 		rot += i;
1457 		stripe_index = rot % map->num_stripes;
1458 		if (stripe_index == num)
1459 			return 0;
1460 		if (stripe_index < num)
1461 			j++;
1462 	}
1463 	*offset = last_offset + btrfs_stripe_nr_to_offset(j);
1464 	return 1;
1465 }
1466 
1467 /*
1468  * Return 0 if the extent item range covers any byte of the range.
1469  * Return <0 if the extent item is before @search_start.
1470  * Return >0 if the extent item is after @start_start + @search_len.
1471  */
compare_extent_item_range(struct btrfs_path * path,u64 search_start,u64 search_len)1472 static int compare_extent_item_range(struct btrfs_path *path,
1473 				     u64 search_start, u64 search_len)
1474 {
1475 	struct btrfs_fs_info *fs_info = path->nodes[0]->fs_info;
1476 	u64 len;
1477 	struct btrfs_key key;
1478 
1479 	btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
1480 	ASSERT(key.type == BTRFS_EXTENT_ITEM_KEY ||
1481 	       key.type == BTRFS_METADATA_ITEM_KEY);
1482 	if (key.type == BTRFS_METADATA_ITEM_KEY)
1483 		len = fs_info->nodesize;
1484 	else
1485 		len = key.offset;
1486 
1487 	if (key.objectid + len <= search_start)
1488 		return -1;
1489 	if (key.objectid >= search_start + search_len)
1490 		return 1;
1491 	return 0;
1492 }
1493 
1494 /*
1495  * Locate one extent item which covers any byte in range
1496  * [@search_start, @search_start + @search_length)
1497  *
1498  * If the path is not initialized, we will initialize the search by doing
1499  * a btrfs_search_slot().
1500  * If the path is already initialized, we will use the path as the initial
1501  * slot, to avoid duplicated btrfs_search_slot() calls.
1502  *
1503  * NOTE: If an extent item starts before @search_start, we will still
1504  * return the extent item. This is for data extent crossing stripe boundary.
1505  *
1506  * Return 0 if we found such extent item, and @path will point to the extent item.
1507  * Return >0 if no such extent item can be found, and @path will be released.
1508  * Return <0 if hit fatal error, and @path will be released.
1509  */
find_first_extent_item(struct btrfs_root * extent_root,struct btrfs_path * path,u64 search_start,u64 search_len)1510 static int find_first_extent_item(struct btrfs_root *extent_root,
1511 				  struct btrfs_path *path,
1512 				  u64 search_start, u64 search_len)
1513 {
1514 	struct btrfs_fs_info *fs_info = extent_root->fs_info;
1515 	struct btrfs_key key;
1516 	int ret;
1517 
1518 	/* Continue using the existing path */
1519 	if (path->nodes[0])
1520 		goto search_forward;
1521 
1522 	key.objectid = search_start;
1523 	if (btrfs_fs_incompat(fs_info, SKINNY_METADATA))
1524 		key.type = BTRFS_METADATA_ITEM_KEY;
1525 	else
1526 		key.type = BTRFS_EXTENT_ITEM_KEY;
1527 	key.offset = (u64)-1;
1528 
1529 	ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0);
1530 	if (ret < 0)
1531 		return ret;
1532 	if (unlikely(ret == 0)) {
1533 		/*
1534 		 * Key with offset -1 found, there would have to exist an extent
1535 		 * item with such offset, but this is out of the valid range.
1536 		 */
1537 		btrfs_release_path(path);
1538 		return -EUCLEAN;
1539 	}
1540 
1541 	/*
1542 	 * Here we intentionally pass 0 as @min_objectid, as there could be
1543 	 * an extent item starting before @search_start.
1544 	 */
1545 	ret = btrfs_previous_extent_item(extent_root, path, 0);
1546 	if (ret < 0)
1547 		return ret;
1548 	/*
1549 	 * No matter whether we have found an extent item, the next loop will
1550 	 * properly do every check on the key.
1551 	 */
1552 search_forward:
1553 	while (true) {
1554 		btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
1555 		if (key.objectid >= search_start + search_len)
1556 			break;
1557 		if (key.type != BTRFS_METADATA_ITEM_KEY &&
1558 		    key.type != BTRFS_EXTENT_ITEM_KEY)
1559 			goto next;
1560 
1561 		ret = compare_extent_item_range(path, search_start, search_len);
1562 		if (ret == 0)
1563 			return ret;
1564 		if (ret > 0)
1565 			break;
1566 next:
1567 		ret = btrfs_next_item(extent_root, path);
1568 		if (ret) {
1569 			/* Either no more items or a fatal error. */
1570 			btrfs_release_path(path);
1571 			return ret;
1572 		}
1573 	}
1574 	btrfs_release_path(path);
1575 	return 1;
1576 }
1577 
get_extent_info(struct btrfs_path * path,u64 * extent_start_ret,u64 * size_ret,u64 * flags_ret,u64 * generation_ret)1578 static void get_extent_info(struct btrfs_path *path, u64 *extent_start_ret,
1579 			    u64 *size_ret, u64 *flags_ret, u64 *generation_ret)
1580 {
1581 	struct btrfs_key key;
1582 	struct btrfs_extent_item *ei;
1583 
1584 	btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
1585 	ASSERT(key.type == BTRFS_METADATA_ITEM_KEY ||
1586 	       key.type == BTRFS_EXTENT_ITEM_KEY);
1587 	*extent_start_ret = key.objectid;
1588 	if (key.type == BTRFS_METADATA_ITEM_KEY)
1589 		*size_ret = path->nodes[0]->fs_info->nodesize;
1590 	else
1591 		*size_ret = key.offset;
1592 	ei = btrfs_item_ptr(path->nodes[0], path->slots[0], struct btrfs_extent_item);
1593 	*flags_ret = btrfs_extent_flags(path->nodes[0], ei);
1594 	*generation_ret = btrfs_extent_generation(path->nodes[0], ei);
1595 }
1596 
sync_write_pointer_for_zoned(struct scrub_ctx * sctx,u64 logical,u64 physical,u64 physical_end)1597 static int sync_write_pointer_for_zoned(struct scrub_ctx *sctx, u64 logical,
1598 					u64 physical, u64 physical_end)
1599 {
1600 	struct btrfs_fs_info *fs_info = sctx->fs_info;
1601 	int ret = 0;
1602 
1603 	if (!btrfs_is_zoned(fs_info))
1604 		return 0;
1605 
1606 	mutex_lock(&sctx->wr_lock);
1607 	if (sctx->write_pointer < physical_end) {
1608 		ret = btrfs_sync_zone_write_pointer(sctx->wr_tgtdev, logical,
1609 						    physical,
1610 						    sctx->write_pointer);
1611 		if (ret)
1612 			btrfs_err(fs_info, "scrub: zoned: failed to recover write pointer");
1613 	}
1614 	mutex_unlock(&sctx->wr_lock);
1615 	btrfs_dev_clear_zone_empty(sctx->wr_tgtdev, physical);
1616 
1617 	return ret;
1618 }
1619 
fill_one_extent_info(struct btrfs_fs_info * fs_info,struct scrub_stripe * stripe,u64 extent_start,u64 extent_len,u64 extent_flags,u64 extent_gen)1620 static void fill_one_extent_info(struct btrfs_fs_info *fs_info,
1621 				 struct scrub_stripe *stripe,
1622 				 u64 extent_start, u64 extent_len,
1623 				 u64 extent_flags, u64 extent_gen)
1624 {
1625 	for (u64 cur_logical = max(stripe->logical, extent_start);
1626 	     cur_logical < min(stripe->logical + BTRFS_STRIPE_LEN,
1627 			       extent_start + extent_len);
1628 	     cur_logical += fs_info->sectorsize) {
1629 		const int nr_sector = (cur_logical - stripe->logical) >>
1630 				      fs_info->sectorsize_bits;
1631 		struct scrub_sector_verification *sector =
1632 						&stripe->sectors[nr_sector];
1633 
1634 		scrub_bitmap_set_bit_has_extent(stripe, nr_sector);
1635 		if (extent_flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
1636 			scrub_bitmap_set_bit_is_metadata(stripe, nr_sector);
1637 			sector->generation = extent_gen;
1638 		}
1639 	}
1640 }
1641 
scrub_stripe_reset_bitmaps(struct scrub_stripe * stripe)1642 static void scrub_stripe_reset_bitmaps(struct scrub_stripe *stripe)
1643 {
1644 	ASSERT(stripe->nr_sectors);
1645 	bitmap_zero(stripe->bitmaps, scrub_bitmap_nr_last * stripe->nr_sectors);
1646 }
1647 
1648 /*
1649  * Locate one stripe which has at least one extent in its range.
1650  *
1651  * Return 0 if found such stripe, and store its info into @stripe.
1652  * Return >0 if there is no such stripe in the specified range.
1653  * Return <0 for error.
1654  */
scrub_find_fill_first_stripe(struct btrfs_block_group * bg,struct btrfs_path * extent_path,struct btrfs_path * csum_path,struct btrfs_device * dev,u64 physical,int mirror_num,u64 logical_start,u32 logical_len,struct scrub_stripe * stripe)1655 static int scrub_find_fill_first_stripe(struct btrfs_block_group *bg,
1656 					struct btrfs_path *extent_path,
1657 					struct btrfs_path *csum_path,
1658 					struct btrfs_device *dev, u64 physical,
1659 					int mirror_num, u64 logical_start,
1660 					u32 logical_len,
1661 					struct scrub_stripe *stripe)
1662 {
1663 	struct btrfs_fs_info *fs_info = bg->fs_info;
1664 	struct btrfs_root *extent_root = btrfs_extent_root(fs_info, bg->start);
1665 	struct btrfs_root *csum_root = btrfs_csum_root(fs_info, bg->start);
1666 	const u64 logical_end = logical_start + logical_len;
1667 	u64 cur_logical = logical_start;
1668 	u64 stripe_end;
1669 	u64 extent_start;
1670 	u64 extent_len;
1671 	u64 extent_flags;
1672 	u64 extent_gen;
1673 	int ret;
1674 
1675 	if (unlikely(!extent_root || !csum_root)) {
1676 		btrfs_err(fs_info, "scrub: no valid extent or csum root found");
1677 		return -EUCLEAN;
1678 	}
1679 	memset(stripe->sectors, 0, sizeof(struct scrub_sector_verification) *
1680 				   stripe->nr_sectors);
1681 	scrub_stripe_reset_bitmaps(stripe);
1682 
1683 	/* The range must be inside the bg. */
1684 	ASSERT(logical_start >= bg->start && logical_end <= bg->start + bg->length);
1685 
1686 	ret = find_first_extent_item(extent_root, extent_path, logical_start,
1687 				     logical_len);
1688 	/* Either error or not found. */
1689 	if (ret)
1690 		goto out;
1691 	get_extent_info(extent_path, &extent_start, &extent_len, &extent_flags,
1692 			&extent_gen);
1693 	if (extent_flags & BTRFS_EXTENT_FLAG_TREE_BLOCK)
1694 		stripe->nr_meta_extents++;
1695 	if (extent_flags & BTRFS_EXTENT_FLAG_DATA)
1696 		stripe->nr_data_extents++;
1697 	cur_logical = max(extent_start, cur_logical);
1698 
1699 	/*
1700 	 * Round down to stripe boundary.
1701 	 *
1702 	 * The extra calculation against bg->start is to handle block groups
1703 	 * whose logical bytenr is not BTRFS_STRIPE_LEN aligned.
1704 	 */
1705 	stripe->logical = round_down(cur_logical - bg->start, BTRFS_STRIPE_LEN) +
1706 			  bg->start;
1707 	stripe->physical = physical + stripe->logical - logical_start;
1708 	stripe->dev = dev;
1709 	stripe->bg = bg;
1710 	stripe->mirror_num = mirror_num;
1711 	stripe_end = stripe->logical + BTRFS_STRIPE_LEN - 1;
1712 
1713 	/* Fill the first extent info into stripe->sectors[] array. */
1714 	fill_one_extent_info(fs_info, stripe, extent_start, extent_len,
1715 			     extent_flags, extent_gen);
1716 	cur_logical = extent_start + extent_len;
1717 
1718 	/* Fill the extent info for the remaining sectors. */
1719 	while (cur_logical <= stripe_end) {
1720 		ret = find_first_extent_item(extent_root, extent_path, cur_logical,
1721 					     stripe_end - cur_logical + 1);
1722 		if (ret < 0)
1723 			goto out;
1724 		if (ret > 0) {
1725 			ret = 0;
1726 			break;
1727 		}
1728 		get_extent_info(extent_path, &extent_start, &extent_len,
1729 				&extent_flags, &extent_gen);
1730 		if (extent_flags & BTRFS_EXTENT_FLAG_TREE_BLOCK)
1731 			stripe->nr_meta_extents++;
1732 		if (extent_flags & BTRFS_EXTENT_FLAG_DATA)
1733 			stripe->nr_data_extents++;
1734 		fill_one_extent_info(fs_info, stripe, extent_start, extent_len,
1735 				     extent_flags, extent_gen);
1736 		cur_logical = extent_start + extent_len;
1737 	}
1738 
1739 	/* Now fill the data csum. */
1740 	if (bg->flags & BTRFS_BLOCK_GROUP_DATA) {
1741 		int sector_nr;
1742 		unsigned long csum_bitmap = 0;
1743 
1744 		/* Csum space should have already been allocated. */
1745 		ASSERT(stripe->csums);
1746 
1747 		/*
1748 		 * Our csum bitmap should be large enough, as BTRFS_STRIPE_LEN
1749 		 * should contain at most 16 sectors.
1750 		 */
1751 		ASSERT(BITS_PER_LONG >= BTRFS_STRIPE_LEN >> fs_info->sectorsize_bits);
1752 
1753 		ret = btrfs_lookup_csums_bitmap(csum_root, csum_path,
1754 						stripe->logical, stripe_end,
1755 						stripe->csums, &csum_bitmap);
1756 		if (ret < 0)
1757 			goto out;
1758 		if (ret > 0)
1759 			ret = 0;
1760 
1761 		for_each_set_bit(sector_nr, &csum_bitmap, stripe->nr_sectors) {
1762 			stripe->sectors[sector_nr].csum = stripe->csums +
1763 				sector_nr * fs_info->csum_size;
1764 		}
1765 	}
1766 	set_bit(SCRUB_STRIPE_FLAG_INITIALIZED, &stripe->state);
1767 out:
1768 	return ret;
1769 }
1770 
scrub_reset_stripe(struct scrub_stripe * stripe)1771 static void scrub_reset_stripe(struct scrub_stripe *stripe)
1772 {
1773 	scrub_stripe_reset_bitmaps(stripe);
1774 
1775 	stripe->nr_meta_extents = 0;
1776 	stripe->nr_data_extents = 0;
1777 	stripe->state = 0;
1778 
1779 	for (int i = 0; i < stripe->nr_sectors; i++) {
1780 		stripe->sectors[i].csum = NULL;
1781 		stripe->sectors[i].generation = 0;
1782 	}
1783 }
1784 
stripe_length(const struct scrub_stripe * stripe)1785 static u32 stripe_length(const struct scrub_stripe *stripe)
1786 {
1787 	ASSERT(stripe->bg);
1788 
1789 	return min(BTRFS_STRIPE_LEN,
1790 		   stripe->bg->start + stripe->bg->length - stripe->logical);
1791 }
1792 
scrub_submit_extent_sector_read(struct scrub_stripe * stripe)1793 static void scrub_submit_extent_sector_read(struct scrub_stripe *stripe)
1794 {
1795 	struct btrfs_fs_info *fs_info = stripe->bg->fs_info;
1796 	struct btrfs_bio *bbio = NULL;
1797 	unsigned int nr_sectors = stripe_length(stripe) >> fs_info->sectorsize_bits;
1798 	const unsigned long has_extent = scrub_bitmap_read_has_extent(stripe);
1799 	u64 stripe_len = BTRFS_STRIPE_LEN;
1800 	int mirror = stripe->mirror_num;
1801 	int i;
1802 
1803 	atomic_inc(&stripe->pending_io);
1804 
1805 	for_each_set_bit(i, &has_extent, stripe->nr_sectors) {
1806 		/* We're beyond the chunk boundary, no need to read anymore. */
1807 		if (i >= nr_sectors)
1808 			break;
1809 
1810 		/* The current sector cannot be merged, submit the bio. */
1811 		if (bbio &&
1812 		    ((i > 0 && !test_bit(i - 1, &has_extent)) ||
1813 		     bbio->bio.bi_iter.bi_size >= stripe_len)) {
1814 			ASSERT(bbio->bio.bi_iter.bi_size);
1815 			atomic_inc(&stripe->pending_io);
1816 			btrfs_submit_bbio(bbio, mirror);
1817 			bbio = NULL;
1818 		}
1819 
1820 		if (!bbio) {
1821 			struct btrfs_io_stripe io_stripe = {};
1822 			struct btrfs_io_context *bioc = NULL;
1823 			const u64 logical = stripe->logical +
1824 					    (i << fs_info->sectorsize_bits);
1825 			int ret;
1826 
1827 			io_stripe.rst_search_commit_root = true;
1828 			stripe_len = (nr_sectors - i) << fs_info->sectorsize_bits;
1829 			/*
1830 			 * For RST cases, we need to manually split the bbio to
1831 			 * follow the RST boundary.
1832 			 */
1833 			ret = btrfs_map_block(fs_info, BTRFS_MAP_READ, logical,
1834 					      &stripe_len, &bioc, &io_stripe, &mirror);
1835 			btrfs_put_bioc(bioc);
1836 			if (ret < 0) {
1837 				if (ret != -ENODATA) {
1838 					/*
1839 					 * Earlier btrfs_get_raid_extent_offset()
1840 					 * returned -ENODATA, which means there's
1841 					 * no entry for the corresponding range
1842 					 * in the stripe tree.  But if it's in
1843 					 * the extent tree, then it's a preallocated
1844 					 * extent and not an error.
1845 					 */
1846 					scrub_bitmap_set_bit_io_error(stripe, i);
1847 					scrub_bitmap_set_bit_error(stripe, i);
1848 				}
1849 				continue;
1850 			}
1851 
1852 			bbio = btrfs_bio_alloc(stripe->nr_sectors, REQ_OP_READ,
1853 					       fs_info, scrub_read_endio, stripe);
1854 			bbio->bio.bi_iter.bi_sector = logical >> SECTOR_SHIFT;
1855 		}
1856 
1857 		scrub_bio_add_sector(bbio, stripe, i);
1858 	}
1859 
1860 	if (bbio) {
1861 		ASSERT(bbio->bio.bi_iter.bi_size);
1862 		atomic_inc(&stripe->pending_io);
1863 		btrfs_submit_bbio(bbio, mirror);
1864 	}
1865 
1866 	if (atomic_dec_and_test(&stripe->pending_io)) {
1867 		wake_up(&stripe->io_wait);
1868 		INIT_WORK(&stripe->work, scrub_stripe_read_repair_worker);
1869 		queue_work(stripe->bg->fs_info->scrub_workers, &stripe->work);
1870 	}
1871 }
1872 
scrub_submit_initial_read(struct scrub_ctx * sctx,struct scrub_stripe * stripe)1873 static void scrub_submit_initial_read(struct scrub_ctx *sctx,
1874 				      struct scrub_stripe *stripe)
1875 {
1876 	struct btrfs_fs_info *fs_info = sctx->fs_info;
1877 	struct btrfs_bio *bbio;
1878 	const u32 min_folio_shift = PAGE_SHIFT + fs_info->block_min_order;
1879 	unsigned int nr_sectors = stripe_length(stripe) >> fs_info->sectorsize_bits;
1880 	int mirror = stripe->mirror_num;
1881 
1882 	ASSERT(stripe->bg);
1883 	ASSERT(stripe->mirror_num > 0);
1884 	ASSERT(test_bit(SCRUB_STRIPE_FLAG_INITIALIZED, &stripe->state));
1885 
1886 	if (btrfs_need_stripe_tree_update(fs_info, stripe->bg->flags)) {
1887 		scrub_submit_extent_sector_read(stripe);
1888 		return;
1889 	}
1890 
1891 	bbio = btrfs_bio_alloc(BTRFS_STRIPE_LEN >> min_folio_shift, REQ_OP_READ, fs_info,
1892 			       scrub_read_endio, stripe);
1893 
1894 	bbio->bio.bi_iter.bi_sector = stripe->logical >> SECTOR_SHIFT;
1895 	/* Read the whole range inside the chunk boundary. */
1896 	for (unsigned int cur = 0; cur < nr_sectors; cur++)
1897 		scrub_bio_add_sector(bbio, stripe, cur);
1898 	atomic_inc(&stripe->pending_io);
1899 
1900 	/*
1901 	 * For dev-replace, either user asks to avoid the source dev, or
1902 	 * the device is missing, we try the next mirror instead.
1903 	 */
1904 	if (sctx->is_dev_replace &&
1905 	    (fs_info->dev_replace.cont_reading_from_srcdev_mode ==
1906 	     BTRFS_DEV_REPLACE_ITEM_CONT_READING_FROM_SRCDEV_MODE_AVOID ||
1907 	     !stripe->dev->bdev)) {
1908 		int num_copies = btrfs_num_copies(fs_info, stripe->bg->start,
1909 						  stripe->bg->length);
1910 
1911 		mirror = calc_next_mirror(mirror, num_copies);
1912 	}
1913 	btrfs_submit_bbio(bbio, mirror);
1914 }
1915 
stripe_has_metadata_error(struct scrub_stripe * stripe)1916 static bool stripe_has_metadata_error(struct scrub_stripe *stripe)
1917 {
1918 	const unsigned long error = scrub_bitmap_read_error(stripe);
1919 	int i;
1920 
1921 	for_each_set_bit(i, &error, stripe->nr_sectors) {
1922 		if (scrub_bitmap_test_bit_is_metadata(stripe, i)) {
1923 			struct btrfs_fs_info *fs_info = stripe->bg->fs_info;
1924 
1925 			btrfs_err(fs_info,
1926 		    "scrub: stripe %llu has unrepaired metadata sector at logical %llu",
1927 				  stripe->logical,
1928 				  stripe->logical + (i << fs_info->sectorsize_bits));
1929 			return true;
1930 		}
1931 	}
1932 	return false;
1933 }
1934 
submit_initial_group_read(struct scrub_ctx * sctx,unsigned int first_slot,unsigned int nr_stripes)1935 static void submit_initial_group_read(struct scrub_ctx *sctx,
1936 				      unsigned int first_slot,
1937 				      unsigned int nr_stripes)
1938 {
1939 	struct blk_plug plug;
1940 
1941 	ASSERT(first_slot < SCRUB_TOTAL_STRIPES);
1942 	ASSERT(first_slot + nr_stripes <= SCRUB_TOTAL_STRIPES);
1943 
1944 	scrub_throttle_dev_io(sctx, sctx->stripes[0].dev,
1945 			      btrfs_stripe_nr_to_offset(nr_stripes));
1946 	blk_start_plug(&plug);
1947 	for (int i = 0; i < nr_stripes; i++) {
1948 		struct scrub_stripe *stripe = &sctx->stripes[first_slot + i];
1949 
1950 		/* Those stripes should be initialized. */
1951 		ASSERT(test_bit(SCRUB_STRIPE_FLAG_INITIALIZED, &stripe->state));
1952 		scrub_submit_initial_read(sctx, stripe);
1953 	}
1954 	blk_finish_plug(&plug);
1955 }
1956 
flush_scrub_stripes(struct scrub_ctx * sctx)1957 static int flush_scrub_stripes(struct scrub_ctx *sctx)
1958 {
1959 	struct btrfs_fs_info *fs_info = sctx->fs_info;
1960 	struct scrub_stripe *stripe;
1961 	const int nr_stripes = sctx->cur_stripe;
1962 	int ret = 0;
1963 
1964 	if (!nr_stripes)
1965 		return 0;
1966 
1967 	ASSERT(test_bit(SCRUB_STRIPE_FLAG_INITIALIZED, &sctx->stripes[0].state));
1968 
1969 	/* Submit the stripes which are populated but not submitted. */
1970 	if (nr_stripes % SCRUB_STRIPES_PER_GROUP) {
1971 		const int first_slot = round_down(nr_stripes, SCRUB_STRIPES_PER_GROUP);
1972 
1973 		submit_initial_group_read(sctx, first_slot, nr_stripes - first_slot);
1974 	}
1975 
1976 	for (int i = 0; i < nr_stripes; i++) {
1977 		stripe = &sctx->stripes[i];
1978 
1979 		wait_event(stripe->repair_wait,
1980 			   test_bit(SCRUB_STRIPE_FLAG_REPAIR_DONE, &stripe->state));
1981 	}
1982 
1983 	/* Submit for dev-replace. */
1984 	if (sctx->is_dev_replace) {
1985 		/*
1986 		 * For dev-replace, if we know there is something wrong with
1987 		 * metadata, we should immediately abort.
1988 		 */
1989 		for (int i = 0; i < nr_stripes; i++) {
1990 			if (unlikely(stripe_has_metadata_error(&sctx->stripes[i]))) {
1991 				ret = -EIO;
1992 				goto out;
1993 			}
1994 		}
1995 		for (int i = 0; i < nr_stripes; i++) {
1996 			unsigned long good;
1997 			unsigned long has_extent;
1998 			unsigned long error;
1999 
2000 			stripe = &sctx->stripes[i];
2001 
2002 			ASSERT(stripe->dev == fs_info->dev_replace.srcdev);
2003 
2004 			has_extent = scrub_bitmap_read_has_extent(stripe);
2005 			error = scrub_bitmap_read_error(stripe);
2006 			bitmap_andnot(&good, &has_extent, &error, stripe->nr_sectors);
2007 			scrub_write_sectors(sctx, stripe, good, true);
2008 		}
2009 	}
2010 
2011 	/* Wait for the above writebacks to finish. */
2012 	for (int i = 0; i < nr_stripes; i++) {
2013 		stripe = &sctx->stripes[i];
2014 
2015 		wait_scrub_stripe_io(stripe);
2016 		spin_lock(&sctx->stat_lock);
2017 		sctx->stat.last_physical = stripe->physical + stripe_length(stripe);
2018 		spin_unlock(&sctx->stat_lock);
2019 		scrub_reset_stripe(stripe);
2020 	}
2021 out:
2022 	sctx->cur_stripe = 0;
2023 	return ret;
2024 }
2025 
raid56_scrub_wait_endio(struct bio * bio)2026 static void raid56_scrub_wait_endio(struct bio *bio)
2027 {
2028 	complete(bio->bi_private);
2029 }
2030 
queue_scrub_stripe(struct scrub_ctx * sctx,struct btrfs_block_group * bg,struct btrfs_device * dev,int mirror_num,u64 logical,u32 length,u64 physical,u64 * found_logical_ret)2031 static int queue_scrub_stripe(struct scrub_ctx *sctx, struct btrfs_block_group *bg,
2032 			      struct btrfs_device *dev, int mirror_num,
2033 			      u64 logical, u32 length, u64 physical,
2034 			      u64 *found_logical_ret)
2035 {
2036 	struct scrub_stripe *stripe;
2037 	int ret;
2038 
2039 	/*
2040 	 * There should always be one slot left, as caller filling the last
2041 	 * slot should flush them all.
2042 	 */
2043 	ASSERT(sctx->cur_stripe < SCRUB_TOTAL_STRIPES);
2044 
2045 	/* @found_logical_ret must be specified. */
2046 	ASSERT(found_logical_ret);
2047 
2048 	stripe = &sctx->stripes[sctx->cur_stripe];
2049 	scrub_reset_stripe(stripe);
2050 	ret = scrub_find_fill_first_stripe(bg, &sctx->extent_path,
2051 					   &sctx->csum_path, dev, physical,
2052 					   mirror_num, logical, length, stripe);
2053 	/* Either >0 as no more extents or <0 for error. */
2054 	if (ret)
2055 		return ret;
2056 	*found_logical_ret = stripe->logical;
2057 	sctx->cur_stripe++;
2058 
2059 	/* We filled one group, submit it. */
2060 	if (sctx->cur_stripe % SCRUB_STRIPES_PER_GROUP == 0) {
2061 		const int first_slot = sctx->cur_stripe - SCRUB_STRIPES_PER_GROUP;
2062 
2063 		submit_initial_group_read(sctx, first_slot, SCRUB_STRIPES_PER_GROUP);
2064 	}
2065 
2066 	/* Last slot used, flush them all. */
2067 	if (sctx->cur_stripe == SCRUB_TOTAL_STRIPES)
2068 		return flush_scrub_stripes(sctx);
2069 	return 0;
2070 }
2071 
scrub_raid56_parity_stripe(struct scrub_ctx * sctx,struct btrfs_device * scrub_dev,struct btrfs_block_group * bg,struct btrfs_chunk_map * map,u64 full_stripe_start)2072 static int scrub_raid56_parity_stripe(struct scrub_ctx *sctx,
2073 				      struct btrfs_device *scrub_dev,
2074 				      struct btrfs_block_group *bg,
2075 				      struct btrfs_chunk_map *map,
2076 				      u64 full_stripe_start)
2077 {
2078 	DECLARE_COMPLETION_ONSTACK(io_done);
2079 	struct btrfs_fs_info *fs_info = sctx->fs_info;
2080 	struct btrfs_raid_bio *rbio;
2081 	struct btrfs_io_context *bioc = NULL;
2082 	struct btrfs_path extent_path = { 0 };
2083 	struct btrfs_path csum_path = { 0 };
2084 	struct bio *bio;
2085 	struct scrub_stripe *stripe;
2086 	bool all_empty = true;
2087 	const int data_stripes = nr_data_stripes(map);
2088 	unsigned long extent_bitmap = 0;
2089 	u64 length = btrfs_stripe_nr_to_offset(data_stripes);
2090 	int ret;
2091 
2092 	ASSERT(sctx->raid56_data_stripes);
2093 
2094 	/*
2095 	 * For data stripe search, we cannot reuse the same extent/csum paths,
2096 	 * as the data stripe bytenr may be smaller than previous extent.  Thus
2097 	 * we have to use our own extent/csum paths.
2098 	 */
2099 	extent_path.search_commit_root = 1;
2100 	extent_path.skip_locking = 1;
2101 	csum_path.search_commit_root = 1;
2102 	csum_path.skip_locking = 1;
2103 
2104 	for (int i = 0; i < data_stripes; i++) {
2105 		int stripe_index;
2106 		int rot;
2107 		u64 physical;
2108 
2109 		stripe = &sctx->raid56_data_stripes[i];
2110 		rot = div_u64(full_stripe_start - bg->start,
2111 			      data_stripes) >> BTRFS_STRIPE_LEN_SHIFT;
2112 		stripe_index = (i + rot) % map->num_stripes;
2113 		physical = map->stripes[stripe_index].physical +
2114 			   btrfs_stripe_nr_to_offset(rot);
2115 
2116 		scrub_reset_stripe(stripe);
2117 		set_bit(SCRUB_STRIPE_FLAG_NO_REPORT, &stripe->state);
2118 		ret = scrub_find_fill_first_stripe(bg, &extent_path, &csum_path,
2119 				map->stripes[stripe_index].dev, physical, 1,
2120 				full_stripe_start + btrfs_stripe_nr_to_offset(i),
2121 				BTRFS_STRIPE_LEN, stripe);
2122 		if (ret < 0)
2123 			goto out;
2124 		/*
2125 		 * No extent in this data stripe, need to manually mark them
2126 		 * initialized to make later read submission happy.
2127 		 */
2128 		if (ret > 0) {
2129 			stripe->logical = full_stripe_start +
2130 					  btrfs_stripe_nr_to_offset(i);
2131 			stripe->dev = map->stripes[stripe_index].dev;
2132 			stripe->mirror_num = 1;
2133 			set_bit(SCRUB_STRIPE_FLAG_INITIALIZED, &stripe->state);
2134 		}
2135 	}
2136 
2137 	/* Check if all data stripes are empty. */
2138 	for (int i = 0; i < data_stripes; i++) {
2139 		stripe = &sctx->raid56_data_stripes[i];
2140 		if (!scrub_bitmap_empty_has_extent(stripe)) {
2141 			all_empty = false;
2142 			break;
2143 		}
2144 	}
2145 	if (all_empty) {
2146 		ret = 0;
2147 		goto out;
2148 	}
2149 
2150 	for (int i = 0; i < data_stripes; i++) {
2151 		stripe = &sctx->raid56_data_stripes[i];
2152 		scrub_submit_initial_read(sctx, stripe);
2153 	}
2154 	for (int i = 0; i < data_stripes; i++) {
2155 		stripe = &sctx->raid56_data_stripes[i];
2156 
2157 		wait_event(stripe->repair_wait,
2158 			   test_bit(SCRUB_STRIPE_FLAG_REPAIR_DONE, &stripe->state));
2159 	}
2160 	/* For now, no zoned support for RAID56. */
2161 	ASSERT(!btrfs_is_zoned(sctx->fs_info));
2162 
2163 	/*
2164 	 * Now all data stripes are properly verified. Check if we have any
2165 	 * unrepaired, if so abort immediately or we could further corrupt the
2166 	 * P/Q stripes.
2167 	 *
2168 	 * During the loop, also populate extent_bitmap.
2169 	 */
2170 	for (int i = 0; i < data_stripes; i++) {
2171 		unsigned long error;
2172 		unsigned long has_extent;
2173 
2174 		stripe = &sctx->raid56_data_stripes[i];
2175 
2176 		error = scrub_bitmap_read_error(stripe);
2177 		has_extent = scrub_bitmap_read_has_extent(stripe);
2178 
2179 		/*
2180 		 * We should only check the errors where there is an extent.
2181 		 * As we may hit an empty data stripe while it's missing.
2182 		 */
2183 		bitmap_and(&error, &error, &has_extent, stripe->nr_sectors);
2184 		if (unlikely(!bitmap_empty(&error, stripe->nr_sectors))) {
2185 			btrfs_err(fs_info,
2186 "scrub: unrepaired sectors detected, full stripe %llu data stripe %u errors %*pbl",
2187 				  full_stripe_start, i, stripe->nr_sectors,
2188 				  &error);
2189 			ret = -EIO;
2190 			goto out;
2191 		}
2192 		bitmap_or(&extent_bitmap, &extent_bitmap, &has_extent,
2193 			  stripe->nr_sectors);
2194 	}
2195 
2196 	/* Now we can check and regenerate the P/Q stripe. */
2197 	bio = bio_alloc(NULL, 1, REQ_OP_READ, GFP_NOFS);
2198 	bio->bi_iter.bi_sector = full_stripe_start >> SECTOR_SHIFT;
2199 	bio->bi_private = &io_done;
2200 	bio->bi_end_io = raid56_scrub_wait_endio;
2201 
2202 	btrfs_bio_counter_inc_blocked(fs_info);
2203 	ret = btrfs_map_block(fs_info, BTRFS_MAP_WRITE, full_stripe_start,
2204 			      &length, &bioc, NULL, NULL);
2205 	if (ret < 0) {
2206 		btrfs_put_bioc(bioc);
2207 		btrfs_bio_counter_dec(fs_info);
2208 		goto out;
2209 	}
2210 	rbio = raid56_parity_alloc_scrub_rbio(bio, bioc, scrub_dev, &extent_bitmap,
2211 				BTRFS_STRIPE_LEN >> fs_info->sectorsize_bits);
2212 	btrfs_put_bioc(bioc);
2213 	if (!rbio) {
2214 		ret = -ENOMEM;
2215 		btrfs_bio_counter_dec(fs_info);
2216 		goto out;
2217 	}
2218 	/* Use the recovered stripes as cache to avoid read them from disk again. */
2219 	for (int i = 0; i < data_stripes; i++) {
2220 		stripe = &sctx->raid56_data_stripes[i];
2221 
2222 		raid56_parity_cache_data_folios(rbio, stripe->folios,
2223 				full_stripe_start + (i << BTRFS_STRIPE_LEN_SHIFT));
2224 	}
2225 	raid56_parity_submit_scrub_rbio(rbio);
2226 	wait_for_completion_io(&io_done);
2227 	ret = blk_status_to_errno(bio->bi_status);
2228 	bio_put(bio);
2229 	btrfs_bio_counter_dec(fs_info);
2230 
2231 	btrfs_release_path(&extent_path);
2232 	btrfs_release_path(&csum_path);
2233 out:
2234 	return ret;
2235 }
2236 
2237 /*
2238  * Scrub one range which can only has simple mirror based profile.
2239  * (Including all range in SINGLE/DUP/RAID1/RAID1C*, and each stripe in
2240  *  RAID0/RAID10).
2241  *
2242  * Since we may need to handle a subset of block group, we need @logical_start
2243  * and @logical_length parameter.
2244  */
scrub_simple_mirror(struct scrub_ctx * sctx,struct btrfs_block_group * bg,u64 logical_start,u64 logical_length,struct btrfs_device * device,u64 physical,int mirror_num)2245 static int scrub_simple_mirror(struct scrub_ctx *sctx,
2246 			       struct btrfs_block_group *bg,
2247 			       u64 logical_start, u64 logical_length,
2248 			       struct btrfs_device *device,
2249 			       u64 physical, int mirror_num)
2250 {
2251 	struct btrfs_fs_info *fs_info = sctx->fs_info;
2252 	const u64 logical_end = logical_start + logical_length;
2253 	u64 cur_logical = logical_start;
2254 	int ret = 0;
2255 
2256 	/* The range must be inside the bg */
2257 	ASSERT(logical_start >= bg->start && logical_end <= bg->start + bg->length);
2258 
2259 	/* Go through each extent items inside the logical range */
2260 	while (cur_logical < logical_end) {
2261 		u64 found_logical = U64_MAX;
2262 		u64 cur_physical = physical + cur_logical - logical_start;
2263 
2264 		/* Canceled? */
2265 		if (atomic_read(&fs_info->scrub_cancel_req) ||
2266 		    atomic_read(&sctx->cancel_req)) {
2267 			ret = -ECANCELED;
2268 			break;
2269 		}
2270 		/* Paused? */
2271 		if (atomic_read(&fs_info->scrub_pause_req)) {
2272 			/* Push queued extents */
2273 			scrub_blocked_if_needed(fs_info);
2274 		}
2275 		/* Block group removed? */
2276 		spin_lock(&bg->lock);
2277 		if (test_bit(BLOCK_GROUP_FLAG_REMOVED, &bg->runtime_flags)) {
2278 			spin_unlock(&bg->lock);
2279 			ret = 0;
2280 			break;
2281 		}
2282 		spin_unlock(&bg->lock);
2283 
2284 		ret = queue_scrub_stripe(sctx, bg, device, mirror_num,
2285 					 cur_logical, logical_end - cur_logical,
2286 					 cur_physical, &found_logical);
2287 		if (ret > 0) {
2288 			/* No more extent, just update the accounting */
2289 			spin_lock(&sctx->stat_lock);
2290 			sctx->stat.last_physical = physical + logical_length;
2291 			spin_unlock(&sctx->stat_lock);
2292 			ret = 0;
2293 			break;
2294 		}
2295 		if (ret < 0)
2296 			break;
2297 
2298 		/* queue_scrub_stripe() returned 0, @found_logical must be updated. */
2299 		ASSERT(found_logical != U64_MAX);
2300 		cur_logical = found_logical + BTRFS_STRIPE_LEN;
2301 
2302 		/* Don't hold CPU for too long time */
2303 		cond_resched();
2304 	}
2305 	return ret;
2306 }
2307 
2308 /* Calculate the full stripe length for simple stripe based profiles */
simple_stripe_full_stripe_len(const struct btrfs_chunk_map * map)2309 static u64 simple_stripe_full_stripe_len(const struct btrfs_chunk_map *map)
2310 {
2311 	ASSERT(map->type & (BTRFS_BLOCK_GROUP_RAID0 |
2312 			    BTRFS_BLOCK_GROUP_RAID10));
2313 
2314 	return btrfs_stripe_nr_to_offset(map->num_stripes / map->sub_stripes);
2315 }
2316 
2317 /* Get the logical bytenr for the stripe */
simple_stripe_get_logical(struct btrfs_chunk_map * map,struct btrfs_block_group * bg,int stripe_index)2318 static u64 simple_stripe_get_logical(struct btrfs_chunk_map *map,
2319 				     struct btrfs_block_group *bg,
2320 				     int stripe_index)
2321 {
2322 	ASSERT(map->type & (BTRFS_BLOCK_GROUP_RAID0 |
2323 			    BTRFS_BLOCK_GROUP_RAID10));
2324 	ASSERT(stripe_index < map->num_stripes);
2325 
2326 	/*
2327 	 * (stripe_index / sub_stripes) gives how many data stripes we need to
2328 	 * skip.
2329 	 */
2330 	return btrfs_stripe_nr_to_offset(stripe_index / map->sub_stripes) +
2331 	       bg->start;
2332 }
2333 
2334 /* Get the mirror number for the stripe */
simple_stripe_mirror_num(struct btrfs_chunk_map * map,int stripe_index)2335 static int simple_stripe_mirror_num(struct btrfs_chunk_map *map, int stripe_index)
2336 {
2337 	ASSERT(map->type & (BTRFS_BLOCK_GROUP_RAID0 |
2338 			    BTRFS_BLOCK_GROUP_RAID10));
2339 	ASSERT(stripe_index < map->num_stripes);
2340 
2341 	/* For RAID0, it's fixed to 1, for RAID10 it's 0,1,0,1... */
2342 	return stripe_index % map->sub_stripes + 1;
2343 }
2344 
scrub_simple_stripe(struct scrub_ctx * sctx,struct btrfs_block_group * bg,struct btrfs_chunk_map * map,struct btrfs_device * device,int stripe_index)2345 static int scrub_simple_stripe(struct scrub_ctx *sctx,
2346 			       struct btrfs_block_group *bg,
2347 			       struct btrfs_chunk_map *map,
2348 			       struct btrfs_device *device,
2349 			       int stripe_index)
2350 {
2351 	const u64 logical_increment = simple_stripe_full_stripe_len(map);
2352 	const u64 orig_logical = simple_stripe_get_logical(map, bg, stripe_index);
2353 	const u64 orig_physical = map->stripes[stripe_index].physical;
2354 	const int mirror_num = simple_stripe_mirror_num(map, stripe_index);
2355 	u64 cur_logical = orig_logical;
2356 	u64 cur_physical = orig_physical;
2357 	int ret = 0;
2358 
2359 	while (cur_logical < bg->start + bg->length) {
2360 		/*
2361 		 * Inside each stripe, RAID0 is just SINGLE, and RAID10 is
2362 		 * just RAID1, so we can reuse scrub_simple_mirror() to scrub
2363 		 * this stripe.
2364 		 */
2365 		ret = scrub_simple_mirror(sctx, bg, cur_logical,
2366 					  BTRFS_STRIPE_LEN, device, cur_physical,
2367 					  mirror_num);
2368 		if (ret)
2369 			return ret;
2370 		/* Skip to next stripe which belongs to the target device */
2371 		cur_logical += logical_increment;
2372 		/* For physical offset, we just go to next stripe */
2373 		cur_physical += BTRFS_STRIPE_LEN;
2374 	}
2375 	return ret;
2376 }
2377 
scrub_stripe(struct scrub_ctx * sctx,struct btrfs_block_group * bg,struct btrfs_chunk_map * map,struct btrfs_device * scrub_dev,int stripe_index)2378 static noinline_for_stack int scrub_stripe(struct scrub_ctx *sctx,
2379 					   struct btrfs_block_group *bg,
2380 					   struct btrfs_chunk_map *map,
2381 					   struct btrfs_device *scrub_dev,
2382 					   int stripe_index)
2383 {
2384 	struct btrfs_fs_info *fs_info = sctx->fs_info;
2385 	const u64 profile = map->type & BTRFS_BLOCK_GROUP_PROFILE_MASK;
2386 	const u64 chunk_logical = bg->start;
2387 	int ret;
2388 	int ret2;
2389 	u64 physical = map->stripes[stripe_index].physical;
2390 	const u64 dev_stripe_len = btrfs_calc_stripe_length(map);
2391 	const u64 physical_end = physical + dev_stripe_len;
2392 	u64 logical;
2393 	u64 logic_end;
2394 	/* The logical increment after finishing one stripe */
2395 	u64 increment;
2396 	/* Offset inside the chunk */
2397 	u64 offset;
2398 	u64 stripe_logical;
2399 
2400 	/* Extent_path should be released by now. */
2401 	ASSERT(sctx->extent_path.nodes[0] == NULL);
2402 
2403 	scrub_blocked_if_needed(fs_info);
2404 
2405 	if (sctx->is_dev_replace &&
2406 	    btrfs_dev_is_sequential(sctx->wr_tgtdev, physical)) {
2407 		mutex_lock(&sctx->wr_lock);
2408 		sctx->write_pointer = physical;
2409 		mutex_unlock(&sctx->wr_lock);
2410 	}
2411 
2412 	/* Prepare the extra data stripes used by RAID56. */
2413 	if (profile & BTRFS_BLOCK_GROUP_RAID56_MASK) {
2414 		ASSERT(sctx->raid56_data_stripes == NULL);
2415 
2416 		sctx->raid56_data_stripes = kcalloc(nr_data_stripes(map),
2417 						    sizeof(struct scrub_stripe),
2418 						    GFP_KERNEL);
2419 		if (!sctx->raid56_data_stripes) {
2420 			ret = -ENOMEM;
2421 			goto out;
2422 		}
2423 		for (int i = 0; i < nr_data_stripes(map); i++) {
2424 			ret = init_scrub_stripe(fs_info,
2425 						&sctx->raid56_data_stripes[i]);
2426 			if (ret < 0)
2427 				goto out;
2428 			sctx->raid56_data_stripes[i].bg = bg;
2429 			sctx->raid56_data_stripes[i].sctx = sctx;
2430 		}
2431 	}
2432 	/*
2433 	 * There used to be a big double loop to handle all profiles using the
2434 	 * same routine, which grows larger and more gross over time.
2435 	 *
2436 	 * So here we handle each profile differently, so simpler profiles
2437 	 * have simpler scrubbing function.
2438 	 */
2439 	if (!(profile & (BTRFS_BLOCK_GROUP_RAID0 | BTRFS_BLOCK_GROUP_RAID10 |
2440 			 BTRFS_BLOCK_GROUP_RAID56_MASK))) {
2441 		/*
2442 		 * Above check rules out all complex profile, the remaining
2443 		 * profiles are SINGLE|DUP|RAID1|RAID1C*, which is simple
2444 		 * mirrored duplication without stripe.
2445 		 *
2446 		 * Only @physical and @mirror_num needs to calculated using
2447 		 * @stripe_index.
2448 		 */
2449 		ret = scrub_simple_mirror(sctx, bg, bg->start, bg->length,
2450 				scrub_dev, map->stripes[stripe_index].physical,
2451 				stripe_index + 1);
2452 		offset = 0;
2453 		goto out;
2454 	}
2455 	if (profile & (BTRFS_BLOCK_GROUP_RAID0 | BTRFS_BLOCK_GROUP_RAID10)) {
2456 		ret = scrub_simple_stripe(sctx, bg, map, scrub_dev, stripe_index);
2457 		offset = btrfs_stripe_nr_to_offset(stripe_index / map->sub_stripes);
2458 		goto out;
2459 	}
2460 
2461 	/* Only RAID56 goes through the old code */
2462 	ASSERT(map->type & BTRFS_BLOCK_GROUP_RAID56_MASK);
2463 	ret = 0;
2464 
2465 	/* Calculate the logical end of the stripe */
2466 	get_raid56_logic_offset(physical_end, stripe_index,
2467 				map, &logic_end, NULL);
2468 	logic_end += chunk_logical;
2469 
2470 	/* Initialize @offset in case we need to go to out: label */
2471 	get_raid56_logic_offset(physical, stripe_index, map, &offset, NULL);
2472 	increment = btrfs_stripe_nr_to_offset(nr_data_stripes(map));
2473 
2474 	/*
2475 	 * Due to the rotation, for RAID56 it's better to iterate each stripe
2476 	 * using their physical offset.
2477 	 */
2478 	while (physical < physical_end) {
2479 		ret = get_raid56_logic_offset(physical, stripe_index, map,
2480 					      &logical, &stripe_logical);
2481 		logical += chunk_logical;
2482 		if (ret) {
2483 			/* it is parity strip */
2484 			stripe_logical += chunk_logical;
2485 			ret = scrub_raid56_parity_stripe(sctx, scrub_dev, bg,
2486 							 map, stripe_logical);
2487 			spin_lock(&sctx->stat_lock);
2488 			sctx->stat.last_physical = min(physical + BTRFS_STRIPE_LEN,
2489 						       physical_end);
2490 			spin_unlock(&sctx->stat_lock);
2491 			if (ret)
2492 				goto out;
2493 			goto next;
2494 		}
2495 
2496 		/*
2497 		 * Now we're at a data stripe, scrub each extents in the range.
2498 		 *
2499 		 * At this stage, if we ignore the repair part, inside each data
2500 		 * stripe it is no different than SINGLE profile.
2501 		 * We can reuse scrub_simple_mirror() here, as the repair part
2502 		 * is still based on @mirror_num.
2503 		 */
2504 		ret = scrub_simple_mirror(sctx, bg, logical, BTRFS_STRIPE_LEN,
2505 					  scrub_dev, physical, 1);
2506 		if (ret < 0)
2507 			goto out;
2508 next:
2509 		logical += increment;
2510 		physical += BTRFS_STRIPE_LEN;
2511 		spin_lock(&sctx->stat_lock);
2512 		sctx->stat.last_physical = physical;
2513 		spin_unlock(&sctx->stat_lock);
2514 	}
2515 out:
2516 	ret2 = flush_scrub_stripes(sctx);
2517 	if (!ret)
2518 		ret = ret2;
2519 	btrfs_release_path(&sctx->extent_path);
2520 	btrfs_release_path(&sctx->csum_path);
2521 
2522 	if (sctx->raid56_data_stripes) {
2523 		for (int i = 0; i < nr_data_stripes(map); i++)
2524 			release_scrub_stripe(&sctx->raid56_data_stripes[i]);
2525 		kfree(sctx->raid56_data_stripes);
2526 		sctx->raid56_data_stripes = NULL;
2527 	}
2528 
2529 	if (sctx->is_dev_replace && ret >= 0) {
2530 		int ret2;
2531 
2532 		ret2 = sync_write_pointer_for_zoned(sctx,
2533 				chunk_logical + offset,
2534 				map->stripes[stripe_index].physical,
2535 				physical_end);
2536 		if (ret2)
2537 			ret = ret2;
2538 	}
2539 
2540 	return ret < 0 ? ret : 0;
2541 }
2542 
scrub_chunk(struct scrub_ctx * sctx,struct btrfs_block_group * bg,struct btrfs_device * scrub_dev,u64 dev_offset,u64 dev_extent_len)2543 static noinline_for_stack int scrub_chunk(struct scrub_ctx *sctx,
2544 					  struct btrfs_block_group *bg,
2545 					  struct btrfs_device *scrub_dev,
2546 					  u64 dev_offset,
2547 					  u64 dev_extent_len)
2548 {
2549 	struct btrfs_fs_info *fs_info = sctx->fs_info;
2550 	struct btrfs_chunk_map *map;
2551 	int i;
2552 	int ret = 0;
2553 
2554 	map = btrfs_find_chunk_map(fs_info, bg->start, bg->length);
2555 	if (!map) {
2556 		/*
2557 		 * Might have been an unused block group deleted by the cleaner
2558 		 * kthread or relocation.
2559 		 */
2560 		spin_lock(&bg->lock);
2561 		if (!test_bit(BLOCK_GROUP_FLAG_REMOVED, &bg->runtime_flags))
2562 			ret = -EINVAL;
2563 		spin_unlock(&bg->lock);
2564 
2565 		return ret;
2566 	}
2567 	if (map->start != bg->start)
2568 		goto out;
2569 	if (map->chunk_len < dev_extent_len)
2570 		goto out;
2571 
2572 	for (i = 0; i < map->num_stripes; ++i) {
2573 		if (map->stripes[i].dev->bdev == scrub_dev->bdev &&
2574 		    map->stripes[i].physical == dev_offset) {
2575 			ret = scrub_stripe(sctx, bg, map, scrub_dev, i);
2576 			if (ret)
2577 				goto out;
2578 		}
2579 	}
2580 out:
2581 	btrfs_free_chunk_map(map);
2582 
2583 	return ret;
2584 }
2585 
finish_extent_writes_for_zoned(struct btrfs_root * root,struct btrfs_block_group * cache)2586 static int finish_extent_writes_for_zoned(struct btrfs_root *root,
2587 					  struct btrfs_block_group *cache)
2588 {
2589 	struct btrfs_fs_info *fs_info = cache->fs_info;
2590 
2591 	if (!btrfs_is_zoned(fs_info))
2592 		return 0;
2593 
2594 	btrfs_wait_block_group_reservations(cache);
2595 	btrfs_wait_nocow_writers(cache);
2596 	btrfs_wait_ordered_roots(fs_info, U64_MAX, cache);
2597 
2598 	return btrfs_commit_current_transaction(root);
2599 }
2600 
2601 static noinline_for_stack
scrub_enumerate_chunks(struct scrub_ctx * sctx,struct btrfs_device * scrub_dev,u64 start,u64 end)2602 int scrub_enumerate_chunks(struct scrub_ctx *sctx,
2603 			   struct btrfs_device *scrub_dev, u64 start, u64 end)
2604 {
2605 	struct btrfs_dev_extent *dev_extent = NULL;
2606 	BTRFS_PATH_AUTO_FREE(path);
2607 	struct btrfs_fs_info *fs_info = sctx->fs_info;
2608 	struct btrfs_root *root = fs_info->dev_root;
2609 	u64 chunk_offset;
2610 	int ret = 0;
2611 	int ro_set;
2612 	int slot;
2613 	struct extent_buffer *l;
2614 	struct btrfs_key key;
2615 	struct btrfs_key found_key;
2616 	struct btrfs_block_group *cache;
2617 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
2618 
2619 	path = btrfs_alloc_path();
2620 	if (!path)
2621 		return -ENOMEM;
2622 
2623 	path->reada = READA_FORWARD;
2624 	path->search_commit_root = 1;
2625 	path->skip_locking = 1;
2626 
2627 	key.objectid = scrub_dev->devid;
2628 	key.type = BTRFS_DEV_EXTENT_KEY;
2629 	key.offset = 0ull;
2630 
2631 	while (1) {
2632 		u64 dev_extent_len;
2633 
2634 		ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
2635 		if (ret < 0)
2636 			break;
2637 		if (ret > 0) {
2638 			if (path->slots[0] >=
2639 			    btrfs_header_nritems(path->nodes[0])) {
2640 				ret = btrfs_next_leaf(root, path);
2641 				if (ret < 0)
2642 					break;
2643 				if (ret > 0) {
2644 					ret = 0;
2645 					break;
2646 				}
2647 			} else {
2648 				ret = 0;
2649 			}
2650 		}
2651 
2652 		l = path->nodes[0];
2653 		slot = path->slots[0];
2654 
2655 		btrfs_item_key_to_cpu(l, &found_key, slot);
2656 
2657 		if (found_key.objectid != scrub_dev->devid)
2658 			break;
2659 
2660 		if (found_key.type != BTRFS_DEV_EXTENT_KEY)
2661 			break;
2662 
2663 		if (found_key.offset >= end)
2664 			break;
2665 
2666 		if (found_key.offset < key.offset)
2667 			break;
2668 
2669 		dev_extent = btrfs_item_ptr(l, slot, struct btrfs_dev_extent);
2670 		dev_extent_len = btrfs_dev_extent_length(l, dev_extent);
2671 
2672 		if (found_key.offset + dev_extent_len <= start)
2673 			goto skip;
2674 
2675 		chunk_offset = btrfs_dev_extent_chunk_offset(l, dev_extent);
2676 
2677 		/*
2678 		 * get a reference on the corresponding block group to prevent
2679 		 * the chunk from going away while we scrub it
2680 		 */
2681 		cache = btrfs_lookup_block_group(fs_info, chunk_offset);
2682 
2683 		/* some chunks are removed but not committed to disk yet,
2684 		 * continue scrubbing */
2685 		if (!cache)
2686 			goto skip;
2687 
2688 		ASSERT(cache->start <= chunk_offset);
2689 		/*
2690 		 * We are using the commit root to search for device extents, so
2691 		 * that means we could have found a device extent item from a
2692 		 * block group that was deleted in the current transaction. The
2693 		 * logical start offset of the deleted block group, stored at
2694 		 * @chunk_offset, might be part of the logical address range of
2695 		 * a new block group (which uses different physical extents).
2696 		 * In this case btrfs_lookup_block_group() has returned the new
2697 		 * block group, and its start address is less than @chunk_offset.
2698 		 *
2699 		 * We skip such new block groups, because it's pointless to
2700 		 * process them, as we won't find their extents because we search
2701 		 * for them using the commit root of the extent tree. For a device
2702 		 * replace it's also fine to skip it, we won't miss copying them
2703 		 * to the target device because we have the write duplication
2704 		 * setup through the regular write path (by btrfs_map_block()),
2705 		 * and we have committed a transaction when we started the device
2706 		 * replace, right after setting up the device replace state.
2707 		 */
2708 		if (cache->start < chunk_offset) {
2709 			btrfs_put_block_group(cache);
2710 			goto skip;
2711 		}
2712 
2713 		if (sctx->is_dev_replace && btrfs_is_zoned(fs_info)) {
2714 			if (!test_bit(BLOCK_GROUP_FLAG_TO_COPY, &cache->runtime_flags)) {
2715 				btrfs_put_block_group(cache);
2716 				goto skip;
2717 			}
2718 		}
2719 
2720 		/*
2721 		 * Make sure that while we are scrubbing the corresponding block
2722 		 * group doesn't get its logical address and its device extents
2723 		 * reused for another block group, which can possibly be of a
2724 		 * different type and different profile. We do this to prevent
2725 		 * false error detections and crashes due to bogus attempts to
2726 		 * repair extents.
2727 		 */
2728 		spin_lock(&cache->lock);
2729 		if (test_bit(BLOCK_GROUP_FLAG_REMOVED, &cache->runtime_flags)) {
2730 			spin_unlock(&cache->lock);
2731 			btrfs_put_block_group(cache);
2732 			goto skip;
2733 		}
2734 		btrfs_freeze_block_group(cache);
2735 		spin_unlock(&cache->lock);
2736 
2737 		/*
2738 		 * we need call btrfs_inc_block_group_ro() with scrubs_paused,
2739 		 * to avoid deadlock caused by:
2740 		 * btrfs_inc_block_group_ro()
2741 		 * -> btrfs_wait_for_commit()
2742 		 * -> btrfs_commit_transaction()
2743 		 * -> btrfs_scrub_pause()
2744 		 */
2745 		scrub_pause_on(fs_info);
2746 
2747 		/*
2748 		 * Don't do chunk preallocation for scrub.
2749 		 *
2750 		 * This is especially important for SYSTEM bgs, or we can hit
2751 		 * -EFBIG from btrfs_finish_chunk_alloc() like:
2752 		 * 1. The only SYSTEM bg is marked RO.
2753 		 *    Since SYSTEM bg is small, that's pretty common.
2754 		 * 2. New SYSTEM bg will be allocated
2755 		 *    Due to regular version will allocate new chunk.
2756 		 * 3. New SYSTEM bg is empty and will get cleaned up
2757 		 *    Before cleanup really happens, it's marked RO again.
2758 		 * 4. Empty SYSTEM bg get scrubbed
2759 		 *    We go back to 2.
2760 		 *
2761 		 * This can easily boost the amount of SYSTEM chunks if cleaner
2762 		 * thread can't be triggered fast enough, and use up all space
2763 		 * of btrfs_super_block::sys_chunk_array
2764 		 *
2765 		 * While for dev replace, we need to try our best to mark block
2766 		 * group RO, to prevent race between:
2767 		 * - Write duplication
2768 		 *   Contains latest data
2769 		 * - Scrub copy
2770 		 *   Contains data from commit tree
2771 		 *
2772 		 * If target block group is not marked RO, nocow writes can
2773 		 * be overwritten by scrub copy, causing data corruption.
2774 		 * So for dev-replace, it's not allowed to continue if a block
2775 		 * group is not RO.
2776 		 */
2777 		ret = btrfs_inc_block_group_ro(cache, sctx->is_dev_replace);
2778 		if (!ret && sctx->is_dev_replace) {
2779 			ret = finish_extent_writes_for_zoned(root, cache);
2780 			if (ret) {
2781 				btrfs_dec_block_group_ro(cache);
2782 				scrub_pause_off(fs_info);
2783 				btrfs_put_block_group(cache);
2784 				break;
2785 			}
2786 		}
2787 
2788 		if (ret == 0) {
2789 			ro_set = 1;
2790 		} else if (ret == -ENOSPC && !sctx->is_dev_replace &&
2791 			   !(cache->flags & BTRFS_BLOCK_GROUP_RAID56_MASK)) {
2792 			/*
2793 			 * btrfs_inc_block_group_ro return -ENOSPC when it
2794 			 * failed in creating new chunk for metadata.
2795 			 * It is not a problem for scrub, because
2796 			 * metadata are always cowed, and our scrub paused
2797 			 * commit_transactions.
2798 			 *
2799 			 * For RAID56 chunks, we have to mark them read-only
2800 			 * for scrub, as later we would use our own cache
2801 			 * out of RAID56 realm.
2802 			 * Thus we want the RAID56 bg to be marked RO to
2803 			 * prevent RMW from screwing up out cache.
2804 			 */
2805 			ro_set = 0;
2806 		} else if (ret == -ETXTBSY) {
2807 			btrfs_warn(fs_info,
2808 	     "scrub: skipping scrub of block group %llu due to active swapfile",
2809 				   cache->start);
2810 			scrub_pause_off(fs_info);
2811 			ret = 0;
2812 			goto skip_unfreeze;
2813 		} else {
2814 			btrfs_warn(fs_info, "scrub: failed setting block group ro: %d",
2815 				   ret);
2816 			btrfs_unfreeze_block_group(cache);
2817 			btrfs_put_block_group(cache);
2818 			scrub_pause_off(fs_info);
2819 			break;
2820 		}
2821 
2822 		/*
2823 		 * Now the target block is marked RO, wait for nocow writes to
2824 		 * finish before dev-replace.
2825 		 * COW is fine, as COW never overwrites extents in commit tree.
2826 		 */
2827 		if (sctx->is_dev_replace) {
2828 			btrfs_wait_nocow_writers(cache);
2829 			btrfs_wait_ordered_roots(fs_info, U64_MAX, cache);
2830 		}
2831 
2832 		scrub_pause_off(fs_info);
2833 		down_write(&dev_replace->rwsem);
2834 		dev_replace->cursor_right = found_key.offset + dev_extent_len;
2835 		dev_replace->cursor_left = found_key.offset;
2836 		dev_replace->item_needs_writeback = 1;
2837 		up_write(&dev_replace->rwsem);
2838 
2839 		ret = scrub_chunk(sctx, cache, scrub_dev, found_key.offset,
2840 				  dev_extent_len);
2841 		if (sctx->is_dev_replace &&
2842 		    !btrfs_finish_block_group_to_copy(dev_replace->srcdev,
2843 						      cache, found_key.offset))
2844 			ro_set = 0;
2845 
2846 		down_write(&dev_replace->rwsem);
2847 		dev_replace->cursor_left = dev_replace->cursor_right;
2848 		dev_replace->item_needs_writeback = 1;
2849 		up_write(&dev_replace->rwsem);
2850 
2851 		if (ro_set)
2852 			btrfs_dec_block_group_ro(cache);
2853 
2854 		/*
2855 		 * We might have prevented the cleaner kthread from deleting
2856 		 * this block group if it was already unused because we raced
2857 		 * and set it to RO mode first. So add it back to the unused
2858 		 * list, otherwise it might not ever be deleted unless a manual
2859 		 * balance is triggered or it becomes used and unused again.
2860 		 */
2861 		spin_lock(&cache->lock);
2862 		if (!test_bit(BLOCK_GROUP_FLAG_REMOVED, &cache->runtime_flags) &&
2863 		    !cache->ro && cache->reserved == 0 && cache->used == 0) {
2864 			spin_unlock(&cache->lock);
2865 			if (btrfs_test_opt(fs_info, DISCARD_ASYNC))
2866 				btrfs_discard_queue_work(&fs_info->discard_ctl,
2867 							 cache);
2868 			else
2869 				btrfs_mark_bg_unused(cache);
2870 		} else {
2871 			spin_unlock(&cache->lock);
2872 		}
2873 skip_unfreeze:
2874 		btrfs_unfreeze_block_group(cache);
2875 		btrfs_put_block_group(cache);
2876 		if (ret)
2877 			break;
2878 		if (unlikely(sctx->is_dev_replace &&
2879 			     atomic64_read(&dev_replace->num_write_errors) > 0)) {
2880 			ret = -EIO;
2881 			break;
2882 		}
2883 		if (sctx->stat.malloc_errors > 0) {
2884 			ret = -ENOMEM;
2885 			break;
2886 		}
2887 skip:
2888 		key.offset = found_key.offset + dev_extent_len;
2889 		btrfs_release_path(path);
2890 	}
2891 
2892 	return ret;
2893 }
2894 
scrub_one_super(struct scrub_ctx * sctx,struct btrfs_device * dev,struct page * page,u64 physical,u64 generation)2895 static int scrub_one_super(struct scrub_ctx *sctx, struct btrfs_device *dev,
2896 			   struct page *page, u64 physical, u64 generation)
2897 {
2898 	struct btrfs_fs_info *fs_info = sctx->fs_info;
2899 	struct btrfs_super_block *sb = page_address(page);
2900 	int ret;
2901 
2902 	ret = bdev_rw_virt(dev->bdev, physical >> SECTOR_SHIFT, sb,
2903 			BTRFS_SUPER_INFO_SIZE, REQ_OP_READ);
2904 	if (ret < 0)
2905 		return ret;
2906 	ret = btrfs_check_super_csum(fs_info, sb);
2907 	if (unlikely(ret != 0)) {
2908 		btrfs_err_rl(fs_info,
2909 		  "scrub: super block at physical %llu devid %llu has bad csum",
2910 			physical, dev->devid);
2911 		return -EIO;
2912 	}
2913 	if (unlikely(btrfs_super_generation(sb) != generation)) {
2914 		btrfs_err_rl(fs_info,
2915 "scrub: super block at physical %llu devid %llu has bad generation %llu expect %llu",
2916 			     physical, dev->devid,
2917 			     btrfs_super_generation(sb), generation);
2918 		return -EUCLEAN;
2919 	}
2920 
2921 	return btrfs_validate_super(fs_info, sb, -1);
2922 }
2923 
scrub_supers(struct scrub_ctx * sctx,struct btrfs_device * scrub_dev)2924 static noinline_for_stack int scrub_supers(struct scrub_ctx *sctx,
2925 					   struct btrfs_device *scrub_dev)
2926 {
2927 	int	i;
2928 	u64	bytenr;
2929 	u64	gen;
2930 	int ret = 0;
2931 	struct page *page;
2932 	struct btrfs_fs_info *fs_info = sctx->fs_info;
2933 
2934 	if (BTRFS_FS_ERROR(fs_info))
2935 		return -EROFS;
2936 
2937 	page = alloc_page(GFP_KERNEL);
2938 	if (!page) {
2939 		spin_lock(&sctx->stat_lock);
2940 		sctx->stat.malloc_errors++;
2941 		spin_unlock(&sctx->stat_lock);
2942 		return -ENOMEM;
2943 	}
2944 
2945 	/* Seed devices of a new filesystem has their own generation. */
2946 	if (scrub_dev->fs_devices != fs_info->fs_devices)
2947 		gen = scrub_dev->generation;
2948 	else
2949 		gen = btrfs_get_last_trans_committed(fs_info);
2950 
2951 	for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) {
2952 		ret = btrfs_sb_log_location(scrub_dev, i, 0, &bytenr);
2953 		if (ret == -ENOENT)
2954 			break;
2955 
2956 		if (ret) {
2957 			spin_lock(&sctx->stat_lock);
2958 			sctx->stat.super_errors++;
2959 			spin_unlock(&sctx->stat_lock);
2960 			continue;
2961 		}
2962 
2963 		if (bytenr + BTRFS_SUPER_INFO_SIZE >
2964 		    scrub_dev->commit_total_bytes)
2965 			break;
2966 		if (!btrfs_check_super_location(scrub_dev, bytenr))
2967 			continue;
2968 
2969 		ret = scrub_one_super(sctx, scrub_dev, page, bytenr, gen);
2970 		if (ret) {
2971 			spin_lock(&sctx->stat_lock);
2972 			sctx->stat.super_errors++;
2973 			spin_unlock(&sctx->stat_lock);
2974 		}
2975 	}
2976 	__free_page(page);
2977 	return 0;
2978 }
2979 
scrub_workers_put(struct btrfs_fs_info * fs_info)2980 static void scrub_workers_put(struct btrfs_fs_info *fs_info)
2981 {
2982 	if (refcount_dec_and_mutex_lock(&fs_info->scrub_workers_refcnt,
2983 					&fs_info->scrub_lock)) {
2984 		struct workqueue_struct *scrub_workers = fs_info->scrub_workers;
2985 
2986 		fs_info->scrub_workers = NULL;
2987 		mutex_unlock(&fs_info->scrub_lock);
2988 
2989 		if (scrub_workers)
2990 			destroy_workqueue(scrub_workers);
2991 	}
2992 }
2993 
2994 /*
2995  * get a reference count on fs_info->scrub_workers. start worker if necessary
2996  */
scrub_workers_get(struct btrfs_fs_info * fs_info)2997 static noinline_for_stack int scrub_workers_get(struct btrfs_fs_info *fs_info)
2998 {
2999 	struct workqueue_struct *scrub_workers = NULL;
3000 	unsigned int flags = WQ_FREEZABLE | WQ_UNBOUND;
3001 	int max_active = fs_info->thread_pool_size;
3002 	int ret = -ENOMEM;
3003 
3004 	if (refcount_inc_not_zero(&fs_info->scrub_workers_refcnt))
3005 		return 0;
3006 
3007 	scrub_workers = alloc_workqueue("btrfs-scrub", flags, max_active);
3008 	if (!scrub_workers)
3009 		return -ENOMEM;
3010 
3011 	mutex_lock(&fs_info->scrub_lock);
3012 	if (refcount_read(&fs_info->scrub_workers_refcnt) == 0) {
3013 		ASSERT(fs_info->scrub_workers == NULL);
3014 		fs_info->scrub_workers = scrub_workers;
3015 		refcount_set(&fs_info->scrub_workers_refcnt, 1);
3016 		mutex_unlock(&fs_info->scrub_lock);
3017 		return 0;
3018 	}
3019 	/* Other thread raced in and created the workers for us */
3020 	refcount_inc(&fs_info->scrub_workers_refcnt);
3021 	mutex_unlock(&fs_info->scrub_lock);
3022 
3023 	ret = 0;
3024 
3025 	destroy_workqueue(scrub_workers);
3026 	return ret;
3027 }
3028 
btrfs_scrub_dev(struct btrfs_fs_info * fs_info,u64 devid,u64 start,u64 end,struct btrfs_scrub_progress * progress,bool readonly,bool is_dev_replace)3029 int btrfs_scrub_dev(struct btrfs_fs_info *fs_info, u64 devid, u64 start,
3030 		    u64 end, struct btrfs_scrub_progress *progress,
3031 		    bool readonly, bool is_dev_replace)
3032 {
3033 	struct btrfs_dev_lookup_args args = { .devid = devid };
3034 	struct scrub_ctx *sctx;
3035 	int ret;
3036 	struct btrfs_device *dev;
3037 	unsigned int nofs_flag;
3038 	bool need_commit = false;
3039 
3040 	if (btrfs_fs_closing(fs_info))
3041 		return -EAGAIN;
3042 
3043 	/* At mount time we have ensured nodesize is in the range of [4K, 64K]. */
3044 	ASSERT(fs_info->nodesize <= BTRFS_STRIPE_LEN);
3045 
3046 	/*
3047 	 * SCRUB_MAX_SECTORS_PER_BLOCK is calculated using the largest possible
3048 	 * value (max nodesize / min sectorsize), thus nodesize should always
3049 	 * be fine.
3050 	 */
3051 	ASSERT(fs_info->nodesize <=
3052 	       SCRUB_MAX_SECTORS_PER_BLOCK << fs_info->sectorsize_bits);
3053 
3054 	/* Allocate outside of device_list_mutex */
3055 	sctx = scrub_setup_ctx(fs_info, is_dev_replace);
3056 	if (IS_ERR(sctx))
3057 		return PTR_ERR(sctx);
3058 
3059 	ret = scrub_workers_get(fs_info);
3060 	if (ret)
3061 		goto out_free_ctx;
3062 
3063 	mutex_lock(&fs_info->fs_devices->device_list_mutex);
3064 	dev = btrfs_find_device(fs_info->fs_devices, &args);
3065 	if (!dev || (test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state) &&
3066 		     !is_dev_replace)) {
3067 		mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3068 		ret = -ENODEV;
3069 		goto out;
3070 	}
3071 
3072 	if (!is_dev_replace && !readonly &&
3073 	    !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state)) {
3074 		mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3075 		btrfs_err(fs_info,
3076 			"scrub: devid %llu: filesystem on %s is not writable",
3077 				 devid, btrfs_dev_name(dev));
3078 		ret = -EROFS;
3079 		goto out;
3080 	}
3081 
3082 	mutex_lock(&fs_info->scrub_lock);
3083 	if (unlikely(!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &dev->dev_state) ||
3084 		     test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &dev->dev_state))) {
3085 		mutex_unlock(&fs_info->scrub_lock);
3086 		mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3087 		ret = -EIO;
3088 		goto out;
3089 	}
3090 
3091 	down_read(&fs_info->dev_replace.rwsem);
3092 	if (dev->scrub_ctx ||
3093 	    (!is_dev_replace &&
3094 	     btrfs_dev_replace_is_ongoing(&fs_info->dev_replace))) {
3095 		up_read(&fs_info->dev_replace.rwsem);
3096 		mutex_unlock(&fs_info->scrub_lock);
3097 		mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3098 		ret = -EINPROGRESS;
3099 		goto out;
3100 	}
3101 	up_read(&fs_info->dev_replace.rwsem);
3102 
3103 	sctx->readonly = readonly;
3104 	dev->scrub_ctx = sctx;
3105 	mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3106 
3107 	/*
3108 	 * checking @scrub_pause_req here, we can avoid
3109 	 * race between committing transaction and scrubbing.
3110 	 */
3111 	__scrub_blocked_if_needed(fs_info);
3112 	atomic_inc(&fs_info->scrubs_running);
3113 	mutex_unlock(&fs_info->scrub_lock);
3114 
3115 	/*
3116 	 * In order to avoid deadlock with reclaim when there is a transaction
3117 	 * trying to pause scrub, make sure we use GFP_NOFS for all the
3118 	 * allocations done at btrfs_scrub_sectors() and scrub_sectors_for_parity()
3119 	 * invoked by our callees. The pausing request is done when the
3120 	 * transaction commit starts, and it blocks the transaction until scrub
3121 	 * is paused (done at specific points at scrub_stripe() or right above
3122 	 * before incrementing fs_info->scrubs_running).
3123 	 */
3124 	nofs_flag = memalloc_nofs_save();
3125 	if (!is_dev_replace) {
3126 		u64 old_super_errors;
3127 
3128 		spin_lock(&sctx->stat_lock);
3129 		old_super_errors = sctx->stat.super_errors;
3130 		spin_unlock(&sctx->stat_lock);
3131 
3132 		btrfs_info(fs_info, "scrub: started on devid %llu", devid);
3133 		/*
3134 		 * by holding device list mutex, we can
3135 		 * kick off writing super in log tree sync.
3136 		 */
3137 		mutex_lock(&fs_info->fs_devices->device_list_mutex);
3138 		ret = scrub_supers(sctx, dev);
3139 		mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3140 
3141 		spin_lock(&sctx->stat_lock);
3142 		/*
3143 		 * Super block errors found, but we can not commit transaction
3144 		 * at current context, since btrfs_commit_transaction() needs
3145 		 * to pause the current running scrub (hold by ourselves).
3146 		 */
3147 		if (sctx->stat.super_errors > old_super_errors && !sctx->readonly)
3148 			need_commit = true;
3149 		spin_unlock(&sctx->stat_lock);
3150 	}
3151 
3152 	if (!ret)
3153 		ret = scrub_enumerate_chunks(sctx, dev, start, end);
3154 	memalloc_nofs_restore(nofs_flag);
3155 
3156 	atomic_dec(&fs_info->scrubs_running);
3157 	wake_up(&fs_info->scrub_pause_wait);
3158 
3159 	if (progress)
3160 		memcpy(progress, &sctx->stat, sizeof(*progress));
3161 
3162 	if (!is_dev_replace)
3163 		btrfs_info(fs_info, "scrub: %s on devid %llu with status: %d",
3164 			ret ? "not finished" : "finished", devid, ret);
3165 
3166 	mutex_lock(&fs_info->scrub_lock);
3167 	dev->scrub_ctx = NULL;
3168 	mutex_unlock(&fs_info->scrub_lock);
3169 
3170 	scrub_workers_put(fs_info);
3171 	scrub_put_ctx(sctx);
3172 
3173 	/*
3174 	 * We found some super block errors before, now try to force a
3175 	 * transaction commit, as scrub has finished.
3176 	 */
3177 	if (need_commit) {
3178 		struct btrfs_trans_handle *trans;
3179 
3180 		trans = btrfs_start_transaction(fs_info->tree_root, 0);
3181 		if (IS_ERR(trans)) {
3182 			ret = PTR_ERR(trans);
3183 			btrfs_err(fs_info,
3184 	"scrub: failed to start transaction to fix super block errors: %d", ret);
3185 			return ret;
3186 		}
3187 		ret = btrfs_commit_transaction(trans);
3188 		if (ret < 0)
3189 			btrfs_err(fs_info,
3190 	"scrub: failed to commit transaction to fix super block errors: %d", ret);
3191 	}
3192 	return ret;
3193 out:
3194 	scrub_workers_put(fs_info);
3195 out_free_ctx:
3196 	scrub_free_ctx(sctx);
3197 
3198 	return ret;
3199 }
3200 
btrfs_scrub_pause(struct btrfs_fs_info * fs_info)3201 void btrfs_scrub_pause(struct btrfs_fs_info *fs_info)
3202 {
3203 	mutex_lock(&fs_info->scrub_lock);
3204 	atomic_inc(&fs_info->scrub_pause_req);
3205 	while (atomic_read(&fs_info->scrubs_paused) !=
3206 	       atomic_read(&fs_info->scrubs_running)) {
3207 		mutex_unlock(&fs_info->scrub_lock);
3208 		wait_event(fs_info->scrub_pause_wait,
3209 			   atomic_read(&fs_info->scrubs_paused) ==
3210 			   atomic_read(&fs_info->scrubs_running));
3211 		mutex_lock(&fs_info->scrub_lock);
3212 	}
3213 	mutex_unlock(&fs_info->scrub_lock);
3214 }
3215 
btrfs_scrub_continue(struct btrfs_fs_info * fs_info)3216 void btrfs_scrub_continue(struct btrfs_fs_info *fs_info)
3217 {
3218 	atomic_dec(&fs_info->scrub_pause_req);
3219 	wake_up(&fs_info->scrub_pause_wait);
3220 }
3221 
btrfs_scrub_cancel(struct btrfs_fs_info * fs_info)3222 int btrfs_scrub_cancel(struct btrfs_fs_info *fs_info)
3223 {
3224 	mutex_lock(&fs_info->scrub_lock);
3225 	if (!atomic_read(&fs_info->scrubs_running)) {
3226 		mutex_unlock(&fs_info->scrub_lock);
3227 		return -ENOTCONN;
3228 	}
3229 
3230 	atomic_inc(&fs_info->scrub_cancel_req);
3231 	while (atomic_read(&fs_info->scrubs_running)) {
3232 		mutex_unlock(&fs_info->scrub_lock);
3233 		wait_event(fs_info->scrub_pause_wait,
3234 			   atomic_read(&fs_info->scrubs_running) == 0);
3235 		mutex_lock(&fs_info->scrub_lock);
3236 	}
3237 	atomic_dec(&fs_info->scrub_cancel_req);
3238 	mutex_unlock(&fs_info->scrub_lock);
3239 
3240 	return 0;
3241 }
3242 
btrfs_scrub_cancel_dev(struct btrfs_device * dev)3243 int btrfs_scrub_cancel_dev(struct btrfs_device *dev)
3244 {
3245 	struct btrfs_fs_info *fs_info = dev->fs_info;
3246 	struct scrub_ctx *sctx;
3247 
3248 	mutex_lock(&fs_info->scrub_lock);
3249 	sctx = dev->scrub_ctx;
3250 	if (!sctx) {
3251 		mutex_unlock(&fs_info->scrub_lock);
3252 		return -ENOTCONN;
3253 	}
3254 	atomic_inc(&sctx->cancel_req);
3255 	while (dev->scrub_ctx) {
3256 		mutex_unlock(&fs_info->scrub_lock);
3257 		wait_event(fs_info->scrub_pause_wait,
3258 			   dev->scrub_ctx == NULL);
3259 		mutex_lock(&fs_info->scrub_lock);
3260 	}
3261 	mutex_unlock(&fs_info->scrub_lock);
3262 
3263 	return 0;
3264 }
3265 
btrfs_scrub_progress(struct btrfs_fs_info * fs_info,u64 devid,struct btrfs_scrub_progress * progress)3266 int btrfs_scrub_progress(struct btrfs_fs_info *fs_info, u64 devid,
3267 			 struct btrfs_scrub_progress *progress)
3268 {
3269 	struct btrfs_dev_lookup_args args = { .devid = devid };
3270 	struct btrfs_device *dev;
3271 	struct scrub_ctx *sctx = NULL;
3272 
3273 	mutex_lock(&fs_info->fs_devices->device_list_mutex);
3274 	dev = btrfs_find_device(fs_info->fs_devices, &args);
3275 	if (dev)
3276 		sctx = dev->scrub_ctx;
3277 	if (sctx)
3278 		memcpy(progress, &sctx->stat, sizeof(*progress));
3279 	mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3280 
3281 	return dev ? (sctx ? 0 : -ENOTCONN) : -ENODEV;
3282 }
3283