xref: /linux/fs/xfs/scrub/cow_repair.c (revision 02c8e8d78792164df615a8c54411501e91fc7b64)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * Copyright (C) 2022-2023 Oracle.  All Rights Reserved.
4  * Author: Darrick J. Wong <djwong@kernel.org>
5  */
6 #include "xfs_platform.h"
7 #include "xfs_fs.h"
8 #include "xfs_shared.h"
9 #include "xfs_format.h"
10 #include "xfs_trans_resv.h"
11 #include "xfs_mount.h"
12 #include "xfs_defer.h"
13 #include "xfs_btree.h"
14 #include "xfs_log_format.h"
15 #include "xfs_trans.h"
16 #include "xfs_inode.h"
17 #include "xfs_inode_fork.h"
18 #include "xfs_alloc.h"
19 #include "xfs_bmap.h"
20 #include "xfs_rmap.h"
21 #include "xfs_refcount.h"
22 #include "xfs_quota.h"
23 #include "xfs_ialloc.h"
24 #include "xfs_ag.h"
25 #include "xfs_error.h"
26 #include "xfs_errortag.h"
27 #include "xfs_icache.h"
28 #include "xfs_refcount_btree.h"
29 #include "xfs_rtalloc.h"
30 #include "xfs_rtbitmap.h"
31 #include "xfs_rtgroup.h"
32 #include "xfs_bmap_util.h"
33 #include "scrub/xfs_scrub.h"
34 #include "scrub/scrub.h"
35 #include "scrub/common.h"
36 #include "scrub/trace.h"
37 #include "scrub/repair.h"
38 #include "scrub/bitmap.h"
39 #include "scrub/off_bitmap.h"
40 #include "scrub/fsb_bitmap.h"
41 #include "scrub/rtb_bitmap.h"
42 #include "scrub/reap.h"
43 
44 /*
45  * CoW Fork Mapping Repair
46  * =======================
47  *
48  * Although CoW staging extents are owned by incore CoW inode forks, on disk
49  * they are owned by the refcount btree.  The ondisk metadata does not record
50  * any ownership information, which limits what we can do to repair the
51  * mappings in the CoW fork.  At most, we can replace ifork mappings that lack
52  * an entry in the refcount btree or are described by a reverse mapping record
53  * whose owner is not OWN_COW.
54  *
55  * Replacing extents is also tricky -- we can't touch written CoW fork extents
56  * since they are undergoing writeback, and delalloc extents do not require
57  * repair since they only exist incore.  Hence the most we can do is find the
58  * bad parts of unwritten mappings, allocate a replacement set of blocks, and
59  * replace the incore mapping.  We use the regular reaping process to unmap
60  * or free the discarded blocks, as appropriate.
61  */
62 struct xrep_cow {
63 	struct xfs_scrub	*sc;
64 
65 	/* Bitmap of file offset ranges that need replacing. */
66 	struct xoff_bitmap	bad_fileoffs;
67 
68 	/* Bitmap of fsblocks that were removed from the CoW fork. */
69 	union {
70 		struct xfsb_bitmap	old_cowfork_fsblocks;
71 		struct xrtb_bitmap	old_cowfork_rtblocks;
72 	};
73 
74 	/* CoW fork mappings used to scan for bad CoW staging extents. */
75 	struct xfs_bmbt_irec	irec;
76 
77 	/* refcount btree block number of irec.br_startblock */
78 	unsigned int		irec_startbno;
79 
80 	/* refcount btree block number of the next refcount record we expect */
81 	unsigned int		next_bno;
82 };
83 
84 /*
85  * Mark the part of the file range that corresponds to the given physical
86  * space.  Caller must ensure that the physical range is within xc->irec.
87  */
88 STATIC int
89 xrep_cow_mark_file_range(
90 	struct xrep_cow		*xc,
91 	xfs_fsblock_t		startblock,
92 	xfs_filblks_t		blockcount)
93 {
94 	xfs_fileoff_t		startoff;
95 
96 	startoff = xc->irec.br_startoff +
97 				(startblock - xc->irec.br_startblock);
98 
99 	trace_xrep_cow_mark_file_range(xc->sc->ip, startblock, startoff,
100 			blockcount);
101 
102 	return xoff_bitmap_set(&xc->bad_fileoffs, startoff, blockcount);
103 }
104 
105 /*
106  * Trim @src to fit within the CoW fork mapping being examined, and put the
107  * result in @dst.
108  */
109 static inline void
110 xrep_cow_trim_refcount(
111 	struct xrep_cow			*xc,
112 	struct xfs_refcount_irec	*dst,
113 	const struct xfs_refcount_irec	*src)
114 {
115 	unsigned int			adj;
116 
117 	memcpy(dst, src, sizeof(*dst));
118 
119 	if (dst->rc_startblock < xc->irec_startbno) {
120 		adj = xc->irec_startbno - dst->rc_startblock;
121 		dst->rc_blockcount -= adj;
122 		dst->rc_startblock += adj;
123 	}
124 
125 	if (dst->rc_startblock + dst->rc_blockcount >
126 	    xc->irec_startbno + xc->irec.br_blockcount) {
127 		adj = (dst->rc_startblock + dst->rc_blockcount) -
128 		      (xc->irec_startbno + xc->irec.br_blockcount);
129 		dst->rc_blockcount -= adj;
130 	}
131 }
132 
133 /* Mark any shared CoW staging extents. */
134 STATIC int
135 xrep_cow_mark_shared_staging(
136 	struct xfs_btree_cur		*cur,
137 	const struct xfs_refcount_irec	*rec,
138 	void				*priv)
139 {
140 	struct xrep_cow			*xc = priv;
141 	struct xfs_refcount_irec	rrec;
142 
143 	if (!xfs_refcount_check_domain(rec) ||
144 	    rec->rc_domain != XFS_REFC_DOMAIN_SHARED)
145 		return -EFSCORRUPTED;
146 
147 	xrep_cow_trim_refcount(xc, &rrec, rec);
148 
149 	return xrep_cow_mark_file_range(xc,
150 			xfs_gbno_to_fsb(cur->bc_group, rrec.rc_startblock),
151 			rrec.rc_blockcount);
152 }
153 
154 /*
155  * Mark any portion of the CoW fork file offset range where there is not a CoW
156  * staging extent record in the refcountbt, and keep a record of where we did
157  * find correct refcountbt records.  Staging records are always cleaned out at
158  * mount time, so any two inodes trying to map the same staging area would have
159  * already taken the fs down due to refcount btree verifier errors.  Hence this
160  * inode should be the sole creator of the staging extent records ondisk.
161  */
162 STATIC int
163 xrep_cow_mark_missing_staging(
164 	struct xfs_btree_cur		*cur,
165 	const struct xfs_refcount_irec	*rec,
166 	void				*priv)
167 {
168 	struct xrep_cow			*xc = priv;
169 	struct xfs_refcount_irec	rrec;
170 	int				error;
171 
172 	if (!xfs_refcount_check_domain(rec) ||
173 	    rec->rc_domain != XFS_REFC_DOMAIN_COW)
174 		return -EFSCORRUPTED;
175 
176 	xrep_cow_trim_refcount(xc, &rrec, rec);
177 
178 	if (xc->next_bno >= rrec.rc_startblock)
179 		goto next;
180 
181 	error = xrep_cow_mark_file_range(xc,
182 			xfs_gbno_to_fsb(cur->bc_group, xc->next_bno),
183 			rrec.rc_startblock - xc->next_bno);
184 	if (error)
185 		return error;
186 
187 next:
188 	xc->next_bno = rrec.rc_startblock + rrec.rc_blockcount;
189 	return 0;
190 }
191 
192 /*
193  * Mark any area that does not correspond to a CoW staging rmap.  These are
194  * cross-linked areas that must be avoided.
195  */
196 STATIC int
197 xrep_cow_mark_missing_staging_rmap(
198 	struct xfs_btree_cur		*cur,
199 	const struct xfs_rmap_irec	*rec,
200 	void				*priv)
201 {
202 	struct xrep_cow			*xc = priv;
203 	const xfs_agblock_t		rec_end =
204 			rec->rm_startblock + rec->rm_blockcount;
205 	int				error = 0;
206 
207 	/* Was there a gap since the last rmap? */
208 	if (xc->next_bno < rec->rm_startblock) {
209 		error = xrep_cow_mark_file_range(xc,
210 				xfs_gbno_to_fsb(cur->bc_group, xc->next_bno),
211 				rec->rm_startblock - xc->next_bno);
212 		if (error)
213 			return error;
214 	}
215 
216 	if (rec->rm_owner != XFS_RMAP_OWN_COW) {
217 		const xfs_agblock_t	bad_bno =
218 				max(xc->irec_startbno, rec->rm_startblock);
219 		const xfs_agblock_t	irec_end =
220 				xc->irec_startbno + xc->irec.br_blockcount;
221 		const xfs_extlen_t	bad_end = min(irec_end, rec_end);
222 
223 		error = xrep_cow_mark_file_range(xc,
224 				xfs_gbno_to_fsb(cur->bc_group, bad_bno),
225 				bad_end - bad_bno);
226 		if (error)
227 			return error;
228 	}
229 
230 	xc->next_bno = max(xc->next_bno, rec_end);
231 	return error;
232 }
233 
234 /*
235  * Trim the start and end of the current mapping by up to 1/4 of the length
236  * and mark that as "bad" to test the cow fork repair mechanism.
237  */
238 static inline int
239 xrep_cow_debug_replacement(
240 	struct xrep_cow		*xc)
241 {
242 	xfs_fsblock_t		fsbno = xc->irec.br_startblock;
243 	xfs_extlen_t		len = xc->irec.br_blockcount;
244 	uint32_t		trim;
245 
246 	/* get_random_u32_below requires a nonzero argument */
247 	trim = len > 4 ? get_random_u32_below(len / 4) : 0;
248 	len -= trim;
249 
250 	trim = len > 4 ? get_random_u32_below(len / 4) : 0;
251 	fsbno += trim;
252 	len -= trim;
253 
254 	return xrep_cow_mark_file_range(xc, fsbno, len);
255 }
256 
257 /*
258  * Find any part of the CoW fork mapping that isn't a single-owner CoW staging
259  * extent and mark the corresponding part of the file range in the bitmap.
260  */
261 STATIC int
262 xrep_cow_find_bad(
263 	struct xrep_cow			*xc)
264 {
265 	struct xfs_refcount_irec	rc_low = { 0 };
266 	struct xfs_refcount_irec	rc_high = { 0 };
267 	struct xfs_rmap_irec		rm_low = { 0 };
268 	struct xfs_rmap_irec		rm_high = { 0 };
269 	struct xfs_perag		*pag;
270 	struct xfs_scrub		*sc = xc->sc;
271 	xfs_agnumber_t			agno;
272 	xfs_agblock_t			irec_end;
273 	int				error;
274 
275 	agno = XFS_FSB_TO_AGNO(sc->mp, xc->irec.br_startblock);
276 	xc->irec_startbno = XFS_FSB_TO_AGBNO(sc->mp, xc->irec.br_startblock);
277 	irec_end = xc->irec_startbno + xc->irec.br_blockcount;
278 
279 	pag = xfs_perag_get(sc->mp, agno);
280 	if (!pag)
281 		return -EFSCORRUPTED;
282 
283 	error = xrep_ag_init(sc, pag);
284 	if (error)
285 		goto out_pag;
286 
287 	/* Mark any CoW fork extents that are shared. */
288 	rc_low.rc_startblock = xc->irec_startbno;
289 	rc_high.rc_startblock = irec_end - 1;
290 	rc_low.rc_domain = rc_high.rc_domain = XFS_REFC_DOMAIN_SHARED;
291 	error = xfs_refcount_query_range(sc->sa.refc_cur, &rc_low, &rc_high,
292 			xrep_cow_mark_shared_staging, xc);
293 	if (error)
294 		goto out_sa;
295 
296 	/* Make sure there are CoW staging extents for the whole mapping. */
297 	rc_low.rc_startblock = xc->irec_startbno;
298 	rc_high.rc_startblock = irec_end - 1;
299 	rc_low.rc_domain = rc_high.rc_domain = XFS_REFC_DOMAIN_COW;
300 	xc->next_bno = xc->irec_startbno;
301 	error = xfs_refcount_query_range(sc->sa.refc_cur, &rc_low, &rc_high,
302 			xrep_cow_mark_missing_staging, xc);
303 	if (error)
304 		goto out_sa;
305 
306 	if (xc->next_bno < irec_end) {
307 		error = xrep_cow_mark_file_range(xc,
308 				xfs_agbno_to_fsb(pag, xc->next_bno),
309 				irec_end - xc->next_bno);
310 		if (error)
311 			goto out_sa;
312 	}
313 
314 	/* Mark any area has an rmap that isn't a COW staging extent. */
315 	rm_low.rm_startblock = xc->irec_startbno;
316 	memset(&rm_high, 0xFF, sizeof(rm_high));
317 	rm_high.rm_startblock = xc->irec_startbno + xc->irec.br_blockcount - 1;
318 	xc->next_bno = xc->irec_startbno;
319 	error = xfs_rmap_query_range(sc->sa.rmap_cur, &rm_low, &rm_high,
320 			xrep_cow_mark_missing_staging_rmap, xc);
321 	if (error)
322 		goto out_sa;
323 
324 	if (xc->next_bno < irec_end) {
325 		error = xrep_cow_mark_file_range(xc,
326 				xfs_agbno_to_fsb(pag, xc->next_bno),
327 				irec_end - xc->next_bno);
328 		if (error)
329 			goto out_sa;
330 	}
331 
332 	/*
333 	 * If userspace is forcing us to rebuild the CoW fork or someone turned
334 	 * on the debugging knob, replace everything in the CoW fork.
335 	 */
336 	if (XFS_TEST_ERROR(sc->mp, XFS_ERRTAG_FORCE_SCRUB_REPAIR))
337 		error = xrep_cow_debug_replacement(xc);
338 	else if (sc->sm->sm_flags & XFS_SCRUB_IFLAG_FORCE_REBUILD)
339 		error = xrep_cow_mark_file_range(xc, xc->irec.br_startblock,
340 				xc->irec.br_blockcount);
341 
342 out_sa:
343 	xchk_ag_free(sc);
344 out_pag:
345 	xfs_perag_put(pag);
346 	return error;
347 }
348 
349 /*
350  * Find any part of the CoW fork mapping that isn't a single-owner CoW staging
351  * extent and mark the corresponding part of the file range in the bitmap.
352  */
353 STATIC int
354 xrep_cow_find_bad_rt(
355 	struct xrep_cow			*xc)
356 {
357 	struct xfs_refcount_irec	rc_low = { 0 };
358 	struct xfs_refcount_irec	rc_high = { 0 };
359 	struct xfs_rmap_irec		rm_low = { 0 };
360 	struct xfs_rmap_irec		rm_high = { 0 };
361 	struct xfs_scrub		*sc = xc->sc;
362 	struct xfs_rtgroup		*rtg;
363 	xfs_agblock_t			irec_end;
364 	int				error = 0;
365 
366 	xc->irec_startbno = xfs_rtb_to_rgbno(sc->mp, xc->irec.br_startblock);
367 	irec_end = xc->irec_startbno + xc->irec.br_blockcount;
368 
369 	rtg = xfs_rtgroup_get(sc->mp,
370 			xfs_rtb_to_rgno(sc->mp, xc->irec.br_startblock));
371 	if (!rtg)
372 		return -EFSCORRUPTED;
373 
374 	error = xrep_rtgroup_init(sc, rtg, XFS_RTGLOCK_RMAP |
375 					   XFS_RTGLOCK_REFCOUNT);
376 	if (error)
377 		goto out_rtg;
378 
379 	/* Mark any CoW fork extents that are shared. */
380 	rc_low.rc_startblock = xc->irec_startbno;
381 	rc_high.rc_startblock = irec_end - 1;
382 	rc_low.rc_domain = rc_high.rc_domain = XFS_REFC_DOMAIN_SHARED;
383 	error = xfs_refcount_query_range(sc->sr.refc_cur, &rc_low, &rc_high,
384 			xrep_cow_mark_shared_staging, xc);
385 	if (error)
386 		goto out_sr;
387 
388 	/* Make sure there are CoW staging extents for the whole mapping. */
389 	rc_low.rc_startblock = xc->irec_startbno;
390 	rc_high.rc_startblock = irec_end - 1;
391 	rc_low.rc_domain = rc_high.rc_domain = XFS_REFC_DOMAIN_COW;
392 	xc->next_bno = xc->irec_startbno;
393 	error = xfs_refcount_query_range(sc->sr.refc_cur, &rc_low, &rc_high,
394 			xrep_cow_mark_missing_staging, xc);
395 	if (error)
396 		goto out_sr;
397 
398 	if (xc->next_bno < irec_end) {
399 		error = xrep_cow_mark_file_range(xc,
400 				xfs_rgbno_to_rtb(rtg, xc->next_bno),
401 				irec_end - xc->next_bno);
402 		if (error)
403 			goto out_sr;
404 	}
405 
406 	/* Mark any area has an rmap that isn't a COW staging extent. */
407 	rm_low.rm_startblock = xc->irec_startbno;
408 	memset(&rm_high, 0xFF, sizeof(rm_high));
409 	rm_high.rm_startblock = irec_end - 1;
410 	xc->next_bno = xc->irec_startbno;
411 	error = xfs_rmap_query_range(sc->sr.rmap_cur, &rm_low, &rm_high,
412 			xrep_cow_mark_missing_staging_rmap, xc);
413 	if (error)
414 		goto out_sr;
415 
416 	if (xc->next_bno < irec_end) {
417 		error = xrep_cow_mark_file_range(xc,
418 				xfs_rgbno_to_rtb(rtg, xc->next_bno),
419 				irec_end - xc->next_bno);
420 		if (error)
421 			goto out_sr;
422 	}
423 
424 	/*
425 	 * If userspace is forcing us to rebuild the CoW fork or someone
426 	 * turned on the debugging knob, replace everything in the
427 	 * CoW fork and then scan for staging extents in the refcountbt.
428 	 */
429 	if (XFS_TEST_ERROR(sc->mp, XFS_ERRTAG_FORCE_SCRUB_REPAIR))
430 		error = xrep_cow_debug_replacement(xc);
431 	else if (sc->sm->sm_flags & XFS_SCRUB_IFLAG_FORCE_REBUILD)
432 		error = xrep_cow_mark_file_range(xc, xc->irec.br_startblock,
433 				xc->irec.br_blockcount);
434 
435 out_sr:
436 	xchk_rtgroup_btcur_free(sc);
437 	xchk_rtgroup_free(sc);
438 out_rtg:
439 	xfs_rtgroup_put(rtg);
440 	return error;
441 }
442 
443 /*
444  * Allocate a replacement CoW staging extent of up to the given number of
445  * blocks, and fill out the mapping.
446  */
447 STATIC int
448 xrep_cow_alloc(
449 	struct xfs_scrub	*sc,
450 	struct xfs_bmbt_irec	*del)
451 {
452 	struct xfs_alloc_arg	args = {
453 		.tp		= sc->tp,
454 		.mp		= sc->mp,
455 		.oinfo		= XFS_RMAP_OINFO_SKIP_UPDATE,
456 		.minlen		= 1,
457 		.maxlen		= del->br_blockcount,
458 		.prod		= 1,
459 		.resv		= XFS_AG_RESV_NONE,
460 		.datatype	= XFS_ALLOC_USERDATA,
461 	};
462 	int			error;
463 
464 	error = xfs_trans_reserve_more(sc->tp, del->br_blockcount, 0);
465 	if (error)
466 		return error;
467 
468 	error = xfs_alloc_vextent_start_ag(&args, XFS_INODE_TO_FSB(sc->ip));
469 	if (error)
470 		return error;
471 	if (args.fsbno == NULLFSBLOCK)
472 		return -ENOSPC;
473 
474 	xfs_refcount_alloc_cow_extent(sc->tp, false, args.fsbno, args.len);
475 
476 	del->br_startblock = args.fsbno;
477 	del->br_blockcount = args.len;
478 	return 0;
479 }
480 
481 /*
482  * Allocate a replacement rt CoW staging extent of up to the given number of
483  * blocks, and fill out the mapping.
484  */
485 STATIC int
486 xrep_cow_alloc_rt(
487 	struct xfs_scrub	*sc,
488 	struct xfs_bmbt_irec	*del)
489 {
490 	xfs_fsblock_t		fsbno;
491 	xfs_rtxlen_t		maxrtx =
492 		min(U32_MAX, xfs_blen_to_rtbxlen(sc->mp, del->br_blockcount));
493 	xfs_extlen_t		len;
494 	int			error;
495 
496 	error = xfs_trans_reserve_more(sc->tp, 0, maxrtx);
497 	if (error)
498 		return error;
499 
500 	error = xfs_rtallocate_rtgs(sc->tp, NULLRTBLOCK, 1, maxrtx, 1, false,
501 			false, &fsbno, &len);
502 	if (error)
503 		return error;
504 
505 	xfs_refcount_alloc_cow_extent(sc->tp, true, fsbno, len);
506 
507 	del->br_startblock = fsbno;
508 	del->br_blockcount = len;
509 	return 0;
510 }
511 
512 /*
513  * Look up the current CoW fork mapping so that we only allocate enough to
514  * replace a single mapping.  If we don't find a mapping that covers the start
515  * of the file range, or we find a delalloc or written extent, something is
516  * seriously wrong, since we didn't drop the ILOCK.
517  */
518 static inline int
519 xrep_cow_find_mapping(
520 	struct xrep_cow		*xc,
521 	struct xfs_iext_cursor	*icur,
522 	xfs_fileoff_t		badoff,
523 	xfs_extlen_t		badlen,
524 	struct xfs_bmbt_irec	*got,
525 	struct xfs_bmbt_irec	*rep)
526 {
527 	struct xfs_inode	*ip = xc->sc->ip;
528 	struct xfs_ifork	*ifp = xfs_ifork_ptr(ip, XFS_COW_FORK);
529 
530 	if (!xfs_iext_lookup_extent(ip, ifp, badoff, icur, got))
531 		goto bad;
532 	memcpy(rep, got, sizeof(*rep));
533 
534 	if (got->br_startoff > badoff)
535 		goto bad;
536 
537 	if (isnullstartblock(got->br_startblock))
538 		goto bad;
539 
540 	if (xfs_bmap_is_written_extent(got))
541 		goto bad;
542 
543 	if (got->br_startoff < badoff) {
544 		const int64_t	delta = badoff - got->br_startoff;
545 
546 		rep->br_blockcount -= delta;
547 		rep->br_startoff += delta;
548 		rep->br_startblock += delta;
549 	}
550 
551 	if (got->br_startoff + got->br_blockcount > badoff + badlen) {
552 		const int64_t	delta = (got->br_startoff + got->br_blockcount) -
553 					(badoff + badlen);
554 
555 		rep->br_blockcount -= delta;
556 	}
557 
558 	if (got->br_blockcount == 0)
559 		goto bad;
560 
561 	return 0;
562 bad:
563 	ASSERT(0);
564 	return -EFSCORRUPTED;
565 }
566 
567 /*
568  * Replace the unwritten CoW staging extent backing the given file range with a
569  * new space extent that isn't as problematic.
570  */
571 STATIC int
572 xrep_cow_replace_range(
573 	struct xrep_cow		*xc,
574 	xfs_fileoff_t		startoff,
575 	xfs_extlen_t		*blockcount)
576 {
577 	struct xfs_iext_cursor	icur;
578 	struct xfs_bmbt_irec	got, rep;
579 	struct xfs_scrub	*sc = xc->sc;
580 	xfs_fsblock_t		old_fsbno;
581 	int			error;
582 
583 	/*
584 	 * Put the existing CoW fork mapping in @got, and put in @rep the
585 	 * contents of @got trimmed to @startoff/@blockcount.  We only want
586 	 * to replace the bad region, and only one mapping at a time.
587 	 */
588 	error = xrep_cow_find_mapping(xc, &icur, startoff, *blockcount, &got,
589 			&rep);
590 	if (error)
591 		return error;
592 	old_fsbno = rep.br_startblock;
593 
594 	/*
595 	 * Allocate a replacement extent.  If we don't fill all the blocks,
596 	 * shorten the quantity that will be deleted in this step.
597 	 */
598 	if (XFS_IS_REALTIME_INODE(sc->ip))
599 		error = xrep_cow_alloc_rt(sc, &rep);
600 	else
601 		error = xrep_cow_alloc(sc, &rep);
602 	if (error)
603 		return error;
604 
605 	/*
606 	 * Replace the old mapping with the new one, and commit the metadata
607 	 * changes made so far.
608 	 */
609 	xfs_bmap_replace_cow_mapping(sc->ip, &icur, &got, &rep);
610 
611 	xfs_inode_set_cowblocks_tag(sc->ip);
612 	error = xfs_defer_finish(&sc->tp);
613 	if (error)
614 		return error;
615 
616 	/* Note the old CoW staging extents; we'll reap them all later. */
617 	if (XFS_IS_REALTIME_INODE(sc->ip))
618 		error = xrtb_bitmap_set(&xc->old_cowfork_rtblocks, old_fsbno,
619 				rep.br_blockcount);
620 	else
621 		error = xfsb_bitmap_set(&xc->old_cowfork_fsblocks, old_fsbno,
622 				rep.br_blockcount);
623 	if (error)
624 		return error;
625 
626 	*blockcount = rep.br_blockcount;
627 	return 0;
628 }
629 
630 /*
631  * Replace a bad part of an unwritten CoW staging extent with a fresh delalloc
632  * reservation.
633  */
634 STATIC int
635 xrep_cow_replace(
636 	uint64_t		startoff,
637 	uint64_t		blockcount,
638 	void			*priv)
639 {
640 	struct xrep_cow		*xc = priv;
641 	int			error = 0;
642 
643 	while (blockcount > 0) {
644 		xfs_extlen_t	len = min_t(xfs_filblks_t, blockcount,
645 					    XFS_MAX_BMBT_EXTLEN);
646 
647 		error = xrep_cow_replace_range(xc, startoff, &len);
648 		if (error)
649 			break;
650 
651 		blockcount -= len;
652 		startoff += len;
653 	}
654 
655 	return error;
656 }
657 
658 /*
659  * Repair an inode's CoW fork.  The CoW fork is an in-core structure, so
660  * there's no btree to rebuid.  Instead, we replace any mappings that are
661  * cross-linked or lack ondisk CoW fork records in the refcount btree.
662  */
663 int
664 xrep_bmap_cow(
665 	struct xfs_scrub	*sc)
666 {
667 	struct xrep_cow		*xc;
668 	struct xfs_iext_cursor	icur;
669 	struct xfs_ifork	*ifp = xfs_ifork_ptr(sc->ip, XFS_COW_FORK);
670 	int			error;
671 
672 	if (!xfs_has_rmapbt(sc->mp) || !xfs_has_reflink(sc->mp))
673 		return -EOPNOTSUPP;
674 
675 	if (!ifp)
676 		return 0;
677 
678 	/*
679 	 * Realtime files with large extent sizes are not supported because
680 	 * we could encounter an CoW mapping that has been partially written
681 	 * out *and* requires replacement, and there's no solution to that.
682 	 */
683 	if (xfs_inode_has_bigrtalloc(sc->ip))
684 		return -EOPNOTSUPP;
685 
686 	/* Metadata inodes aren't supposed to have data on the rt volume. */
687 	if (xfs_is_metadir_inode(sc->ip) && XFS_IS_REALTIME_INODE(sc->ip))
688 		return -EOPNOTSUPP;
689 
690 	/*
691 	 * If we're somehow not in extents format, then reinitialize it to
692 	 * an empty extent mapping fork and exit.
693 	 */
694 	if (ifp->if_format != XFS_DINODE_FMT_EXTENTS) {
695 		ifp->if_format = XFS_DINODE_FMT_EXTENTS;
696 		ifp->if_nextents = 0;
697 		return 0;
698 	}
699 
700 	xc = kzalloc_obj(struct xrep_cow, XCHK_GFP_FLAGS);
701 	if (!xc)
702 		return -ENOMEM;
703 
704 	xfs_trans_ijoin(sc->tp, sc->ip, 0);
705 
706 	xc->sc = sc;
707 	xoff_bitmap_init(&xc->bad_fileoffs);
708 	if (XFS_IS_REALTIME_INODE(sc->ip))
709 		xrtb_bitmap_init(&xc->old_cowfork_rtblocks);
710 	else
711 		xfsb_bitmap_init(&xc->old_cowfork_fsblocks);
712 
713 	for_each_xfs_iext(ifp, &icur, &xc->irec) {
714 		if (xchk_should_terminate(sc, &error))
715 			goto out_bitmap;
716 
717 		/*
718 		 * delalloc reservations only exist incore, so there is no
719 		 * ondisk metadata that we can examine.  Hence we leave them
720 		 * alone.
721 		 */
722 		if (isnullstartblock(xc->irec.br_startblock))
723 			continue;
724 
725 		/*
726 		 * COW fork extents are only in the written state if writeback
727 		 * is actively writing to disk.  We cannot restart the write
728 		 * at a different disk address since we've already issued the
729 		 * IO, so we leave these alone and hope for the best.
730 		 */
731 		if (xfs_bmap_is_written_extent(&xc->irec))
732 			continue;
733 
734 		if (XFS_IS_REALTIME_INODE(sc->ip))
735 			error = xrep_cow_find_bad_rt(xc);
736 		else
737 			error = xrep_cow_find_bad(xc);
738 		if (error)
739 			goto out_bitmap;
740 	}
741 
742 	/* Replace any bad unwritten mappings with fresh reservations. */
743 	error = xoff_bitmap_walk(&xc->bad_fileoffs, xrep_cow_replace, xc);
744 	if (error)
745 		goto out_bitmap;
746 
747 	/*
748 	 * Reap as many of the old CoW blocks as we can.  They are owned ondisk
749 	 * by the refcount btree, not the inode, so it is correct to treat them
750 	 * like inode metadata.
751 	 */
752 	if (XFS_IS_REALTIME_INODE(sc->ip))
753 		error = xrep_reap_rtblocks(sc, &xc->old_cowfork_rtblocks,
754 				&XFS_RMAP_OINFO_COW);
755 	else
756 		error = xrep_reap_fsblocks(sc, &xc->old_cowfork_fsblocks,
757 				&XFS_RMAP_OINFO_COW);
758 	if (error)
759 		goto out_bitmap;
760 
761 out_bitmap:
762 	if (XFS_IS_REALTIME_INODE(sc->ip))
763 		xrtb_bitmap_destroy(&xc->old_cowfork_rtblocks);
764 	else
765 		xfsb_bitmap_destroy(&xc->old_cowfork_fsblocks);
766 	xoff_bitmap_destroy(&xc->bad_fileoffs);
767 	kfree(xc);
768 	return error;
769 }
770