1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (C) 2020-2023 Oracle. All Rights Reserved. 4 * Author: Darrick J. Wong <djwong@kernel.org> 5 */ 6 #include "xfs_platform.h" 7 #include "xfs_fs.h" 8 #include "xfs_shared.h" 9 #include "xfs_format.h" 10 #include "xfs_trans_resv.h" 11 #include "xfs_mount.h" 12 #include "xfs_btree.h" 13 #include "xfs_log_format.h" 14 #include "xfs_trans.h" 15 #include "xfs_rtalloc.h" 16 #include "xfs_inode.h" 17 #include "xfs_bit.h" 18 #include "xfs_bmap.h" 19 #include "xfs_bmap_btree.h" 20 #include "xfs_rmap.h" 21 #include "xfs_rtrmap_btree.h" 22 #include "xfs_exchmaps.h" 23 #include "xfs_rtbitmap.h" 24 #include "xfs_rtgroup.h" 25 #include "xfs_extent_busy.h" 26 #include "xfs_refcount.h" 27 #include "scrub/scrub.h" 28 #include "scrub/common.h" 29 #include "scrub/trace.h" 30 #include "scrub/repair.h" 31 #include "scrub/xfile.h" 32 #include "scrub/tempfile.h" 33 #include "scrub/tempexch.h" 34 #include "scrub/reap.h" 35 #include "scrub/rtbitmap.h" 36 37 /* rt bitmap content repairs */ 38 39 /* 40 * Reserve enough blocks to write out a completely new bitmap file, plus twice 41 * as many blocks as we would need if we can only allocate one block per data 42 * fork mapping. This should cover the preallocation of the temporary file and 43 * exchanging the extent mappings. 44 * 45 * We cannot use xfs_exchmaps_estimate because we have not yet constructed the 46 * replacement bitmap and therefore do not know how many extents it will use. 47 * By the time we do, we will have a dirty transaction (which we cannot drop 48 * because we cannot drop the rtbitmap ILOCK) and cannot ask for more 49 * reservation. 50 */ 51 static inline unsigned long long 52 xrep_rtbitmap_calc_blocks(struct xfs_mount *mp, unsigned long long blocks) 53 { 54 return blocks + (xfs_bmbt_calc_size(mp, blocks) * 2); 55 } 56 57 /* Set up to repair the realtime bitmap for this group. */ 58 int 59 xrep_setup_rtbitmap( 60 struct xfs_scrub *sc, 61 struct xchk_rtbitmap *rtb) 62 { 63 struct xfs_mount *mp = sc->mp; 64 unsigned long long blocks = mp->m_sb.sb_rbmblocks; 65 int error; 66 67 error = xrep_tempfile_create(sc, S_IFREG); 68 if (error) 69 return error; 70 71 /* Create an xfile to hold our reconstructed bitmap. */ 72 error = xfile_create("realtime bitmap file", 73 blocks * mp->m_sb.sb_blocksize, &sc->xfile); 74 if (error) 75 return error; 76 77 blocks = xrep_rtbitmap_calc_blocks(mp, mp->m_sb.sb_rbmblocks); 78 if (blocks > UINT_MAX) 79 return -EOPNOTSUPP; 80 81 rtb->resblks += blocks; 82 return 0; 83 } 84 85 static inline xrep_wordoff_t 86 rtx_to_wordoff( 87 struct xfs_mount *mp, 88 xfs_rtxnum_t rtx) 89 { 90 return rtx >> XFS_NBWORDLOG; 91 } 92 93 static inline xrep_wordcnt_t 94 rtxlen_to_wordcnt( 95 xfs_rtxlen_t rtxlen) 96 { 97 return rtxlen >> XFS_NBWORDLOG; 98 } 99 100 /* Helper functions to record rtwords in an xfile. */ 101 102 static inline int 103 xfbmp_load( 104 struct xchk_rtbitmap *rtb, 105 xrep_wordoff_t wordoff, 106 xfs_rtword_t *word) 107 { 108 union xfs_rtword_raw urk; 109 int error; 110 111 ASSERT(xfs_has_rtgroups(rtb->sc->mp)); 112 113 error = xfile_load(rtb->sc->xfile, &urk, 114 sizeof(union xfs_rtword_raw), 115 wordoff << XFS_WORDLOG); 116 if (error) 117 return error; 118 119 *word = be32_to_cpu(urk.rtg); 120 return 0; 121 } 122 123 static inline int 124 xfbmp_store( 125 struct xchk_rtbitmap *rtb, 126 xrep_wordoff_t wordoff, 127 const xfs_rtword_t word) 128 { 129 union xfs_rtword_raw urk; 130 131 ASSERT(xfs_has_rtgroups(rtb->sc->mp)); 132 133 urk.rtg = cpu_to_be32(word); 134 return xfile_store(rtb->sc->xfile, &urk, 135 sizeof(union xfs_rtword_raw), 136 wordoff << XFS_WORDLOG); 137 } 138 139 static inline int 140 xfbmp_copyin( 141 struct xchk_rtbitmap *rtb, 142 xrep_wordoff_t wordoff, 143 const union xfs_rtword_raw *word, 144 xrep_wordcnt_t nr_words) 145 { 146 return xfile_store(rtb->sc->xfile, word, nr_words << XFS_WORDLOG, 147 wordoff << XFS_WORDLOG); 148 } 149 150 static inline int 151 xfbmp_copyout( 152 struct xchk_rtbitmap *rtb, 153 xrep_wordoff_t wordoff, 154 union xfs_rtword_raw *word, 155 xrep_wordcnt_t nr_words) 156 { 157 return xfile_load(rtb->sc->xfile, word, nr_words << XFS_WORDLOG, 158 wordoff << XFS_WORDLOG); 159 } 160 161 /* Perform a logical OR operation on an rtword in the incore bitmap. */ 162 static int 163 xrep_rtbitmap_or( 164 struct xchk_rtbitmap *rtb, 165 xrep_wordoff_t wordoff, 166 xfs_rtword_t mask) 167 { 168 xfs_rtword_t word; 169 int error; 170 171 error = xfbmp_load(rtb, wordoff, &word); 172 if (error) 173 return error; 174 175 trace_xrep_rtbitmap_or(rtb->sc->mp, wordoff, mask, word); 176 177 return xfbmp_store(rtb, wordoff, word | mask); 178 } 179 180 /* 181 * Mark as free every rt extent between the next rt block we expected to see 182 * in the rtrmap records and the given rt block. 183 */ 184 STATIC int 185 xrep_rtbitmap_mark_free( 186 struct xchk_rtbitmap *rtb, 187 xfs_rgblock_t rgbno) 188 { 189 struct xfs_mount *mp = rtb->sc->mp; 190 struct xchk_rt *sr = &rtb->sc->sr; 191 struct xfs_rtgroup *rtg = sr->rtg; 192 xfs_rtxnum_t startrtx; 193 xfs_rtxnum_t nextrtx; 194 xrep_wordoff_t wordoff, nextwordoff; 195 unsigned int bit; 196 unsigned int bufwsize; 197 xfs_extlen_t mod; 198 xfs_rtword_t mask; 199 enum xbtree_recpacking outcome; 200 int error; 201 202 if (!xfs_verify_rgbext(rtg, rtb->next_rgbno, rgbno - rtb->next_rgbno)) 203 return -EFSCORRUPTED; 204 205 /* 206 * Convert rt blocks to rt extents The block range we find must be 207 * aligned to an rtextent boundary on both ends. 208 */ 209 startrtx = xfs_rgbno_to_rtx(mp, rtb->next_rgbno); 210 mod = xfs_rgbno_to_rtxoff(mp, rtb->next_rgbno); 211 if (mod) 212 return -EFSCORRUPTED; 213 214 nextrtx = xfs_rgbno_to_rtx(mp, rgbno - 1) + 1; 215 mod = xfs_rgbno_to_rtxoff(mp, rgbno - 1); 216 if (mod != mp->m_sb.sb_rextsize - 1) 217 return -EFSCORRUPTED; 218 219 /* Must not be shared or CoW staging. */ 220 if (sr->refc_cur) { 221 error = xfs_refcount_has_records(sr->refc_cur, 222 XFS_REFC_DOMAIN_SHARED, rtb->next_rgbno, 223 rgbno - rtb->next_rgbno, &outcome); 224 if (error) 225 return error; 226 if (outcome != XBTREE_RECPACKING_EMPTY) 227 return -EFSCORRUPTED; 228 229 error = xfs_refcount_has_records(sr->refc_cur, 230 XFS_REFC_DOMAIN_COW, rtb->next_rgbno, 231 rgbno - rtb->next_rgbno, &outcome); 232 if (error) 233 return error; 234 if (outcome != XBTREE_RECPACKING_EMPTY) 235 return -EFSCORRUPTED; 236 } 237 238 trace_xrep_rtbitmap_record_free(mp, startrtx, nextrtx - 1); 239 240 /* Set bits as needed to round startrtx up to the nearest word. */ 241 bit = startrtx & XREP_RTBMP_WORDMASK; 242 if (bit) { 243 xfs_rtblock_t len = nextrtx - startrtx; 244 unsigned int lastbit; 245 246 lastbit = min(bit + len, XFS_NBWORD); 247 mask = (((xfs_rtword_t)1 << (lastbit - bit)) - 1) << bit; 248 249 error = xrep_rtbitmap_or(rtb, rtx_to_wordoff(mp, startrtx), 250 mask); 251 if (error || lastbit - bit == len) 252 return error; 253 startrtx += XFS_NBWORD - bit; 254 } 255 256 /* Set bits as needed to round nextrtx down to the nearest word. */ 257 bit = nextrtx & XREP_RTBMP_WORDMASK; 258 if (bit) { 259 mask = ((xfs_rtword_t)1 << bit) - 1; 260 261 error = xrep_rtbitmap_or(rtb, rtx_to_wordoff(mp, nextrtx), 262 mask); 263 if (error || startrtx + bit == nextrtx) 264 return error; 265 nextrtx -= bit; 266 } 267 268 trace_xrep_rtbitmap_record_free_bulk(mp, startrtx, nextrtx - 1); 269 270 /* Set all the words in between, up to a whole fs block at once. */ 271 wordoff = rtx_to_wordoff(mp, startrtx); 272 nextwordoff = rtx_to_wordoff(mp, nextrtx); 273 bufwsize = mp->m_sb.sb_blocksize >> XFS_WORDLOG; 274 275 while (wordoff < nextwordoff) { 276 xrep_wordoff_t rem; 277 xrep_wordcnt_t wordcnt; 278 279 wordcnt = min_t(xrep_wordcnt_t, nextwordoff - wordoff, 280 bufwsize); 281 282 /* 283 * Try to keep us aligned to the rtwords buffer to reduce the 284 * number of xfile writes. 285 */ 286 rem = wordoff & (bufwsize - 1); 287 if (rem) 288 wordcnt = min_t(xrep_wordcnt_t, wordcnt, 289 bufwsize - rem); 290 291 error = xfbmp_copyin(rtb, wordoff, rtb->words, wordcnt); 292 if (error) 293 return error; 294 295 wordoff += wordcnt; 296 } 297 298 return 0; 299 } 300 301 /* Set free space in the rtbitmap based on rtrmapbt records. */ 302 STATIC int 303 xrep_rtbitmap_walk_rtrmap( 304 struct xfs_btree_cur *cur, 305 const struct xfs_rmap_irec *rec, 306 void *priv) 307 { 308 struct xchk_rtbitmap *rtb = priv; 309 int error = 0; 310 311 if (xchk_should_terminate(rtb->sc, &error)) 312 return error; 313 314 if (rtb->next_rgbno < rec->rm_startblock) { 315 error = xrep_rtbitmap_mark_free(rtb, rec->rm_startblock); 316 if (error) 317 return error; 318 } 319 320 rtb->next_rgbno = max(rtb->next_rgbno, 321 rec->rm_startblock + rec->rm_blockcount); 322 return 0; 323 } 324 325 /* 326 * Walk the rtrmapbt to find all the gaps between records, and mark the gaps 327 * in the realtime bitmap that we're computing. 328 */ 329 STATIC int 330 xrep_rtbitmap_find_freespace( 331 struct xchk_rtbitmap *rtb) 332 { 333 struct xfs_scrub *sc = rtb->sc; 334 struct xfs_mount *mp = sc->mp; 335 struct xfs_rtgroup *rtg = sc->sr.rtg; 336 uint64_t blockcount; 337 int error; 338 339 /* Prepare a buffer of ones so that we can accelerate bulk setting. */ 340 memset(rtb->words, 0xFF, mp->m_sb.sb_blocksize); 341 342 xrep_rtgroup_btcur_init(sc, &sc->sr); 343 error = xfs_rmap_query_all(sc->sr.rmap_cur, xrep_rtbitmap_walk_rtrmap, 344 rtb); 345 if (error) 346 goto out; 347 348 /* 349 * Mark as free every possible rt extent from the last one we saw to 350 * the end of the rt group. 351 */ 352 blockcount = rtg->rtg_extents * mp->m_sb.sb_rextsize; 353 if (rtb->next_rgbno < blockcount) { 354 error = xrep_rtbitmap_mark_free(rtb, blockcount); 355 if (error) 356 goto out; 357 } 358 359 out: 360 xchk_rtgroup_btcur_free(&sc->sr); 361 return error; 362 } 363 364 static int 365 xrep_rtbitmap_prep_buf( 366 struct xfs_scrub *sc, 367 struct xfs_buf *bp, 368 void *data) 369 { 370 struct xchk_rtbitmap *rtb = data; 371 struct xfs_mount *mp = sc->mp; 372 union xfs_rtword_raw *ondisk; 373 int error; 374 375 rtb->args.mp = sc->mp; 376 rtb->args.tp = sc->tp; 377 rtb->args.rbmbp = bp; 378 ondisk = xfs_rbmblock_wordptr(&rtb->args, 0); 379 rtb->args.rbmbp = NULL; 380 381 error = xfbmp_copyout(rtb, rtb->prep_wordoff, ondisk, 382 mp->m_blockwsize); 383 if (error) 384 return error; 385 386 if (xfs_has_rtgroups(sc->mp)) { 387 struct xfs_rtbuf_blkinfo *hdr = bp->b_addr; 388 389 hdr->rt_magic = cpu_to_be32(XFS_RTBITMAP_MAGIC); 390 hdr->rt_owner = cpu_to_be64(I_INO(sc->ip)); 391 hdr->rt_blkno = cpu_to_be64(xfs_buf_daddr(bp)); 392 hdr->rt_lsn = 0; 393 uuid_copy(&hdr->rt_uuid, &sc->mp->m_sb.sb_meta_uuid); 394 bp->b_ops = &xfs_rtbitmap_buf_ops; 395 } else { 396 bp->b_ops = &xfs_rtbuf_ops; 397 } 398 399 rtb->prep_wordoff += mp->m_blockwsize; 400 xfs_trans_buf_set_type(sc->tp, bp, XFS_BLFT_RTBITMAP_BUF); 401 return 0; 402 } 403 404 /* 405 * Make sure that the given range of the data fork of the realtime file is 406 * mapped to written blocks. The caller must ensure that the inode is joined 407 * to the transaction. 408 */ 409 STATIC int 410 xrep_rtbitmap_data_mappings( 411 struct xfs_scrub *sc, 412 xfs_filblks_t len) 413 { 414 struct xfs_bmbt_irec map; 415 xfs_fileoff_t off = 0; 416 int error; 417 418 ASSERT(sc->ip != NULL); 419 420 while (off < len) { 421 int nmaps = 1; 422 423 /* 424 * If we have a real extent mapping this block then we're 425 * in ok shape. 426 */ 427 error = xfs_bmapi_read(sc->ip, off, len - off, &map, &nmaps, 428 XFS_DATA_FORK); 429 if (error) 430 return error; 431 if (nmaps == 0) { 432 ASSERT(nmaps != 0); 433 return -EFSCORRUPTED; 434 } 435 436 /* 437 * Written extents are ok. Holes are not filled because we 438 * do not know the freespace information. 439 */ 440 if (xfs_bmap_is_written_extent(&map) || 441 map.br_startblock == HOLESTARTBLOCK) { 442 off = map.br_startoff + map.br_blockcount; 443 continue; 444 } 445 446 /* 447 * If we find a delalloc reservation then something is very 448 * very wrong. Bail out. 449 */ 450 if (map.br_startblock == DELAYSTARTBLOCK) 451 return -EFSCORRUPTED; 452 453 /* Make sure we're really converting an unwritten extent. */ 454 if (map.br_state != XFS_EXT_UNWRITTEN) { 455 ASSERT(map.br_state == XFS_EXT_UNWRITTEN); 456 return -EFSCORRUPTED; 457 } 458 459 /* Make sure this block has a real zeroed extent mapped. */ 460 nmaps = 1; 461 error = xfs_bmapi_write(sc->tp, sc->ip, map.br_startoff, 462 map.br_blockcount, 463 XFS_BMAPI_CONVERT | XFS_BMAPI_ZERO, 464 0, &map, &nmaps); 465 if (error) 466 return error; 467 468 /* Commit new extent and all deferred work. */ 469 error = xrep_defer_finish(sc); 470 if (error) 471 return error; 472 473 off = map.br_startoff + map.br_blockcount; 474 } 475 476 return 0; 477 } 478 479 /* Fix broken rt volume geometry. */ 480 STATIC int 481 xrep_rtbitmap_geometry( 482 struct xfs_scrub *sc, 483 struct xchk_rtbitmap *rtb) 484 { 485 struct xfs_mount *mp = sc->mp; 486 struct xfs_trans *tp = sc->tp; 487 488 /* Superblock fields */ 489 if (mp->m_sb.sb_rextents != rtb->rextents) 490 xfs_trans_mod_sb(sc->tp, XFS_TRANS_SB_REXTENTS, 491 rtb->rextents - mp->m_sb.sb_rextents); 492 493 if (mp->m_sb.sb_rbmblocks != rtb->rbmblocks) 494 xfs_trans_mod_sb(tp, XFS_TRANS_SB_RBMBLOCKS, 495 rtb->rbmblocks - mp->m_sb.sb_rbmblocks); 496 497 if (mp->m_sb.sb_rextslog != rtb->rextslog) 498 xfs_trans_mod_sb(tp, XFS_TRANS_SB_REXTSLOG, 499 rtb->rextslog - mp->m_sb.sb_rextslog); 500 501 /* Fix broken isize */ 502 sc->ip->i_disk_size = roundup_64(sc->ip->i_disk_size, 503 mp->m_sb.sb_blocksize); 504 505 if (sc->ip->i_disk_size < XFS_FSB_TO_B(mp, rtb->rbmblocks)) 506 sc->ip->i_disk_size = XFS_FSB_TO_B(mp, rtb->rbmblocks); 507 508 xfs_trans_log_inode(sc->tp, sc->ip, XFS_ILOG_CORE); 509 return xrep_roll_trans(sc); 510 } 511 512 /* Repair the realtime bitmap file metadata. */ 513 int 514 xrep_rtbitmap( 515 struct xfs_scrub *sc) 516 { 517 struct xchk_rtbitmap *rtb = sc->buf; 518 struct xfs_mount *mp = sc->mp; 519 struct xfs_group *xg = rtg_group(sc->sr.rtg); 520 unsigned long long blocks; 521 unsigned int busy_gen; 522 int error; 523 524 /* We require the realtime rmapbt to rebuild anything. */ 525 if (!xfs_has_rtrmapbt(sc->mp)) 526 return -EOPNOTSUPP; 527 /* We require atomic file exchange range to rebuild anything. */ 528 if (!xfs_has_exchange_range(sc->mp)) 529 return -EOPNOTSUPP; 530 531 /* Impossibly large rtbitmap means we can't touch the filesystem. */ 532 if (rtb->rbmblocks > U32_MAX) 533 return 0; 534 535 /* 536 * If the size of the rt bitmap file is larger than what we reserved, 537 * figure out if we need to adjust the block reservation in the 538 * transaction. 539 */ 540 blocks = xrep_rtbitmap_calc_blocks(mp, rtb->rbmblocks); 541 if (blocks > UINT_MAX) 542 return -EOPNOTSUPP; 543 if (blocks > rtb->resblks) { 544 uint64_t delta = blocks - rtb->resblks; 545 546 if (delta > UINT_MAX) 547 return -EOPNOTSUPP; 548 549 error = xfs_trans_reserve_more(sc->tp, delta, 0); 550 if (error) 551 return error; 552 553 rtb->resblks += delta; 554 } 555 556 /* Fix inode core and forks. */ 557 error = xrep_metadata_inode_forks(sc); 558 if (error) 559 return error; 560 561 xfs_trans_ijoin(sc->tp, sc->ip, 0); 562 563 /* Ensure no unwritten extents. */ 564 error = xrep_rtbitmap_data_mappings(sc, rtb->rbmblocks); 565 if (error) 566 return error; 567 568 /* 569 * Fix inconsistent bitmap geometry. This function returns with a 570 * clean scrub transaction. 571 */ 572 error = xrep_rtbitmap_geometry(sc, rtb); 573 if (error) 574 return error; 575 576 /* 577 * Make sure the busy extent list is clear because we can't put extents 578 * on there twice. 579 */ 580 if (!xfs_extent_busy_list_empty(xg, &busy_gen)) { 581 error = xfs_extent_busy_flush(sc->tp, xg, busy_gen, 0); 582 if (error) 583 return error; 584 } 585 586 /* 587 * Generate the new rtbitmap data. We don't need the rtbmp information 588 * once this call is finished. 589 */ 590 error = xrep_rtbitmap_find_freespace(rtb); 591 if (error) 592 return error; 593 594 /* 595 * Try to take ILOCK_EXCL of the temporary file. We had better be the 596 * only ones holding onto this inode, but we can't block while holding 597 * the rtbitmap file's ILOCK_EXCL. 598 */ 599 while (!xrep_tempfile_ilock_nowait(sc)) { 600 if (xchk_should_terminate(sc, &error)) 601 return error; 602 delay(1); 603 } 604 605 /* 606 * Make sure we have space allocated for the part of the bitmap 607 * file that corresponds to this group. We already joined sc->ip. 608 */ 609 xfs_trans_ijoin(sc->tp, sc->tempip, 0); 610 error = xrep_tempfile_prealloc(sc, 0, rtb->rbmblocks); 611 if (error) 612 return error; 613 614 /* Last chance to abort before we start committing fixes. */ 615 if (xchk_should_terminate(sc, &error)) 616 return error; 617 618 /* Copy the bitmap file that we generated. */ 619 error = xrep_tempfile_copyin(sc, 0, rtb->rbmblocks, 620 xrep_rtbitmap_prep_buf, rtb); 621 if (error) 622 return error; 623 error = xrep_tempfile_set_isize(sc, 624 XFS_FSB_TO_B(sc->mp, sc->mp->m_sb.sb_rbmblocks)); 625 if (error) 626 return error; 627 628 /* 629 * Now exchange the data fork contents. We're done with the temporary 630 * buffer, so we can reuse it for the tempfile exchmaps information. 631 */ 632 error = xrep_tempexch_trans_reserve(sc, XFS_DATA_FORK, 0, 633 rtb->rbmblocks, &rtb->tempexch); 634 if (error) 635 return error; 636 637 error = xrep_tempexch_contents(sc, &rtb->tempexch); 638 if (error) 639 return error; 640 641 /* Free the old rtbitmap blocks if they're not in use. */ 642 return xrep_reap_ifork(sc, sc->tempip, XFS_DATA_FORK); 643 } 644