1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (c) 2020-2024 Oracle. All Rights Reserved. 4 * Author: Darrick J. Wong <djwong@kernel.org> 5 */ 6 #include "xfs_platform.h" 7 #include "xfs_fs.h" 8 #include "xfs_shared.h" 9 #include "xfs_format.h" 10 #include "xfs_log_format.h" 11 #include "xfs_trans_resv.h" 12 #include "xfs_mount.h" 13 #include "xfs_defer.h" 14 #include "xfs_inode.h" 15 #include "xfs_trans.h" 16 #include "xfs_bmap.h" 17 #include "xfs_icache.h" 18 #include "xfs_quota.h" 19 #include "xfs_exchmaps.h" 20 #include "xfs_trace.h" 21 #include "xfs_bmap_btree.h" 22 #include "xfs_trans_space.h" 23 #include "xfs_error.h" 24 #include "xfs_errortag.h" 25 #include "xfs_health.h" 26 #include "xfs_exchmaps_item.h" 27 #include "xfs_da_format.h" 28 #include "xfs_da_btree.h" 29 #include "xfs_attr_leaf.h" 30 #include "xfs_attr.h" 31 #include "xfs_dir2_priv.h" 32 #include "xfs_dir2.h" 33 #include "xfs_symlink_remote.h" 34 35 struct kmem_cache *xfs_exchmaps_intent_cache; 36 37 /* bmbt mappings adjacent to a pair of records. */ 38 struct xfs_exchmaps_adjacent { 39 struct xfs_bmbt_irec left1; 40 struct xfs_bmbt_irec right1; 41 struct xfs_bmbt_irec left2; 42 struct xfs_bmbt_irec right2; 43 }; 44 45 #define ADJACENT_INIT { \ 46 .left1 = { .br_startblock = HOLESTARTBLOCK }, \ 47 .right1 = { .br_startblock = HOLESTARTBLOCK }, \ 48 .left2 = { .br_startblock = HOLESTARTBLOCK }, \ 49 .right2 = { .br_startblock = HOLESTARTBLOCK }, \ 50 } 51 52 /* Information to reset CoW fork state after an exchange. */ 53 54 /* 55 * If the reflink flag is set on either inode, make sure it has an incore CoW 56 * fork, since all reflink inodes must have them. If there's a CoW fork and it 57 * has mappings in it, make sure the inodes are tagged appropriately so that 58 * speculative preallocations can be GC'd if we run low of space. 59 */ 60 static inline void 61 xfs_exchmaps_ensure_cowfork( 62 struct xfs_inode *ip) 63 { 64 struct xfs_ifork *cfork; 65 66 if (xfs_is_reflink_inode(ip)) 67 xfs_ifork_init_cow(ip); 68 69 cfork = xfs_ifork_ptr(ip, XFS_COW_FORK); 70 if (!cfork) 71 return; 72 if (cfork->if_bytes > 0) 73 xfs_inode_set_cowblocks_tag(ip); 74 else 75 xfs_inode_clear_cowblocks_tag(ip); 76 } 77 78 /* 79 * Adjust the on-disk inode size upwards if needed so that we never add 80 * mappings into the file past EOF. This is crucial so that log recovery won't 81 * get confused by the sudden appearance of post-eof mappings. 82 */ 83 STATIC void 84 xfs_exchmaps_update_size( 85 struct xfs_trans *tp, 86 struct xfs_inode *ip, 87 struct xfs_bmbt_irec *imap, 88 xfs_fsize_t new_isize) 89 { 90 struct xfs_mount *mp = tp->t_mountp; 91 xfs_fsize_t len; 92 93 if (new_isize < 0) 94 return; 95 96 len = min(XFS_FSB_TO_B(mp, imap->br_startoff + imap->br_blockcount), 97 new_isize); 98 99 if (len <= ip->i_disk_size) 100 return; 101 102 trace_xfs_exchmaps_update_inode_size(ip, len); 103 104 ip->i_disk_size = len; 105 xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); 106 } 107 108 /* Advance the incore state tracking after exchanging a mapping. */ 109 static inline void 110 xmi_advance( 111 struct xfs_exchmaps_intent *xmi, 112 const struct xfs_bmbt_irec *irec) 113 { 114 xmi->xmi_startoff1 += irec->br_blockcount; 115 xmi->xmi_startoff2 += irec->br_blockcount; 116 xmi->xmi_blockcount -= irec->br_blockcount; 117 } 118 119 /* Do we still have more mappings to exchange? */ 120 static inline bool 121 xmi_has_more_exchange_work(const struct xfs_exchmaps_intent *xmi) 122 { 123 return xmi->xmi_blockcount > 0; 124 } 125 126 /* Do we have post-operation cleanups to perform? */ 127 static inline bool 128 xmi_has_postop_work(const struct xfs_exchmaps_intent *xmi) 129 { 130 return xmi->xmi_flags & __XFS_EXCHMAPS_INO2_SHORTFORM; 131 } 132 133 /* Check all mappings to make sure we can actually exchange them. */ 134 int 135 xfs_exchmaps_check_forks( 136 struct xfs_mount *mp, 137 const struct xfs_exchmaps_req *req) 138 { 139 struct xfs_ifork *ifp1, *ifp2; 140 int whichfork = xfs_exchmaps_reqfork(req); 141 142 /* No fork? */ 143 ifp1 = xfs_ifork_ptr(req->ip1, whichfork); 144 ifp2 = xfs_ifork_ptr(req->ip2, whichfork); 145 if (!ifp1 || !ifp2) 146 return -EINVAL; 147 148 /* We don't know how to exchange local format forks. */ 149 if (ifp1->if_format == XFS_DINODE_FMT_LOCAL || 150 ifp2->if_format == XFS_DINODE_FMT_LOCAL) 151 return -EINVAL; 152 153 return 0; 154 } 155 156 #ifdef CONFIG_XFS_QUOTA 157 /* Log the actual updates to the quota accounting. */ 158 static inline void 159 xfs_exchmaps_update_quota( 160 struct xfs_trans *tp, 161 struct xfs_exchmaps_intent *xmi, 162 struct xfs_bmbt_irec *irec1, 163 struct xfs_bmbt_irec *irec2) 164 { 165 int64_t ip1_delta = 0, ip2_delta = 0; 166 unsigned int qflag; 167 168 qflag = XFS_IS_REALTIME_INODE(xmi->xmi_ip1) ? XFS_TRANS_DQ_RTBCOUNT : 169 XFS_TRANS_DQ_BCOUNT; 170 171 if (xfs_bmap_is_real_extent(irec1)) { 172 ip1_delta -= irec1->br_blockcount; 173 ip2_delta += irec1->br_blockcount; 174 } 175 176 if (xfs_bmap_is_real_extent(irec2)) { 177 ip1_delta += irec2->br_blockcount; 178 ip2_delta -= irec2->br_blockcount; 179 } 180 181 xfs_trans_mod_dquot_byino(tp, xmi->xmi_ip1, qflag, ip1_delta); 182 xfs_trans_mod_dquot_byino(tp, xmi->xmi_ip2, qflag, ip2_delta); 183 } 184 #else 185 # define xfs_exchmaps_update_quota(tp, xmi, irec1, irec2) ((void)0) 186 #endif 187 188 /* Decide if we want to skip this mapping from file1. */ 189 static inline bool 190 xfs_exchmaps_can_skip_mapping( 191 struct xfs_exchmaps_intent *xmi, 192 struct xfs_bmbt_irec *irec) 193 { 194 struct xfs_mount *mp = xmi->xmi_ip1->i_mount; 195 196 /* Do not skip this mapping if the caller did not tell us to. */ 197 if (!(xmi->xmi_flags & XFS_EXCHMAPS_INO1_WRITTEN)) 198 return false; 199 200 /* Do not skip mapped, written mappings. */ 201 if (xfs_bmap_is_written_extent(irec)) 202 return false; 203 204 /* 205 * The mapping is unwritten or a hole. It cannot be a delalloc 206 * reservation because we already excluded those. It cannot be an 207 * unwritten extent with dirty page cache because we flushed the page 208 * cache. For files where the allocation unit is 1FSB (files on the 209 * data dev, rt files if the extent size is 1FSB), we can safely 210 * skip this mapping. 211 */ 212 if (!xfs_inode_has_bigrtalloc(xmi->xmi_ip1)) 213 return true; 214 215 /* 216 * For a realtime file with a multi-fsb allocation unit, the decision 217 * is trickier because we can only swap full allocation units. 218 * Unwritten mappings can appear in the middle of an rtx if the rtx is 219 * partially written, but they can also appear for preallocations. 220 * 221 * If the mapping is a hole, skip it entirely. Holes should align with 222 * rtx boundaries. 223 */ 224 if (!xfs_bmap_is_real_extent(irec)) 225 return true; 226 227 /* 228 * All mappings below this point are unwritten. 229 * 230 * - If the beginning is not aligned to an rtx, trim the end of the 231 * mapping so that it does not cross an rtx boundary, and swap it. 232 * 233 * - If both ends are aligned to an rtx, skip the entire mapping. 234 */ 235 if (!isaligned_64(irec->br_startoff, mp->m_sb.sb_rextsize)) { 236 xfs_fileoff_t new_end; 237 238 new_end = roundup_64(irec->br_startoff, mp->m_sb.sb_rextsize); 239 irec->br_blockcount = min(irec->br_blockcount, 240 new_end - irec->br_startoff); 241 return false; 242 } 243 if (isaligned_64(irec->br_blockcount, mp->m_sb.sb_rextsize)) 244 return true; 245 246 /* 247 * All mappings below this point are unwritten, start on an rtx 248 * boundary, and do not end on an rtx boundary. 249 * 250 * - If the mapping is longer than one rtx, trim the end of the mapping 251 * down to an rtx boundary and skip it. 252 * 253 * - The mapping is shorter than one rtx. Swap it. 254 */ 255 if (irec->br_blockcount > mp->m_sb.sb_rextsize) { 256 xfs_fileoff_t new_end; 257 258 new_end = rounddown_64(irec->br_startoff + irec->br_blockcount, 259 mp->m_sb.sb_rextsize); 260 irec->br_blockcount = new_end - irec->br_startoff; 261 return true; 262 } 263 264 return false; 265 } 266 267 /* 268 * Walk forward through the file ranges in @xmi until we find two different 269 * mappings to exchange. If there is work to do, return the mappings; 270 * otherwise we've reached the end of the range and xmi_blockcount will be 271 * zero. 272 * 273 * If the walk skips over a pair of mappings to the same storage, save them as 274 * the left records in @adj (if provided) so that the simulation phase can 275 * avoid an extra lookup. 276 */ 277 static int 278 xfs_exchmaps_find_mappings( 279 struct xfs_exchmaps_intent *xmi, 280 struct xfs_bmbt_irec *irec1, 281 struct xfs_bmbt_irec *irec2, 282 struct xfs_exchmaps_adjacent *adj) 283 { 284 int nimaps; 285 int bmap_flags; 286 int error; 287 288 bmap_flags = xfs_bmapi_aflag(xfs_exchmaps_whichfork(xmi)); 289 290 for (; xmi_has_more_exchange_work(xmi); xmi_advance(xmi, irec1)) { 291 /* Read mapping from the first file */ 292 nimaps = 1; 293 error = xfs_bmapi_read(xmi->xmi_ip1, xmi->xmi_startoff1, 294 xmi->xmi_blockcount, irec1, &nimaps, 295 bmap_flags); 296 if (error) 297 return error; 298 if (nimaps != 1 || 299 irec1->br_startblock == DELAYSTARTBLOCK || 300 irec1->br_startoff != xmi->xmi_startoff1) { 301 /* 302 * We should never get no mapping or a delalloc mapping 303 * or something that doesn't match what we asked for, 304 * since the caller flushed both inodes and we hold the 305 * ILOCKs for both inodes. 306 */ 307 ASSERT(0); 308 return -EINVAL; 309 } 310 311 if (xfs_exchmaps_can_skip_mapping(xmi, irec1)) { 312 trace_xfs_exchmaps_mapping1_skip(xmi->xmi_ip1, irec1); 313 continue; 314 } 315 316 /* Read mapping from the second file */ 317 nimaps = 1; 318 error = xfs_bmapi_read(xmi->xmi_ip2, xmi->xmi_startoff2, 319 irec1->br_blockcount, irec2, &nimaps, 320 bmap_flags); 321 if (error) 322 return error; 323 if (nimaps != 1 || 324 irec2->br_startblock == DELAYSTARTBLOCK || 325 irec2->br_startoff != xmi->xmi_startoff2) { 326 /* 327 * We should never get no mapping or a delalloc mapping 328 * or something that doesn't match what we asked for, 329 * since the caller flushed both inodes and we hold the 330 * ILOCKs for both inodes. 331 */ 332 ASSERT(0); 333 return -EINVAL; 334 } 335 336 /* 337 * We can only exchange as many blocks as the smaller of the 338 * two mapping maps. 339 */ 340 irec1->br_blockcount = min(irec1->br_blockcount, 341 irec2->br_blockcount); 342 343 trace_xfs_exchmaps_mapping1(xmi->xmi_ip1, irec1); 344 trace_xfs_exchmaps_mapping2(xmi->xmi_ip2, irec2); 345 346 /* We found something to exchange, so return it. */ 347 if (irec1->br_startblock != irec2->br_startblock) 348 return 0; 349 350 /* 351 * Two mappings pointing to the same physical block must not 352 * have different states; that's filesystem corruption. Move 353 * on to the next mapping if they're both holes or both point 354 * to the same physical space extent. 355 */ 356 if (irec1->br_state != irec2->br_state) { 357 xfs_bmap_mark_sick(xmi->xmi_ip1, 358 xfs_exchmaps_whichfork(xmi)); 359 xfs_bmap_mark_sick(xmi->xmi_ip2, 360 xfs_exchmaps_whichfork(xmi)); 361 return -EFSCORRUPTED; 362 } 363 364 /* 365 * Save the mappings if we're estimating work and skipping 366 * these identical mappings. 367 */ 368 if (adj) { 369 memcpy(&adj->left1, irec1, sizeof(*irec1)); 370 memcpy(&adj->left2, irec2, sizeof(*irec2)); 371 } 372 } 373 374 return 0; 375 } 376 377 /* Exchange these two mappings. */ 378 static void 379 xfs_exchmaps_one_step( 380 struct xfs_trans *tp, 381 struct xfs_exchmaps_intent *xmi, 382 struct xfs_bmbt_irec *irec1, 383 struct xfs_bmbt_irec *irec2) 384 { 385 int whichfork = xfs_exchmaps_whichfork(xmi); 386 387 xfs_exchmaps_update_quota(tp, xmi, irec1, irec2); 388 389 /* Remove both mappings. */ 390 xfs_bmap_unmap_extent(tp, xmi->xmi_ip1, whichfork, irec1); 391 xfs_bmap_unmap_extent(tp, xmi->xmi_ip2, whichfork, irec2); 392 393 /* 394 * Re-add both mappings. We exchange the file offsets between the two 395 * maps and add the opposite map, which has the effect of filling the 396 * logical offsets we just unmapped, but with the physical mapping 397 * information exchanged. 398 */ 399 swap(irec1->br_startoff, irec2->br_startoff); 400 xfs_bmap_map_extent(tp, xmi->xmi_ip1, whichfork, irec2); 401 xfs_bmap_map_extent(tp, xmi->xmi_ip2, whichfork, irec1); 402 403 /* Make sure we're not adding mappings past EOF. */ 404 if (whichfork == XFS_DATA_FORK) { 405 xfs_exchmaps_update_size(tp, xmi->xmi_ip1, irec2, 406 xmi->xmi_isize1); 407 xfs_exchmaps_update_size(tp, xmi->xmi_ip2, irec1, 408 xmi->xmi_isize2); 409 } 410 411 /* 412 * Advance our cursor and exit. The caller (either defer ops or log 413 * recovery) will log the XMD item, and if *blockcount is nonzero, it 414 * will log a new XMI item for the remainder and call us back. 415 */ 416 xmi_advance(xmi, irec1); 417 } 418 419 /* Convert inode2's leaf attr fork back to shortform, if possible.. */ 420 STATIC int 421 xfs_exchmaps_attr_to_sf( 422 struct xfs_trans *tp, 423 struct xfs_exchmaps_intent *xmi) 424 { 425 struct xfs_da_args args = { 426 .dp = xmi->xmi_ip2, 427 .geo = tp->t_mountp->m_attr_geo, 428 .whichfork = XFS_ATTR_FORK, 429 .trans = tp, 430 .owner = I_INO(xmi->xmi_ip2), 431 }; 432 struct xfs_buf *bp; 433 int forkoff; 434 int error; 435 436 if (!xfs_attr_is_leaf(xmi->xmi_ip2)) 437 return 0; 438 439 error = xfs_attr3_leaf_read(tp, xmi->xmi_ip2, I_INO(xmi->xmi_ip2), 0, 440 &bp); 441 if (error) 442 return error; 443 444 forkoff = xfs_attr_shortform_allfit(bp, xmi->xmi_ip2); 445 if (forkoff == 0) 446 return 0; 447 448 return xfs_attr3_leaf_to_shortform(bp, &args, forkoff); 449 } 450 451 /* Convert inode2's block dir fork back to shortform, if possible.. */ 452 STATIC int 453 xfs_exchmaps_dir_to_sf( 454 struct xfs_trans *tp, 455 struct xfs_exchmaps_intent *xmi) 456 { 457 struct xfs_da_args args = { 458 .dp = xmi->xmi_ip2, 459 .geo = tp->t_mountp->m_dir_geo, 460 .whichfork = XFS_DATA_FORK, 461 .trans = tp, 462 .owner = I_INO(xmi->xmi_ip2), 463 }; 464 struct xfs_dir2_sf_hdr sfh; 465 struct xfs_buf *bp; 466 int size; 467 int error = 0; 468 469 if (xfs_dir2_format(&args, &error) != XFS_DIR2_FMT_BLOCK) 470 return error; 471 472 error = xfs_dir3_block_read(tp, xmi->xmi_ip2, I_INO(xmi->xmi_ip2), &bp); 473 if (error) 474 return error; 475 476 size = xfs_dir2_block_sfsize(xmi->xmi_ip2, bp->b_addr, &sfh); 477 if (size > xfs_inode_data_fork_size(xmi->xmi_ip2)) 478 return 0; 479 480 return xfs_dir2_block_to_sf(&args, bp, size, &sfh); 481 } 482 483 /* Convert inode2's remote symlink target back to shortform, if possible. */ 484 STATIC int 485 xfs_exchmaps_link_to_sf( 486 struct xfs_trans *tp, 487 struct xfs_exchmaps_intent *xmi) 488 { 489 struct xfs_inode *ip = xmi->xmi_ip2; 490 struct xfs_ifork *ifp = xfs_ifork_ptr(ip, XFS_DATA_FORK); 491 char *buf; 492 int error; 493 494 if (ifp->if_format == XFS_DINODE_FMT_LOCAL || 495 ip->i_disk_size > xfs_inode_data_fork_size(ip)) 496 return 0; 497 498 /* Read the current symlink target into a buffer. */ 499 buf = kmalloc(ip->i_disk_size + 1, 500 GFP_KERNEL | __GFP_NOLOCKDEP | __GFP_NOFAIL); 501 if (!buf) { 502 ASSERT(0); 503 return -ENOMEM; 504 } 505 506 error = xfs_symlink_remote_read(ip, buf); 507 if (error) 508 goto free; 509 510 /* Remove the blocks. */ 511 error = xfs_symlink_remote_truncate(tp, ip); 512 if (error) 513 goto free; 514 515 /* Convert fork to local format and log our changes. */ 516 xfs_idestroy_fork(ifp); 517 ifp->if_bytes = 0; 518 ifp->if_format = XFS_DINODE_FMT_LOCAL; 519 xfs_init_local_fork(ip, XFS_DATA_FORK, buf, ip->i_disk_size); 520 xfs_trans_log_inode(tp, ip, XFS_ILOG_DDATA | XFS_ILOG_CORE); 521 free: 522 kfree(buf); 523 return error; 524 } 525 526 /* Finish whatever work might come after an exchange operation. */ 527 static int 528 xfs_exchmaps_do_postop_work( 529 struct xfs_trans *tp, 530 struct xfs_exchmaps_intent *xmi) 531 { 532 if (xmi->xmi_flags & __XFS_EXCHMAPS_INO2_SHORTFORM) { 533 int error = 0; 534 535 if (xmi->xmi_flags & XFS_EXCHMAPS_ATTR_FORK) 536 error = xfs_exchmaps_attr_to_sf(tp, xmi); 537 else if (S_ISDIR(VFS_I(xmi->xmi_ip2)->i_mode)) 538 error = xfs_exchmaps_dir_to_sf(tp, xmi); 539 else if (S_ISLNK(VFS_I(xmi->xmi_ip2)->i_mode)) 540 error = xfs_exchmaps_link_to_sf(tp, xmi); 541 xmi->xmi_flags &= ~__XFS_EXCHMAPS_INO2_SHORTFORM; 542 if (error) 543 return error; 544 } 545 546 return 0; 547 } 548 549 /* Finish one step in a mapping exchange operation, possibly relogging. */ 550 int 551 xfs_exchmaps_finish_one( 552 struct xfs_trans *tp, 553 struct xfs_exchmaps_intent *xmi) 554 { 555 struct xfs_bmbt_irec irec1, irec2; 556 int error; 557 558 if (xmi_has_more_exchange_work(xmi)) { 559 /* 560 * If the operation state says that some range of the files 561 * have not yet been exchanged, look for mappings in that range 562 * to exchange. If we find some mappings, exchange them. 563 */ 564 error = xfs_exchmaps_find_mappings(xmi, &irec1, &irec2, NULL); 565 if (error) 566 return error; 567 568 if (xmi_has_more_exchange_work(xmi)) 569 xfs_exchmaps_one_step(tp, xmi, &irec1, &irec2); 570 571 /* 572 * If the caller asked us to exchange the file sizes after the 573 * exchange and either we just exchanged the last mappings in 574 * the range or we didn't find anything to exchange, update the 575 * ondisk file sizes. 576 */ 577 if ((xmi->xmi_flags & XFS_EXCHMAPS_SET_SIZES) && 578 !xmi_has_more_exchange_work(xmi)) { 579 xmi->xmi_ip1->i_disk_size = xmi->xmi_isize1; 580 xmi->xmi_ip2->i_disk_size = xmi->xmi_isize2; 581 582 xfs_trans_log_inode(tp, xmi->xmi_ip1, XFS_ILOG_CORE); 583 xfs_trans_log_inode(tp, xmi->xmi_ip2, XFS_ILOG_CORE); 584 } 585 } else if (xmi_has_postop_work(xmi)) { 586 /* 587 * Now that we're finished with the exchange operation, 588 * complete the post-op cleanup work. 589 */ 590 error = xfs_exchmaps_do_postop_work(tp, xmi); 591 if (error) 592 return error; 593 } 594 595 if (XFS_TEST_ERROR(tp->t_mountp, XFS_ERRTAG_EXCHMAPS_FINISH_ONE)) 596 return -EIO; 597 598 /* If we still have work to do, ask for a new transaction. */ 599 if (xmi_has_more_exchange_work(xmi) || xmi_has_postop_work(xmi)) { 600 trace_xfs_exchmaps_defer(tp->t_mountp, xmi); 601 return -EAGAIN; 602 } 603 604 /* 605 * If we reach here, we've finished all the exchange work and the post 606 * operation work. The last thing we need to do before returning to 607 * the caller is to make sure that COW forks are set up correctly. 608 */ 609 if (!(xmi->xmi_flags & XFS_EXCHMAPS_ATTR_FORK)) { 610 xfs_exchmaps_ensure_cowfork(xmi->xmi_ip1); 611 xfs_exchmaps_ensure_cowfork(xmi->xmi_ip2); 612 } 613 614 return 0; 615 } 616 617 /* 618 * Compute the amount of bmbt blocks we should reserve for each file. In the 619 * worst case, each exchange will fill a hole with a new mapping, which could 620 * result in a btree split every time we add a new leaf block. 621 */ 622 static inline uint64_t 623 xfs_exchmaps_bmbt_blocks( 624 struct xfs_mount *mp, 625 const struct xfs_exchmaps_req *req) 626 { 627 return howmany_64(req->nr_exchanges, 628 XFS_MAX_CONTIG_BMAPS_PER_BLOCK(mp)) * 629 XFS_EXTENTADD_SPACE_RES(mp, xfs_exchmaps_reqfork(req)); 630 } 631 632 /* Compute the space we should reserve for the rmap btree expansions. */ 633 static inline uint64_t 634 xfs_exchmaps_rmapbt_blocks( 635 struct xfs_mount *mp, 636 const struct xfs_exchmaps_req *req) 637 { 638 if (!xfs_has_rmapbt(mp)) 639 return 0; 640 if (XFS_IS_REALTIME_INODE(req->ip1)) 641 return howmany_64(req->nr_exchanges, 642 XFS_MAX_CONTIG_RTRMAPS_PER_BLOCK(mp)) * 643 XFS_RTRMAPADD_SPACE_RES(mp); 644 645 return howmany_64(req->nr_exchanges, 646 XFS_MAX_CONTIG_RMAPS_PER_BLOCK(mp)) * 647 XFS_RMAPADD_SPACE_RES(mp); 648 } 649 650 /* Estimate the bmbt and rmapbt overhead required to exchange mappings. */ 651 int 652 xfs_exchmaps_estimate_overhead( 653 struct xfs_exchmaps_req *req) 654 { 655 struct xfs_mount *mp = req->ip1->i_mount; 656 xfs_filblks_t bmbt_blocks; 657 xfs_filblks_t rmapbt_blocks; 658 xfs_filblks_t resblks = req->resblks; 659 660 /* 661 * Compute the number of bmbt and rmapbt blocks we might need to handle 662 * the estimated number of exchanges. 663 */ 664 bmbt_blocks = xfs_exchmaps_bmbt_blocks(mp, req); 665 rmapbt_blocks = xfs_exchmaps_rmapbt_blocks(mp, req); 666 667 trace_xfs_exchmaps_overhead(mp, bmbt_blocks, rmapbt_blocks); 668 669 /* Make sure the change in file block count doesn't overflow. */ 670 if (check_add_overflow(req->ip1_bcount, bmbt_blocks, &req->ip1_bcount)) 671 return -EFBIG; 672 if (check_add_overflow(req->ip2_bcount, bmbt_blocks, &req->ip2_bcount)) 673 return -EFBIG; 674 675 /* 676 * Add together the number of blocks we need to handle btree growth, 677 * then add it to the number of blocks we need to reserve to this 678 * transaction. 679 */ 680 if (check_add_overflow(resblks, bmbt_blocks, &resblks)) 681 return -ENOSPC; 682 if (check_add_overflow(resblks, bmbt_blocks, &resblks)) 683 return -ENOSPC; 684 if (check_add_overflow(resblks, rmapbt_blocks, &resblks)) 685 return -ENOSPC; 686 if (check_add_overflow(resblks, rmapbt_blocks, &resblks)) 687 return -ENOSPC; 688 689 /* Can't actually reserve more than UINT_MAX blocks. */ 690 if (resblks > UINT_MAX) 691 return -ENOSPC; 692 693 req->resblks = resblks; 694 trace_xfs_exchmaps_final_estimate(req); 695 return 0; 696 } 697 698 /* Decide if we can merge two real mappings. */ 699 static inline bool 700 xmi_can_merge( 701 const struct xfs_bmbt_irec *b1, 702 const struct xfs_bmbt_irec *b2) 703 { 704 /* Don't merge holes. */ 705 if (b1->br_startblock == HOLESTARTBLOCK || 706 b2->br_startblock == HOLESTARTBLOCK) 707 return false; 708 709 /* We don't merge holes. */ 710 if (!xfs_bmap_is_real_extent(b1) || !xfs_bmap_is_real_extent(b2)) 711 return false; 712 713 if (b1->br_startoff + b1->br_blockcount == b2->br_startoff && 714 b1->br_startblock + b1->br_blockcount == b2->br_startblock && 715 b1->br_state == b2->br_state && 716 b1->br_blockcount + b2->br_blockcount <= XFS_MAX_BMBT_EXTLEN) 717 return true; 718 719 return false; 720 } 721 722 /* 723 * Decide if we can merge three mappings. Caller must ensure all three 724 * mappings must not be holes or delalloc reservations. 725 */ 726 static inline bool 727 xmi_can_merge_all( 728 const struct xfs_bmbt_irec *l, 729 const struct xfs_bmbt_irec *m, 730 const struct xfs_bmbt_irec *r) 731 { 732 xfs_filblks_t new_len; 733 734 new_len = l->br_blockcount + m->br_blockcount + r->br_blockcount; 735 return new_len <= XFS_MAX_BMBT_EXTLEN; 736 } 737 738 #define CLEFT_CONTIG 0x01 739 #define CRIGHT_CONTIG 0x02 740 #define CHOLE 0x04 741 #define CBOTH_CONTIG (CLEFT_CONTIG | CRIGHT_CONTIG) 742 743 #define NLEFT_CONTIG 0x10 744 #define NRIGHT_CONTIG 0x20 745 #define NHOLE 0x40 746 #define NBOTH_CONTIG (NLEFT_CONTIG | NRIGHT_CONTIG) 747 748 /* Estimate the effect of a single exchange on mapping count. */ 749 static inline int 750 xmi_delta_nextents_step( 751 struct xfs_mount *mp, 752 const struct xfs_bmbt_irec *left, 753 const struct xfs_bmbt_irec *curr, 754 const struct xfs_bmbt_irec *new, 755 const struct xfs_bmbt_irec *right) 756 { 757 bool lhole, rhole, chole, nhole; 758 unsigned int state = 0; 759 int ret = 0; 760 761 lhole = left->br_startblock == HOLESTARTBLOCK; 762 rhole = right->br_startblock == HOLESTARTBLOCK; 763 chole = curr->br_startblock == HOLESTARTBLOCK; 764 nhole = new->br_startblock == HOLESTARTBLOCK; 765 766 if (chole) 767 state |= CHOLE; 768 if (!lhole && !chole && xmi_can_merge(left, curr)) 769 state |= CLEFT_CONTIG; 770 if (!rhole && !chole && xmi_can_merge(curr, right)) 771 state |= CRIGHT_CONTIG; 772 if ((state & CBOTH_CONTIG) == CBOTH_CONTIG && 773 !xmi_can_merge_all(left, curr, right)) 774 state &= ~CRIGHT_CONTIG; 775 776 if (nhole) 777 state |= NHOLE; 778 if (!lhole && !nhole && xmi_can_merge(left, new)) 779 state |= NLEFT_CONTIG; 780 if (!rhole && !nhole && xmi_can_merge(new, right)) 781 state |= NRIGHT_CONTIG; 782 if ((state & NBOTH_CONTIG) == NBOTH_CONTIG && 783 !xmi_can_merge_all(left, new, right)) 784 state &= ~NRIGHT_CONTIG; 785 786 switch (state & (CLEFT_CONTIG | CRIGHT_CONTIG | CHOLE)) { 787 case CLEFT_CONTIG | CRIGHT_CONTIG: 788 /* 789 * left/curr/right are the same mapping, so deleting curr 790 * causes 2 new mappings to be created. 791 */ 792 ret += 2; 793 break; 794 case 0: 795 /* 796 * curr is not contiguous with any mapping, so we remove curr 797 * completely 798 */ 799 ret--; 800 break; 801 case CHOLE: 802 /* hole, do nothing */ 803 break; 804 case CLEFT_CONTIG: 805 case CRIGHT_CONTIG: 806 /* trim either left or right, no change */ 807 break; 808 } 809 810 switch (state & (NLEFT_CONTIG | NRIGHT_CONTIG | NHOLE)) { 811 case NLEFT_CONTIG | NRIGHT_CONTIG: 812 /* 813 * left/curr/right will become the same mapping, so adding 814 * curr causes the deletion of right. 815 */ 816 ret--; 817 break; 818 case 0: 819 /* new is not contiguous with any mapping */ 820 ret++; 821 break; 822 case NHOLE: 823 /* hole, do nothing. */ 824 break; 825 case NLEFT_CONTIG: 826 case NRIGHT_CONTIG: 827 /* new is absorbed into left or right, no change */ 828 break; 829 } 830 831 trace_xfs_exchmaps_delta_nextents_step(mp, left, curr, new, right, ret, 832 state); 833 return ret; 834 } 835 836 /* Make sure we don't overflow the extent (mapping) counters. */ 837 static inline int 838 xmi_ensure_delta_nextents( 839 struct xfs_exchmaps_req *req, 840 struct xfs_inode *ip, 841 int64_t delta) 842 { 843 struct xfs_mount *mp = ip->i_mount; 844 int whichfork = xfs_exchmaps_reqfork(req); 845 struct xfs_ifork *ifp = xfs_ifork_ptr(ip, whichfork); 846 uint64_t new_nextents; 847 xfs_extnum_t max_nextents; 848 849 if (delta < 0) 850 return 0; 851 852 /* 853 * It's always an error if the delta causes integer overflow. delta 854 * needs an explicit cast here to avoid warnings about implicit casts 855 * coded into the overflow check. 856 */ 857 if (check_add_overflow(ifp->if_nextents, (uint64_t)delta, 858 &new_nextents)) 859 return -EFBIG; 860 861 if (XFS_TEST_ERROR(mp, XFS_ERRTAG_REDUCE_MAX_IEXTENTS) && 862 new_nextents > 10) 863 return -EFBIG; 864 865 /* 866 * We always promote both inodes to have large extent counts if the 867 * superblock feature is enabled, so we only need to check against the 868 * theoretical maximum. 869 */ 870 max_nextents = xfs_iext_max_nextents(xfs_has_large_extent_counts(mp), 871 whichfork); 872 if (new_nextents > max_nextents) 873 return -EFBIG; 874 875 return 0; 876 } 877 878 /* Find the next mapping after irec. */ 879 static inline int 880 xmi_next( 881 struct xfs_inode *ip, 882 int bmap_flags, 883 const struct xfs_bmbt_irec *irec, 884 struct xfs_bmbt_irec *nrec) 885 { 886 xfs_fileoff_t off; 887 xfs_filblks_t blockcount; 888 int nimaps = 1; 889 int error; 890 891 off = irec->br_startoff + irec->br_blockcount; 892 blockcount = XFS_MAX_FILEOFF - off; 893 error = xfs_bmapi_read(ip, off, blockcount, nrec, &nimaps, bmap_flags); 894 if (error) 895 return error; 896 if (nrec->br_startblock == DELAYSTARTBLOCK || 897 nrec->br_startoff != off) { 898 /* 899 * If we don't get the mapping we want, return a zero-length 900 * mapping, which our estimator function will pretend is a hole. 901 * We shouldn't get delalloc reservations. 902 */ 903 nrec->br_startblock = HOLESTARTBLOCK; 904 } 905 906 return 0; 907 } 908 909 int __init 910 xfs_exchmaps_intent_init_cache(void) 911 { 912 xfs_exchmaps_intent_cache = kmem_cache_create("xfs_exchmaps_intent", 913 sizeof(struct xfs_exchmaps_intent), 914 0, 0, NULL); 915 916 return xfs_exchmaps_intent_cache != NULL ? 0 : -ENOMEM; 917 } 918 919 void 920 xfs_exchmaps_intent_destroy_cache(void) 921 { 922 kmem_cache_destroy(xfs_exchmaps_intent_cache); 923 xfs_exchmaps_intent_cache = NULL; 924 } 925 926 /* 927 * Allocate and initialize a new incore intent item from a request. 928 */ 929 struct xfs_exchmaps_intent * 930 xfs_exchmaps_init_intent( 931 const struct xfs_exchmaps_req *req) 932 { 933 struct xfs_exchmaps_intent *xmi; 934 struct xfs_mount *mp = req->ip1->i_mount; 935 936 xmi = kmem_cache_zalloc(xfs_exchmaps_intent_cache, 937 GFP_NOFS | __GFP_NOFAIL); 938 INIT_LIST_HEAD(&xmi->xmi_list); 939 xmi->xmi_ip1 = req->ip1; 940 xmi->xmi_ip2 = req->ip2; 941 xmi->xmi_startoff1 = req->startoff1; 942 xmi->xmi_startoff2 = req->startoff2; 943 xmi->xmi_blockcount = req->blockcount; 944 xmi->xmi_isize1 = xmi->xmi_isize2 = -1; 945 xmi->xmi_flags = req->flags & XFS_EXCHMAPS_PARAMS; 946 947 if (xfs_exchmaps_whichfork(xmi) == XFS_ATTR_FORK) { 948 xmi->xmi_flags |= __XFS_EXCHMAPS_INO2_SHORTFORM; 949 return xmi; 950 } 951 952 /* 953 * If the caller wanted, set each file's size to that file's exchange 954 * offset + length exchanged from the other file because the ranges in 955 * each file might be different lengths. 956 */ 957 if (req->flags & XFS_EXCHMAPS_SET_SIZES) { 958 loff_t off1 = XFS_FSB_TO_B(mp, xmi->xmi_startoff1); 959 loff_t off2 = XFS_FSB_TO_B(mp, xmi->xmi_startoff2); 960 961 xmi->xmi_flags |= XFS_EXCHMAPS_SET_SIZES; 962 xmi->xmi_isize1 = off1 + (req->ip2->i_disk_size - off2); 963 xmi->xmi_isize2 = off2 + (req->ip1->i_disk_size - off1); 964 } 965 966 if (S_ISDIR(VFS_I(xmi->xmi_ip2)->i_mode) || 967 S_ISLNK(VFS_I(xmi->xmi_ip2)->i_mode)) 968 xmi->xmi_flags |= __XFS_EXCHMAPS_INO2_SHORTFORM; 969 970 return xmi; 971 } 972 973 /* 974 * Estimate the number of exchange operations and the number of file blocks 975 * in each file that will be affected by the exchange operation. 976 */ 977 int 978 xfs_exchmaps_estimate( 979 struct xfs_exchmaps_req *req) 980 { 981 struct xfs_exchmaps_intent *xmi; 982 struct xfs_bmbt_irec irec1, irec2; 983 struct xfs_exchmaps_adjacent adj = ADJACENT_INIT; 984 xfs_filblks_t ip1_blocks = 0, ip2_blocks = 0; 985 int64_t d_nexts1, d_nexts2; 986 int bmap_flags; 987 int error; 988 989 ASSERT(!(req->flags & ~XFS_EXCHMAPS_PARAMS)); 990 991 bmap_flags = xfs_bmapi_aflag(xfs_exchmaps_reqfork(req)); 992 xmi = xfs_exchmaps_init_intent(req); 993 994 /* 995 * To guard against the possibility of overflowing the extent counters, 996 * we have to estimate an upper bound on the potential increase in that 997 * counter. We can split the mapping at each end of the range, and for 998 * each step of the exchange we can split the mapping that we're 999 * working on if the mappings do not align. 1000 */ 1001 d_nexts1 = d_nexts2 = 3; 1002 1003 while (xmi_has_more_exchange_work(xmi)) { 1004 /* 1005 * Walk through the file ranges until we find something to 1006 * exchange. Because we're simulating the exchange, pass in 1007 * adj to capture skipped mappings for correct estimation of 1008 * bmbt record merges. 1009 */ 1010 error = xfs_exchmaps_find_mappings(xmi, &irec1, &irec2, &adj); 1011 if (error) 1012 goto out_free; 1013 if (!xmi_has_more_exchange_work(xmi)) 1014 break; 1015 1016 /* Update accounting. */ 1017 if (xfs_bmap_is_real_extent(&irec1)) 1018 ip1_blocks += irec1.br_blockcount; 1019 if (xfs_bmap_is_real_extent(&irec2)) 1020 ip2_blocks += irec2.br_blockcount; 1021 req->nr_exchanges++; 1022 1023 /* Read the next mappings from both files. */ 1024 error = xmi_next(req->ip1, bmap_flags, &irec1, &adj.right1); 1025 if (error) 1026 goto out_free; 1027 1028 error = xmi_next(req->ip2, bmap_flags, &irec2, &adj.right2); 1029 if (error) 1030 goto out_free; 1031 1032 /* Update extent count deltas. */ 1033 d_nexts1 += xmi_delta_nextents_step(req->ip1->i_mount, 1034 &adj.left1, &irec1, &irec2, &adj.right1); 1035 1036 d_nexts2 += xmi_delta_nextents_step(req->ip1->i_mount, 1037 &adj.left2, &irec2, &irec1, &adj.right2); 1038 1039 /* Now pretend we exchanged the mappings. */ 1040 if (xmi_can_merge(&adj.left2, &irec1)) 1041 adj.left2.br_blockcount += irec1.br_blockcount; 1042 else 1043 memcpy(&adj.left2, &irec1, sizeof(irec1)); 1044 1045 if (xmi_can_merge(&adj.left1, &irec2)) 1046 adj.left1.br_blockcount += irec2.br_blockcount; 1047 else 1048 memcpy(&adj.left1, &irec2, sizeof(irec2)); 1049 1050 xmi_advance(xmi, &irec1); 1051 } 1052 1053 /* Account for the blocks that are being exchanged. */ 1054 if (XFS_IS_REALTIME_INODE(req->ip1) && 1055 xfs_exchmaps_reqfork(req) == XFS_DATA_FORK) { 1056 req->ip1_rtbcount = ip1_blocks; 1057 req->ip2_rtbcount = ip2_blocks; 1058 } else { 1059 req->ip1_bcount = ip1_blocks; 1060 req->ip2_bcount = ip2_blocks; 1061 } 1062 1063 /* 1064 * Make sure that both forks have enough slack left in their extent 1065 * counters that the exchange operation will not overflow. 1066 */ 1067 trace_xfs_exchmaps_delta_nextents(req, d_nexts1, d_nexts2); 1068 if (req->ip1 == req->ip2) { 1069 error = xmi_ensure_delta_nextents(req, req->ip1, 1070 d_nexts1 + d_nexts2); 1071 } else { 1072 error = xmi_ensure_delta_nextents(req, req->ip1, d_nexts1); 1073 if (error) 1074 goto out_free; 1075 error = xmi_ensure_delta_nextents(req, req->ip2, d_nexts2); 1076 } 1077 if (error) 1078 goto out_free; 1079 1080 trace_xfs_exchmaps_initial_estimate(req); 1081 error = xfs_exchmaps_estimate_overhead(req); 1082 out_free: 1083 kmem_cache_free(xfs_exchmaps_intent_cache, xmi); 1084 return error; 1085 } 1086 1087 /* Set the reflink flag before an operation. */ 1088 static inline void 1089 xfs_exchmaps_set_reflink( 1090 struct xfs_trans *tp, 1091 struct xfs_inode *ip) 1092 { 1093 trace_xfs_reflink_set_inode_flag(ip); 1094 1095 ip->i_diflags2 |= XFS_DIFLAG2_REFLINK; 1096 xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); 1097 } 1098 1099 /* 1100 * If either file has shared blocks and we're exchanging data forks, we must 1101 * flag the other file as having shared blocks so that we get the shared-block 1102 * rmap functions if we need to fix up the rmaps. 1103 */ 1104 void 1105 xfs_exchmaps_ensure_reflink( 1106 struct xfs_trans *tp, 1107 const struct xfs_exchmaps_intent *xmi) 1108 { 1109 unsigned int rs = 0; 1110 1111 if (xfs_is_reflink_inode(xmi->xmi_ip1)) 1112 rs |= 1; 1113 if (xfs_is_reflink_inode(xmi->xmi_ip2)) 1114 rs |= 2; 1115 1116 if ((rs & 1) && !xfs_is_reflink_inode(xmi->xmi_ip2)) 1117 xfs_exchmaps_set_reflink(tp, xmi->xmi_ip2); 1118 1119 if ((rs & 2) && !xfs_is_reflink_inode(xmi->xmi_ip1)) 1120 xfs_exchmaps_set_reflink(tp, xmi->xmi_ip1); 1121 } 1122 1123 /* Set the large extent count flag before an operation if needed. */ 1124 static inline void 1125 xfs_exchmaps_ensure_large_extent_counts( 1126 struct xfs_trans *tp, 1127 struct xfs_inode *ip) 1128 { 1129 if (xfs_inode_has_large_extent_counts(ip)) 1130 return; 1131 1132 ip->i_diflags2 |= XFS_DIFLAG2_NREXT64; 1133 xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); 1134 } 1135 1136 /* Widen the extent counter fields of both inodes if necessary. */ 1137 void 1138 xfs_exchmaps_upgrade_extent_counts( 1139 struct xfs_trans *tp, 1140 const struct xfs_exchmaps_intent *xmi) 1141 { 1142 if (!xfs_has_large_extent_counts(tp->t_mountp)) 1143 return; 1144 1145 xfs_exchmaps_ensure_large_extent_counts(tp, xmi->xmi_ip1); 1146 xfs_exchmaps_ensure_large_extent_counts(tp, xmi->xmi_ip2); 1147 } 1148 1149 /* 1150 * Schedule an exchange a range of mappings from one inode to another. 1151 * 1152 * The use of file mapping exchange log intent items ensures the operation can 1153 * be resumed even if the system goes down. The caller must commit the 1154 * transaction to start the work. 1155 * 1156 * The caller must ensure the inodes must be joined to the transaction and 1157 * ILOCKd; they will still be joined to the transaction at exit. 1158 */ 1159 void 1160 xfs_exchange_mappings( 1161 struct xfs_trans *tp, 1162 const struct xfs_exchmaps_req *req) 1163 { 1164 struct xfs_exchmaps_intent *xmi; 1165 1166 BUILD_BUG_ON(XFS_EXCHMAPS_INTERNAL_FLAGS & XFS_EXCHMAPS_LOGGED_FLAGS); 1167 1168 xfs_assert_ilocked(req->ip1, XFS_ILOCK_EXCL); 1169 xfs_assert_ilocked(req->ip2, XFS_ILOCK_EXCL); 1170 ASSERT(!(req->flags & ~XFS_EXCHMAPS_LOGGED_FLAGS)); 1171 if (req->flags & XFS_EXCHMAPS_SET_SIZES) 1172 ASSERT(!(req->flags & XFS_EXCHMAPS_ATTR_FORK)); 1173 ASSERT(xfs_has_exchange_range(tp->t_mountp)); 1174 1175 if (req->blockcount == 0) 1176 return; 1177 1178 xmi = xfs_exchmaps_init_intent(req); 1179 xfs_exchmaps_defer_add(tp, xmi); 1180 xfs_exchmaps_ensure_reflink(tp, xmi); 1181 xfs_exchmaps_upgrade_extent_counts(tp, xmi); 1182 } 1183