1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (c) 2020-2024 Oracle. All Rights Reserved. 4 * Author: Darrick J. Wong <djwong@kernel.org> 5 */ 6 #include "xfs_platform.h" 7 #include "xfs_fs.h" 8 #include "xfs_shared.h" 9 #include "xfs_format.h" 10 #include "xfs_log_format.h" 11 #include "xfs_trans_resv.h" 12 #include "xfs_mount.h" 13 #include "xfs_defer.h" 14 #include "xfs_inode.h" 15 #include "xfs_trans.h" 16 #include "xfs_bmap.h" 17 #include "xfs_icache.h" 18 #include "xfs_quota.h" 19 #include "xfs_exchmaps.h" 20 #include "xfs_trace.h" 21 #include "xfs_bmap_btree.h" 22 #include "xfs_trans_space.h" 23 #include "xfs_error.h" 24 #include "xfs_errortag.h" 25 #include "xfs_health.h" 26 #include "xfs_exchmaps_item.h" 27 #include "xfs_da_format.h" 28 #include "xfs_da_btree.h" 29 #include "xfs_attr_leaf.h" 30 #include "xfs_attr.h" 31 #include "xfs_dir2_priv.h" 32 #include "xfs_dir2.h" 33 #include "xfs_symlink_remote.h" 34 35 struct kmem_cache *xfs_exchmaps_intent_cache; 36 37 /* bmbt mappings adjacent to a pair of records. */ 38 struct xfs_exchmaps_adjacent { 39 struct xfs_bmbt_irec left1; 40 struct xfs_bmbt_irec right1; 41 struct xfs_bmbt_irec left2; 42 struct xfs_bmbt_irec right2; 43 }; 44 45 #define ADJACENT_INIT { \ 46 .left1 = { .br_startblock = HOLESTARTBLOCK }, \ 47 .right1 = { .br_startblock = HOLESTARTBLOCK }, \ 48 .left2 = { .br_startblock = HOLESTARTBLOCK }, \ 49 .right2 = { .br_startblock = HOLESTARTBLOCK }, \ 50 } 51 52 /* Information to reset CoW fork state after an exchange. */ 53 54 /* 55 * If the reflink flag is set on either inode, make sure it has an incore CoW 56 * fork, since all reflink inodes must have them. If there's a CoW fork and it 57 * has mappings in it, make sure the inodes are tagged appropriately so that 58 * speculative preallocations can be GC'd if we run low of space. 59 */ 60 static inline void 61 xfs_exchmaps_ensure_cowfork( 62 struct xfs_inode *ip) 63 { 64 struct xfs_ifork *cfork; 65 66 if (xfs_is_reflink_inode(ip)) 67 xfs_ifork_init_cow(ip); 68 69 cfork = xfs_ifork_ptr(ip, XFS_COW_FORK); 70 if (!cfork) 71 return; 72 if (cfork->if_bytes > 0) 73 xfs_inode_set_cowblocks_tag(ip); 74 else 75 xfs_inode_clear_cowblocks_tag(ip); 76 } 77 78 /* 79 * Adjust the on-disk inode size upwards if needed so that we never add 80 * mappings into the file past EOF. This is crucial so that log recovery won't 81 * get confused by the sudden appearance of post-eof mappings. 82 */ 83 STATIC void 84 xfs_exchmaps_update_size( 85 struct xfs_trans *tp, 86 struct xfs_inode *ip, 87 struct xfs_bmbt_irec *imap, 88 xfs_fsize_t new_isize) 89 { 90 struct xfs_mount *mp = tp->t_mountp; 91 xfs_fsize_t len; 92 93 if (new_isize < 0) 94 return; 95 96 len = min(XFS_FSB_TO_B(mp, imap->br_startoff + imap->br_blockcount), 97 new_isize); 98 99 if (len <= ip->i_disk_size) 100 return; 101 102 trace_xfs_exchmaps_update_inode_size(ip, len); 103 104 ip->i_disk_size = len; 105 xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); 106 } 107 108 /* Advance the incore state tracking after exchanging a mapping. */ 109 static inline void 110 xmi_advance( 111 struct xfs_exchmaps_intent *xmi, 112 const struct xfs_bmbt_irec *irec) 113 { 114 xmi->xmi_startoff1 += irec->br_blockcount; 115 xmi->xmi_startoff2 += irec->br_blockcount; 116 xmi->xmi_blockcount -= irec->br_blockcount; 117 } 118 119 /* Do we still have more mappings to exchange? */ 120 static inline bool 121 xmi_has_more_exchange_work(const struct xfs_exchmaps_intent *xmi) 122 { 123 return xmi->xmi_blockcount > 0; 124 } 125 126 /* Do we have post-operation cleanups to perform? */ 127 static inline bool 128 xmi_has_postop_work(const struct xfs_exchmaps_intent *xmi) 129 { 130 return xmi->xmi_flags & __XFS_EXCHMAPS_INO2_SHORTFORM; 131 } 132 133 /* Check all mappings to make sure we can actually exchange them. */ 134 int 135 xfs_exchmaps_check_forks( 136 const struct xfs_exchmaps_req *req) 137 { 138 struct xfs_ifork *ifp1, *ifp2; 139 int whichfork = xfs_exchmaps_reqfork(req); 140 141 /* No fork? */ 142 ifp1 = xfs_ifork_ptr(req->ip1, whichfork); 143 ifp2 = xfs_ifork_ptr(req->ip2, whichfork); 144 if (!ifp1 || !ifp2) 145 return -EINVAL; 146 147 /* We don't know how to exchange local format forks. */ 148 if (ifp1->if_format == XFS_DINODE_FMT_LOCAL || 149 ifp2->if_format == XFS_DINODE_FMT_LOCAL) 150 return -EINVAL; 151 152 return 0; 153 } 154 155 #ifdef CONFIG_XFS_QUOTA 156 /* Log the actual updates to the quota accounting. */ 157 static inline void 158 xfs_exchmaps_update_quota( 159 struct xfs_trans *tp, 160 struct xfs_exchmaps_intent *xmi, 161 struct xfs_bmbt_irec *irec1, 162 struct xfs_bmbt_irec *irec2) 163 { 164 int64_t ip1_delta = 0, ip2_delta = 0; 165 unsigned int qflag; 166 167 qflag = XFS_IS_REALTIME_INODE(xmi->xmi_ip1) ? XFS_TRANS_DQ_RTBCOUNT : 168 XFS_TRANS_DQ_BCOUNT; 169 170 if (xfs_bmap_is_real_extent(irec1)) { 171 ip1_delta -= irec1->br_blockcount; 172 ip2_delta += irec1->br_blockcount; 173 } 174 175 if (xfs_bmap_is_real_extent(irec2)) { 176 ip1_delta += irec2->br_blockcount; 177 ip2_delta -= irec2->br_blockcount; 178 } 179 180 xfs_trans_mod_dquot_byino(tp, xmi->xmi_ip1, qflag, ip1_delta); 181 xfs_trans_mod_dquot_byino(tp, xmi->xmi_ip2, qflag, ip2_delta); 182 } 183 #else 184 # define xfs_exchmaps_update_quota(tp, xmi, irec1, irec2) ((void)0) 185 #endif 186 187 /* Decide if we want to skip this mapping from file1. */ 188 static inline bool 189 xfs_exchmaps_can_skip_mapping( 190 struct xfs_exchmaps_intent *xmi, 191 struct xfs_bmbt_irec *irec) 192 { 193 struct xfs_mount *mp = xmi->xmi_ip1->i_mount; 194 195 /* Do not skip this mapping if the caller did not tell us to. */ 196 if (!(xmi->xmi_flags & XFS_EXCHMAPS_INO1_WRITTEN)) 197 return false; 198 199 /* Do not skip mapped, written mappings. */ 200 if (xfs_bmap_is_written_extent(irec)) 201 return false; 202 203 /* 204 * The mapping is unwritten or a hole. It cannot be a delalloc 205 * reservation because we already excluded those. It cannot be an 206 * unwritten extent with dirty page cache because we flushed the page 207 * cache. For files where the allocation unit is 1FSB (files on the 208 * data dev, rt files if the extent size is 1FSB), we can safely 209 * skip this mapping. 210 */ 211 if (!xfs_inode_has_bigrtalloc(xmi->xmi_ip1)) 212 return true; 213 214 /* 215 * For a realtime file with a multi-fsb allocation unit, the decision 216 * is trickier because we can only swap full allocation units. 217 * Unwritten mappings can appear in the middle of an rtx if the rtx is 218 * partially written, but they can also appear for preallocations. 219 * 220 * If the mapping is a hole, skip it entirely. Holes should align with 221 * rtx boundaries. 222 */ 223 if (!xfs_bmap_is_real_extent(irec)) 224 return true; 225 226 /* 227 * All mappings below this point are unwritten. 228 * 229 * - If the beginning is not aligned to an rtx, trim the end of the 230 * mapping so that it does not cross an rtx boundary, and swap it. 231 * 232 * - If both ends are aligned to an rtx, skip the entire mapping. 233 */ 234 if (!isaligned_64(irec->br_startoff, mp->m_sb.sb_rextsize)) { 235 xfs_fileoff_t new_end; 236 237 new_end = roundup_64(irec->br_startoff, mp->m_sb.sb_rextsize); 238 irec->br_blockcount = min(irec->br_blockcount, 239 new_end - irec->br_startoff); 240 return false; 241 } 242 if (isaligned_64(irec->br_blockcount, mp->m_sb.sb_rextsize)) 243 return true; 244 245 /* 246 * All mappings below this point are unwritten, start on an rtx 247 * boundary, and do not end on an rtx boundary. 248 * 249 * - If the mapping is longer than one rtx, trim the end of the mapping 250 * down to an rtx boundary and skip it. 251 * 252 * - The mapping is shorter than one rtx. Swap it. 253 */ 254 if (irec->br_blockcount > mp->m_sb.sb_rextsize) { 255 xfs_fileoff_t new_end; 256 257 new_end = rounddown_64(irec->br_startoff + irec->br_blockcount, 258 mp->m_sb.sb_rextsize); 259 irec->br_blockcount = new_end - irec->br_startoff; 260 return true; 261 } 262 263 return false; 264 } 265 266 /* 267 * Walk forward through the file ranges in @xmi until we find two different 268 * mappings to exchange. If there is work to do, return the mappings; 269 * otherwise we've reached the end of the range and xmi_blockcount will be 270 * zero. 271 * 272 * If the walk skips over a pair of mappings to the same storage, save them as 273 * the left records in @adj (if provided) so that the simulation phase can 274 * avoid an extra lookup. 275 */ 276 static int 277 xfs_exchmaps_find_mappings( 278 struct xfs_exchmaps_intent *xmi, 279 struct xfs_bmbt_irec *irec1, 280 struct xfs_bmbt_irec *irec2, 281 struct xfs_exchmaps_adjacent *adj) 282 { 283 int nimaps; 284 int bmap_flags; 285 int error; 286 287 bmap_flags = xfs_bmapi_aflag(xfs_exchmaps_whichfork(xmi)); 288 289 for (; xmi_has_more_exchange_work(xmi); xmi_advance(xmi, irec1)) { 290 /* Read mapping from the first file */ 291 nimaps = 1; 292 error = xfs_bmapi_read(xmi->xmi_ip1, xmi->xmi_startoff1, 293 xmi->xmi_blockcount, irec1, &nimaps, 294 bmap_flags); 295 if (error) 296 return error; 297 if (nimaps != 1 || 298 irec1->br_startblock == DELAYSTARTBLOCK || 299 irec1->br_startoff != xmi->xmi_startoff1) { 300 /* 301 * We should never get no mapping or a delalloc mapping 302 * or something that doesn't match what we asked for, 303 * since the caller flushed both inodes and we hold the 304 * ILOCKs for both inodes. 305 */ 306 ASSERT(0); 307 return -EINVAL; 308 } 309 310 if (xfs_exchmaps_can_skip_mapping(xmi, irec1)) { 311 trace_xfs_exchmaps_mapping1_skip(xmi->xmi_ip1, irec1); 312 continue; 313 } 314 315 /* Read mapping from the second file */ 316 nimaps = 1; 317 error = xfs_bmapi_read(xmi->xmi_ip2, xmi->xmi_startoff2, 318 irec1->br_blockcount, irec2, &nimaps, 319 bmap_flags); 320 if (error) 321 return error; 322 if (nimaps != 1 || 323 irec2->br_startblock == DELAYSTARTBLOCK || 324 irec2->br_startoff != xmi->xmi_startoff2) { 325 /* 326 * We should never get no mapping or a delalloc mapping 327 * or something that doesn't match what we asked for, 328 * since the caller flushed both inodes and we hold the 329 * ILOCKs for both inodes. 330 */ 331 ASSERT(0); 332 return -EINVAL; 333 } 334 335 /* 336 * We can only exchange as many blocks as the smaller of the 337 * two mapping maps. 338 */ 339 irec1->br_blockcount = min(irec1->br_blockcount, 340 irec2->br_blockcount); 341 342 trace_xfs_exchmaps_mapping1(xmi->xmi_ip1, irec1); 343 trace_xfs_exchmaps_mapping2(xmi->xmi_ip2, irec2); 344 345 /* We found something to exchange, so return it. */ 346 if (irec1->br_startblock != irec2->br_startblock) 347 return 0; 348 349 /* 350 * Two mappings pointing to the same physical block must not 351 * have different states; that's filesystem corruption. Move 352 * on to the next mapping if they're both holes or both point 353 * to the same physical space extent. 354 */ 355 if (irec1->br_state != irec2->br_state) { 356 xfs_bmap_mark_sick(xmi->xmi_ip1, 357 xfs_exchmaps_whichfork(xmi)); 358 xfs_bmap_mark_sick(xmi->xmi_ip2, 359 xfs_exchmaps_whichfork(xmi)); 360 return -EFSCORRUPTED; 361 } 362 363 /* 364 * Save the mappings if we're estimating work and skipping 365 * these identical mappings. 366 */ 367 if (adj) { 368 memcpy(&adj->left1, irec1, sizeof(*irec1)); 369 memcpy(&adj->left2, irec2, sizeof(*irec2)); 370 } 371 } 372 373 return 0; 374 } 375 376 /* Exchange these two mappings. */ 377 static void 378 xfs_exchmaps_one_step( 379 struct xfs_trans *tp, 380 struct xfs_exchmaps_intent *xmi, 381 struct xfs_bmbt_irec *irec1, 382 struct xfs_bmbt_irec *irec2) 383 { 384 int whichfork = xfs_exchmaps_whichfork(xmi); 385 386 xfs_exchmaps_update_quota(tp, xmi, irec1, irec2); 387 388 /* Remove both mappings. */ 389 xfs_bmap_unmap_extent(tp, xmi->xmi_ip1, whichfork, irec1); 390 xfs_bmap_unmap_extent(tp, xmi->xmi_ip2, whichfork, irec2); 391 392 /* 393 * Re-add both mappings. We exchange the file offsets between the two 394 * maps and add the opposite map, which has the effect of filling the 395 * logical offsets we just unmapped, but with the physical mapping 396 * information exchanged. 397 */ 398 swap(irec1->br_startoff, irec2->br_startoff); 399 xfs_bmap_map_extent(tp, xmi->xmi_ip1, whichfork, irec2); 400 xfs_bmap_map_extent(tp, xmi->xmi_ip2, whichfork, irec1); 401 402 /* Make sure we're not adding mappings past EOF. */ 403 if (whichfork == XFS_DATA_FORK) { 404 xfs_exchmaps_update_size(tp, xmi->xmi_ip1, irec2, 405 xmi->xmi_isize1); 406 xfs_exchmaps_update_size(tp, xmi->xmi_ip2, irec1, 407 xmi->xmi_isize2); 408 } 409 410 /* 411 * Advance our cursor and exit. The caller (either defer ops or log 412 * recovery) will log the XMD item, and if *blockcount is nonzero, it 413 * will log a new XMI item for the remainder and call us back. 414 */ 415 xmi_advance(xmi, irec1); 416 } 417 418 /* Convert inode2's leaf attr fork back to shortform, if possible.. */ 419 STATIC int 420 xfs_exchmaps_attr_to_sf( 421 struct xfs_trans *tp, 422 struct xfs_exchmaps_intent *xmi) 423 { 424 struct xfs_da_args args = { 425 .dp = xmi->xmi_ip2, 426 .geo = tp->t_mountp->m_attr_geo, 427 .whichfork = XFS_ATTR_FORK, 428 .trans = tp, 429 .owner = I_INO(xmi->xmi_ip2), 430 }; 431 struct xfs_buf *bp; 432 int forkoff; 433 int error; 434 435 if (!xfs_attr_is_leaf(xmi->xmi_ip2)) 436 return 0; 437 438 error = xfs_attr3_leaf_read(tp, xmi->xmi_ip2, I_INO(xmi->xmi_ip2), 0, 439 &bp); 440 if (error) 441 return error; 442 443 forkoff = xfs_attr_shortform_allfit(bp, xmi->xmi_ip2); 444 if (forkoff == 0) 445 return 0; 446 447 return xfs_attr3_leaf_to_shortform(bp, &args, forkoff); 448 } 449 450 /* Convert inode2's block dir fork back to shortform, if possible.. */ 451 STATIC int 452 xfs_exchmaps_dir_to_sf( 453 struct xfs_trans *tp, 454 struct xfs_exchmaps_intent *xmi) 455 { 456 struct xfs_da_args args = { 457 .dp = xmi->xmi_ip2, 458 .geo = tp->t_mountp->m_dir_geo, 459 .whichfork = XFS_DATA_FORK, 460 .trans = tp, 461 .owner = I_INO(xmi->xmi_ip2), 462 }; 463 struct xfs_dir2_sf_hdr sfh; 464 struct xfs_buf *bp; 465 int size; 466 int error = 0; 467 468 if (xfs_dir2_format(&args, &error) != XFS_DIR2_FMT_BLOCK) 469 return error; 470 471 error = xfs_dir3_block_read(tp, xmi->xmi_ip2, I_INO(xmi->xmi_ip2), &bp); 472 if (error) 473 return error; 474 475 size = xfs_dir2_block_sfsize(xmi->xmi_ip2, bp->b_addr, &sfh); 476 if (size > xfs_inode_data_fork_size(xmi->xmi_ip2)) 477 return 0; 478 479 return xfs_dir2_block_to_sf(&args, bp, size, &sfh); 480 } 481 482 /* Convert inode2's remote symlink target back to shortform, if possible. */ 483 STATIC int 484 xfs_exchmaps_link_to_sf( 485 struct xfs_trans *tp, 486 struct xfs_exchmaps_intent *xmi) 487 { 488 struct xfs_inode *ip = xmi->xmi_ip2; 489 struct xfs_ifork *ifp = xfs_ifork_ptr(ip, XFS_DATA_FORK); 490 char *buf; 491 int error; 492 493 if (ifp->if_format == XFS_DINODE_FMT_LOCAL || 494 ip->i_disk_size > xfs_inode_data_fork_size(ip)) 495 return 0; 496 497 /* Read the current symlink target into a buffer. */ 498 buf = kmalloc(ip->i_disk_size + 1, 499 GFP_KERNEL | __GFP_NOLOCKDEP | __GFP_NOFAIL); 500 if (!buf) { 501 ASSERT(0); 502 return -ENOMEM; 503 } 504 505 error = xfs_symlink_remote_read(ip, buf); 506 if (error) 507 goto free; 508 509 /* Remove the blocks. */ 510 error = xfs_symlink_remote_truncate(tp, ip); 511 if (error) 512 goto free; 513 514 /* Convert fork to local format and log our changes. */ 515 xfs_idestroy_fork(ifp); 516 ifp->if_bytes = 0; 517 ifp->if_format = XFS_DINODE_FMT_LOCAL; 518 xfs_init_local_fork(ip, XFS_DATA_FORK, buf, ip->i_disk_size); 519 xfs_trans_log_inode(tp, ip, XFS_ILOG_DDATA | XFS_ILOG_CORE); 520 free: 521 kfree(buf); 522 return error; 523 } 524 525 /* Finish whatever work might come after an exchange operation. */ 526 static int 527 xfs_exchmaps_do_postop_work( 528 struct xfs_trans *tp, 529 struct xfs_exchmaps_intent *xmi) 530 { 531 if (xmi->xmi_flags & __XFS_EXCHMAPS_INO2_SHORTFORM) { 532 int error = 0; 533 534 if (xmi->xmi_flags & XFS_EXCHMAPS_ATTR_FORK) 535 error = xfs_exchmaps_attr_to_sf(tp, xmi); 536 else if (S_ISDIR(VFS_I(xmi->xmi_ip2)->i_mode)) 537 error = xfs_exchmaps_dir_to_sf(tp, xmi); 538 else if (S_ISLNK(VFS_I(xmi->xmi_ip2)->i_mode)) 539 error = xfs_exchmaps_link_to_sf(tp, xmi); 540 xmi->xmi_flags &= ~__XFS_EXCHMAPS_INO2_SHORTFORM; 541 if (error) 542 return error; 543 } 544 545 return 0; 546 } 547 548 /* Finish one step in a mapping exchange operation, possibly relogging. */ 549 int 550 xfs_exchmaps_finish_one( 551 struct xfs_trans *tp, 552 struct xfs_exchmaps_intent *xmi) 553 { 554 struct xfs_bmbt_irec irec1, irec2; 555 int error; 556 557 if (xmi_has_more_exchange_work(xmi)) { 558 /* 559 * If the operation state says that some range of the files 560 * have not yet been exchanged, look for mappings in that range 561 * to exchange. If we find some mappings, exchange them. 562 */ 563 error = xfs_exchmaps_find_mappings(xmi, &irec1, &irec2, NULL); 564 if (error) 565 return error; 566 567 if (xmi_has_more_exchange_work(xmi)) 568 xfs_exchmaps_one_step(tp, xmi, &irec1, &irec2); 569 570 /* 571 * If the caller asked us to exchange the file sizes after the 572 * exchange and either we just exchanged the last mappings in 573 * the range or we didn't find anything to exchange, update the 574 * ondisk file sizes. 575 */ 576 if ((xmi->xmi_flags & XFS_EXCHMAPS_SET_SIZES) && 577 !xmi_has_more_exchange_work(xmi)) { 578 xmi->xmi_ip1->i_disk_size = xmi->xmi_isize1; 579 xmi->xmi_ip2->i_disk_size = xmi->xmi_isize2; 580 581 xfs_trans_log_inode(tp, xmi->xmi_ip1, XFS_ILOG_CORE); 582 xfs_trans_log_inode(tp, xmi->xmi_ip2, XFS_ILOG_CORE); 583 } 584 } else if (xmi_has_postop_work(xmi)) { 585 /* 586 * Now that we're finished with the exchange operation, 587 * complete the post-op cleanup work. 588 */ 589 error = xfs_exchmaps_do_postop_work(tp, xmi); 590 if (error) 591 return error; 592 } 593 594 if (XFS_TEST_ERROR(tp->t_mountp, XFS_ERRTAG_EXCHMAPS_FINISH_ONE)) 595 return -EIO; 596 597 /* If we still have work to do, ask for a new transaction. */ 598 if (xmi_has_more_exchange_work(xmi) || xmi_has_postop_work(xmi)) { 599 trace_xfs_exchmaps_defer(tp->t_mountp, xmi); 600 return -EAGAIN; 601 } 602 603 /* 604 * If we reach here, we've finished all the exchange work and the post 605 * operation work. The last thing we need to do before returning to 606 * the caller is to make sure that COW forks are set up correctly. 607 */ 608 if (!(xmi->xmi_flags & XFS_EXCHMAPS_ATTR_FORK)) { 609 xfs_exchmaps_ensure_cowfork(xmi->xmi_ip1); 610 xfs_exchmaps_ensure_cowfork(xmi->xmi_ip2); 611 } 612 613 return 0; 614 } 615 616 /* 617 * Compute the amount of bmbt blocks we should reserve for each file. In the 618 * worst case, each exchange will fill a hole with a new mapping, which could 619 * result in a btree split every time we add a new leaf block. 620 */ 621 static inline uint64_t 622 xfs_exchmaps_bmbt_blocks( 623 struct xfs_mount *mp, 624 const struct xfs_exchmaps_req *req) 625 { 626 return howmany_64(req->nr_exchanges, 627 XFS_MAX_CONTIG_BMAPS_PER_BLOCK(mp)) * 628 XFS_EXTENTADD_SPACE_RES(mp, xfs_exchmaps_reqfork(req)); 629 } 630 631 /* Compute the space we should reserve for the rmap btree expansions. */ 632 static inline uint64_t 633 xfs_exchmaps_rmapbt_blocks( 634 struct xfs_mount *mp, 635 const struct xfs_exchmaps_req *req) 636 { 637 if (!xfs_has_rmapbt(mp)) 638 return 0; 639 if (XFS_IS_REALTIME_INODE(req->ip1)) 640 return howmany_64(req->nr_exchanges, 641 XFS_MAX_CONTIG_RTRMAPS_PER_BLOCK(mp)) * 642 XFS_RTRMAPADD_SPACE_RES(mp); 643 644 return howmany_64(req->nr_exchanges, 645 XFS_MAX_CONTIG_RMAPS_PER_BLOCK(mp)) * 646 XFS_RMAPADD_SPACE_RES(mp); 647 } 648 649 /* Estimate the bmbt and rmapbt overhead required to exchange mappings. */ 650 int 651 xfs_exchmaps_estimate_overhead( 652 struct xfs_exchmaps_req *req) 653 { 654 struct xfs_mount *mp = req->ip1->i_mount; 655 xfs_filblks_t bmbt_blocks; 656 xfs_filblks_t rmapbt_blocks; 657 xfs_filblks_t resblks = req->resblks; 658 659 /* 660 * Compute the number of bmbt and rmapbt blocks we might need to handle 661 * the estimated number of exchanges. 662 */ 663 bmbt_blocks = xfs_exchmaps_bmbt_blocks(mp, req); 664 rmapbt_blocks = xfs_exchmaps_rmapbt_blocks(mp, req); 665 666 trace_xfs_exchmaps_overhead(mp, bmbt_blocks, rmapbt_blocks); 667 668 /* Make sure the change in file block count doesn't overflow. */ 669 if (check_add_overflow(req->ip1_bcount, bmbt_blocks, &req->ip1_bcount)) 670 return -EFBIG; 671 if (check_add_overflow(req->ip2_bcount, bmbt_blocks, &req->ip2_bcount)) 672 return -EFBIG; 673 674 /* 675 * Add together the number of blocks we need to handle btree growth, 676 * then add it to the number of blocks we need to reserve to this 677 * transaction. 678 */ 679 if (check_add_overflow(resblks, bmbt_blocks, &resblks)) 680 return -ENOSPC; 681 if (check_add_overflow(resblks, bmbt_blocks, &resblks)) 682 return -ENOSPC; 683 if (check_add_overflow(resblks, rmapbt_blocks, &resblks)) 684 return -ENOSPC; 685 if (check_add_overflow(resblks, rmapbt_blocks, &resblks)) 686 return -ENOSPC; 687 688 /* Can't actually reserve more than UINT_MAX blocks. */ 689 if (resblks > UINT_MAX) 690 return -ENOSPC; 691 692 req->resblks = resblks; 693 trace_xfs_exchmaps_final_estimate(req); 694 return 0; 695 } 696 697 /* Decide if we can merge two real mappings. */ 698 static inline bool 699 xmi_can_merge( 700 const struct xfs_bmbt_irec *b1, 701 const struct xfs_bmbt_irec *b2) 702 { 703 /* Don't merge holes. */ 704 if (b1->br_startblock == HOLESTARTBLOCK || 705 b2->br_startblock == HOLESTARTBLOCK) 706 return false; 707 708 /* We don't merge holes. */ 709 if (!xfs_bmap_is_real_extent(b1) || !xfs_bmap_is_real_extent(b2)) 710 return false; 711 712 if (b1->br_startoff + b1->br_blockcount == b2->br_startoff && 713 b1->br_startblock + b1->br_blockcount == b2->br_startblock && 714 b1->br_state == b2->br_state && 715 b1->br_blockcount + b2->br_blockcount <= XFS_MAX_BMBT_EXTLEN) 716 return true; 717 718 return false; 719 } 720 721 /* 722 * Decide if we can merge three mappings. Caller must ensure all three 723 * mappings must not be holes or delalloc reservations. 724 */ 725 static inline bool 726 xmi_can_merge_all( 727 const struct xfs_bmbt_irec *l, 728 const struct xfs_bmbt_irec *m, 729 const struct xfs_bmbt_irec *r) 730 { 731 xfs_filblks_t new_len; 732 733 new_len = l->br_blockcount + m->br_blockcount + r->br_blockcount; 734 return new_len <= XFS_MAX_BMBT_EXTLEN; 735 } 736 737 #define CLEFT_CONTIG 0x01 738 #define CRIGHT_CONTIG 0x02 739 #define CHOLE 0x04 740 #define CBOTH_CONTIG (CLEFT_CONTIG | CRIGHT_CONTIG) 741 742 #define NLEFT_CONTIG 0x10 743 #define NRIGHT_CONTIG 0x20 744 #define NHOLE 0x40 745 #define NBOTH_CONTIG (NLEFT_CONTIG | NRIGHT_CONTIG) 746 747 /* Estimate the effect of a single exchange on mapping count. */ 748 static inline int 749 xmi_delta_nextents_step( 750 struct xfs_mount *mp, 751 const struct xfs_bmbt_irec *left, 752 const struct xfs_bmbt_irec *curr, 753 const struct xfs_bmbt_irec *new, 754 const struct xfs_bmbt_irec *right) 755 { 756 bool lhole, rhole, chole, nhole; 757 unsigned int state = 0; 758 int ret = 0; 759 760 lhole = left->br_startblock == HOLESTARTBLOCK; 761 rhole = right->br_startblock == HOLESTARTBLOCK; 762 chole = curr->br_startblock == HOLESTARTBLOCK; 763 nhole = new->br_startblock == HOLESTARTBLOCK; 764 765 if (chole) 766 state |= CHOLE; 767 if (!lhole && !chole && xmi_can_merge(left, curr)) 768 state |= CLEFT_CONTIG; 769 if (!rhole && !chole && xmi_can_merge(curr, right)) 770 state |= CRIGHT_CONTIG; 771 if ((state & CBOTH_CONTIG) == CBOTH_CONTIG && 772 !xmi_can_merge_all(left, curr, right)) 773 state &= ~CRIGHT_CONTIG; 774 775 if (nhole) 776 state |= NHOLE; 777 if (!lhole && !nhole && xmi_can_merge(left, new)) 778 state |= NLEFT_CONTIG; 779 if (!rhole && !nhole && xmi_can_merge(new, right)) 780 state |= NRIGHT_CONTIG; 781 if ((state & NBOTH_CONTIG) == NBOTH_CONTIG && 782 !xmi_can_merge_all(left, new, right)) 783 state &= ~NRIGHT_CONTIG; 784 785 switch (state & (CLEFT_CONTIG | CRIGHT_CONTIG | CHOLE)) { 786 case CLEFT_CONTIG | CRIGHT_CONTIG: 787 /* 788 * left/curr/right are the same mapping, so deleting curr 789 * causes 2 new mappings to be created. 790 */ 791 ret += 2; 792 break; 793 case 0: 794 /* 795 * curr is not contiguous with any mapping, so we remove curr 796 * completely 797 */ 798 ret--; 799 break; 800 case CHOLE: 801 /* hole, do nothing */ 802 break; 803 case CLEFT_CONTIG: 804 case CRIGHT_CONTIG: 805 /* trim either left or right, no change */ 806 break; 807 } 808 809 switch (state & (NLEFT_CONTIG | NRIGHT_CONTIG | NHOLE)) { 810 case NLEFT_CONTIG | NRIGHT_CONTIG: 811 /* 812 * left/curr/right will become the same mapping, so adding 813 * curr causes the deletion of right. 814 */ 815 ret--; 816 break; 817 case 0: 818 /* new is not contiguous with any mapping */ 819 ret++; 820 break; 821 case NHOLE: 822 /* hole, do nothing. */ 823 break; 824 case NLEFT_CONTIG: 825 case NRIGHT_CONTIG: 826 /* new is absorbed into left or right, no change */ 827 break; 828 } 829 830 trace_xfs_exchmaps_delta_nextents_step(mp, left, curr, new, right, ret, 831 state); 832 return ret; 833 } 834 835 /* Make sure we don't overflow the extent (mapping) counters. */ 836 static inline int 837 xmi_ensure_delta_nextents( 838 struct xfs_exchmaps_req *req, 839 struct xfs_inode *ip, 840 int64_t delta) 841 { 842 struct xfs_mount *mp = ip->i_mount; 843 int whichfork = xfs_exchmaps_reqfork(req); 844 struct xfs_ifork *ifp = xfs_ifork_ptr(ip, whichfork); 845 uint64_t new_nextents; 846 xfs_extnum_t max_nextents; 847 848 if (delta < 0) 849 return 0; 850 851 /* 852 * It's always an error if the delta causes integer overflow. delta 853 * needs an explicit cast here to avoid warnings about implicit casts 854 * coded into the overflow check. 855 */ 856 if (check_add_overflow(ifp->if_nextents, (uint64_t)delta, 857 &new_nextents)) 858 return -EFBIG; 859 860 if (XFS_TEST_ERROR(mp, XFS_ERRTAG_REDUCE_MAX_IEXTENTS) && 861 new_nextents > 10) 862 return -EFBIG; 863 864 /* 865 * We always promote both inodes to have large extent counts if the 866 * superblock feature is enabled, so we only need to check against the 867 * theoretical maximum. 868 */ 869 max_nextents = xfs_iext_max_nextents(xfs_has_large_extent_counts(mp), 870 whichfork); 871 if (new_nextents > max_nextents) 872 return -EFBIG; 873 874 return 0; 875 } 876 877 /* Find the next mapping after irec. */ 878 static inline int 879 xmi_next( 880 struct xfs_inode *ip, 881 int bmap_flags, 882 const struct xfs_bmbt_irec *irec, 883 struct xfs_bmbt_irec *nrec) 884 { 885 xfs_fileoff_t off; 886 xfs_filblks_t blockcount; 887 int nimaps = 1; 888 int error; 889 890 off = irec->br_startoff + irec->br_blockcount; 891 blockcount = XFS_MAX_FILEOFF - off; 892 error = xfs_bmapi_read(ip, off, blockcount, nrec, &nimaps, bmap_flags); 893 if (error) 894 return error; 895 if (nrec->br_startblock == DELAYSTARTBLOCK || 896 nrec->br_startoff != off) { 897 /* 898 * If we don't get the mapping we want, return a zero-length 899 * mapping, which our estimator function will pretend is a hole. 900 * We shouldn't get delalloc reservations. 901 */ 902 nrec->br_startblock = HOLESTARTBLOCK; 903 } 904 905 return 0; 906 } 907 908 int __init 909 xfs_exchmaps_intent_init_cache(void) 910 { 911 xfs_exchmaps_intent_cache = kmem_cache_create("xfs_exchmaps_intent", 912 sizeof(struct xfs_exchmaps_intent), 913 0, 0, NULL); 914 915 return xfs_exchmaps_intent_cache != NULL ? 0 : -ENOMEM; 916 } 917 918 void 919 xfs_exchmaps_intent_destroy_cache(void) 920 { 921 kmem_cache_destroy(xfs_exchmaps_intent_cache); 922 xfs_exchmaps_intent_cache = NULL; 923 } 924 925 /* 926 * Allocate and initialize a new incore intent item from a request. 927 */ 928 struct xfs_exchmaps_intent * 929 xfs_exchmaps_init_intent( 930 const struct xfs_exchmaps_req *req) 931 { 932 struct xfs_exchmaps_intent *xmi; 933 struct xfs_mount *mp = req->ip1->i_mount; 934 935 xmi = kmem_cache_zalloc(xfs_exchmaps_intent_cache, 936 GFP_NOFS | __GFP_NOFAIL); 937 INIT_LIST_HEAD(&xmi->xmi_list); 938 xmi->xmi_ip1 = req->ip1; 939 xmi->xmi_ip2 = req->ip2; 940 xmi->xmi_startoff1 = req->startoff1; 941 xmi->xmi_startoff2 = req->startoff2; 942 xmi->xmi_blockcount = req->blockcount; 943 xmi->xmi_isize1 = xmi->xmi_isize2 = -1; 944 xmi->xmi_flags = req->flags & XFS_EXCHMAPS_PARAMS; 945 946 if (xfs_exchmaps_whichfork(xmi) == XFS_ATTR_FORK) { 947 xmi->xmi_flags |= __XFS_EXCHMAPS_INO2_SHORTFORM; 948 return xmi; 949 } 950 951 /* 952 * If the caller wanted, set each file's size to that file's exchange 953 * offset + length exchanged from the other file because the ranges in 954 * each file might be different lengths. 955 */ 956 if (req->flags & XFS_EXCHMAPS_SET_SIZES) { 957 loff_t off1 = XFS_FSB_TO_B(mp, xmi->xmi_startoff1); 958 loff_t off2 = XFS_FSB_TO_B(mp, xmi->xmi_startoff2); 959 960 xmi->xmi_flags |= XFS_EXCHMAPS_SET_SIZES; 961 xmi->xmi_isize1 = off1 + (req->ip2->i_disk_size - off2); 962 xmi->xmi_isize2 = off2 + (req->ip1->i_disk_size - off1); 963 } 964 965 if (S_ISDIR(VFS_I(xmi->xmi_ip2)->i_mode) || 966 S_ISLNK(VFS_I(xmi->xmi_ip2)->i_mode)) 967 xmi->xmi_flags |= __XFS_EXCHMAPS_INO2_SHORTFORM; 968 969 return xmi; 970 } 971 972 /* 973 * Estimate the number of exchange operations and the number of file blocks 974 * in each file that will be affected by the exchange operation. 975 */ 976 int 977 xfs_exchmaps_estimate( 978 struct xfs_exchmaps_req *req) 979 { 980 struct xfs_exchmaps_intent *xmi; 981 struct xfs_bmbt_irec irec1, irec2; 982 struct xfs_exchmaps_adjacent adj = ADJACENT_INIT; 983 xfs_filblks_t ip1_blocks = 0, ip2_blocks = 0; 984 int64_t d_nexts1, d_nexts2; 985 int bmap_flags; 986 int error; 987 988 ASSERT(!(req->flags & ~XFS_EXCHMAPS_PARAMS)); 989 990 bmap_flags = xfs_bmapi_aflag(xfs_exchmaps_reqfork(req)); 991 xmi = xfs_exchmaps_init_intent(req); 992 993 /* 994 * To guard against the possibility of overflowing the extent counters, 995 * we have to estimate an upper bound on the potential increase in that 996 * counter. We can split the mapping at each end of the range, and for 997 * each step of the exchange we can split the mapping that we're 998 * working on if the mappings do not align. 999 */ 1000 d_nexts1 = d_nexts2 = 3; 1001 1002 while (xmi_has_more_exchange_work(xmi)) { 1003 /* 1004 * Walk through the file ranges until we find something to 1005 * exchange. Because we're simulating the exchange, pass in 1006 * adj to capture skipped mappings for correct estimation of 1007 * bmbt record merges. 1008 */ 1009 error = xfs_exchmaps_find_mappings(xmi, &irec1, &irec2, &adj); 1010 if (error) 1011 goto out_free; 1012 if (!xmi_has_more_exchange_work(xmi)) 1013 break; 1014 1015 /* Update accounting. */ 1016 if (xfs_bmap_is_real_extent(&irec1)) 1017 ip1_blocks += irec1.br_blockcount; 1018 if (xfs_bmap_is_real_extent(&irec2)) 1019 ip2_blocks += irec2.br_blockcount; 1020 req->nr_exchanges++; 1021 1022 /* Read the next mappings from both files. */ 1023 error = xmi_next(req->ip1, bmap_flags, &irec1, &adj.right1); 1024 if (error) 1025 goto out_free; 1026 1027 error = xmi_next(req->ip2, bmap_flags, &irec2, &adj.right2); 1028 if (error) 1029 goto out_free; 1030 1031 /* Update extent count deltas. */ 1032 d_nexts1 += xmi_delta_nextents_step(req->ip1->i_mount, 1033 &adj.left1, &irec1, &irec2, &adj.right1); 1034 1035 d_nexts2 += xmi_delta_nextents_step(req->ip1->i_mount, 1036 &adj.left2, &irec2, &irec1, &adj.right2); 1037 1038 /* Now pretend we exchanged the mappings. */ 1039 if (xmi_can_merge(&adj.left2, &irec1)) 1040 adj.left2.br_blockcount += irec1.br_blockcount; 1041 else 1042 memcpy(&adj.left2, &irec1, sizeof(irec1)); 1043 1044 if (xmi_can_merge(&adj.left1, &irec2)) 1045 adj.left1.br_blockcount += irec2.br_blockcount; 1046 else 1047 memcpy(&adj.left1, &irec2, sizeof(irec2)); 1048 1049 xmi_advance(xmi, &irec1); 1050 } 1051 1052 /* Account for the blocks that are being exchanged. */ 1053 if (XFS_IS_REALTIME_INODE(req->ip1) && 1054 xfs_exchmaps_reqfork(req) == XFS_DATA_FORK) { 1055 req->ip1_rtbcount = ip1_blocks; 1056 req->ip2_rtbcount = ip2_blocks; 1057 } else { 1058 req->ip1_bcount = ip1_blocks; 1059 req->ip2_bcount = ip2_blocks; 1060 } 1061 1062 /* 1063 * Make sure that both forks have enough slack left in their extent 1064 * counters that the exchange operation will not overflow. 1065 */ 1066 trace_xfs_exchmaps_delta_nextents(req, d_nexts1, d_nexts2); 1067 if (req->ip1 == req->ip2) { 1068 error = xmi_ensure_delta_nextents(req, req->ip1, 1069 d_nexts1 + d_nexts2); 1070 } else { 1071 error = xmi_ensure_delta_nextents(req, req->ip1, d_nexts1); 1072 if (error) 1073 goto out_free; 1074 error = xmi_ensure_delta_nextents(req, req->ip2, d_nexts2); 1075 } 1076 if (error) 1077 goto out_free; 1078 1079 trace_xfs_exchmaps_initial_estimate(req); 1080 error = xfs_exchmaps_estimate_overhead(req); 1081 out_free: 1082 kmem_cache_free(xfs_exchmaps_intent_cache, xmi); 1083 return error; 1084 } 1085 1086 /* Set the reflink flag before an operation. */ 1087 static inline void 1088 xfs_exchmaps_set_reflink( 1089 struct xfs_trans *tp, 1090 struct xfs_inode *ip) 1091 { 1092 trace_xfs_reflink_set_inode_flag(ip); 1093 1094 ip->i_diflags2 |= XFS_DIFLAG2_REFLINK; 1095 xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); 1096 } 1097 1098 /* 1099 * If either file has shared blocks and we're exchanging data forks, we must 1100 * flag the other file as having shared blocks so that we get the shared-block 1101 * rmap functions if we need to fix up the rmaps. 1102 */ 1103 void 1104 xfs_exchmaps_ensure_reflink( 1105 struct xfs_trans *tp, 1106 const struct xfs_exchmaps_intent *xmi) 1107 { 1108 unsigned int rs = 0; 1109 1110 if (xfs_is_reflink_inode(xmi->xmi_ip1)) 1111 rs |= 1; 1112 if (xfs_is_reflink_inode(xmi->xmi_ip2)) 1113 rs |= 2; 1114 1115 if ((rs & 1) && !xfs_is_reflink_inode(xmi->xmi_ip2)) 1116 xfs_exchmaps_set_reflink(tp, xmi->xmi_ip2); 1117 1118 if ((rs & 2) && !xfs_is_reflink_inode(xmi->xmi_ip1)) 1119 xfs_exchmaps_set_reflink(tp, xmi->xmi_ip1); 1120 } 1121 1122 /* Set the large extent count flag before an operation if needed. */ 1123 static inline void 1124 xfs_exchmaps_ensure_large_extent_counts( 1125 struct xfs_trans *tp, 1126 struct xfs_inode *ip) 1127 { 1128 if (xfs_inode_has_large_extent_counts(ip)) 1129 return; 1130 1131 ip->i_diflags2 |= XFS_DIFLAG2_NREXT64; 1132 xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); 1133 } 1134 1135 /* Widen the extent counter fields of both inodes if necessary. */ 1136 void 1137 xfs_exchmaps_upgrade_extent_counts( 1138 struct xfs_trans *tp, 1139 const struct xfs_exchmaps_intent *xmi) 1140 { 1141 if (!xfs_has_large_extent_counts(tp->t_mountp)) 1142 return; 1143 1144 xfs_exchmaps_ensure_large_extent_counts(tp, xmi->xmi_ip1); 1145 xfs_exchmaps_ensure_large_extent_counts(tp, xmi->xmi_ip2); 1146 } 1147 1148 /* 1149 * Schedule an exchange a range of mappings from one inode to another. 1150 * 1151 * The use of file mapping exchange log intent items ensures the operation can 1152 * be resumed even if the system goes down. The caller must commit the 1153 * transaction to start the work. 1154 * 1155 * The caller must ensure the inodes must be joined to the transaction and 1156 * ILOCKd; they will still be joined to the transaction at exit. 1157 */ 1158 void 1159 xfs_exchange_mappings( 1160 struct xfs_trans *tp, 1161 const struct xfs_exchmaps_req *req) 1162 { 1163 struct xfs_exchmaps_intent *xmi; 1164 1165 BUILD_BUG_ON(XFS_EXCHMAPS_INTERNAL_FLAGS & XFS_EXCHMAPS_LOGGED_FLAGS); 1166 1167 xfs_assert_ilocked(req->ip1, XFS_ILOCK_EXCL); 1168 xfs_assert_ilocked(req->ip2, XFS_ILOCK_EXCL); 1169 ASSERT(!(req->flags & ~XFS_EXCHMAPS_LOGGED_FLAGS)); 1170 if (req->flags & XFS_EXCHMAPS_SET_SIZES) 1171 ASSERT(!(req->flags & XFS_EXCHMAPS_ATTR_FORK)); 1172 ASSERT(xfs_has_exchange_range(tp->t_mountp)); 1173 1174 if (req->blockcount == 0) 1175 return; 1176 1177 xmi = xfs_exchmaps_init_intent(req); 1178 xfs_exchmaps_defer_add(tp, xmi); 1179 xfs_exchmaps_ensure_reflink(tp, xmi); 1180 xfs_exchmaps_upgrade_extent_counts(tp, xmi); 1181 } 1182