1 // SPDX-License-Identifier: LGPL-2.1 2 /* 3 * Copyright IBM Corporation, 2007 4 * Author Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com> 5 * 6 */ 7 8 #include <linux/slab.h> 9 #include "ext4_jbd2.h" 10 #include "ext4_extents.h" 11 12 /* 13 * The contiguous blocks details which can be 14 * represented by a single extent 15 */ 16 struct migrate_struct { 17 ext4_lblk_t first_block, last_block, curr_block; 18 ext4_fsblk_t first_pblock, last_pblock; 19 }; 20 21 static int finish_range(handle_t *handle, struct inode *inode, 22 struct migrate_struct *lb) 23 24 { 25 int retval = 0, needed; 26 struct ext4_extent newext; 27 struct ext4_ext_path *path; 28 if (lb->first_pblock == 0) 29 return 0; 30 31 /* Add the extent to temp inode*/ 32 newext.ee_block = cpu_to_le32(lb->first_block); 33 newext.ee_len = cpu_to_le16(lb->last_block - lb->first_block + 1); 34 ext4_ext_store_pblock(&newext, lb->first_pblock); 35 /* Locking only for convenience since we are operating on temp inode */ 36 down_write(&EXT4_I(inode)->i_data_sem); 37 path = ext4_find_extent(inode, lb->first_block, NULL, 0); 38 if (IS_ERR(path)) { 39 retval = PTR_ERR(path); 40 goto err_out; 41 } 42 43 /* 44 * Calculate the credit needed to inserting this extent 45 * Since we are doing this in loop we may accumulate extra 46 * credit. But below we try to not accumulate too much 47 * of them by restarting the journal. 48 */ 49 needed = ext4_ext_calc_credits_for_single_extent(inode, 50 lb->last_block - lb->first_block + 1, path); 51 52 retval = ext4_datasem_ensure_credits(handle, inode, needed, needed, 0); 53 if (retval < 0) 54 goto err_out; 55 path = ext4_ext_insert_extent(handle, inode, path, &newext, 0); 56 if (IS_ERR(path)) 57 retval = PTR_ERR(path); 58 err_out: 59 up_write((&EXT4_I(inode)->i_data_sem)); 60 ext4_free_ext_path(path); 61 lb->first_pblock = 0; 62 return retval; 63 } 64 65 static int update_extent_range(handle_t *handle, struct inode *inode, 66 ext4_fsblk_t pblock, struct migrate_struct *lb) 67 { 68 int retval; 69 /* 70 * See if we can add on to the existing range (if it exists) 71 */ 72 if (lb->first_pblock && 73 (lb->last_pblock+1 == pblock) && 74 (lb->last_block+1 == lb->curr_block)) { 75 lb->last_pblock = pblock; 76 lb->last_block = lb->curr_block; 77 lb->curr_block++; 78 return 0; 79 } 80 /* 81 * Start a new range. 82 */ 83 retval = finish_range(handle, inode, lb); 84 lb->first_pblock = lb->last_pblock = pblock; 85 lb->first_block = lb->last_block = lb->curr_block; 86 lb->curr_block++; 87 return retval; 88 } 89 90 static int update_ind_extent_range(handle_t *handle, struct inode *inode, 91 ext4_fsblk_t pblock, 92 struct migrate_struct *lb) 93 { 94 struct buffer_head *bh; 95 __le32 *i_data; 96 int i, retval = 0; 97 unsigned long max_entries = inode->i_sb->s_blocksize >> 2; 98 99 bh = ext4_sb_bread(inode->i_sb, pblock, 0); 100 if (IS_ERR(bh)) 101 return PTR_ERR(bh); 102 103 i_data = (__le32 *)bh->b_data; 104 for (i = 0; i < max_entries; i++) { 105 if (i_data[i]) { 106 retval = update_extent_range(handle, inode, 107 le32_to_cpu(i_data[i]), lb); 108 if (retval) 109 break; 110 } else { 111 lb->curr_block++; 112 } 113 } 114 put_bh(bh); 115 return retval; 116 117 } 118 119 static int update_dind_extent_range(handle_t *handle, struct inode *inode, 120 ext4_fsblk_t pblock, 121 struct migrate_struct *lb) 122 { 123 struct buffer_head *bh; 124 __le32 *i_data; 125 int i, retval = 0; 126 unsigned long max_entries = inode->i_sb->s_blocksize >> 2; 127 128 bh = ext4_sb_bread(inode->i_sb, pblock, 0); 129 if (IS_ERR(bh)) 130 return PTR_ERR(bh); 131 132 i_data = (__le32 *)bh->b_data; 133 for (i = 0; i < max_entries; i++) { 134 if (i_data[i]) { 135 retval = update_ind_extent_range(handle, inode, 136 le32_to_cpu(i_data[i]), lb); 137 if (retval) 138 break; 139 } else { 140 /* Only update the file block number */ 141 lb->curr_block += max_entries; 142 } 143 } 144 put_bh(bh); 145 return retval; 146 147 } 148 149 static int update_tind_extent_range(handle_t *handle, struct inode *inode, 150 ext4_fsblk_t pblock, 151 struct migrate_struct *lb) 152 { 153 struct buffer_head *bh; 154 __le32 *i_data; 155 int i, retval = 0; 156 unsigned long max_entries = inode->i_sb->s_blocksize >> 2; 157 158 bh = ext4_sb_bread(inode->i_sb, pblock, 0); 159 if (IS_ERR(bh)) 160 return PTR_ERR(bh); 161 162 i_data = (__le32 *)bh->b_data; 163 for (i = 0; i < max_entries; i++) { 164 if (i_data[i]) { 165 retval = update_dind_extent_range(handle, inode, 166 le32_to_cpu(i_data[i]), lb); 167 if (retval) 168 break; 169 } else { 170 /* Only update the file block number */ 171 lb->curr_block += max_entries * max_entries; 172 } 173 } 174 put_bh(bh); 175 return retval; 176 177 } 178 179 static int free_dind_blocks(handle_t *handle, 180 struct inode *inode, __le32 i_data) 181 { 182 int i; 183 __le32 *tmp_idata; 184 struct buffer_head *bh; 185 struct super_block *sb = inode->i_sb; 186 unsigned long max_entries = inode->i_sb->s_blocksize >> 2; 187 int err; 188 189 bh = ext4_sb_bread(sb, le32_to_cpu(i_data), 0); 190 if (IS_ERR(bh)) 191 return PTR_ERR(bh); 192 193 tmp_idata = (__le32 *)bh->b_data; 194 for (i = 0; i < max_entries; i++) { 195 if (tmp_idata[i]) { 196 err = ext4_journal_ensure_credits(handle, 197 EXT4_RESERVE_TRANS_BLOCKS, 198 ext4_free_metadata_revoke_credits(sb, 1)); 199 if (err < 0) { 200 put_bh(bh); 201 return err; 202 } 203 ext4_free_blocks(handle, inode, NULL, 204 le32_to_cpu(tmp_idata[i]), 1, 205 EXT4_FREE_BLOCKS_METADATA | 206 EXT4_FREE_BLOCKS_FORGET); 207 } 208 } 209 put_bh(bh); 210 err = ext4_journal_ensure_credits(handle, EXT4_RESERVE_TRANS_BLOCKS, 211 ext4_free_metadata_revoke_credits(sb, 1)); 212 if (err < 0) 213 return err; 214 ext4_free_blocks(handle, inode, NULL, le32_to_cpu(i_data), 1, 215 EXT4_FREE_BLOCKS_METADATA | 216 EXT4_FREE_BLOCKS_FORGET); 217 return 0; 218 } 219 220 static int free_tind_blocks(handle_t *handle, 221 struct inode *inode, __le32 i_data) 222 { 223 int i, retval = 0; 224 __le32 *tmp_idata; 225 struct buffer_head *bh; 226 unsigned long max_entries = inode->i_sb->s_blocksize >> 2; 227 228 bh = ext4_sb_bread(inode->i_sb, le32_to_cpu(i_data), 0); 229 if (IS_ERR(bh)) 230 return PTR_ERR(bh); 231 232 tmp_idata = (__le32 *)bh->b_data; 233 for (i = 0; i < max_entries; i++) { 234 if (tmp_idata[i]) { 235 retval = free_dind_blocks(handle, 236 inode, tmp_idata[i]); 237 if (retval) { 238 put_bh(bh); 239 return retval; 240 } 241 } 242 } 243 put_bh(bh); 244 retval = ext4_journal_ensure_credits(handle, EXT4_RESERVE_TRANS_BLOCKS, 245 ext4_free_metadata_revoke_credits(inode->i_sb, 1)); 246 if (retval < 0) 247 return retval; 248 ext4_free_blocks(handle, inode, NULL, le32_to_cpu(i_data), 1, 249 EXT4_FREE_BLOCKS_METADATA | 250 EXT4_FREE_BLOCKS_FORGET); 251 return 0; 252 } 253 254 static int free_ind_block(handle_t *handle, struct inode *inode, __le32 *i_data) 255 { 256 int retval; 257 258 /* ei->i_data[EXT4_IND_BLOCK] */ 259 if (i_data[0]) { 260 retval = ext4_journal_ensure_credits(handle, 261 EXT4_RESERVE_TRANS_BLOCKS, 262 ext4_free_metadata_revoke_credits(inode->i_sb, 1)); 263 if (retval < 0) 264 return retval; 265 ext4_free_blocks(handle, inode, NULL, 266 le32_to_cpu(i_data[0]), 1, 267 EXT4_FREE_BLOCKS_METADATA | 268 EXT4_FREE_BLOCKS_FORGET); 269 } 270 271 /* ei->i_data[EXT4_DIND_BLOCK] */ 272 if (i_data[1]) { 273 retval = free_dind_blocks(handle, inode, i_data[1]); 274 if (retval) 275 return retval; 276 } 277 278 /* ei->i_data[EXT4_TIND_BLOCK] */ 279 if (i_data[2]) { 280 retval = free_tind_blocks(handle, inode, i_data[2]); 281 if (retval) 282 return retval; 283 } 284 return 0; 285 } 286 287 static int ext4_ext_swap_inode_data(handle_t *handle, struct inode *inode, 288 struct inode *tmp_inode) 289 { 290 int retval, retval2 = 0; 291 __le32 i_data[3]; 292 struct ext4_inode_info *ei = EXT4_I(inode); 293 struct ext4_inode_info *tmp_ei = EXT4_I(tmp_inode); 294 295 /* 296 * One credit accounted for writing the 297 * i_data field of the original inode 298 */ 299 retval = ext4_journal_ensure_credits(handle, 1, 0); 300 if (retval < 0) 301 goto err_out; 302 303 i_data[0] = ei->i_data[EXT4_IND_BLOCK]; 304 i_data[1] = ei->i_data[EXT4_DIND_BLOCK]; 305 i_data[2] = ei->i_data[EXT4_TIND_BLOCK]; 306 307 down_write(&EXT4_I(inode)->i_data_sem); 308 /* 309 * if EXT4_STATE_EXT_MIGRATE is cleared a block allocation 310 * happened after we started the migrate. We need to 311 * fail the migrate 312 */ 313 if (!ext4_test_inode_state(inode, EXT4_STATE_EXT_MIGRATE)) { 314 retval = -EAGAIN; 315 up_write(&EXT4_I(inode)->i_data_sem); 316 goto err_out; 317 } else 318 ext4_clear_inode_state(inode, EXT4_STATE_EXT_MIGRATE); 319 /* 320 * We have the extent map build with the tmp inode. 321 * Now copy the i_data across 322 */ 323 ext4_set_inode_flag(inode, EXT4_INODE_EXTENTS); 324 memcpy(ei->i_data, tmp_ei->i_data, sizeof(ei->i_data)); 325 326 /* 327 * Update i_blocks with the new blocks that got 328 * allocated while adding extents for extent index 329 * blocks. 330 * 331 * While converting to extents we need not 332 * update the original inode i_blocks for extent blocks 333 * via quota APIs. The quota update happened via tmp_inode already. 334 */ 335 spin_lock(&inode->i_lock); 336 inode->i_blocks += tmp_inode->i_blocks; 337 spin_unlock(&inode->i_lock); 338 up_write(&EXT4_I(inode)->i_data_sem); 339 340 /* 341 * We mark the inode dirty after, because we decrement the 342 * i_blocks when freeing the indirect meta-data blocks 343 */ 344 retval = free_ind_block(handle, inode, i_data); 345 retval2 = ext4_mark_inode_dirty(handle, inode); 346 if (unlikely(retval2 && !retval)) 347 retval = retval2; 348 349 err_out: 350 return retval; 351 } 352 353 static int free_ext_idx(handle_t *handle, struct inode *inode, 354 struct ext4_extent_idx *ix) 355 { 356 int i, retval = 0; 357 ext4_fsblk_t block; 358 struct buffer_head *bh; 359 struct ext4_extent_header *eh; 360 361 block = ext4_idx_pblock(ix); 362 bh = ext4_sb_bread(inode->i_sb, block, 0); 363 if (IS_ERR(bh)) 364 return PTR_ERR(bh); 365 366 eh = (struct ext4_extent_header *)bh->b_data; 367 if (eh->eh_depth != 0) { 368 ix = EXT_FIRST_INDEX(eh); 369 for (i = 0; i < le16_to_cpu(eh->eh_entries); i++, ix++) { 370 retval = free_ext_idx(handle, inode, ix); 371 if (retval) { 372 put_bh(bh); 373 return retval; 374 } 375 } 376 } 377 put_bh(bh); 378 retval = ext4_journal_ensure_credits(handle, EXT4_RESERVE_TRANS_BLOCKS, 379 ext4_free_metadata_revoke_credits(inode->i_sb, 1)); 380 if (retval < 0) 381 return retval; 382 ext4_free_blocks(handle, inode, NULL, block, 1, 383 EXT4_FREE_BLOCKS_METADATA | EXT4_FREE_BLOCKS_FORGET); 384 return 0; 385 } 386 387 /* 388 * Free the extent meta data blocks only 389 */ 390 static int free_ext_block(handle_t *handle, struct inode *inode) 391 { 392 int i, retval = 0; 393 struct ext4_inode_info *ei = EXT4_I(inode); 394 struct ext4_extent_header *eh = (struct ext4_extent_header *)ei->i_data; 395 struct ext4_extent_idx *ix; 396 if (eh->eh_depth == 0) 397 /* 398 * No extra blocks allocated for extent meta data 399 */ 400 return 0; 401 ix = EXT_FIRST_INDEX(eh); 402 for (i = 0; i < le16_to_cpu(eh->eh_entries); i++, ix++) { 403 retval = free_ext_idx(handle, inode, ix); 404 if (retval) 405 return retval; 406 } 407 return retval; 408 } 409 410 int ext4_ext_migrate(struct inode *inode) 411 { 412 handle_t *handle; 413 int retval = 0, i; 414 __le32 *i_data; 415 struct ext4_inode_info *ei; 416 struct inode *tmp_inode = NULL; 417 struct migrate_struct lb; 418 unsigned long max_entries; 419 __u32 goal, tmp_csum_seed; 420 uid_t owner[2]; 421 int alloc_ctx; 422 423 /* 424 * If the filesystem does not support extents, or the inode 425 * already is extent-based, error out. 426 */ 427 if (!ext4_has_feature_extents(inode->i_sb) || 428 ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS) || 429 ext4_has_inline_data(inode)) 430 return -EINVAL; 431 432 if (S_ISLNK(inode->i_mode) && inode->i_blocks == 0) 433 /* 434 * don't migrate fast symlink 435 */ 436 return retval; 437 438 alloc_ctx = ext4_writepages_down_write(inode->i_sb); 439 440 /* 441 * Worst case we can touch the allocation bitmaps and a block 442 * group descriptor block. We do need to worry about 443 * credits for modifying the quota inode. 444 */ 445 handle = ext4_journal_start(inode, EXT4_HT_MIGRATE, 446 3 + EXT4_MAXQUOTAS_TRANS_BLOCKS(inode->i_sb)); 447 448 if (IS_ERR(handle)) { 449 retval = PTR_ERR(handle); 450 goto out_unlock; 451 } 452 goal = (((inode->i_ino - 1) / EXT4_INODES_PER_GROUP(inode->i_sb)) * 453 EXT4_INODES_PER_GROUP(inode->i_sb)) + 1; 454 owner[0] = i_uid_read(inode); 455 owner[1] = i_gid_read(inode); 456 tmp_inode = ext4_new_inode(handle, d_inode(inode->i_sb->s_root), 457 S_IFREG, NULL, goal, owner, 0); 458 if (IS_ERR(tmp_inode)) { 459 retval = PTR_ERR(tmp_inode); 460 ext4_journal_stop(handle); 461 goto out_unlock; 462 } 463 /* 464 * Use the correct seed for checksum (i.e. the seed from 'inode'). This 465 * is so that the metadata blocks will have the correct checksum after 466 * the migration. 467 */ 468 ei = EXT4_I(inode); 469 tmp_csum_seed = EXT4_I(tmp_inode)->i_csum_seed; 470 EXT4_I(tmp_inode)->i_csum_seed = ei->i_csum_seed; 471 i_size_write(tmp_inode, i_size_read(inode)); 472 /* 473 * Set the i_nlink to zero so it will be deleted later 474 * when we drop inode reference. 475 */ 476 clear_nlink(tmp_inode); 477 478 ext4_ext_tree_init(handle, tmp_inode); 479 ext4_journal_stop(handle); 480 481 /* 482 * start with one credit accounted for 483 * superblock modification. 484 * 485 * For the tmp_inode we already have committed the 486 * transaction that created the inode. Later as and 487 * when we add extents we extent the journal 488 */ 489 /* 490 * Even though we take i_rwsem we can still cause block 491 * allocation via mmap write to holes. If we have allocated 492 * new blocks we fail migrate. New block allocation will 493 * clear EXT4_STATE_EXT_MIGRATE flag. The flag is updated 494 * with i_data_sem held to prevent racing with block 495 * allocation. 496 */ 497 down_read(&EXT4_I(inode)->i_data_sem); 498 ext4_set_inode_state(inode, EXT4_STATE_EXT_MIGRATE); 499 up_read((&EXT4_I(inode)->i_data_sem)); 500 501 handle = ext4_journal_start(inode, EXT4_HT_MIGRATE, 1); 502 if (IS_ERR(handle)) { 503 retval = PTR_ERR(handle); 504 goto out_tmp_inode; 505 } 506 507 i_data = ei->i_data; 508 memset(&lb, 0, sizeof(lb)); 509 510 /* 32 bit block address 4 bytes */ 511 max_entries = inode->i_sb->s_blocksize >> 2; 512 for (i = 0; i < EXT4_NDIR_BLOCKS; i++) { 513 if (i_data[i]) { 514 retval = update_extent_range(handle, tmp_inode, 515 le32_to_cpu(i_data[i]), &lb); 516 if (retval) 517 goto err_out; 518 } else 519 lb.curr_block++; 520 } 521 if (i_data[EXT4_IND_BLOCK]) { 522 retval = update_ind_extent_range(handle, tmp_inode, 523 le32_to_cpu(i_data[EXT4_IND_BLOCK]), &lb); 524 if (retval) 525 goto err_out; 526 } else 527 lb.curr_block += max_entries; 528 if (i_data[EXT4_DIND_BLOCK]) { 529 retval = update_dind_extent_range(handle, tmp_inode, 530 le32_to_cpu(i_data[EXT4_DIND_BLOCK]), &lb); 531 if (retval) 532 goto err_out; 533 } else 534 lb.curr_block += max_entries * max_entries; 535 if (i_data[EXT4_TIND_BLOCK]) { 536 retval = update_tind_extent_range(handle, tmp_inode, 537 le32_to_cpu(i_data[EXT4_TIND_BLOCK]), &lb); 538 if (retval) 539 goto err_out; 540 } 541 /* 542 * Build the last extent 543 */ 544 retval = finish_range(handle, tmp_inode, &lb); 545 err_out: 546 if (retval) 547 /* 548 * Failure case delete the extent information with the 549 * tmp_inode 550 */ 551 free_ext_block(handle, tmp_inode); 552 else { 553 retval = ext4_ext_swap_inode_data(handle, inode, tmp_inode); 554 if (retval) 555 /* 556 * if we fail to swap inode data free the extent 557 * details of the tmp inode 558 */ 559 free_ext_block(handle, tmp_inode); 560 } 561 562 /* We mark the tmp_inode dirty via ext4_ext_tree_init. */ 563 retval = ext4_journal_ensure_credits(handle, 1, 0); 564 if (retval < 0) 565 goto out_stop; 566 /* 567 * Mark the tmp_inode as of size zero 568 */ 569 i_size_write(tmp_inode, 0); 570 571 /* 572 * set the i_blocks count to zero 573 * so that the ext4_evict_inode() does the 574 * right job 575 * 576 * We don't need to take the i_lock because 577 * the inode is not visible to user space. 578 */ 579 tmp_inode->i_blocks = 0; 580 EXT4_I(tmp_inode)->i_csum_seed = tmp_csum_seed; 581 582 /* Reset the extent details */ 583 ext4_ext_tree_init(handle, tmp_inode); 584 out_stop: 585 ext4_journal_stop(handle); 586 out_tmp_inode: 587 unlock_new_inode(tmp_inode); 588 iput(tmp_inode); 589 out_unlock: 590 ext4_writepages_up_write(inode->i_sb, alloc_ctx); 591 return retval; 592 } 593 594 /* 595 * Migrate a simple extent-based inode to use the i_blocks[] array 596 */ 597 int ext4_ind_migrate(struct inode *inode) 598 { 599 struct ext4_extent_header *eh; 600 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); 601 struct ext4_super_block *es = sbi->s_es; 602 struct ext4_inode_info *ei = EXT4_I(inode); 603 struct ext4_extent *ex; 604 unsigned int i, len; 605 ext4_lblk_t start, end; 606 ext4_fsblk_t blk; 607 handle_t *handle; 608 int ret, ret2 = 0; 609 int alloc_ctx; 610 611 if (!ext4_has_feature_extents(inode->i_sb) || 612 (!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) 613 return -EINVAL; 614 615 if (ext4_has_feature_bigalloc(inode->i_sb)) 616 return -EOPNOTSUPP; 617 618 /* 619 * In order to get correct extent info, force all delayed allocation 620 * blocks to be allocated, otherwise delayed allocation blocks may not 621 * be reflected and bypass the checks on extent header. 622 */ 623 if (test_opt(inode->i_sb, DELALLOC)) 624 ext4_alloc_da_blocks(inode); 625 626 alloc_ctx = ext4_writepages_down_write(inode->i_sb); 627 628 handle = ext4_journal_start(inode, EXT4_HT_MIGRATE, 1); 629 if (IS_ERR(handle)) { 630 ret = PTR_ERR(handle); 631 goto out_unlock; 632 } 633 634 down_write(&EXT4_I(inode)->i_data_sem); 635 ret = ext4_ext_check_inode(inode); 636 if (ret) 637 goto errout; 638 639 eh = ext_inode_hdr(inode); 640 ex = EXT_FIRST_EXTENT(eh); 641 if (ext4_blocks_count(es) > EXT4_MAX_BLOCK_FILE_PHYS || 642 eh->eh_depth != 0 || le16_to_cpu(eh->eh_entries) > 1) { 643 ret = -EOPNOTSUPP; 644 goto errout; 645 } 646 if (eh->eh_entries == 0) 647 blk = len = start = end = 0; 648 else { 649 len = le16_to_cpu(ex->ee_len); 650 blk = ext4_ext_pblock(ex); 651 start = le32_to_cpu(ex->ee_block); 652 end = start + len - 1; 653 if (end >= EXT4_NDIR_BLOCKS) { 654 ret = -EOPNOTSUPP; 655 goto errout; 656 } 657 } 658 659 ext4_clear_inode_flag(inode, EXT4_INODE_EXTENTS); 660 memset(ei->i_data, 0, sizeof(ei->i_data)); 661 for (i = start; i <= end; i++) 662 ei->i_data[i] = cpu_to_le32(blk++); 663 ret2 = ext4_mark_inode_dirty(handle, inode); 664 if (unlikely(ret2 && !ret)) 665 ret = ret2; 666 errout: 667 up_write(&EXT4_I(inode)->i_data_sem); 668 ext4_journal_stop(handle); 669 out_unlock: 670 ext4_writepages_up_write(inode->i_sb, alloc_ctx); 671 return ret; 672 } 673