Lines Matching refs:sctx
369 static void inconsistent_snapshot_error(struct send_ctx *sctx, in inconsistent_snapshot_error() argument
394 btrfs_err(sctx->send_root->fs_info, in inconsistent_snapshot_error()
396 result_string, what, sctx->cmp_key->objectid, in inconsistent_snapshot_error()
397 btrfs_root_id(sctx->send_root), in inconsistent_snapshot_error()
398 (sctx->parent_root ? btrfs_root_id(sctx->parent_root) : 0)); in inconsistent_snapshot_error()
402 static bool proto_cmd_ok(const struct send_ctx *sctx, int cmd) in proto_cmd_ok() argument
404 switch (sctx->proto) { in proto_cmd_ok()
412 static int is_waiting_for_move(struct send_ctx *sctx, u64 ino);
415 get_waiting_dir_move(struct send_ctx *sctx, u64 ino);
417 static int is_waiting_for_rm(struct send_ctx *sctx, u64 dir_ino, u64 gen);
419 static int need_send_hole(struct send_ctx *sctx) in need_send_hole() argument
421 return (sctx->parent_root && !sctx->cur_inode_new && in need_send_hole()
422 !sctx->cur_inode_new_gen && !sctx->cur_inode_deleted && in need_send_hole()
423 S_ISREG(sctx->cur_inode_mode)); in need_send_hole()
659 static int tlv_put(struct send_ctx *sctx, u16 attr, const void *data, int len) in tlv_put() argument
663 int left = sctx->send_max_size - sctx->send_size; in tlv_put()
665 if (WARN_ON_ONCE(sctx->put_data)) in tlv_put()
671 hdr = (struct btrfs_tlv_header *) (sctx->send_buf + sctx->send_size); in tlv_put()
675 sctx->send_size += total_len; in tlv_put()
681 static int tlv_put_u##bits(struct send_ctx *sctx, \
685 return tlv_put(sctx, attr, &__tmp, sizeof(__tmp)); \
692 static int tlv_put_string(struct send_ctx *sctx, u16 attr, in tlv_put_string() argument
697 return tlv_put(sctx, attr, str, len); in tlv_put_string()
700 static int tlv_put_uuid(struct send_ctx *sctx, u16 attr, in tlv_put_uuid() argument
703 return tlv_put(sctx, attr, uuid, BTRFS_UUID_SIZE); in tlv_put_uuid()
706 static int tlv_put_btrfs_timespec(struct send_ctx *sctx, u16 attr, in tlv_put_btrfs_timespec() argument
712 return tlv_put(sctx, attr, &bts, sizeof(bts)); in tlv_put_btrfs_timespec()
716 #define TLV_PUT(sctx, attrtype, data, attrlen) \ argument
718 ret = tlv_put(sctx, attrtype, data, attrlen); \
723 #define TLV_PUT_INT(sctx, attrtype, bits, value) \ argument
725 ret = tlv_put_u##bits(sctx, attrtype, value); \
730 #define TLV_PUT_U8(sctx, attrtype, data) TLV_PUT_INT(sctx, attrtype, 8, data) argument
731 #define TLV_PUT_U16(sctx, attrtype, data) TLV_PUT_INT(sctx, attrtype, 16, data) argument
732 #define TLV_PUT_U32(sctx, attrtype, data) TLV_PUT_INT(sctx, attrtype, 32, data) argument
733 #define TLV_PUT_U64(sctx, attrtype, data) TLV_PUT_INT(sctx, attrtype, 64, data) argument
734 #define TLV_PUT_STRING(sctx, attrtype, str, len) \ argument
736 ret = tlv_put_string(sctx, attrtype, str, len); \
740 #define TLV_PUT_PATH(sctx, attrtype, p) \ argument
742 ret = tlv_put_string(sctx, attrtype, p->start, \
747 #define TLV_PUT_UUID(sctx, attrtype, uuid) \ argument
749 ret = tlv_put_uuid(sctx, attrtype, uuid); \
753 #define TLV_PUT_BTRFS_TIMESPEC(sctx, attrtype, eb, ts) \ argument
755 ret = tlv_put_btrfs_timespec(sctx, attrtype, eb, ts); \
760 static int send_header(struct send_ctx *sctx) in send_header() argument
765 hdr.version = cpu_to_le32(sctx->proto); in send_header()
766 return write_buf(sctx->send_filp, &hdr, sizeof(hdr), in send_header()
767 &sctx->send_off); in send_header()
773 static int begin_cmd(struct send_ctx *sctx, int cmd) in begin_cmd() argument
777 if (WARN_ON(!sctx->send_buf)) in begin_cmd()
780 if (unlikely(sctx->send_size != 0)) { in begin_cmd()
781 btrfs_err(sctx->send_root->fs_info, in begin_cmd()
783 cmd, sctx->send_off); in begin_cmd()
787 sctx->send_size += sizeof(*hdr); in begin_cmd()
788 hdr = (struct btrfs_cmd_header *)sctx->send_buf; in begin_cmd()
794 static int send_cmd(struct send_ctx *sctx) in send_cmd() argument
800 hdr = (struct btrfs_cmd_header *)sctx->send_buf; in send_cmd()
801 put_unaligned_le32(sctx->send_size - sizeof(*hdr), &hdr->len); in send_cmd()
804 crc = crc32c(0, (unsigned char *)sctx->send_buf, sctx->send_size); in send_cmd()
807 ret = write_buf(sctx->send_filp, sctx->send_buf, sctx->send_size, in send_cmd()
808 &sctx->send_off); in send_cmd()
810 sctx->send_size = 0; in send_cmd()
811 sctx->put_data = false; in send_cmd()
819 static int send_rename(struct send_ctx *sctx, in send_rename() argument
822 struct btrfs_fs_info *fs_info = sctx->send_root->fs_info; in send_rename()
827 ret = begin_cmd(sctx, BTRFS_SEND_C_RENAME); in send_rename()
831 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, from); in send_rename()
832 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH_TO, to); in send_rename()
834 ret = send_cmd(sctx); in send_rename()
844 static int send_link(struct send_ctx *sctx, in send_link() argument
847 struct btrfs_fs_info *fs_info = sctx->send_root->fs_info; in send_link()
852 ret = begin_cmd(sctx, BTRFS_SEND_C_LINK); in send_link()
856 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path); in send_link()
857 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH_LINK, lnk); in send_link()
859 ret = send_cmd(sctx); in send_link()
869 static int send_unlink(struct send_ctx *sctx, struct fs_path *path) in send_unlink() argument
871 struct btrfs_fs_info *fs_info = sctx->send_root->fs_info; in send_unlink()
876 ret = begin_cmd(sctx, BTRFS_SEND_C_UNLINK); in send_unlink()
880 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path); in send_unlink()
882 ret = send_cmd(sctx); in send_unlink()
892 static int send_rmdir(struct send_ctx *sctx, struct fs_path *path) in send_rmdir() argument
894 struct btrfs_fs_info *fs_info = sctx->send_root->fs_info; in send_rmdir()
899 ret = begin_cmd(sctx, BTRFS_SEND_C_RMDIR); in send_rmdir()
903 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path); in send_rmdir()
905 ret = send_cmd(sctx); in send_rmdir()
1284 struct send_ctx *sctx; member
1342 clone_root = bsearch((void *)(uintptr_t)root_id, bctx->sctx->clone_roots, in iterate_backrefs()
1343 bctx->sctx->clone_roots_cnt, in iterate_backrefs()
1350 if (clone_root->root == bctx->sctx->send_root && in iterate_backrefs()
1359 if (clone_root->root == bctx->sctx->send_root) { in iterate_backrefs()
1375 bctx->sctx->cur_inode_next_write_offset) in iterate_backrefs()
1408 struct send_ctx *sctx = bctx->sctx; in lookup_backref_cache() local
1409 struct btrfs_fs_info *fs_info = sctx->send_root->fs_info; in lookup_backref_cache()
1414 if (sctx->backref_cache.size == 0) in lookup_backref_cache()
1428 if (fs_info->last_reloc_trans > sctx->backref_cache_last_reloc_trans) { in lookup_backref_cache()
1429 btrfs_lru_cache_clear(&sctx->backref_cache); in lookup_backref_cache()
1433 raw_entry = btrfs_lru_cache_lookup(&sctx->backref_cache, key, 0); in lookup_backref_cache()
1448 struct send_ctx *sctx = bctx->sctx; in store_backref_cache() local
1449 struct btrfs_fs_info *fs_info = sctx->send_root->fs_info; in store_backref_cache()
1473 root = bsearch((void *)(uintptr_t)root_id, sctx->clone_roots, in store_backref_cache()
1474 sctx->clone_roots_cnt, sizeof(struct clone_root), in store_backref_cache()
1498 ret = btrfs_lru_cache_store(&sctx->backref_cache, &new_entry->entry, in store_backref_cache()
1512 if (sctx->backref_cache.size == 1) in store_backref_cache()
1513 sctx->backref_cache_last_reloc_trans = fs_info->last_reloc_trans; in store_backref_cache()
1521 const struct send_ctx *sctx = bctx->sctx; in check_extent_item() local
1536 if (refs == 1 && sctx->clone_roots_cnt == 1) in check_extent_item()
1574 static int find_extent_clone(struct send_ctx *sctx, in find_extent_clone() argument
1580 struct btrfs_fs_info *fs_info = sctx->send_root->fs_info; in find_extent_clone()
1619 for (i = 0; i < sctx->clone_roots_cnt; i++) { in find_extent_clone()
1620 cur_clone_root = sctx->clone_roots + i; in find_extent_clone()
1627 backref_ctx.sctx = sctx; in find_extent_clone()
1669 if (sctx->clone_roots_cnt == 1) in find_extent_clone()
1678 if (fs_info->last_reloc_trans > sctx->last_reloc_trans) { in find_extent_clone()
1706 for (i = 0; i < sctx->clone_roots_cnt; i++) { in find_extent_clone()
1707 struct clone_root *clone_root = &sctx->clone_roots[i]; in find_extent_clone()
1812 static int gen_unique_name(struct send_ctx *sctx, in gen_unique_name() argument
1836 di = btrfs_lookup_dir_item(NULL, sctx->send_root, in gen_unique_name()
1850 if (!sctx->parent_root) { in gen_unique_name()
1856 di = btrfs_lookup_dir_item(NULL, sctx->parent_root, in gen_unique_name()
1888 static int get_cur_inode_state(struct send_ctx *sctx, u64 ino, u64 gen, in get_cur_inode_state() argument
1898 ret = get_inode_info(sctx->send_root, ino, &info); in get_cur_inode_state()
1906 if (!sctx->parent_root) { in get_cur_inode_state()
1909 ret = get_inode_info(sctx->parent_root, ino, &info); in get_cur_inode_state()
1922 if (ino < sctx->send_progress) in get_cur_inode_state()
1927 if (ino < sctx->send_progress) in get_cur_inode_state()
1936 if (ino < sctx->send_progress) in get_cur_inode_state()
1945 if (ino < sctx->send_progress) in get_cur_inode_state()
1960 static int is_inode_existent(struct send_ctx *sctx, u64 ino, u64 gen, in is_inode_existent() argument
1968 ret = get_cur_inode_state(sctx, ino, gen, send_gen, parent_gen); in is_inode_existent()
2125 static int will_overwrite_ref(struct send_ctx *sctx, u64 dir, u64 dir_gen, in will_overwrite_ref() argument
2134 if (!sctx->parent_root) in will_overwrite_ref()
2137 ret = is_inode_existent(sctx, dir, dir_gen, NULL, &parent_root_dir_gen); in will_overwrite_ref()
2149 if (sctx->parent_root && dir != BTRFS_FIRST_FREE_OBJECTID && in will_overwrite_ref()
2153 ret = lookup_dir_item_inode(sctx->parent_root, dir, name, name_len, in will_overwrite_ref()
2165 if (other_inode > sctx->send_progress || in will_overwrite_ref()
2166 is_waiting_for_move(sctx, other_inode)) { in will_overwrite_ref()
2167 ret = get_inode_info(sctx->parent_root, other_inode, &info); in will_overwrite_ref()
2187 static int did_overwrite_ref(struct send_ctx *sctx, in did_overwrite_ref() argument
2197 if (!sctx->parent_root) in did_overwrite_ref()
2200 ret = is_inode_existent(sctx, dir, dir_gen, &send_root_dir_gen, NULL); in did_overwrite_ref()
2212 ret = lookup_dir_item_inode(sctx->send_root, dir, name, name_len, in did_overwrite_ref()
2222 ret = get_inode_gen(sctx->send_root, ow_inode, &ow_gen); in did_overwrite_ref()
2237 if (ow_inode < sctx->send_progress) in did_overwrite_ref()
2240 if (ino != sctx->cur_ino && ow_inode == sctx->cur_ino) { in did_overwrite_ref()
2242 ret = get_inode_gen(sctx->send_root, ow_inode, &ow_gen); in did_overwrite_ref()
2246 if (ow_gen == sctx->cur_inode_gen) in did_overwrite_ref()
2258 static int did_overwrite_first_ref(struct send_ctx *sctx, u64 ino, u64 gen) in did_overwrite_first_ref() argument
2265 if (!sctx->parent_root) in did_overwrite_first_ref()
2272 ret = get_first_ref(sctx->parent_root, ino, &dir, &dir_gen, name); in did_overwrite_first_ref()
2276 ret = did_overwrite_ref(sctx, dir, dir_gen, ino, gen, in did_overwrite_first_ref()
2284 static inline struct name_cache_entry *name_cache_search(struct send_ctx *sctx, in name_cache_search() argument
2289 entry = btrfs_lru_cache_lookup(&sctx->name_cache, ino, gen); in name_cache_search()
2304 static int __get_cur_name_and_parent(struct send_ctx *sctx, in __get_cur_name_and_parent() argument
2319 nce = name_cache_search(sctx, ino, gen); in __get_cur_name_and_parent()
2321 if (ino < sctx->send_progress && nce->need_later_update) { in __get_cur_name_and_parent()
2322 btrfs_lru_cache_remove(&sctx->name_cache, &nce->entry); in __get_cur_name_and_parent()
2340 ret = is_inode_existent(sctx, ino, gen, NULL, NULL); in __get_cur_name_and_parent()
2345 ret = gen_unique_name(sctx, ino, gen, dest); in __get_cur_name_and_parent()
2356 if (ino < sctx->send_progress) in __get_cur_name_and_parent()
2357 ret = get_first_ref(sctx->send_root, ino, in __get_cur_name_and_parent()
2360 ret = get_first_ref(sctx->parent_root, ino, in __get_cur_name_and_parent()
2369 ret = did_overwrite_ref(sctx, *parent_ino, *parent_gen, ino, gen, in __get_cur_name_and_parent()
2375 ret = gen_unique_name(sctx, ino, gen, dest); in __get_cur_name_and_parent()
2399 if (ino < sctx->send_progress) in __get_cur_name_and_parent()
2404 nce_ret = btrfs_lru_cache_store(&sctx->name_cache, &nce->entry, GFP_KERNEL); in __get_cur_name_and_parent()
2439 static int get_cur_path(struct send_ctx *sctx, u64 ino, u64 gen, in get_cur_path() argument
2462 if (is_waiting_for_rm(sctx, ino, gen)) { in get_cur_path()
2463 ret = gen_unique_name(sctx, ino, gen, name); in get_cur_path()
2470 wdm = get_waiting_dir_move(sctx, ino); in get_cur_path()
2472 ret = gen_unique_name(sctx, ino, gen, name); in get_cur_path()
2475 ret = get_first_ref(sctx->parent_root, ino, in get_cur_path()
2478 ret = __get_cur_name_and_parent(sctx, ino, gen, in get_cur_path()
2506 static int send_subvol_begin(struct send_ctx *sctx) in send_subvol_begin() argument
2509 struct btrfs_root *send_root = sctx->send_root; in send_subvol_begin()
2510 struct btrfs_root *parent_root = sctx->parent_root; in send_subvol_begin()
2554 ret = begin_cmd(sctx, BTRFS_SEND_C_SNAPSHOT); in send_subvol_begin()
2558 ret = begin_cmd(sctx, BTRFS_SEND_C_SUBVOL); in send_subvol_begin()
2563 TLV_PUT_STRING(sctx, BTRFS_SEND_A_PATH, name, namelen); in send_subvol_begin()
2565 if (!btrfs_is_empty_uuid(sctx->send_root->root_item.received_uuid)) in send_subvol_begin()
2566 TLV_PUT_UUID(sctx, BTRFS_SEND_A_UUID, in send_subvol_begin()
2567 sctx->send_root->root_item.received_uuid); in send_subvol_begin()
2569 TLV_PUT_UUID(sctx, BTRFS_SEND_A_UUID, in send_subvol_begin()
2570 sctx->send_root->root_item.uuid); in send_subvol_begin()
2572 TLV_PUT_U64(sctx, BTRFS_SEND_A_CTRANSID, in send_subvol_begin()
2573 btrfs_root_ctransid(&sctx->send_root->root_item)); in send_subvol_begin()
2576 TLV_PUT_UUID(sctx, BTRFS_SEND_A_CLONE_UUID, in send_subvol_begin()
2579 TLV_PUT_UUID(sctx, BTRFS_SEND_A_CLONE_UUID, in send_subvol_begin()
2581 TLV_PUT_U64(sctx, BTRFS_SEND_A_CLONE_CTRANSID, in send_subvol_begin()
2582 btrfs_root_ctransid(&sctx->parent_root->root_item)); in send_subvol_begin()
2585 ret = send_cmd(sctx); in send_subvol_begin()
2594 static int send_truncate(struct send_ctx *sctx, u64 ino, u64 gen, u64 size) in send_truncate() argument
2596 struct btrfs_fs_info *fs_info = sctx->send_root->fs_info; in send_truncate()
2606 ret = begin_cmd(sctx, BTRFS_SEND_C_TRUNCATE); in send_truncate()
2610 ret = get_cur_path(sctx, ino, gen, p); in send_truncate()
2613 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p); in send_truncate()
2614 TLV_PUT_U64(sctx, BTRFS_SEND_A_SIZE, size); in send_truncate()
2616 ret = send_cmd(sctx); in send_truncate()
2624 static int send_chmod(struct send_ctx *sctx, u64 ino, u64 gen, u64 mode) in send_chmod() argument
2626 struct btrfs_fs_info *fs_info = sctx->send_root->fs_info; in send_chmod()
2636 ret = begin_cmd(sctx, BTRFS_SEND_C_CHMOD); in send_chmod()
2640 ret = get_cur_path(sctx, ino, gen, p); in send_chmod()
2643 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p); in send_chmod()
2644 TLV_PUT_U64(sctx, BTRFS_SEND_A_MODE, mode & 07777); in send_chmod()
2646 ret = send_cmd(sctx); in send_chmod()
2654 static int send_fileattr(struct send_ctx *sctx, u64 ino, u64 gen, u64 fileattr) in send_fileattr() argument
2656 struct btrfs_fs_info *fs_info = sctx->send_root->fs_info; in send_fileattr()
2660 if (sctx->proto < 2) in send_fileattr()
2669 ret = begin_cmd(sctx, BTRFS_SEND_C_FILEATTR); in send_fileattr()
2673 ret = get_cur_path(sctx, ino, gen, p); in send_fileattr()
2676 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p); in send_fileattr()
2677 TLV_PUT_U64(sctx, BTRFS_SEND_A_FILEATTR, fileattr); in send_fileattr()
2679 ret = send_cmd(sctx); in send_fileattr()
2687 static int send_chown(struct send_ctx *sctx, u64 ino, u64 gen, u64 uid, u64 gid) in send_chown() argument
2689 struct btrfs_fs_info *fs_info = sctx->send_root->fs_info; in send_chown()
2700 ret = begin_cmd(sctx, BTRFS_SEND_C_CHOWN); in send_chown()
2704 ret = get_cur_path(sctx, ino, gen, p); in send_chown()
2707 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p); in send_chown()
2708 TLV_PUT_U64(sctx, BTRFS_SEND_A_UID, uid); in send_chown()
2709 TLV_PUT_U64(sctx, BTRFS_SEND_A_GID, gid); in send_chown()
2711 ret = send_cmd(sctx); in send_chown()
2719 static int send_utimes(struct send_ctx *sctx, u64 ino, u64 gen) in send_utimes() argument
2721 struct btrfs_fs_info *fs_info = sctx->send_root->fs_info; in send_utimes()
2745 ret = btrfs_search_slot(NULL, sctx->send_root, &key, path, 0, 0); in send_utimes()
2755 ret = begin_cmd(sctx, BTRFS_SEND_C_UTIMES); in send_utimes()
2759 ret = get_cur_path(sctx, ino, gen, p); in send_utimes()
2762 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p); in send_utimes()
2763 TLV_PUT_BTRFS_TIMESPEC(sctx, BTRFS_SEND_A_ATIME, eb, &ii->atime); in send_utimes()
2764 TLV_PUT_BTRFS_TIMESPEC(sctx, BTRFS_SEND_A_MTIME, eb, &ii->mtime); in send_utimes()
2765 TLV_PUT_BTRFS_TIMESPEC(sctx, BTRFS_SEND_A_CTIME, eb, &ii->ctime); in send_utimes()
2766 if (sctx->proto >= 2) in send_utimes()
2767 TLV_PUT_BTRFS_TIMESPEC(sctx, BTRFS_SEND_A_OTIME, eb, &ii->otime); in send_utimes()
2769 ret = send_cmd(sctx); in send_utimes()
2788 static int cache_dir_utimes(struct send_ctx *sctx, u64 dir, u64 gen) in cache_dir_utimes() argument
2793 entry = btrfs_lru_cache_lookup(&sctx->dir_utimes_cache, dir, gen); in cache_dir_utimes()
2800 return send_utimes(sctx, dir, gen); in cache_dir_utimes()
2805 ret = btrfs_lru_cache_store(&sctx->dir_utimes_cache, entry, GFP_KERNEL); in cache_dir_utimes()
2809 return send_utimes(sctx, dir, gen); in cache_dir_utimes()
2815 static int trim_dir_utimes_cache(struct send_ctx *sctx) in trim_dir_utimes_cache() argument
2817 while (sctx->dir_utimes_cache.size > SEND_MAX_DIR_UTIMES_CACHE_SIZE) { in trim_dir_utimes_cache()
2821 lru = btrfs_lru_cache_lru_entry(&sctx->dir_utimes_cache); in trim_dir_utimes_cache()
2824 ret = send_utimes(sctx, lru->key, lru->gen); in trim_dir_utimes_cache()
2828 btrfs_lru_cache_remove(&sctx->dir_utimes_cache, lru); in trim_dir_utimes_cache()
2839 static int send_create_inode(struct send_ctx *sctx, u64 ino) in send_create_inode() argument
2841 struct btrfs_fs_info *fs_info = sctx->send_root->fs_info; in send_create_inode()
2856 if (ino != sctx->cur_ino) { in send_create_inode()
2857 ret = get_inode_info(sctx->send_root, ino, &info); in send_create_inode()
2864 gen = sctx->cur_inode_gen; in send_create_inode()
2865 mode = sctx->cur_inode_mode; in send_create_inode()
2866 rdev = sctx->cur_inode_rdev; in send_create_inode()
2882 btrfs_warn(sctx->send_root->fs_info, "unexpected inode type %o", in send_create_inode()
2888 ret = begin_cmd(sctx, cmd); in send_create_inode()
2892 ret = gen_unique_name(sctx, ino, gen, p); in send_create_inode()
2896 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p); in send_create_inode()
2897 TLV_PUT_U64(sctx, BTRFS_SEND_A_INO, ino); in send_create_inode()
2901 ret = read_symlink(sctx->send_root, ino, p); in send_create_inode()
2904 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH_LINK, p); in send_create_inode()
2907 TLV_PUT_U64(sctx, BTRFS_SEND_A_RDEV, new_encode_dev(rdev)); in send_create_inode()
2908 TLV_PUT_U64(sctx, BTRFS_SEND_A_MODE, mode); in send_create_inode()
2911 ret = send_cmd(sctx); in send_create_inode()
2922 static void cache_dir_created(struct send_ctx *sctx, u64 dir) in cache_dir_created() argument
2934 ret = btrfs_lru_cache_store(&sctx->dir_created_cache, entry, GFP_KERNEL); in cache_dir_created()
2944 static int did_create_dir(struct send_ctx *sctx, u64 dir) in did_create_dir() argument
2954 if (btrfs_lru_cache_lookup(&sctx->dir_created_cache, dir, 0)) in did_create_dir()
2965 btrfs_for_each_slot(sctx->send_root, &key, &found_key, path, iter_ret) { in did_create_dir()
2978 di_key.objectid < sctx->send_progress) { in did_create_dir()
2980 cache_dir_created(sctx, dir); in did_create_dir()
2998 static int send_create_inode_if_needed(struct send_ctx *sctx) in send_create_inode_if_needed() argument
3002 if (S_ISDIR(sctx->cur_inode_mode)) { in send_create_inode_if_needed()
3003 ret = did_create_dir(sctx, sctx->cur_ino); in send_create_inode_if_needed()
3010 ret = send_create_inode(sctx, sctx->cur_ino); in send_create_inode_if_needed()
3012 if (ret == 0 && S_ISDIR(sctx->cur_inode_mode)) in send_create_inode_if_needed()
3013 cache_dir_created(sctx, sctx->cur_ino); in send_create_inode_if_needed()
3083 static void free_recorded_refs(struct send_ctx *sctx) in free_recorded_refs() argument
3085 __free_recorded_refs(&sctx->new_refs); in free_recorded_refs()
3086 __free_recorded_refs(&sctx->deleted_refs); in free_recorded_refs()
3094 static int orphanize_inode(struct send_ctx *sctx, u64 ino, u64 gen, in orphanize_inode() argument
3104 ret = gen_unique_name(sctx, ino, gen, orphan); in orphanize_inode()
3108 ret = send_rename(sctx, path, orphan); in orphanize_inode()
3115 static struct orphan_dir_info *add_orphan_dir_info(struct send_ctx *sctx, in add_orphan_dir_info() argument
3118 struct rb_node **p = &sctx->orphan_dirs.rb_node; in add_orphan_dir_info()
3146 rb_insert_color(&odi->node, &sctx->orphan_dirs); in add_orphan_dir_info()
3150 static struct orphan_dir_info *get_orphan_dir_info(struct send_ctx *sctx, in get_orphan_dir_info() argument
3153 struct rb_node *n = sctx->orphan_dirs.rb_node; in get_orphan_dir_info()
3172 static int is_waiting_for_rm(struct send_ctx *sctx, u64 dir_ino, u64 gen) in is_waiting_for_rm() argument
3174 struct orphan_dir_info *odi = get_orphan_dir_info(sctx, dir_ino, gen); in is_waiting_for_rm()
3179 static void free_orphan_dir_info(struct send_ctx *sctx, in free_orphan_dir_info() argument
3184 rb_erase(&odi->node, &sctx->orphan_dirs); in free_orphan_dir_info()
3193 static int can_rmdir(struct send_ctx *sctx, u64 dir, u64 dir_gen) in can_rmdir() argument
3197 struct btrfs_root *root = sctx->parent_root; in can_rmdir()
3213 odi = get_orphan_dir_info(sctx, dir, dir_gen); in can_rmdir()
3214 if (odi && sctx->cur_ino < odi->dir_high_seq_ino) in can_rmdir()
3258 if (sctx->cur_ino < dir_high_seq_ino) { in can_rmdir()
3284 dm = get_waiting_dir_move(sctx, loc.objectid); in can_rmdir()
3292 if (loc.objectid > sctx->cur_ino) { in can_rmdir()
3301 free_orphan_dir_info(sctx, odi); in can_rmdir()
3312 odi = add_orphan_dir_info(sctx, dir, dir_gen); in can_rmdir()
3325 static int is_waiting_for_move(struct send_ctx *sctx, u64 ino) in is_waiting_for_move() argument
3327 struct waiting_dir_move *entry = get_waiting_dir_move(sctx, ino); in is_waiting_for_move()
3332 static int add_waiting_dir_move(struct send_ctx *sctx, u64 ino, bool orphanized) in add_waiting_dir_move() argument
3334 struct rb_node **p = &sctx->waiting_dir_moves.rb_node; in add_waiting_dir_move()
3360 rb_insert_color(&dm->node, &sctx->waiting_dir_moves); in add_waiting_dir_move()
3365 get_waiting_dir_move(struct send_ctx *sctx, u64 ino) in get_waiting_dir_move() argument
3367 struct rb_node *n = sctx->waiting_dir_moves.rb_node; in get_waiting_dir_move()
3382 static void free_waiting_dir_move(struct send_ctx *sctx, in free_waiting_dir_move() argument
3387 rb_erase(&dm->node, &sctx->waiting_dir_moves); in free_waiting_dir_move()
3391 static int add_pending_dir_move(struct send_ctx *sctx, in add_pending_dir_move() argument
3399 struct rb_node **p = &sctx->pending_dir_moves.rb_node; in add_pending_dir_move()
3440 ret = add_waiting_dir_move(sctx, pm->ino, is_orphan); in add_pending_dir_move()
3448 rb_insert_color(&pm->node, &sctx->pending_dir_moves); in add_pending_dir_move()
3459 static struct pending_dir_move *get_pending_dir_moves(struct send_ctx *sctx, in get_pending_dir_moves() argument
3462 struct rb_node *n = sctx->pending_dir_moves.rb_node; in get_pending_dir_moves()
3477 static int path_loop(struct send_ctx *sctx, struct fs_path *name, in path_loop() argument
3489 if (is_waiting_for_rm(sctx, ino, gen)) in path_loop()
3491 if (is_waiting_for_move(sctx, ino)) { in path_loop()
3494 ret = get_first_ref(sctx->parent_root, ino, in path_loop()
3497 ret = __get_cur_name_and_parent(sctx, ino, gen, in path_loop()
3519 static int apply_dir_move(struct send_ctx *sctx, struct pending_dir_move *pm) in apply_dir_move() argument
3524 u64 orig_progress = sctx->send_progress; in apply_dir_move()
3541 dm = get_waiting_dir_move(sctx, pm->ino); in apply_dir_move()
3546 free_waiting_dir_move(sctx, dm); in apply_dir_move()
3549 ret = gen_unique_name(sctx, pm->ino, in apply_dir_move()
3552 ret = get_first_ref(sctx->parent_root, pm->ino, in apply_dir_move()
3556 ret = get_cur_path(sctx, parent_ino, parent_gen, in apply_dir_move()
3565 sctx->send_progress = sctx->cur_ino + 1; in apply_dir_move()
3566 ret = path_loop(sctx, name, pm->ino, pm->gen, &ancestor); in apply_dir_move()
3572 ret = add_pending_dir_move(sctx, pm->ino, pm->gen, ancestor, in apply_dir_move()
3578 dm = get_waiting_dir_move(sctx, pm->ino); in apply_dir_move()
3588 ret = get_cur_path(sctx, pm->ino, pm->gen, to_path); in apply_dir_move()
3592 ret = send_rename(sctx, from_path, to_path); in apply_dir_move()
3600 odi = get_orphan_dir_info(sctx, rmdir_ino, rmdir_gen); in apply_dir_move()
3607 ret = can_rmdir(sctx, rmdir_ino, gen); in apply_dir_move()
3618 ret = get_cur_path(sctx, rmdir_ino, gen, name); in apply_dir_move()
3621 ret = send_rmdir(sctx, name); in apply_dir_move()
3627 ret = cache_dir_utimes(sctx, pm->ino, pm->gen); in apply_dir_move()
3639 ret = get_inode_info(sctx->send_root, cur->dir, NULL); in apply_dir_move()
3647 ret = cache_dir_utimes(sctx, cur->dir, cur->dir_gen); in apply_dir_move()
3656 sctx->send_progress = orig_progress; in apply_dir_move()
3661 static void free_pending_move(struct send_ctx *sctx, struct pending_dir_move *m) in free_pending_move() argument
3666 rb_erase(&m->node, &sctx->pending_dir_moves); in free_pending_move()
3671 static void tail_append_pending_moves(struct send_ctx *sctx, in tail_append_pending_moves() argument
3684 rb_erase(&moves->node, &sctx->pending_dir_moves); in tail_append_pending_moves()
3689 static int apply_children_dir_moves(struct send_ctx *sctx) in apply_children_dir_moves() argument
3693 u64 parent_ino = sctx->cur_ino; in apply_children_dir_moves()
3696 pm = get_pending_dir_moves(sctx, parent_ino); in apply_children_dir_moves()
3700 tail_append_pending_moves(sctx, pm, &stack); in apply_children_dir_moves()
3705 ret = apply_dir_move(sctx, pm); in apply_children_dir_moves()
3706 free_pending_move(sctx, pm); in apply_children_dir_moves()
3709 pm = get_pending_dir_moves(sctx, parent_ino); in apply_children_dir_moves()
3711 tail_append_pending_moves(sctx, pm, &stack); in apply_children_dir_moves()
3718 free_pending_move(sctx, pm); in apply_children_dir_moves()
3759 static int wait_for_dest_dir_move(struct send_ctx *sctx, in wait_for_dest_dir_move() argument
3772 if (RB_EMPTY_ROOT(&sctx->waiting_dir_moves)) in wait_for_dest_dir_move()
3783 ret = btrfs_search_slot(NULL, sctx->parent_root, &key, path, 0, 0); in wait_for_dest_dir_move()
3811 ret = get_inode_gen(sctx->parent_root, di_key.objectid, &left_gen); in wait_for_dest_dir_move()
3814 ret = get_inode_gen(sctx->send_root, di_key.objectid, &right_gen); in wait_for_dest_dir_move()
3827 wdm = get_waiting_dir_move(sctx, di_key.objectid); in wait_for_dest_dir_move()
3829 ret = add_pending_dir_move(sctx, in wait_for_dest_dir_move()
3830 sctx->cur_ino, in wait_for_dest_dir_move()
3831 sctx->cur_inode_gen, in wait_for_dest_dir_move()
3833 &sctx->new_refs, in wait_for_dest_dir_move()
3834 &sctx->deleted_refs, in wait_for_dest_dir_move()
3964 static int wait_for_parent_move(struct send_ctx *sctx, in wait_for_parent_move() argument
3993 if (is_waiting_for_move(sctx, ino)) { in wait_for_parent_move()
4004 ret = is_ancestor(sctx->parent_root, in wait_for_parent_move()
4005 sctx->cur_ino, sctx->cur_inode_gen, in wait_for_parent_move()
4014 ret = get_first_ref(sctx->send_root, ino, &parent_ino_after, in wait_for_parent_move()
4018 ret = get_first_ref(sctx->parent_root, ino, &parent_ino_before, in wait_for_parent_move()
4029 if (ino > sctx->cur_ino && in wait_for_parent_move()
4034 ret = get_inode_gen(sctx->parent_root, ino, &parent_ino_gen); in wait_for_parent_move()
4051 ret = add_pending_dir_move(sctx, in wait_for_parent_move()
4052 sctx->cur_ino, in wait_for_parent_move()
4053 sctx->cur_inode_gen, in wait_for_parent_move()
4055 &sctx->new_refs, in wait_for_parent_move()
4056 &sctx->deleted_refs, in wait_for_parent_move()
4065 static int update_ref_path(struct send_ctx *sctx, struct recorded_ref *ref) in update_ref_path() argument
4078 ret = get_cur_path(sctx, ref->dir, ref->dir_gen, new_path); in update_ref_path()
4136 static int refresh_ref_path(struct send_ctx *sctx, struct recorded_ref *ref) in refresh_ref_path() argument
4146 ret = get_cur_path(sctx, ref->dir, ref->dir_gen, ref->full_path); in refresh_ref_path()
4164 static int process_recorded_refs(struct send_ctx *sctx, int *pending_move) in process_recorded_refs() argument
4166 struct btrfs_fs_info *fs_info = sctx->send_root->fs_info; in process_recorded_refs()
4182 btrfs_debug(fs_info, "process_recorded_refs %llu", sctx->cur_ino); in process_recorded_refs()
4188 if (unlikely(sctx->cur_ino <= BTRFS_FIRST_FREE_OBJECTID)) { in process_recorded_refs()
4191 sctx->cur_ino); in process_recorded_refs()
4213 if (!sctx->cur_inode_new) { in process_recorded_refs()
4214 ret = did_overwrite_first_ref(sctx, sctx->cur_ino, in process_recorded_refs()
4215 sctx->cur_inode_gen); in process_recorded_refs()
4221 if (sctx->cur_inode_new || did_overwrite) { in process_recorded_refs()
4222 ret = gen_unique_name(sctx, sctx->cur_ino, in process_recorded_refs()
4223 sctx->cur_inode_gen, valid_path); in process_recorded_refs()
4228 ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, in process_recorded_refs()
4272 list_for_each_entry(cur, &sctx->new_refs, list) { in process_recorded_refs()
4273 ret = get_cur_inode_state(sctx, cur->dir, cur->dir_gen, NULL, NULL); in process_recorded_refs()
4285 ret = will_overwrite_ref(sctx, cur->dir, cur->dir_gen, in process_recorded_refs()
4291 ret = is_first_ref(sctx->parent_root, in process_recorded_refs()
4301 ret = refresh_ref_path(sctx, cur); in process_recorded_refs()
4306 ret = orphanize_inode(sctx, ow_inode, ow_gen, in process_recorded_refs()
4319 wdm = get_waiting_dir_move(sctx, ow_inode); in process_recorded_refs()
4333 nce = name_cache_search(sctx, ow_inode, ow_gen); in process_recorded_refs()
4335 btrfs_lru_cache_remove(&sctx->name_cache, in process_recorded_refs()
4345 ret = is_ancestor(sctx->parent_root, in process_recorded_refs()
4347 sctx->cur_ino, NULL); in process_recorded_refs()
4351 ret = get_cur_path(sctx, sctx->cur_ino, in process_recorded_refs()
4352 sctx->cur_inode_gen, in process_recorded_refs()
4365 ret = refresh_ref_path(sctx, cur); in process_recorded_refs()
4369 ret = send_unlink(sctx, cur->full_path); in process_recorded_refs()
4377 list_for_each_entry(cur, &sctx->new_refs, list) { in process_recorded_refs()
4385 ret = get_cur_inode_state(sctx, cur->dir, cur->dir_gen, NULL, NULL); in process_recorded_refs()
4394 list_for_each_entry(cur2, &sctx->new_refs, list) { in process_recorded_refs()
4408 ret = did_create_dir(sctx, cur->dir); in process_recorded_refs()
4412 ret = send_create_inode(sctx, cur->dir); in process_recorded_refs()
4415 cache_dir_created(sctx, cur->dir); in process_recorded_refs()
4419 if (S_ISDIR(sctx->cur_inode_mode) && sctx->parent_root) { in process_recorded_refs()
4420 ret = wait_for_dest_dir_move(sctx, cur, is_orphan); in process_recorded_refs()
4429 if (S_ISDIR(sctx->cur_inode_mode) && sctx->parent_root && in process_recorded_refs()
4431 ret = wait_for_parent_move(sctx, cur, is_orphan); in process_recorded_refs()
4446 ret = send_rename(sctx, valid_path, cur->full_path); in process_recorded_refs()
4454 if (S_ISDIR(sctx->cur_inode_mode)) { in process_recorded_refs()
4460 ret = send_rename(sctx, valid_path, in process_recorded_refs()
4476 ret = update_ref_path(sctx, cur); in process_recorded_refs()
4480 ret = send_link(sctx, cur->full_path, in process_recorded_refs()
4491 if (S_ISDIR(sctx->cur_inode_mode) && sctx->cur_inode_deleted) { in process_recorded_refs()
4498 ret = can_rmdir(sctx, sctx->cur_ino, sctx->cur_inode_gen); in process_recorded_refs()
4502 ret = send_rmdir(sctx, valid_path); in process_recorded_refs()
4506 ret = orphanize_inode(sctx, sctx->cur_ino, in process_recorded_refs()
4507 sctx->cur_inode_gen, valid_path); in process_recorded_refs()
4513 list_for_each_entry(cur, &sctx->deleted_refs, list) { in process_recorded_refs()
4518 } else if (S_ISDIR(sctx->cur_inode_mode) && in process_recorded_refs()
4519 !list_empty(&sctx->deleted_refs)) { in process_recorded_refs()
4523 cur = list_entry(sctx->deleted_refs.next, struct recorded_ref, in process_recorded_refs()
4528 } else if (!S_ISDIR(sctx->cur_inode_mode)) { in process_recorded_refs()
4534 list_for_each_entry(cur, &sctx->deleted_refs, list) { in process_recorded_refs()
4535 ret = did_overwrite_ref(sctx, cur->dir, cur->dir_gen, in process_recorded_refs()
4536 sctx->cur_ino, sctx->cur_inode_gen, in process_recorded_refs()
4549 ret = update_ref_path(sctx, cur); in process_recorded_refs()
4553 ret = send_unlink(sctx, cur->full_path); in process_recorded_refs()
4570 ret = send_unlink(sctx, valid_path); in process_recorded_refs()
4588 if (cur->dir > sctx->cur_ino) in process_recorded_refs()
4591 ret = get_cur_inode_state(sctx, cur->dir, cur->dir_gen, NULL, NULL); in process_recorded_refs()
4597 ret = cache_dir_utimes(sctx, cur->dir, cur->dir_gen); in process_recorded_refs()
4602 ret = can_rmdir(sctx, cur->dir, cur->dir_gen); in process_recorded_refs()
4606 ret = get_cur_path(sctx, cur->dir, in process_recorded_refs()
4610 ret = send_rmdir(sctx, valid_path); in process_recorded_refs()
4622 free_recorded_refs(sctx); in process_recorded_refs()
4662 struct send_ctx *sctx) in record_ref_in_tree() argument
4680 ret = get_cur_path(sctx, dir, dir_gen, path); in record_ref_in_tree()
4705 struct send_ctx *sctx = ctx; in record_new_ref_if_needed() local
4711 ret = get_inode_gen(sctx->send_root, dir, &dir_gen); in record_new_ref_if_needed()
4718 node = rb_find(&data, &sctx->rbtree_deleted_refs, rbtree_ref_comp); in record_new_ref_if_needed()
4723 ret = record_ref_in_tree(&sctx->rbtree_new_refs, in record_new_ref_if_needed()
4724 &sctx->new_refs, name, dir, dir_gen, in record_new_ref_if_needed()
4725 sctx); in record_new_ref_if_needed()
4734 struct send_ctx *sctx = ctx; in record_deleted_ref_if_needed() local
4740 ret = get_inode_gen(sctx->parent_root, dir, &dir_gen); in record_deleted_ref_if_needed()
4747 node = rb_find(&data, &sctx->rbtree_new_refs, rbtree_ref_comp); in record_deleted_ref_if_needed()
4752 ret = record_ref_in_tree(&sctx->rbtree_deleted_refs, in record_deleted_ref_if_needed()
4753 &sctx->deleted_refs, name, dir, in record_deleted_ref_if_needed()
4754 dir_gen, sctx); in record_deleted_ref_if_needed()
4760 static int record_new_ref(struct send_ctx *sctx) in record_new_ref() argument
4764 ret = iterate_inode_ref(sctx->send_root, sctx->left_path, in record_new_ref()
4765 sctx->cmp_key, 0, record_new_ref_if_needed, sctx); in record_new_ref()
4774 static int record_deleted_ref(struct send_ctx *sctx) in record_deleted_ref() argument
4778 ret = iterate_inode_ref(sctx->parent_root, sctx->right_path, in record_deleted_ref()
4779 sctx->cmp_key, 0, record_deleted_ref_if_needed, in record_deleted_ref()
4780 sctx); in record_deleted_ref()
4789 static int record_changed_ref(struct send_ctx *sctx) in record_changed_ref() argument
4793 ret = iterate_inode_ref(sctx->send_root, sctx->left_path, in record_changed_ref()
4794 sctx->cmp_key, 0, record_new_ref_if_needed, sctx); in record_changed_ref()
4797 ret = iterate_inode_ref(sctx->parent_root, sctx->right_path, in record_changed_ref()
4798 sctx->cmp_key, 0, record_deleted_ref_if_needed, sctx); in record_changed_ref()
4811 static int process_all_refs(struct send_ctx *sctx, in process_all_refs() argument
4828 root = sctx->send_root; in process_all_refs()
4831 root = sctx->parent_root; in process_all_refs()
4834 btrfs_err(sctx->send_root->fs_info, in process_all_refs()
4840 key.objectid = sctx->cmp_key->objectid; in process_all_refs()
4849 ret = iterate_inode_ref(root, path, &found_key, 0, cb, sctx); in process_all_refs()
4865 ret = process_recorded_refs(sctx, &pending_move); in process_all_refs()
4871 static int send_set_xattr(struct send_ctx *sctx, in send_set_xattr() argument
4878 ret = begin_cmd(sctx, BTRFS_SEND_C_SET_XATTR); in send_set_xattr()
4882 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path); in send_set_xattr()
4883 TLV_PUT_STRING(sctx, BTRFS_SEND_A_XATTR_NAME, name, name_len); in send_set_xattr()
4884 TLV_PUT(sctx, BTRFS_SEND_A_XATTR_DATA, data, data_len); in send_set_xattr()
4886 ret = send_cmd(sctx); in send_set_xattr()
4893 static int send_remove_xattr(struct send_ctx *sctx, in send_remove_xattr() argument
4899 ret = begin_cmd(sctx, BTRFS_SEND_C_REMOVE_XATTR); in send_remove_xattr()
4903 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path); in send_remove_xattr()
4904 TLV_PUT_STRING(sctx, BTRFS_SEND_A_XATTR_NAME, name, name_len); in send_remove_xattr()
4906 ret = send_cmd(sctx); in send_remove_xattr()
4918 struct send_ctx *sctx = ctx; in __process_new_xattr() local
4946 ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, p); in __process_new_xattr()
4950 ret = send_set_xattr(sctx, p, name, name_len, data, data_len); in __process_new_xattr()
4962 struct send_ctx *sctx = ctx; in __process_deleted_xattr() local
4969 ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, p); in __process_deleted_xattr()
4973 ret = send_remove_xattr(sctx, p, name, name_len); in __process_deleted_xattr()
4980 static int process_new_xattr(struct send_ctx *sctx) in process_new_xattr() argument
4984 ret = iterate_dir_item(sctx->send_root, sctx->left_path, in process_new_xattr()
4985 __process_new_xattr, sctx); in process_new_xattr()
4990 static int process_deleted_xattr(struct send_ctx *sctx) in process_deleted_xattr() argument
4992 return iterate_dir_item(sctx->parent_root, sctx->right_path, in process_deleted_xattr()
4993 __process_deleted_xattr, sctx); in process_deleted_xattr()
5058 struct send_ctx *sctx = ctx; in __process_changed_new_xattr() local
5062 ret = find_xattr(sctx->parent_root, sctx->right_path, in __process_changed_new_xattr()
5063 sctx->cmp_key, name, name_len, &found_data, in __process_changed_new_xattr()
5088 struct send_ctx *sctx = ctx; in __process_changed_deleted_xattr() local
5090 ret = find_xattr(sctx->send_root, sctx->left_path, sctx->cmp_key, in __process_changed_deleted_xattr()
5101 static int process_changed_xattr(struct send_ctx *sctx) in process_changed_xattr() argument
5105 ret = iterate_dir_item(sctx->send_root, sctx->left_path, in process_changed_xattr()
5106 __process_changed_new_xattr, sctx); in process_changed_xattr()
5109 ret = iterate_dir_item(sctx->parent_root, sctx->right_path, in process_changed_xattr()
5110 __process_changed_deleted_xattr, sctx); in process_changed_xattr()
5116 static int process_all_new_xattrs(struct send_ctx *sctx) in process_all_new_xattrs() argument
5129 root = sctx->send_root; in process_all_new_xattrs()
5131 key.objectid = sctx->cmp_key->objectid; in process_all_new_xattrs()
5141 ret = iterate_dir_item(root, path, __process_new_xattr, sctx); in process_all_new_xattrs()
5153 static int send_verity(struct send_ctx *sctx, struct fs_path *path, in send_verity() argument
5158 ret = begin_cmd(sctx, BTRFS_SEND_C_ENABLE_VERITY); in send_verity()
5162 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path); in send_verity()
5163 TLV_PUT_U8(sctx, BTRFS_SEND_A_VERITY_ALGORITHM, in send_verity()
5165 TLV_PUT_U32(sctx, BTRFS_SEND_A_VERITY_BLOCK_SIZE, in send_verity()
5167 TLV_PUT(sctx, BTRFS_SEND_A_VERITY_SALT_DATA, desc->salt, in send_verity()
5169 TLV_PUT(sctx, BTRFS_SEND_A_VERITY_SIG_DATA, desc->signature, in send_verity()
5172 ret = send_cmd(sctx); in send_verity()
5179 static int process_verity(struct send_ctx *sctx) in process_verity() argument
5185 inode = btrfs_iget(sctx->cur_ino, sctx->send_root); in process_verity()
5197 if (!sctx->verity_descriptor) { in process_verity()
5198 sctx->verity_descriptor = kvmalloc(FS_VERITY_MAX_DESCRIPTOR_SIZE, in process_verity()
5200 if (!sctx->verity_descriptor) { in process_verity()
5206 ret = btrfs_get_verity_descriptor(inode, sctx->verity_descriptor, ret); in process_verity()
5215 ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, p); in process_verity()
5219 ret = send_verity(sctx, p, sctx->verity_descriptor); in process_verity()
5230 static inline u64 max_send_read_size(const struct send_ctx *sctx) in max_send_read_size() argument
5232 return sctx->send_max_size - SZ_16K; in max_send_read_size()
5235 static int put_data_header(struct send_ctx *sctx, u32 len) in put_data_header() argument
5237 if (WARN_ON_ONCE(sctx->put_data)) in put_data_header()
5239 sctx->put_data = true; in put_data_header()
5240 if (sctx->proto >= 2) { in put_data_header()
5245 if (sctx->send_max_size - sctx->send_size < sizeof(__le16) + len) in put_data_header()
5247 put_unaligned_le16(BTRFS_SEND_A_DATA, sctx->send_buf + sctx->send_size); in put_data_header()
5248 sctx->send_size += sizeof(__le16); in put_data_header()
5252 if (sctx->send_max_size - sctx->send_size < sizeof(*hdr) + len) in put_data_header()
5254 hdr = (struct btrfs_tlv_header *)(sctx->send_buf + sctx->send_size); in put_data_header()
5257 sctx->send_size += sizeof(*hdr); in put_data_header()
5262 static int put_file_data(struct send_ctx *sctx, u64 offset, u32 len) in put_file_data() argument
5264 struct btrfs_root *root = sctx->send_root; in put_file_data()
5270 struct address_space *mapping = sctx->cur_inode->i_mapping; in put_file_data()
5273 ret = put_data_header(sctx, len); in put_file_data()
5287 &sctx->ra, NULL, index, in put_file_data()
5300 page_cache_async_readahead(mapping, &sctx->ra, NULL, folio, in put_file_data()
5310 folio_pos(folio), sctx->cur_ino, in put_file_data()
5311 btrfs_root_id(sctx->send_root)); in put_file_data()
5323 memcpy_from_folio(sctx->send_buf + sctx->send_size, folio, in put_file_data()
5330 sctx->send_size += cur_len; in put_file_data()
5340 static int send_write(struct send_ctx *sctx, u64 offset, u32 len) in send_write() argument
5342 struct btrfs_fs_info *fs_info = sctx->send_root->fs_info; in send_write()
5352 ret = begin_cmd(sctx, BTRFS_SEND_C_WRITE); in send_write()
5356 ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, p); in send_write()
5360 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p); in send_write()
5361 TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset); in send_write()
5362 ret = put_file_data(sctx, offset, len); in send_write()
5366 ret = send_cmd(sctx); in send_write()
5377 static int send_clone(struct send_ctx *sctx, in send_clone() argument
5385 btrfs_debug(sctx->send_root->fs_info, in send_clone()
5394 ret = begin_cmd(sctx, BTRFS_SEND_C_CLONE); in send_clone()
5398 ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, p); in send_clone()
5402 TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset); in send_clone()
5403 TLV_PUT_U64(sctx, BTRFS_SEND_A_CLONE_LEN, len); in send_clone()
5404 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p); in send_clone()
5406 if (clone_root->root == sctx->send_root) { in send_clone()
5407 ret = get_inode_gen(sctx->send_root, clone_root->ino, &gen); in send_clone()
5410 ret = get_cur_path(sctx, clone_root->ino, gen, p); in send_clone()
5427 TLV_PUT_UUID(sctx, BTRFS_SEND_A_CLONE_UUID, in send_clone()
5430 TLV_PUT_UUID(sctx, BTRFS_SEND_A_CLONE_UUID, in send_clone()
5432 TLV_PUT_U64(sctx, BTRFS_SEND_A_CLONE_CTRANSID, in send_clone()
5434 TLV_PUT_PATH(sctx, BTRFS_SEND_A_CLONE_PATH, p); in send_clone()
5435 TLV_PUT_U64(sctx, BTRFS_SEND_A_CLONE_OFFSET, in send_clone()
5438 ret = send_cmd(sctx); in send_clone()
5449 static int send_update_extent(struct send_ctx *sctx, in send_update_extent() argument
5459 ret = begin_cmd(sctx, BTRFS_SEND_C_UPDATE_EXTENT); in send_update_extent()
5463 ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, p); in send_update_extent()
5467 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p); in send_update_extent()
5468 TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset); in send_update_extent()
5469 TLV_PUT_U64(sctx, BTRFS_SEND_A_SIZE, len); in send_update_extent()
5471 ret = send_cmd(sctx); in send_update_extent()
5479 static int send_hole(struct send_ctx *sctx, u64 end) in send_hole() argument
5482 u64 read_size = max_send_read_size(sctx); in send_hole()
5483 u64 offset = sctx->cur_inode_last_extent; in send_hole()
5492 if (offset >= sctx->cur_inode_size) in send_hole()
5499 end = min_t(u64, end, sctx->cur_inode_size); in send_hole()
5501 if (sctx->flags & BTRFS_SEND_FLAG_NO_FILE_DATA) in send_hole()
5502 return send_update_extent(sctx, offset, end - offset); in send_hole()
5507 ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, p); in send_hole()
5513 ret = begin_cmd(sctx, BTRFS_SEND_C_WRITE); in send_hole()
5516 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p); in send_hole()
5517 TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset); in send_hole()
5518 ret = put_data_header(sctx, len); in send_hole()
5521 memset(sctx->send_buf + sctx->send_size, 0, len); in send_hole()
5522 sctx->send_size += len; in send_hole()
5523 ret = send_cmd(sctx); in send_hole()
5528 sctx->cur_inode_next_write_offset = offset; in send_hole()
5534 static int send_encoded_inline_extent(struct send_ctx *sctx, in send_encoded_inline_extent() argument
5538 struct btrfs_root *root = sctx->send_root; in send_encoded_inline_extent()
5549 inode = btrfs_iget(sctx->cur_ino, root); in send_encoded_inline_extent()
5559 ret = begin_cmd(sctx, BTRFS_SEND_C_ENCODED_WRITE); in send_encoded_inline_extent()
5563 ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, fspath); in send_encoded_inline_extent()
5572 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, fspath); in send_encoded_inline_extent()
5573 TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset); in send_encoded_inline_extent()
5574 TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_FILE_LEN, in send_encoded_inline_extent()
5576 TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_LEN, ram_bytes); in send_encoded_inline_extent()
5577 TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_OFFSET, offset - key.offset); in send_encoded_inline_extent()
5582 TLV_PUT_U32(sctx, BTRFS_SEND_A_COMPRESSION, ret); in send_encoded_inline_extent()
5584 ret = put_data_header(sctx, inline_size); in send_encoded_inline_extent()
5587 read_extent_buffer(leaf, sctx->send_buf + sctx->send_size, in send_encoded_inline_extent()
5589 sctx->send_size += inline_size; in send_encoded_inline_extent()
5591 ret = send_cmd(sctx); in send_encoded_inline_extent()
5600 static int send_encoded_extent(struct send_ctx *sctx, struct btrfs_path *path, in send_encoded_extent() argument
5603 struct btrfs_root *root = sctx->send_root; in send_encoded_extent()
5616 inode = btrfs_iget(sctx->cur_ino, root); in send_encoded_extent()
5626 ret = begin_cmd(sctx, BTRFS_SEND_C_ENCODED_WRITE); in send_encoded_extent()
5630 ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, fspath); in send_encoded_extent()
5639 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, fspath); in send_encoded_extent()
5640 TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset); in send_encoded_extent()
5641 TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_FILE_LEN, in send_encoded_extent()
5644 TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_LEN, in send_encoded_extent()
5646 TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_OFFSET, in send_encoded_extent()
5652 TLV_PUT_U32(sctx, BTRFS_SEND_A_COMPRESSION, ret); in send_encoded_extent()
5653 TLV_PUT_U32(sctx, BTRFS_SEND_A_ENCRYPTION, 0); in send_encoded_extent()
5655 ret = put_data_header(sctx, disk_num_bytes); in send_encoded_extent()
5664 data_offset = PAGE_ALIGN(sctx->send_size); in send_encoded_extent()
5665 if (data_offset > sctx->send_max_size || in send_encoded_extent()
5666 sctx->send_max_size - data_offset < disk_num_bytes) { in send_encoded_extent()
5677 sctx->send_buf_pages + in send_encoded_extent()
5683 hdr = (struct btrfs_cmd_header *)sctx->send_buf; in send_encoded_extent()
5684 hdr->len = cpu_to_le32(sctx->send_size + disk_num_bytes - sizeof(*hdr)); in send_encoded_extent()
5686 crc = crc32c(0, sctx->send_buf, sctx->send_size); in send_encoded_extent()
5687 crc = crc32c(crc, sctx->send_buf + data_offset, disk_num_bytes); in send_encoded_extent()
5690 ret = write_buf(sctx->send_filp, sctx->send_buf, sctx->send_size, in send_encoded_extent()
5691 &sctx->send_off); in send_encoded_extent()
5693 ret = write_buf(sctx->send_filp, sctx->send_buf + data_offset, in send_encoded_extent()
5694 disk_num_bytes, &sctx->send_off); in send_encoded_extent()
5696 sctx->send_size = 0; in send_encoded_extent()
5697 sctx->put_data = false; in send_encoded_extent()
5706 static int send_extent_data(struct send_ctx *sctx, struct btrfs_path *path, in send_extent_data() argument
5712 u64 read_size = max_send_read_size(sctx); in send_extent_data()
5715 if (sctx->flags & BTRFS_SEND_FLAG_NO_FILE_DATA) in send_extent_data()
5716 return send_update_extent(sctx, offset, len); in send_extent_data()
5720 if ((sctx->flags & BTRFS_SEND_FLAG_COMPRESSED) && in send_extent_data()
5735 return send_encoded_inline_extent(sctx, path, offset, in send_extent_data()
5739 return send_encoded_extent(sctx, path, offset, len); in send_extent_data()
5743 if (sctx->cur_inode == NULL) { in send_extent_data()
5744 struct btrfs_root *root = sctx->send_root; in send_extent_data()
5746 sctx->cur_inode = btrfs_iget(sctx->cur_ino, root); in send_extent_data()
5747 if (IS_ERR(sctx->cur_inode)) { in send_extent_data()
5748 int err = PTR_ERR(sctx->cur_inode); in send_extent_data()
5750 sctx->cur_inode = NULL; in send_extent_data()
5753 memset(&sctx->ra, 0, sizeof(struct file_ra_state)); in send_extent_data()
5754 file_ra_state_init(&sctx->ra, sctx->cur_inode->i_mapping); in send_extent_data()
5775 sctx->clean_page_cache = (sctx->cur_inode->i_mapping->nrpages == 0); in send_extent_data()
5776 sctx->page_cache_clear_start = round_down(offset, PAGE_SIZE); in send_extent_data()
5783 ret = send_write(sctx, offset + sent, size); in send_extent_data()
5789 if (sctx->clean_page_cache && PAGE_ALIGNED(end)) { in send_extent_data()
5813 truncate_inode_pages_range(&sctx->cur_inode->i_data, in send_extent_data()
5814 sctx->page_cache_clear_start, in send_extent_data()
5816 sctx->page_cache_clear_start = end; in send_extent_data()
5829 static int send_capabilities(struct send_ctx *sctx) in send_capabilities() argument
5844 di = btrfs_lookup_xattr(NULL, sctx->send_root, path, sctx->cur_ino, in send_capabilities()
5864 ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, fspath); in send_capabilities()
5871 ret = send_set_xattr(sctx, fspath, XATTR_NAME_CAPS, in send_capabilities()
5880 static int clone_range(struct send_ctx *sctx, struct btrfs_path *dst_path, in clone_range() argument
5906 len == sctx->send_root->fs_info->sectorsize) in clone_range()
5907 return send_extent_data(sctx, dst_path, offset, len); in clone_range()
6005 ret = send_extent_data(sctx, dst_path, offset, in clone_range()
6070 offset + clone_len < sctx->cur_inode_size) { in clone_range()
6076 ret = send_clone(sctx, offset, slen, in clone_range()
6081 ret = send_extent_data(sctx, dst_path, in clone_range()
6085 ret = send_clone(sctx, offset, clone_len, in clone_range()
6108 ret = send_extent_data(sctx, dst_path, offset, in clone_range()
6130 if (clone_root->root == sctx->send_root && in clone_range()
6131 clone_root->ino == sctx->cur_ino && in clone_range()
6132 clone_root->offset >= sctx->cur_inode_next_write_offset) in clone_range()
6141 ret = send_extent_data(sctx, dst_path, offset, len); in clone_range()
6149 static int send_write_or_clone(struct send_ctx *sctx, in send_write_or_clone() argument
6157 u64 bs = sctx->send_root->fs_info->sectorsize; in send_write_or_clone()
6164 end = min_t(u64, btrfs_file_extent_end(path), sctx->cur_inode_size); in send_write_or_clone()
6181 if (end != sctx->cur_inode_size) in send_write_or_clone()
6203 if (sctx->parent_root != NULL) { in send_write_or_clone()
6204 ret = send_truncate(sctx, sctx->cur_ino, in send_write_or_clone()
6205 sctx->cur_inode_gen, offset); in send_write_or_clone()
6213 ret = send_extent_data(sctx, path, offset, num_bytes); in send_write_or_clone()
6214 sctx->cur_inode_next_write_offset = end; in send_write_or_clone()
6222 ret = clone_range(sctx, path, clone_root, disk_byte, data_offset, offset, in send_write_or_clone()
6224 sctx->cur_inode_next_write_offset = end; in send_write_or_clone()
6228 static int is_extent_unchanged(struct send_ctx *sctx, in is_extent_unchanged() argument
6293 ret = btrfs_search_slot_for_read(sctx->parent_root, &key, path, 0, 0); in is_extent_unchanged()
6383 ret = btrfs_next_item(sctx->parent_root, path); in is_extent_unchanged()
6418 static int get_last_extent(struct send_ctx *sctx, u64 offset) in get_last_extent() argument
6421 struct btrfs_root *root = sctx->send_root; in get_last_extent()
6429 sctx->cur_inode_last_extent = 0; in get_last_extent()
6431 key.objectid = sctx->cur_ino; in get_last_extent()
6439 if (key.objectid != sctx->cur_ino || key.type != BTRFS_EXTENT_DATA_KEY) in get_last_extent()
6442 sctx->cur_inode_last_extent = btrfs_file_extent_end(path); in get_last_extent()
6448 static int range_is_hole_in_parent(struct send_ctx *sctx, in range_is_hole_in_parent() argument
6454 struct btrfs_root *root = sctx->parent_root; in range_is_hole_in_parent()
6462 key.objectid = sctx->cur_ino; in range_is_hole_in_parent()
6487 if (key.objectid < sctx->cur_ino || in range_is_hole_in_parent()
6490 if (key.objectid > sctx->cur_ino || in range_is_hole_in_parent()
6514 static int maybe_send_hole(struct send_ctx *sctx, struct btrfs_path *path, in maybe_send_hole() argument
6519 if (sctx->cur_ino != key->objectid || !need_send_hole(sctx)) in maybe_send_hole()
6532 if ((sctx->cur_inode_last_extent == (u64)-1) || in maybe_send_hole()
6533 (path->slots[0] == 0 && sctx->cur_inode_last_extent < key->offset)) { in maybe_send_hole()
6534 ret = get_last_extent(sctx, key->offset - 1); in maybe_send_hole()
6539 if (sctx->cur_inode_last_extent < key->offset) { in maybe_send_hole()
6540 ret = range_is_hole_in_parent(sctx, in maybe_send_hole()
6541 sctx->cur_inode_last_extent, in maybe_send_hole()
6546 ret = send_hole(sctx, key->offset); in maybe_send_hole()
6550 sctx->cur_inode_last_extent = btrfs_file_extent_end(path); in maybe_send_hole()
6554 static int process_extent(struct send_ctx *sctx, in process_extent() argument
6561 if (S_ISLNK(sctx->cur_inode_mode)) in process_extent()
6564 if (sctx->parent_root && !sctx->cur_inode_new) { in process_extent()
6565 ret = is_extent_unchanged(sctx, path, key); in process_extent()
6600 ret = find_extent_clone(sctx, path, key->objectid, key->offset, in process_extent()
6601 sctx->cur_inode_size, &found_clone); in process_extent()
6605 ret = send_write_or_clone(sctx, path, key, found_clone); in process_extent()
6609 ret = maybe_send_hole(sctx, path, key); in process_extent()
6614 static int process_all_extents(struct send_ctx *sctx) in process_all_extents() argument
6623 root = sctx->send_root; in process_all_extents()
6628 key.objectid = sctx->cmp_key->objectid; in process_all_extents()
6638 ret = process_extent(sctx, path, &found_key); in process_all_extents()
6650 static int process_recorded_refs_if_needed(struct send_ctx *sctx, int at_end, in process_recorded_refs_if_needed() argument
6656 if (sctx->cur_ino == 0) in process_recorded_refs_if_needed()
6658 if (!at_end && sctx->cur_ino == sctx->cmp_key->objectid && in process_recorded_refs_if_needed()
6659 sctx->cmp_key->type <= BTRFS_INODE_EXTREF_KEY) in process_recorded_refs_if_needed()
6661 if (list_empty(&sctx->new_refs) && list_empty(&sctx->deleted_refs)) in process_recorded_refs_if_needed()
6664 ret = process_recorded_refs(sctx, pending_move); in process_recorded_refs_if_needed()
6673 static int finish_inode_if_needed(struct send_ctx *sctx, int at_end) in finish_inode_if_needed() argument
6692 if (sctx->ignore_cur_inode) in finish_inode_if_needed()
6695 ret = process_recorded_refs_if_needed(sctx, at_end, &pending_move, in finish_inode_if_needed()
6713 sctx->send_progress = sctx->cur_ino + 1; in finish_inode_if_needed()
6715 if (sctx->cur_ino == 0 || sctx->cur_inode_deleted) in finish_inode_if_needed()
6717 if (!at_end && sctx->cmp_key->objectid == sctx->cur_ino) in finish_inode_if_needed()
6719 ret = get_inode_info(sctx->send_root, sctx->cur_ino, &info); in finish_inode_if_needed()
6727 if (!sctx->parent_root || sctx->cur_inode_new) { in finish_inode_if_needed()
6729 if (!S_ISLNK(sctx->cur_inode_mode)) in finish_inode_if_needed()
6731 if (sctx->cur_inode_next_write_offset == sctx->cur_inode_size) in finish_inode_if_needed()
6736 ret = get_inode_info(sctx->parent_root, sctx->cur_ino, &info); in finish_inode_if_needed()
6747 if (!S_ISLNK(sctx->cur_inode_mode) && left_mode != right_mode) in finish_inode_if_needed()
6749 if (!S_ISLNK(sctx->cur_inode_mode) && left_fileattr != right_fileattr) in finish_inode_if_needed()
6751 if ((old_size == sctx->cur_inode_size) || in finish_inode_if_needed()
6752 (sctx->cur_inode_size > old_size && in finish_inode_if_needed()
6753 sctx->cur_inode_next_write_offset == sctx->cur_inode_size)) in finish_inode_if_needed()
6757 if (S_ISREG(sctx->cur_inode_mode)) { in finish_inode_if_needed()
6758 if (need_send_hole(sctx)) { in finish_inode_if_needed()
6759 if (sctx->cur_inode_last_extent == (u64)-1 || in finish_inode_if_needed()
6760 sctx->cur_inode_last_extent < in finish_inode_if_needed()
6761 sctx->cur_inode_size) { in finish_inode_if_needed()
6762 ret = get_last_extent(sctx, (u64)-1); in finish_inode_if_needed()
6766 if (sctx->cur_inode_last_extent < sctx->cur_inode_size) { in finish_inode_if_needed()
6767 ret = range_is_hole_in_parent(sctx, in finish_inode_if_needed()
6768 sctx->cur_inode_last_extent, in finish_inode_if_needed()
6769 sctx->cur_inode_size); in finish_inode_if_needed()
6773 ret = send_hole(sctx, sctx->cur_inode_size); in finish_inode_if_needed()
6783 ret = send_truncate(sctx, sctx->cur_ino, in finish_inode_if_needed()
6784 sctx->cur_inode_gen, in finish_inode_if_needed()
6785 sctx->cur_inode_size); in finish_inode_if_needed()
6792 ret = send_chown(sctx, sctx->cur_ino, sctx->cur_inode_gen, in finish_inode_if_needed()
6798 ret = send_chmod(sctx, sctx->cur_ino, sctx->cur_inode_gen, in finish_inode_if_needed()
6804 ret = send_fileattr(sctx, sctx->cur_ino, sctx->cur_inode_gen, in finish_inode_if_needed()
6810 if (proto_cmd_ok(sctx, BTRFS_SEND_C_ENABLE_VERITY) in finish_inode_if_needed()
6811 && sctx->cur_inode_needs_verity) { in finish_inode_if_needed()
6812 ret = process_verity(sctx); in finish_inode_if_needed()
6817 ret = send_capabilities(sctx); in finish_inode_if_needed()
6825 if (!is_waiting_for_move(sctx, sctx->cur_ino)) { in finish_inode_if_needed()
6826 ret = apply_children_dir_moves(sctx); in finish_inode_if_needed()
6836 sctx->send_progress = sctx->cur_ino + 1; in finish_inode_if_needed()
6844 if (S_ISDIR(sctx->cur_inode_mode) && sctx->cur_inode_size > 0) in finish_inode_if_needed()
6845 ret = cache_dir_utimes(sctx, sctx->cur_ino, sctx->cur_inode_gen); in finish_inode_if_needed()
6847 ret = send_utimes(sctx, sctx->cur_ino, sctx->cur_inode_gen); in finish_inode_if_needed()
6855 ret = trim_dir_utimes_cache(sctx); in finish_inode_if_needed()
6860 static void close_current_inode(struct send_ctx *sctx) in close_current_inode() argument
6864 if (sctx->cur_inode == NULL) in close_current_inode()
6867 i_size = i_size_read(sctx->cur_inode); in close_current_inode()
6875 if (sctx->clean_page_cache && sctx->page_cache_clear_start < i_size) in close_current_inode()
6876 truncate_inode_pages_range(&sctx->cur_inode->i_data, in close_current_inode()
6877 sctx->page_cache_clear_start, in close_current_inode()
6880 iput(sctx->cur_inode); in close_current_inode()
6881 sctx->cur_inode = NULL; in close_current_inode()
6884 static int changed_inode(struct send_ctx *sctx, in changed_inode() argument
6888 struct btrfs_key *key = sctx->cmp_key; in changed_inode()
6894 close_current_inode(sctx); in changed_inode()
6896 sctx->cur_ino = key->objectid; in changed_inode()
6897 sctx->cur_inode_new_gen = false; in changed_inode()
6898 sctx->cur_inode_last_extent = (u64)-1; in changed_inode()
6899 sctx->cur_inode_next_write_offset = 0; in changed_inode()
6900 sctx->ignore_cur_inode = false; in changed_inode()
6907 sctx->send_progress = sctx->cur_ino; in changed_inode()
6911 left_ii = btrfs_item_ptr(sctx->left_path->nodes[0], in changed_inode()
6912 sctx->left_path->slots[0], in changed_inode()
6914 left_gen = btrfs_inode_generation(sctx->left_path->nodes[0], in changed_inode()
6917 right_ii = btrfs_item_ptr(sctx->right_path->nodes[0], in changed_inode()
6918 sctx->right_path->slots[0], in changed_inode()
6920 right_gen = btrfs_inode_generation(sctx->right_path->nodes[0], in changed_inode()
6924 right_ii = btrfs_item_ptr(sctx->right_path->nodes[0], in changed_inode()
6925 sctx->right_path->slots[0], in changed_inode()
6928 right_gen = btrfs_inode_generation(sctx->right_path->nodes[0], in changed_inode()
6937 sctx->cur_ino != BTRFS_FIRST_FREE_OBJECTID) in changed_inode()
6938 sctx->cur_inode_new_gen = true; in changed_inode()
6976 if (btrfs_inode_nlink(sctx->left_path->nodes[0], left_ii) == 0) { in changed_inode()
6977 sctx->ignore_cur_inode = true; in changed_inode()
6980 sctx->cur_inode_gen = left_gen; in changed_inode()
6981 sctx->cur_inode_new = true; in changed_inode()
6982 sctx->cur_inode_deleted = false; in changed_inode()
6983 sctx->cur_inode_size = btrfs_inode_size( in changed_inode()
6984 sctx->left_path->nodes[0], left_ii); in changed_inode()
6985 sctx->cur_inode_mode = btrfs_inode_mode( in changed_inode()
6986 sctx->left_path->nodes[0], left_ii); in changed_inode()
6987 sctx->cur_inode_rdev = btrfs_inode_rdev( in changed_inode()
6988 sctx->left_path->nodes[0], left_ii); in changed_inode()
6989 if (sctx->cur_ino != BTRFS_FIRST_FREE_OBJECTID) in changed_inode()
6990 ret = send_create_inode_if_needed(sctx); in changed_inode()
6992 sctx->cur_inode_gen = right_gen; in changed_inode()
6993 sctx->cur_inode_new = false; in changed_inode()
6994 sctx->cur_inode_deleted = true; in changed_inode()
6995 sctx->cur_inode_size = btrfs_inode_size( in changed_inode()
6996 sctx->right_path->nodes[0], right_ii); in changed_inode()
6997 sctx->cur_inode_mode = btrfs_inode_mode( in changed_inode()
6998 sctx->right_path->nodes[0], right_ii); in changed_inode()
7002 new_nlinks = btrfs_inode_nlink(sctx->left_path->nodes[0], left_ii); in changed_inode()
7003 old_nlinks = btrfs_inode_nlink(sctx->right_path->nodes[0], right_ii); in changed_inode()
7005 sctx->ignore_cur_inode = true; in changed_inode()
7008 sctx->cur_inode_new_gen = 1; in changed_inode()
7017 if (sctx->cur_inode_new_gen) { in changed_inode()
7022 sctx->cur_inode_gen = right_gen; in changed_inode()
7023 sctx->cur_inode_new = false; in changed_inode()
7024 sctx->cur_inode_deleted = true; in changed_inode()
7025 sctx->cur_inode_size = btrfs_inode_size( in changed_inode()
7026 sctx->right_path->nodes[0], right_ii); in changed_inode()
7027 sctx->cur_inode_mode = btrfs_inode_mode( in changed_inode()
7028 sctx->right_path->nodes[0], right_ii); in changed_inode()
7029 ret = process_all_refs(sctx, in changed_inode()
7039 sctx->cur_inode_gen = left_gen; in changed_inode()
7040 sctx->cur_inode_new = true; in changed_inode()
7041 sctx->cur_inode_deleted = false; in changed_inode()
7042 sctx->cur_inode_size = btrfs_inode_size( in changed_inode()
7043 sctx->left_path->nodes[0], in changed_inode()
7045 sctx->cur_inode_mode = btrfs_inode_mode( in changed_inode()
7046 sctx->left_path->nodes[0], in changed_inode()
7048 sctx->cur_inode_rdev = btrfs_inode_rdev( in changed_inode()
7049 sctx->left_path->nodes[0], in changed_inode()
7051 ret = send_create_inode_if_needed(sctx); in changed_inode()
7055 ret = process_all_refs(sctx, BTRFS_COMPARE_TREE_NEW); in changed_inode()
7063 sctx->send_progress = sctx->cur_ino + 1; in changed_inode()
7069 ret = process_all_extents(sctx); in changed_inode()
7072 ret = process_all_new_xattrs(sctx); in changed_inode()
7077 sctx->cur_inode_gen = left_gen; in changed_inode()
7078 sctx->cur_inode_new = false; in changed_inode()
7079 sctx->cur_inode_new_gen = false; in changed_inode()
7080 sctx->cur_inode_deleted = false; in changed_inode()
7081 sctx->cur_inode_size = btrfs_inode_size( in changed_inode()
7082 sctx->left_path->nodes[0], left_ii); in changed_inode()
7083 sctx->cur_inode_mode = btrfs_inode_mode( in changed_inode()
7084 sctx->left_path->nodes[0], left_ii); in changed_inode()
7102 static int changed_ref(struct send_ctx *sctx, in changed_ref() argument
7107 if (sctx->cur_ino != sctx->cmp_key->objectid) { in changed_ref()
7108 inconsistent_snapshot_error(sctx, result, "reference"); in changed_ref()
7112 if (!sctx->cur_inode_new_gen && in changed_ref()
7113 sctx->cur_ino != BTRFS_FIRST_FREE_OBJECTID) { in changed_ref()
7115 ret = record_new_ref(sctx); in changed_ref()
7117 ret = record_deleted_ref(sctx); in changed_ref()
7119 ret = record_changed_ref(sctx); in changed_ref()
7130 static int changed_xattr(struct send_ctx *sctx, in changed_xattr() argument
7135 if (sctx->cur_ino != sctx->cmp_key->objectid) { in changed_xattr()
7136 inconsistent_snapshot_error(sctx, result, "xattr"); in changed_xattr()
7140 if (!sctx->cur_inode_new_gen && !sctx->cur_inode_deleted) { in changed_xattr()
7142 ret = process_new_xattr(sctx); in changed_xattr()
7144 ret = process_deleted_xattr(sctx); in changed_xattr()
7146 ret = process_changed_xattr(sctx); in changed_xattr()
7157 static int changed_extent(struct send_ctx *sctx, in changed_extent() argument
7175 if (sctx->cur_ino != sctx->cmp_key->objectid) in changed_extent()
7178 if (!sctx->cur_inode_new_gen && !sctx->cur_inode_deleted) { in changed_extent()
7180 ret = process_extent(sctx, sctx->left_path, in changed_extent()
7181 sctx->cmp_key); in changed_extent()
7187 static int changed_verity(struct send_ctx *sctx, enum btrfs_compare_tree_result result) in changed_verity() argument
7189 if (!sctx->cur_inode_new_gen && !sctx->cur_inode_deleted) { in changed_verity()
7191 sctx->cur_inode_needs_verity = true; in changed_verity()
7196 static int dir_changed(struct send_ctx *sctx, u64 dir) in dir_changed() argument
7201 ret = get_inode_gen(sctx->send_root, dir, &new_gen); in dir_changed()
7205 ret = get_inode_gen(sctx->parent_root, dir, &orig_gen); in dir_changed()
7212 static int compare_refs(struct send_ctx *sctx, struct btrfs_path *path, in compare_refs() argument
7228 ret = dir_changed(sctx, dirid); in compare_refs()
7243 ret = dir_changed(sctx, dirid); in compare_refs()
7260 struct send_ctx *sctx) in changed_cb() argument
7289 lockdep_assert_not_held(&sctx->send_root->fs_info->commit_root_sem); in changed_cb()
7311 ret = compare_refs(sctx, left_path, key); in changed_cb()
7317 return maybe_send_hole(sctx, left_path, key); in changed_cb()
7325 sctx->left_path = left_path; in changed_cb()
7326 sctx->right_path = right_path; in changed_cb()
7327 sctx->cmp_key = key; in changed_cb()
7329 ret = finish_inode_if_needed(sctx, 0); in changed_cb()
7339 ret = changed_inode(sctx, result); in changed_cb()
7340 } else if (!sctx->ignore_cur_inode) { in changed_cb()
7343 ret = changed_ref(sctx, result); in changed_cb()
7345 ret = changed_xattr(sctx, result); in changed_cb()
7347 ret = changed_extent(sctx, result); in changed_cb()
7350 ret = changed_verity(sctx, result); in changed_cb()
7357 static int search_key_again(const struct send_ctx *sctx, in search_key_again() argument
7381 (root == sctx->parent_root ? "parent" : "send"), in search_key_again()
7390 static int full_send_tree(struct send_ctx *sctx) in full_send_tree() argument
7393 struct btrfs_root *send_root = sctx->send_root; in full_send_tree()
7408 sctx->last_reloc_trans = fs_info->last_reloc_trans; in full_send_tree()
7421 BTRFS_COMPARE_TREE_NEW, sctx); in full_send_tree()
7426 if (fs_info->last_reloc_trans > sctx->last_reloc_trans) { in full_send_tree()
7427 sctx->last_reloc_trans = fs_info->last_reloc_trans; in full_send_tree()
7440 ret = search_key_again(sctx, send_root, path, &key); in full_send_tree()
7457 ret = finish_inode_if_needed(sctx, 1); in full_send_tree()
7631 const struct send_ctx *sctx) in restart_after_relocation() argument
7636 lockdep_assert_held_read(&sctx->send_root->fs_info->commit_root_sem); in restart_after_relocation()
7648 ret = search_key_again(sctx, sctx->send_root, left_path, left_key); in restart_after_relocation()
7653 ret = search_key_again(sctx, sctx->parent_root, right_path, right_key); in restart_after_relocation()
7680 root_level = btrfs_header_level(sctx->send_root->commit_root); in restart_after_relocation()
7687 root_level = btrfs_header_level(sctx->parent_root->commit_root); in restart_after_relocation()
7711 struct btrfs_root *right_root, struct send_ctx *sctx) in btrfs_compare_trees() argument
7840 sctx->last_reloc_trans = fs_info->last_reloc_trans; in btrfs_compare_trees()
7850 if (fs_info->last_reloc_trans > sctx->last_reloc_trans) { in btrfs_compare_trees()
7854 sctx); in btrfs_compare_trees()
7857 sctx->last_reloc_trans = fs_info->last_reloc_trans; in btrfs_compare_trees()
7892 sctx); in btrfs_compare_trees()
7905 sctx); in btrfs_compare_trees()
7921 sctx); in btrfs_compare_trees()
7927 sctx); in btrfs_compare_trees()
7940 &left_key, result, sctx); in btrfs_compare_trees()
7996 static int send_subvol(struct send_ctx *sctx) in send_subvol() argument
8000 if (!(sctx->flags & BTRFS_SEND_FLAG_OMIT_STREAM_HEADER)) { in send_subvol()
8001 ret = send_header(sctx); in send_subvol()
8006 ret = send_subvol_begin(sctx); in send_subvol()
8010 if (sctx->parent_root) { in send_subvol()
8011 ret = btrfs_compare_trees(sctx->send_root, sctx->parent_root, sctx); in send_subvol()
8014 ret = finish_inode_if_needed(sctx, 1); in send_subvol()
8018 ret = full_send_tree(sctx); in send_subvol()
8024 free_recorded_refs(sctx); in send_subvol()
8041 static int ensure_commit_roots_uptodate(struct send_ctx *sctx) in ensure_commit_roots_uptodate() argument
8043 struct btrfs_root *root = sctx->parent_root; in ensure_commit_roots_uptodate()
8048 for (int i = 0; i < sctx->clone_roots_cnt; i++) { in ensure_commit_roots_uptodate()
8049 root = sctx->clone_roots[i].root; in ensure_commit_roots_uptodate()
8065 static int flush_delalloc_roots(struct send_ctx *sctx) in flush_delalloc_roots() argument
8067 struct btrfs_root *root = sctx->parent_root; in flush_delalloc_roots()
8078 for (i = 0; i < sctx->clone_roots_cnt; i++) { in flush_delalloc_roots()
8079 root = sctx->clone_roots[i].root; in flush_delalloc_roots()
8117 struct send_ctx *sctx = NULL; in btrfs_ioctl_send() local
8171 sctx = kzalloc(sizeof(struct send_ctx), GFP_KERNEL); in btrfs_ioctl_send()
8172 if (!sctx) { in btrfs_ioctl_send()
8177 INIT_LIST_HEAD(&sctx->new_refs); in btrfs_ioctl_send()
8178 INIT_LIST_HEAD(&sctx->deleted_refs); in btrfs_ioctl_send()
8180 btrfs_lru_cache_init(&sctx->name_cache, SEND_MAX_NAME_CACHE_SIZE); in btrfs_ioctl_send()
8181 btrfs_lru_cache_init(&sctx->backref_cache, SEND_MAX_BACKREF_CACHE_SIZE); in btrfs_ioctl_send()
8182 btrfs_lru_cache_init(&sctx->dir_created_cache, in btrfs_ioctl_send()
8188 btrfs_lru_cache_init(&sctx->dir_utimes_cache, 0); in btrfs_ioctl_send()
8190 sctx->pending_dir_moves = RB_ROOT; in btrfs_ioctl_send()
8191 sctx->waiting_dir_moves = RB_ROOT; in btrfs_ioctl_send()
8192 sctx->orphan_dirs = RB_ROOT; in btrfs_ioctl_send()
8193 sctx->rbtree_new_refs = RB_ROOT; in btrfs_ioctl_send()
8194 sctx->rbtree_deleted_refs = RB_ROOT; in btrfs_ioctl_send()
8196 sctx->flags = arg->flags; in btrfs_ioctl_send()
8204 sctx->proto = arg->version ?: BTRFS_SEND_STREAM_VERSION; in btrfs_ioctl_send()
8206 sctx->proto = 1; in btrfs_ioctl_send()
8208 if ((arg->flags & BTRFS_SEND_FLAG_COMPRESSED) && sctx->proto < 2) { in btrfs_ioctl_send()
8213 sctx->send_filp = fget(arg->send_fd); in btrfs_ioctl_send()
8214 if (!sctx->send_filp || !(sctx->send_filp->f_mode & FMODE_WRITE)) { in btrfs_ioctl_send()
8219 sctx->send_root = send_root; in btrfs_ioctl_send()
8220 sctx->clone_roots_cnt = arg->clone_sources_count; in btrfs_ioctl_send()
8222 if (sctx->proto >= 2) { in btrfs_ioctl_send()
8225 sctx->send_max_size = BTRFS_SEND_BUF_SIZE_V2; in btrfs_ioctl_send()
8226 sctx->send_buf = vmalloc(sctx->send_max_size); in btrfs_ioctl_send()
8227 if (!sctx->send_buf) { in btrfs_ioctl_send()
8231 send_buf_num_pages = sctx->send_max_size >> PAGE_SHIFT; in btrfs_ioctl_send()
8232 sctx->send_buf_pages = kcalloc(send_buf_num_pages, in btrfs_ioctl_send()
8233 sizeof(*sctx->send_buf_pages), in btrfs_ioctl_send()
8235 if (!sctx->send_buf_pages) { in btrfs_ioctl_send()
8240 sctx->send_buf_pages[i] = in btrfs_ioctl_send()
8241 vmalloc_to_page(sctx->send_buf + (i << PAGE_SHIFT)); in btrfs_ioctl_send()
8244 sctx->send_max_size = BTRFS_SEND_BUF_SIZE_V1; in btrfs_ioctl_send()
8245 sctx->send_buf = kvmalloc(sctx->send_max_size, GFP_KERNEL); in btrfs_ioctl_send()
8247 if (!sctx->send_buf) { in btrfs_ioctl_send()
8252 sctx->clone_roots = kvcalloc(arg->clone_sources_count + 1, in btrfs_ioctl_send()
8253 sizeof(*sctx->clone_roots), in btrfs_ioctl_send()
8255 if (!sctx->clone_roots) { in btrfs_ioctl_send()
8302 sctx->clone_roots[i].root = clone_root; in btrfs_ioctl_send()
8310 sctx->parent_root = btrfs_get_fs_root(fs_info, arg->parent_root, in btrfs_ioctl_send()
8312 if (IS_ERR(sctx->parent_root)) { in btrfs_ioctl_send()
8313 ret = PTR_ERR(sctx->parent_root); in btrfs_ioctl_send()
8317 spin_lock(&sctx->parent_root->root_item_lock); in btrfs_ioctl_send()
8318 sctx->parent_root->send_in_progress++; in btrfs_ioctl_send()
8319 if (!btrfs_root_readonly(sctx->parent_root) || in btrfs_ioctl_send()
8320 btrfs_root_dead(sctx->parent_root)) { in btrfs_ioctl_send()
8321 spin_unlock(&sctx->parent_root->root_item_lock); in btrfs_ioctl_send()
8325 if (sctx->parent_root->dedupe_in_progress) { in btrfs_ioctl_send()
8326 dedupe_in_progress_warn(sctx->parent_root); in btrfs_ioctl_send()
8327 spin_unlock(&sctx->parent_root->root_item_lock); in btrfs_ioctl_send()
8331 spin_unlock(&sctx->parent_root->root_item_lock); in btrfs_ioctl_send()
8339 sctx->clone_roots[sctx->clone_roots_cnt++].root = in btrfs_ioctl_send()
8340 btrfs_grab_root(sctx->send_root); in btrfs_ioctl_send()
8343 sort(sctx->clone_roots, sctx->clone_roots_cnt, in btrfs_ioctl_send()
8344 sizeof(*sctx->clone_roots), __clone_root_cmp_sort, in btrfs_ioctl_send()
8348 ret = flush_delalloc_roots(sctx); in btrfs_ioctl_send()
8352 ret = ensure_commit_roots_uptodate(sctx); in btrfs_ioctl_send()
8356 ret = send_subvol(sctx); in btrfs_ioctl_send()
8360 btrfs_lru_cache_for_each_entry_safe(&sctx->dir_utimes_cache, entry, tmp) { in btrfs_ioctl_send()
8361 ret = send_utimes(sctx, entry->key, entry->gen); in btrfs_ioctl_send()
8364 btrfs_lru_cache_remove(&sctx->dir_utimes_cache, entry); in btrfs_ioctl_send()
8367 if (!(sctx->flags & BTRFS_SEND_FLAG_OMIT_END_CMD)) { in btrfs_ioctl_send()
8368 ret = begin_cmd(sctx, BTRFS_SEND_C_END); in btrfs_ioctl_send()
8371 ret = send_cmd(sctx); in btrfs_ioctl_send()
8377 WARN_ON(sctx && !ret && !RB_EMPTY_ROOT(&sctx->pending_dir_moves)); in btrfs_ioctl_send()
8378 while (sctx && !RB_EMPTY_ROOT(&sctx->pending_dir_moves)) { in btrfs_ioctl_send()
8382 n = rb_first(&sctx->pending_dir_moves); in btrfs_ioctl_send()
8389 free_pending_move(sctx, pm2); in btrfs_ioctl_send()
8391 free_pending_move(sctx, pm); in btrfs_ioctl_send()
8394 WARN_ON(sctx && !ret && !RB_EMPTY_ROOT(&sctx->waiting_dir_moves)); in btrfs_ioctl_send()
8395 while (sctx && !RB_EMPTY_ROOT(&sctx->waiting_dir_moves)) { in btrfs_ioctl_send()
8399 n = rb_first(&sctx->waiting_dir_moves); in btrfs_ioctl_send()
8401 rb_erase(&dm->node, &sctx->waiting_dir_moves); in btrfs_ioctl_send()
8405 WARN_ON(sctx && !ret && !RB_EMPTY_ROOT(&sctx->orphan_dirs)); in btrfs_ioctl_send()
8406 while (sctx && !RB_EMPTY_ROOT(&sctx->orphan_dirs)) { in btrfs_ioctl_send()
8410 n = rb_first(&sctx->orphan_dirs); in btrfs_ioctl_send()
8412 free_orphan_dir_info(sctx, odi); in btrfs_ioctl_send()
8416 for (i = 0; i < sctx->clone_roots_cnt; i++) { in btrfs_ioctl_send()
8418 sctx->clone_roots[i].root); in btrfs_ioctl_send()
8419 btrfs_put_root(sctx->clone_roots[i].root); in btrfs_ioctl_send()
8422 for (i = 0; sctx && i < clone_sources_to_rollback; i++) { in btrfs_ioctl_send()
8424 sctx->clone_roots[i].root); in btrfs_ioctl_send()
8425 btrfs_put_root(sctx->clone_roots[i].root); in btrfs_ioctl_send()
8430 if (sctx && !IS_ERR_OR_NULL(sctx->parent_root)) { in btrfs_ioctl_send()
8431 btrfs_root_dec_send_in_progress(sctx->parent_root); in btrfs_ioctl_send()
8432 btrfs_put_root(sctx->parent_root); in btrfs_ioctl_send()
8437 if (sctx) { in btrfs_ioctl_send()
8438 if (sctx->send_filp) in btrfs_ioctl_send()
8439 fput(sctx->send_filp); in btrfs_ioctl_send()
8441 kvfree(sctx->clone_roots); in btrfs_ioctl_send()
8442 kfree(sctx->send_buf_pages); in btrfs_ioctl_send()
8443 kvfree(sctx->send_buf); in btrfs_ioctl_send()
8444 kvfree(sctx->verity_descriptor); in btrfs_ioctl_send()
8446 close_current_inode(sctx); in btrfs_ioctl_send()
8448 btrfs_lru_cache_clear(&sctx->name_cache); in btrfs_ioctl_send()
8449 btrfs_lru_cache_clear(&sctx->backref_cache); in btrfs_ioctl_send()
8450 btrfs_lru_cache_clear(&sctx->dir_created_cache); in btrfs_ioctl_send()
8451 btrfs_lru_cache_clear(&sctx->dir_utimes_cache); in btrfs_ioctl_send()
8453 kfree(sctx); in btrfs_ioctl_send()