1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2012 Alexander Block. All rights reserved.
4 */
5
6 #include <linux/bsearch.h>
7 #include <linux/falloc.h>
8 #include <linux/fs.h>
9 #include <linux/file.h>
10 #include <linux/sort.h>
11 #include <linux/mount.h>
12 #include <linux/xattr.h>
13 #include <linux/posix_acl_xattr.h>
14 #include <linux/radix-tree.h>
15 #include <linux/vmalloc.h>
16 #include <linux/string.h>
17 #include <linux/compat.h>
18 #include <linux/crc32c.h>
19 #include <linux/fsverity.h>
20 #include "send.h"
21 #include "ctree.h"
22 #include "backref.h"
23 #include "locking.h"
24 #include "disk-io.h"
25 #include "btrfs_inode.h"
26 #include "transaction.h"
27 #include "compression.h"
28 #include "print-tree.h"
29 #include "accessors.h"
30 #include "dir-item.h"
31 #include "file-item.h"
32 #include "ioctl.h"
33 #include "verity.h"
34 #include "lru_cache.h"
35
36 /*
37 * Maximum number of references an extent can have in order for us to attempt to
38 * issue clone operations instead of write operations. This currently exists to
39 * avoid hitting limitations of the backreference walking code (taking a lot of
40 * time and using too much memory for extents with large number of references).
41 */
42 #define SEND_MAX_EXTENT_REFS 1024
43
44 /*
45 * A fs_path is a helper to dynamically build path names with unknown size.
46 * It reallocates the internal buffer on demand.
47 * It allows fast adding of path elements on the right side (normal path) and
48 * fast adding to the left side (reversed path). A reversed path can also be
49 * unreversed if needed.
50 */
51 struct fs_path {
52 union {
53 struct {
54 char *start;
55 char *end;
56
57 char *buf;
58 unsigned short buf_len:15;
59 unsigned short reversed:1;
60 char inline_buf[];
61 };
62 /*
63 * Average path length does not exceed 200 bytes, we'll have
64 * better packing in the slab and higher chance to satisfy
65 * an allocation later during send.
66 */
67 char pad[256];
68 };
69 };
70 #define FS_PATH_INLINE_SIZE \
71 (sizeof(struct fs_path) - offsetof(struct fs_path, inline_buf))
72
73
74 /* reused for each extent */
75 struct clone_root {
76 struct btrfs_root *root;
77 u64 ino;
78 u64 offset;
79 u64 num_bytes;
80 bool found_ref;
81 };
82
83 #define SEND_MAX_NAME_CACHE_SIZE 256
84
85 /*
86 * Limit the root_ids array of struct backref_cache_entry to 17 elements.
87 * This makes the size of a cache entry to be exactly 192 bytes on x86_64, which
88 * can be satisfied from the kmalloc-192 slab, without wasting any space.
89 * The most common case is to have a single root for cloning, which corresponds
90 * to the send root. Having the user specify more than 16 clone roots is not
91 * common, and in such rare cases we simply don't use caching if the number of
92 * cloning roots that lead down to a leaf is more than 17.
93 */
94 #define SEND_MAX_BACKREF_CACHE_ROOTS 17
95
96 /*
97 * Max number of entries in the cache.
98 * With SEND_MAX_BACKREF_CACHE_ROOTS as 17, the size in bytes, excluding
99 * maple tree's internal nodes, is 24K.
100 */
101 #define SEND_MAX_BACKREF_CACHE_SIZE 128
102
103 /*
104 * A backref cache entry maps a leaf to a list of IDs of roots from which the
105 * leaf is accessible and we can use for clone operations.
106 * With SEND_MAX_BACKREF_CACHE_ROOTS as 12, each cache entry is 128 bytes (on
107 * x86_64).
108 */
109 struct backref_cache_entry {
110 struct btrfs_lru_cache_entry entry;
111 u64 root_ids[SEND_MAX_BACKREF_CACHE_ROOTS];
112 /* Number of valid elements in the root_ids array. */
113 int num_roots;
114 };
115
116 /* See the comment at lru_cache.h about struct btrfs_lru_cache_entry. */
117 static_assert(offsetof(struct backref_cache_entry, entry) == 0);
118
119 /*
120 * Max number of entries in the cache that stores directories that were already
121 * created. The cache uses raw struct btrfs_lru_cache_entry entries, so it uses
122 * at most 4096 bytes - sizeof(struct btrfs_lru_cache_entry) is 48 bytes, but
123 * the kmalloc-64 slab is used, so we get 4096 bytes (64 bytes * 64).
124 */
125 #define SEND_MAX_DIR_CREATED_CACHE_SIZE 64
126
127 /*
128 * Max number of entries in the cache that stores directories that were already
129 * created. The cache uses raw struct btrfs_lru_cache_entry entries, so it uses
130 * at most 4096 bytes - sizeof(struct btrfs_lru_cache_entry) is 48 bytes, but
131 * the kmalloc-64 slab is used, so we get 4096 bytes (64 bytes * 64).
132 */
133 #define SEND_MAX_DIR_UTIMES_CACHE_SIZE 64
134
135 struct send_ctx {
136 struct file *send_filp;
137 loff_t send_off;
138 char *send_buf;
139 u32 send_size;
140 u32 send_max_size;
141 /*
142 * Whether BTRFS_SEND_A_DATA attribute was already added to current
143 * command (since protocol v2, data must be the last attribute).
144 */
145 bool put_data;
146 struct page **send_buf_pages;
147 u64 flags; /* 'flags' member of btrfs_ioctl_send_args is u64 */
148 /* Protocol version compatibility requested */
149 u32 proto;
150
151 struct btrfs_root *send_root;
152 struct btrfs_root *parent_root;
153 struct clone_root *clone_roots;
154 int clone_roots_cnt;
155
156 /* current state of the compare_tree call */
157 struct btrfs_path *left_path;
158 struct btrfs_path *right_path;
159 struct btrfs_key *cmp_key;
160
161 /*
162 * Keep track of the generation of the last transaction that was used
163 * for relocating a block group. This is periodically checked in order
164 * to detect if a relocation happened since the last check, so that we
165 * don't operate on stale extent buffers for nodes (level >= 1) or on
166 * stale disk_bytenr values of file extent items.
167 */
168 u64 last_reloc_trans;
169
170 /*
171 * infos of the currently processed inode. In case of deleted inodes,
172 * these are the values from the deleted inode.
173 */
174 u64 cur_ino;
175 u64 cur_inode_gen;
176 u64 cur_inode_size;
177 u64 cur_inode_mode;
178 u64 cur_inode_rdev;
179 u64 cur_inode_last_extent;
180 u64 cur_inode_next_write_offset;
181 bool cur_inode_new;
182 bool cur_inode_new_gen;
183 bool cur_inode_deleted;
184 bool ignore_cur_inode;
185 bool cur_inode_needs_verity;
186 void *verity_descriptor;
187
188 u64 send_progress;
189
190 struct list_head new_refs;
191 struct list_head deleted_refs;
192
193 struct btrfs_lru_cache name_cache;
194
195 /*
196 * The inode we are currently processing. It's not NULL only when we
197 * need to issue write commands for data extents from this inode.
198 */
199 struct inode *cur_inode;
200 struct file_ra_state ra;
201 u64 page_cache_clear_start;
202 bool clean_page_cache;
203
204 /*
205 * We process inodes by their increasing order, so if before an
206 * incremental send we reverse the parent/child relationship of
207 * directories such that a directory with a lower inode number was
208 * the parent of a directory with a higher inode number, and the one
209 * becoming the new parent got renamed too, we can't rename/move the
210 * directory with lower inode number when we finish processing it - we
211 * must process the directory with higher inode number first, then
212 * rename/move it and then rename/move the directory with lower inode
213 * number. Example follows.
214 *
215 * Tree state when the first send was performed:
216 *
217 * .
218 * |-- a (ino 257)
219 * |-- b (ino 258)
220 * |
221 * |
222 * |-- c (ino 259)
223 * | |-- d (ino 260)
224 * |
225 * |-- c2 (ino 261)
226 *
227 * Tree state when the second (incremental) send is performed:
228 *
229 * .
230 * |-- a (ino 257)
231 * |-- b (ino 258)
232 * |-- c2 (ino 261)
233 * |-- d2 (ino 260)
234 * |-- cc (ino 259)
235 *
236 * The sequence of steps that lead to the second state was:
237 *
238 * mv /a/b/c/d /a/b/c2/d2
239 * mv /a/b/c /a/b/c2/d2/cc
240 *
241 * "c" has lower inode number, but we can't move it (2nd mv operation)
242 * before we move "d", which has higher inode number.
243 *
244 * So we just memorize which move/rename operations must be performed
245 * later when their respective parent is processed and moved/renamed.
246 */
247
248 /* Indexed by parent directory inode number. */
249 struct rb_root pending_dir_moves;
250
251 /*
252 * Reverse index, indexed by the inode number of a directory that
253 * is waiting for the move/rename of its immediate parent before its
254 * own move/rename can be performed.
255 */
256 struct rb_root waiting_dir_moves;
257
258 /*
259 * A directory that is going to be rm'ed might have a child directory
260 * which is in the pending directory moves index above. In this case,
261 * the directory can only be removed after the move/rename of its child
262 * is performed. Example:
263 *
264 * Parent snapshot:
265 *
266 * . (ino 256)
267 * |-- a/ (ino 257)
268 * |-- b/ (ino 258)
269 * |-- c/ (ino 259)
270 * | |-- x/ (ino 260)
271 * |
272 * |-- y/ (ino 261)
273 *
274 * Send snapshot:
275 *
276 * . (ino 256)
277 * |-- a/ (ino 257)
278 * |-- b/ (ino 258)
279 * |-- YY/ (ino 261)
280 * |-- x/ (ino 260)
281 *
282 * Sequence of steps that lead to the send snapshot:
283 * rm -f /a/b/c/foo.txt
284 * mv /a/b/y /a/b/YY
285 * mv /a/b/c/x /a/b/YY
286 * rmdir /a/b/c
287 *
288 * When the child is processed, its move/rename is delayed until its
289 * parent is processed (as explained above), but all other operations
290 * like update utimes, chown, chgrp, etc, are performed and the paths
291 * that it uses for those operations must use the orphanized name of
292 * its parent (the directory we're going to rm later), so we need to
293 * memorize that name.
294 *
295 * Indexed by the inode number of the directory to be deleted.
296 */
297 struct rb_root orphan_dirs;
298
299 struct rb_root rbtree_new_refs;
300 struct rb_root rbtree_deleted_refs;
301
302 struct btrfs_lru_cache backref_cache;
303 u64 backref_cache_last_reloc_trans;
304
305 struct btrfs_lru_cache dir_created_cache;
306 struct btrfs_lru_cache dir_utimes_cache;
307
308 /* Must be last as it ends in a flexible-array member. */
309 struct fs_path cur_inode_path;
310 };
311
312 struct pending_dir_move {
313 struct rb_node node;
314 struct list_head list;
315 u64 parent_ino;
316 u64 ino;
317 u64 gen;
318 struct list_head update_refs;
319 };
320
321 struct waiting_dir_move {
322 struct rb_node node;
323 u64 ino;
324 /*
325 * There might be some directory that could not be removed because it
326 * was waiting for this directory inode to be moved first. Therefore
327 * after this directory is moved, we can try to rmdir the ino rmdir_ino.
328 */
329 u64 rmdir_ino;
330 u64 rmdir_gen;
331 bool orphanized;
332 };
333
334 struct orphan_dir_info {
335 struct rb_node node;
336 u64 ino;
337 u64 gen;
338 u64 last_dir_index_offset;
339 u64 dir_high_seq_ino;
340 };
341
342 struct name_cache_entry {
343 /*
344 * The key in the entry is an inode number, and the generation matches
345 * the inode's generation.
346 */
347 struct btrfs_lru_cache_entry entry;
348 u64 parent_ino;
349 u64 parent_gen;
350 int ret;
351 int need_later_update;
352 /* Name length without NUL terminator. */
353 int name_len;
354 /* Not NUL terminated. */
355 char name[] __counted_by(name_len) __nonstring;
356 };
357
358 /* See the comment at lru_cache.h about struct btrfs_lru_cache_entry. */
359 static_assert(offsetof(struct name_cache_entry, entry) == 0);
360
361 #define ADVANCE 1
362 #define ADVANCE_ONLY_NEXT -1
363
364 enum btrfs_compare_tree_result {
365 BTRFS_COMPARE_TREE_NEW,
366 BTRFS_COMPARE_TREE_DELETED,
367 BTRFS_COMPARE_TREE_CHANGED,
368 BTRFS_COMPARE_TREE_SAME,
369 };
370
371 __cold
inconsistent_snapshot_error(struct send_ctx * sctx,enum btrfs_compare_tree_result result,const char * what)372 static void inconsistent_snapshot_error(struct send_ctx *sctx,
373 enum btrfs_compare_tree_result result,
374 const char *what)
375 {
376 const char *result_string;
377
378 switch (result) {
379 case BTRFS_COMPARE_TREE_NEW:
380 result_string = "new";
381 break;
382 case BTRFS_COMPARE_TREE_DELETED:
383 result_string = "deleted";
384 break;
385 case BTRFS_COMPARE_TREE_CHANGED:
386 result_string = "updated";
387 break;
388 case BTRFS_COMPARE_TREE_SAME:
389 DEBUG_WARN("no change between trees");
390 result_string = "unchanged";
391 break;
392 default:
393 DEBUG_WARN("unexpected comparison result %d", result);
394 result_string = "unexpected";
395 }
396
397 btrfs_err(sctx->send_root->fs_info,
398 "Send: inconsistent snapshot, found %s %s for inode %llu without updated inode item, send root is %llu, parent root is %llu",
399 result_string, what, sctx->cmp_key->objectid,
400 btrfs_root_id(sctx->send_root),
401 (sctx->parent_root ? btrfs_root_id(sctx->parent_root) : 0));
402 }
403
404 __maybe_unused
proto_cmd_ok(const struct send_ctx * sctx,int cmd)405 static bool proto_cmd_ok(const struct send_ctx *sctx, int cmd)
406 {
407 switch (sctx->proto) {
408 case 1: return cmd <= BTRFS_SEND_C_MAX_V1;
409 case 2: return cmd <= BTRFS_SEND_C_MAX_V2;
410 case 3: return cmd <= BTRFS_SEND_C_MAX_V3;
411 default: return false;
412 }
413 }
414
415 static int is_waiting_for_move(struct send_ctx *sctx, u64 ino);
416
417 static struct waiting_dir_move *
418 get_waiting_dir_move(struct send_ctx *sctx, u64 ino);
419
420 static int is_waiting_for_rm(struct send_ctx *sctx, u64 dir_ino, u64 gen);
421
need_send_hole(struct send_ctx * sctx)422 static int need_send_hole(struct send_ctx *sctx)
423 {
424 return (sctx->parent_root && !sctx->cur_inode_new &&
425 !sctx->cur_inode_new_gen && !sctx->cur_inode_deleted &&
426 S_ISREG(sctx->cur_inode_mode));
427 }
428
fs_path_reset(struct fs_path * p)429 static void fs_path_reset(struct fs_path *p)
430 {
431 if (p->reversed)
432 p->start = p->buf + p->buf_len - 1;
433 else
434 p->start = p->buf;
435
436 p->end = p->start;
437 *p->start = 0;
438 }
439
init_path(struct fs_path * p)440 static void init_path(struct fs_path *p)
441 {
442 p->reversed = 0;
443 p->buf = p->inline_buf;
444 p->buf_len = FS_PATH_INLINE_SIZE;
445 fs_path_reset(p);
446 }
447
fs_path_alloc(void)448 static struct fs_path *fs_path_alloc(void)
449 {
450 struct fs_path *p;
451
452 p = kmalloc(sizeof(*p), GFP_KERNEL);
453 if (!p)
454 return NULL;
455 init_path(p);
456 return p;
457 }
458
fs_path_alloc_reversed(void)459 static struct fs_path *fs_path_alloc_reversed(void)
460 {
461 struct fs_path *p;
462
463 p = fs_path_alloc();
464 if (!p)
465 return NULL;
466 p->reversed = 1;
467 fs_path_reset(p);
468 return p;
469 }
470
fs_path_free(struct fs_path * p)471 static void fs_path_free(struct fs_path *p)
472 {
473 if (!p)
474 return;
475 if (p->buf != p->inline_buf)
476 kfree(p->buf);
477 kfree(p);
478 }
479
fs_path_len(const struct fs_path * p)480 static inline int fs_path_len(const struct fs_path *p)
481 {
482 return p->end - p->start;
483 }
484
fs_path_ensure_buf(struct fs_path * p,int len)485 static int fs_path_ensure_buf(struct fs_path *p, int len)
486 {
487 char *tmp_buf;
488 int path_len;
489 int old_buf_len;
490
491 len++;
492
493 if (p->buf_len >= len)
494 return 0;
495
496 if (WARN_ON(len > PATH_MAX))
497 return -ENAMETOOLONG;
498
499 path_len = fs_path_len(p);
500 old_buf_len = p->buf_len;
501
502 /*
503 * Allocate to the next largest kmalloc bucket size, to let
504 * the fast path happen most of the time.
505 */
506 len = kmalloc_size_roundup(len);
507 /*
508 * First time the inline_buf does not suffice
509 */
510 if (p->buf == p->inline_buf) {
511 tmp_buf = kmalloc(len, GFP_KERNEL);
512 if (tmp_buf)
513 memcpy(tmp_buf, p->buf, old_buf_len);
514 } else {
515 tmp_buf = krealloc(p->buf, len, GFP_KERNEL);
516 }
517 if (!tmp_buf)
518 return -ENOMEM;
519 p->buf = tmp_buf;
520 p->buf_len = len;
521
522 if (p->reversed) {
523 tmp_buf = p->buf + old_buf_len - path_len - 1;
524 p->end = p->buf + p->buf_len - 1;
525 p->start = p->end - path_len;
526 memmove(p->start, tmp_buf, path_len + 1);
527 } else {
528 p->start = p->buf;
529 p->end = p->start + path_len;
530 }
531 return 0;
532 }
533
fs_path_prepare_for_add(struct fs_path * p,int name_len,char ** prepared)534 static int fs_path_prepare_for_add(struct fs_path *p, int name_len,
535 char **prepared)
536 {
537 int ret;
538 int new_len;
539
540 new_len = fs_path_len(p) + name_len;
541 if (p->start != p->end)
542 new_len++;
543 ret = fs_path_ensure_buf(p, new_len);
544 if (ret < 0)
545 return ret;
546
547 if (p->reversed) {
548 if (p->start != p->end)
549 *--p->start = '/';
550 p->start -= name_len;
551 *prepared = p->start;
552 } else {
553 if (p->start != p->end)
554 *p->end++ = '/';
555 *prepared = p->end;
556 p->end += name_len;
557 *p->end = 0;
558 }
559
560 return 0;
561 }
562
fs_path_add(struct fs_path * p,const char * name,int name_len)563 static int fs_path_add(struct fs_path *p, const char *name, int name_len)
564 {
565 int ret;
566 char *prepared;
567
568 ret = fs_path_prepare_for_add(p, name_len, &prepared);
569 if (ret < 0)
570 return ret;
571 memcpy(prepared, name, name_len);
572
573 return 0;
574 }
575
fs_path_add_path(struct fs_path * p,const struct fs_path * p2)576 static inline int fs_path_add_path(struct fs_path *p, const struct fs_path *p2)
577 {
578 return fs_path_add(p, p2->start, fs_path_len(p2));
579 }
580
fs_path_add_from_extent_buffer(struct fs_path * p,struct extent_buffer * eb,unsigned long off,int len)581 static int fs_path_add_from_extent_buffer(struct fs_path *p,
582 struct extent_buffer *eb,
583 unsigned long off, int len)
584 {
585 int ret;
586 char *prepared;
587
588 ret = fs_path_prepare_for_add(p, len, &prepared);
589 if (ret < 0)
590 return ret;
591
592 read_extent_buffer(eb, prepared, off, len);
593
594 return 0;
595 }
596
fs_path_copy(struct fs_path * p,struct fs_path * from)597 static int fs_path_copy(struct fs_path *p, struct fs_path *from)
598 {
599 p->reversed = from->reversed;
600 fs_path_reset(p);
601
602 return fs_path_add_path(p, from);
603 }
604
fs_path_unreverse(struct fs_path * p)605 static void fs_path_unreverse(struct fs_path *p)
606 {
607 char *tmp;
608 int len;
609
610 if (!p->reversed)
611 return;
612
613 tmp = p->start;
614 len = fs_path_len(p);
615 p->start = p->buf;
616 p->end = p->start + len;
617 memmove(p->start, tmp, len + 1);
618 p->reversed = 0;
619 }
620
is_current_inode_path(const struct send_ctx * sctx,const struct fs_path * path)621 static inline bool is_current_inode_path(const struct send_ctx *sctx,
622 const struct fs_path *path)
623 {
624 const struct fs_path *cur = &sctx->cur_inode_path;
625
626 return (strncmp(path->start, cur->start, fs_path_len(cur)) == 0);
627 }
628
alloc_path_for_send(void)629 static struct btrfs_path *alloc_path_for_send(void)
630 {
631 struct btrfs_path *path;
632
633 path = btrfs_alloc_path();
634 if (!path)
635 return NULL;
636 path->search_commit_root = 1;
637 path->skip_locking = 1;
638 path->need_commit_sem = 1;
639 return path;
640 }
641
write_buf(struct file * filp,const void * buf,u32 len,loff_t * off)642 static int write_buf(struct file *filp, const void *buf, u32 len, loff_t *off)
643 {
644 int ret;
645 u32 pos = 0;
646
647 while (pos < len) {
648 ret = kernel_write(filp, buf + pos, len - pos, off);
649 if (ret < 0)
650 return ret;
651 if (unlikely(ret == 0))
652 return -EIO;
653 pos += ret;
654 }
655
656 return 0;
657 }
658
tlv_put(struct send_ctx * sctx,u16 attr,const void * data,int len)659 static int tlv_put(struct send_ctx *sctx, u16 attr, const void *data, int len)
660 {
661 struct btrfs_tlv_header *hdr;
662 int total_len = sizeof(*hdr) + len;
663 int left = sctx->send_max_size - sctx->send_size;
664
665 if (WARN_ON_ONCE(sctx->put_data))
666 return -EINVAL;
667
668 if (unlikely(left < total_len))
669 return -EOVERFLOW;
670
671 hdr = (struct btrfs_tlv_header *) (sctx->send_buf + sctx->send_size);
672 put_unaligned_le16(attr, &hdr->tlv_type);
673 put_unaligned_le16(len, &hdr->tlv_len);
674 memcpy(hdr + 1, data, len);
675 sctx->send_size += total_len;
676
677 return 0;
678 }
679
680 #define TLV_PUT_DEFINE_INT(bits) \
681 static int tlv_put_u##bits(struct send_ctx *sctx, \
682 u##bits attr, u##bits value) \
683 { \
684 __le##bits __tmp = cpu_to_le##bits(value); \
685 return tlv_put(sctx, attr, &__tmp, sizeof(__tmp)); \
686 }
687
688 TLV_PUT_DEFINE_INT(8)
689 TLV_PUT_DEFINE_INT(32)
690 TLV_PUT_DEFINE_INT(64)
691
tlv_put_string(struct send_ctx * sctx,u16 attr,const char * str,int len)692 static int tlv_put_string(struct send_ctx *sctx, u16 attr,
693 const char *str, int len)
694 {
695 if (len == -1)
696 len = strlen(str);
697 return tlv_put(sctx, attr, str, len);
698 }
699
tlv_put_uuid(struct send_ctx * sctx,u16 attr,const u8 * uuid)700 static int tlv_put_uuid(struct send_ctx *sctx, u16 attr,
701 const u8 *uuid)
702 {
703 return tlv_put(sctx, attr, uuid, BTRFS_UUID_SIZE);
704 }
705
tlv_put_btrfs_timespec(struct send_ctx * sctx,u16 attr,struct extent_buffer * eb,struct btrfs_timespec * ts)706 static int tlv_put_btrfs_timespec(struct send_ctx *sctx, u16 attr,
707 struct extent_buffer *eb,
708 struct btrfs_timespec *ts)
709 {
710 struct btrfs_timespec bts;
711 read_extent_buffer(eb, &bts, (unsigned long)ts, sizeof(bts));
712 return tlv_put(sctx, attr, &bts, sizeof(bts));
713 }
714
715
716 #define TLV_PUT(sctx, attrtype, data, attrlen) \
717 do { \
718 ret = tlv_put(sctx, attrtype, data, attrlen); \
719 if (ret < 0) \
720 goto tlv_put_failure; \
721 } while (0)
722
723 #define TLV_PUT_INT(sctx, attrtype, bits, value) \
724 do { \
725 ret = tlv_put_u##bits(sctx, attrtype, value); \
726 if (ret < 0) \
727 goto tlv_put_failure; \
728 } while (0)
729
730 #define TLV_PUT_U8(sctx, attrtype, data) TLV_PUT_INT(sctx, attrtype, 8, data)
731 #define TLV_PUT_U16(sctx, attrtype, data) TLV_PUT_INT(sctx, attrtype, 16, data)
732 #define TLV_PUT_U32(sctx, attrtype, data) TLV_PUT_INT(sctx, attrtype, 32, data)
733 #define TLV_PUT_U64(sctx, attrtype, data) TLV_PUT_INT(sctx, attrtype, 64, data)
734 #define TLV_PUT_STRING(sctx, attrtype, str, len) \
735 do { \
736 ret = tlv_put_string(sctx, attrtype, str, len); \
737 if (ret < 0) \
738 goto tlv_put_failure; \
739 } while (0)
740 #define TLV_PUT_PATH(sctx, attrtype, p) \
741 do { \
742 ret = tlv_put_string(sctx, attrtype, p->start, \
743 fs_path_len((p))); \
744 if (ret < 0) \
745 goto tlv_put_failure; \
746 } while(0)
747 #define TLV_PUT_UUID(sctx, attrtype, uuid) \
748 do { \
749 ret = tlv_put_uuid(sctx, attrtype, uuid); \
750 if (ret < 0) \
751 goto tlv_put_failure; \
752 } while (0)
753 #define TLV_PUT_BTRFS_TIMESPEC(sctx, attrtype, eb, ts) \
754 do { \
755 ret = tlv_put_btrfs_timespec(sctx, attrtype, eb, ts); \
756 if (ret < 0) \
757 goto tlv_put_failure; \
758 } while (0)
759
send_header(struct send_ctx * sctx)760 static int send_header(struct send_ctx *sctx)
761 {
762 struct btrfs_stream_header hdr;
763
764 strscpy(hdr.magic, BTRFS_SEND_STREAM_MAGIC);
765 hdr.version = cpu_to_le32(sctx->proto);
766 return write_buf(sctx->send_filp, &hdr, sizeof(hdr),
767 &sctx->send_off);
768 }
769
770 /*
771 * For each command/item we want to send to userspace, we call this function.
772 */
begin_cmd(struct send_ctx * sctx,int cmd)773 static int begin_cmd(struct send_ctx *sctx, int cmd)
774 {
775 struct btrfs_cmd_header *hdr;
776
777 if (WARN_ON(!sctx->send_buf))
778 return -EINVAL;
779
780 if (unlikely(sctx->send_size != 0)) {
781 btrfs_err(sctx->send_root->fs_info,
782 "send: command header buffer not empty cmd %d offset %llu",
783 cmd, sctx->send_off);
784 return -EINVAL;
785 }
786
787 sctx->send_size += sizeof(*hdr);
788 hdr = (struct btrfs_cmd_header *)sctx->send_buf;
789 put_unaligned_le16(cmd, &hdr->cmd);
790
791 return 0;
792 }
793
send_cmd(struct send_ctx * sctx)794 static int send_cmd(struct send_ctx *sctx)
795 {
796 int ret;
797 struct btrfs_cmd_header *hdr;
798 u32 crc;
799
800 hdr = (struct btrfs_cmd_header *)sctx->send_buf;
801 put_unaligned_le32(sctx->send_size - sizeof(*hdr), &hdr->len);
802 put_unaligned_le32(0, &hdr->crc);
803
804 crc = crc32c(0, (unsigned char *)sctx->send_buf, sctx->send_size);
805 put_unaligned_le32(crc, &hdr->crc);
806
807 ret = write_buf(sctx->send_filp, sctx->send_buf, sctx->send_size,
808 &sctx->send_off);
809
810 sctx->send_size = 0;
811 sctx->put_data = false;
812
813 return ret;
814 }
815
816 /*
817 * Sends a move instruction to user space
818 */
send_rename(struct send_ctx * sctx,struct fs_path * from,struct fs_path * to)819 static int send_rename(struct send_ctx *sctx,
820 struct fs_path *from, struct fs_path *to)
821 {
822 int ret;
823
824 ret = begin_cmd(sctx, BTRFS_SEND_C_RENAME);
825 if (ret < 0)
826 return ret;
827
828 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, from);
829 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH_TO, to);
830
831 ret = send_cmd(sctx);
832
833 tlv_put_failure:
834 return ret;
835 }
836
837 /*
838 * Sends a link instruction to user space
839 */
send_link(struct send_ctx * sctx,struct fs_path * path,struct fs_path * lnk)840 static int send_link(struct send_ctx *sctx,
841 struct fs_path *path, struct fs_path *lnk)
842 {
843 int ret;
844
845 ret = begin_cmd(sctx, BTRFS_SEND_C_LINK);
846 if (ret < 0)
847 return ret;
848
849 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path);
850 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH_LINK, lnk);
851
852 ret = send_cmd(sctx);
853
854 tlv_put_failure:
855 return ret;
856 }
857
858 /*
859 * Sends an unlink instruction to user space
860 */
send_unlink(struct send_ctx * sctx,struct fs_path * path)861 static int send_unlink(struct send_ctx *sctx, struct fs_path *path)
862 {
863 int ret;
864
865 ret = begin_cmd(sctx, BTRFS_SEND_C_UNLINK);
866 if (ret < 0)
867 return ret;
868
869 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path);
870
871 ret = send_cmd(sctx);
872
873 tlv_put_failure:
874 return ret;
875 }
876
877 /*
878 * Sends a rmdir instruction to user space
879 */
send_rmdir(struct send_ctx * sctx,struct fs_path * path)880 static int send_rmdir(struct send_ctx *sctx, struct fs_path *path)
881 {
882 int ret;
883
884 ret = begin_cmd(sctx, BTRFS_SEND_C_RMDIR);
885 if (ret < 0)
886 return ret;
887
888 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path);
889
890 ret = send_cmd(sctx);
891
892 tlv_put_failure:
893 return ret;
894 }
895
896 struct btrfs_inode_info {
897 u64 size;
898 u64 gen;
899 u64 mode;
900 u64 uid;
901 u64 gid;
902 u64 rdev;
903 u64 fileattr;
904 u64 nlink;
905 };
906
907 /*
908 * Helper function to retrieve some fields from an inode item.
909 */
get_inode_info(struct btrfs_root * root,u64 ino,struct btrfs_inode_info * info)910 static int get_inode_info(struct btrfs_root *root, u64 ino,
911 struct btrfs_inode_info *info)
912 {
913 int ret;
914 BTRFS_PATH_AUTO_FREE(path);
915 struct btrfs_inode_item *ii;
916 struct btrfs_key key;
917
918 path = alloc_path_for_send();
919 if (!path)
920 return -ENOMEM;
921
922 key.objectid = ino;
923 key.type = BTRFS_INODE_ITEM_KEY;
924 key.offset = 0;
925 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
926 if (ret) {
927 if (ret > 0)
928 ret = -ENOENT;
929 return ret;
930 }
931
932 if (!info)
933 return 0;
934
935 ii = btrfs_item_ptr(path->nodes[0], path->slots[0],
936 struct btrfs_inode_item);
937 info->size = btrfs_inode_size(path->nodes[0], ii);
938 info->gen = btrfs_inode_generation(path->nodes[0], ii);
939 info->mode = btrfs_inode_mode(path->nodes[0], ii);
940 info->uid = btrfs_inode_uid(path->nodes[0], ii);
941 info->gid = btrfs_inode_gid(path->nodes[0], ii);
942 info->rdev = btrfs_inode_rdev(path->nodes[0], ii);
943 info->nlink = btrfs_inode_nlink(path->nodes[0], ii);
944 /*
945 * Transfer the unchanged u64 value of btrfs_inode_item::flags, that's
946 * otherwise logically split to 32/32 parts.
947 */
948 info->fileattr = btrfs_inode_flags(path->nodes[0], ii);
949
950 return 0;
951 }
952
get_inode_gen(struct btrfs_root * root,u64 ino,u64 * gen)953 static int get_inode_gen(struct btrfs_root *root, u64 ino, u64 *gen)
954 {
955 int ret;
956 struct btrfs_inode_info info = { 0 };
957
958 ASSERT(gen);
959
960 ret = get_inode_info(root, ino, &info);
961 *gen = info.gen;
962 return ret;
963 }
964
965 typedef int (*iterate_inode_ref_t)(u64 dir, struct fs_path *p, void *ctx);
966
967 /*
968 * Helper function to iterate the entries in ONE btrfs_inode_ref or
969 * btrfs_inode_extref.
970 * The iterate callback may return a non zero value to stop iteration. This can
971 * be a negative value for error codes or 1 to simply stop it.
972 *
973 * path must point to the INODE_REF or INODE_EXTREF when called.
974 */
iterate_inode_ref(struct btrfs_root * root,struct btrfs_path * path,struct btrfs_key * found_key,bool resolve,iterate_inode_ref_t iterate,void * ctx)975 static int iterate_inode_ref(struct btrfs_root *root, struct btrfs_path *path,
976 struct btrfs_key *found_key, bool resolve,
977 iterate_inode_ref_t iterate, void *ctx)
978 {
979 struct extent_buffer *eb = path->nodes[0];
980 struct btrfs_inode_ref *iref;
981 struct btrfs_inode_extref *extref;
982 BTRFS_PATH_AUTO_FREE(tmp_path);
983 struct fs_path *p;
984 u32 cur = 0;
985 u32 total;
986 int slot = path->slots[0];
987 u32 name_len;
988 char *start;
989 int ret = 0;
990 u64 dir;
991 unsigned long name_off;
992 unsigned long elem_size;
993 unsigned long ptr;
994
995 p = fs_path_alloc_reversed();
996 if (!p)
997 return -ENOMEM;
998
999 tmp_path = alloc_path_for_send();
1000 if (!tmp_path) {
1001 fs_path_free(p);
1002 return -ENOMEM;
1003 }
1004
1005
1006 if (found_key->type == BTRFS_INODE_REF_KEY) {
1007 ptr = (unsigned long)btrfs_item_ptr(eb, slot,
1008 struct btrfs_inode_ref);
1009 total = btrfs_item_size(eb, slot);
1010 elem_size = sizeof(*iref);
1011 } else {
1012 ptr = btrfs_item_ptr_offset(eb, slot);
1013 total = btrfs_item_size(eb, slot);
1014 elem_size = sizeof(*extref);
1015 }
1016
1017 while (cur < total) {
1018 fs_path_reset(p);
1019
1020 if (found_key->type == BTRFS_INODE_REF_KEY) {
1021 iref = (struct btrfs_inode_ref *)(ptr + cur);
1022 name_len = btrfs_inode_ref_name_len(eb, iref);
1023 name_off = (unsigned long)(iref + 1);
1024 dir = found_key->offset;
1025 } else {
1026 extref = (struct btrfs_inode_extref *)(ptr + cur);
1027 name_len = btrfs_inode_extref_name_len(eb, extref);
1028 name_off = (unsigned long)&extref->name;
1029 dir = btrfs_inode_extref_parent(eb, extref);
1030 }
1031
1032 if (resolve) {
1033 start = btrfs_ref_to_path(root, tmp_path, name_len,
1034 name_off, eb, dir,
1035 p->buf, p->buf_len);
1036 if (IS_ERR(start)) {
1037 ret = PTR_ERR(start);
1038 goto out;
1039 }
1040 if (start < p->buf) {
1041 /* overflow , try again with larger buffer */
1042 ret = fs_path_ensure_buf(p,
1043 p->buf_len + p->buf - start);
1044 if (ret < 0)
1045 goto out;
1046 start = btrfs_ref_to_path(root, tmp_path,
1047 name_len, name_off,
1048 eb, dir,
1049 p->buf, p->buf_len);
1050 if (IS_ERR(start)) {
1051 ret = PTR_ERR(start);
1052 goto out;
1053 }
1054 if (unlikely(start < p->buf)) {
1055 btrfs_err(root->fs_info,
1056 "send: path ref buffer underflow for key (%llu %u %llu)",
1057 found_key->objectid,
1058 found_key->type,
1059 found_key->offset);
1060 ret = -EINVAL;
1061 goto out;
1062 }
1063 }
1064 p->start = start;
1065 } else {
1066 ret = fs_path_add_from_extent_buffer(p, eb, name_off,
1067 name_len);
1068 if (ret < 0)
1069 goto out;
1070 }
1071
1072 cur += elem_size + name_len;
1073 ret = iterate(dir, p, ctx);
1074 if (ret)
1075 goto out;
1076 }
1077
1078 out:
1079 fs_path_free(p);
1080 return ret;
1081 }
1082
1083 typedef int (*iterate_dir_item_t)(int num, struct btrfs_key *di_key,
1084 const char *name, int name_len,
1085 const char *data, int data_len,
1086 void *ctx);
1087
1088 /*
1089 * Helper function to iterate the entries in ONE btrfs_dir_item.
1090 * The iterate callback may return a non zero value to stop iteration. This can
1091 * be a negative value for error codes or 1 to simply stop it.
1092 *
1093 * path must point to the dir item when called.
1094 */
iterate_dir_item(struct btrfs_root * root,struct btrfs_path * path,iterate_dir_item_t iterate,void * ctx)1095 static int iterate_dir_item(struct btrfs_root *root, struct btrfs_path *path,
1096 iterate_dir_item_t iterate, void *ctx)
1097 {
1098 int ret = 0;
1099 struct extent_buffer *eb;
1100 struct btrfs_dir_item *di;
1101 struct btrfs_key di_key;
1102 char *buf = NULL;
1103 int buf_len;
1104 u32 name_len;
1105 u32 data_len;
1106 u32 cur;
1107 u32 len;
1108 u32 total;
1109 int slot;
1110 int num;
1111
1112 /*
1113 * Start with a small buffer (1 page). If later we end up needing more
1114 * space, which can happen for xattrs on a fs with a leaf size greater
1115 * than the page size, attempt to increase the buffer. Typically xattr
1116 * values are small.
1117 */
1118 buf_len = PATH_MAX;
1119 buf = kmalloc(buf_len, GFP_KERNEL);
1120 if (!buf) {
1121 ret = -ENOMEM;
1122 goto out;
1123 }
1124
1125 eb = path->nodes[0];
1126 slot = path->slots[0];
1127 di = btrfs_item_ptr(eb, slot, struct btrfs_dir_item);
1128 cur = 0;
1129 len = 0;
1130 total = btrfs_item_size(eb, slot);
1131
1132 num = 0;
1133 while (cur < total) {
1134 name_len = btrfs_dir_name_len(eb, di);
1135 data_len = btrfs_dir_data_len(eb, di);
1136 btrfs_dir_item_key_to_cpu(eb, di, &di_key);
1137
1138 if (btrfs_dir_ftype(eb, di) == BTRFS_FT_XATTR) {
1139 if (name_len > XATTR_NAME_MAX) {
1140 ret = -ENAMETOOLONG;
1141 goto out;
1142 }
1143 if (name_len + data_len >
1144 BTRFS_MAX_XATTR_SIZE(root->fs_info)) {
1145 ret = -E2BIG;
1146 goto out;
1147 }
1148 } else {
1149 /*
1150 * Path too long
1151 */
1152 if (name_len + data_len > PATH_MAX) {
1153 ret = -ENAMETOOLONG;
1154 goto out;
1155 }
1156 }
1157
1158 if (name_len + data_len > buf_len) {
1159 buf_len = name_len + data_len;
1160 if (is_vmalloc_addr(buf)) {
1161 vfree(buf);
1162 buf = NULL;
1163 } else {
1164 char *tmp = krealloc(buf, buf_len,
1165 GFP_KERNEL | __GFP_NOWARN);
1166
1167 if (!tmp)
1168 kfree(buf);
1169 buf = tmp;
1170 }
1171 if (!buf) {
1172 buf = kvmalloc(buf_len, GFP_KERNEL);
1173 if (!buf) {
1174 ret = -ENOMEM;
1175 goto out;
1176 }
1177 }
1178 }
1179
1180 read_extent_buffer(eb, buf, (unsigned long)(di + 1),
1181 name_len + data_len);
1182
1183 len = sizeof(*di) + name_len + data_len;
1184 di = (struct btrfs_dir_item *)((char *)di + len);
1185 cur += len;
1186
1187 ret = iterate(num, &di_key, buf, name_len, buf + name_len,
1188 data_len, ctx);
1189 if (ret < 0)
1190 goto out;
1191 if (ret) {
1192 ret = 0;
1193 goto out;
1194 }
1195
1196 num++;
1197 }
1198
1199 out:
1200 kvfree(buf);
1201 return ret;
1202 }
1203
__copy_first_ref(u64 dir,struct fs_path * p,void * ctx)1204 static int __copy_first_ref(u64 dir, struct fs_path *p, void *ctx)
1205 {
1206 int ret;
1207 struct fs_path *pt = ctx;
1208
1209 ret = fs_path_copy(pt, p);
1210 if (ret < 0)
1211 return ret;
1212
1213 /* we want the first only */
1214 return 1;
1215 }
1216
1217 /*
1218 * Retrieve the first path of an inode. If an inode has more then one
1219 * ref/hardlink, this is ignored.
1220 */
get_inode_path(struct btrfs_root * root,u64 ino,struct fs_path * path)1221 static int get_inode_path(struct btrfs_root *root,
1222 u64 ino, struct fs_path *path)
1223 {
1224 int ret;
1225 struct btrfs_key key, found_key;
1226 BTRFS_PATH_AUTO_FREE(p);
1227
1228 p = alloc_path_for_send();
1229 if (!p)
1230 return -ENOMEM;
1231
1232 fs_path_reset(path);
1233
1234 key.objectid = ino;
1235 key.type = BTRFS_INODE_REF_KEY;
1236 key.offset = 0;
1237
1238 ret = btrfs_search_slot_for_read(root, &key, p, 1, 0);
1239 if (ret < 0)
1240 return ret;
1241 if (ret)
1242 return 1;
1243
1244 btrfs_item_key_to_cpu(p->nodes[0], &found_key, p->slots[0]);
1245 if (found_key.objectid != ino ||
1246 (found_key.type != BTRFS_INODE_REF_KEY &&
1247 found_key.type != BTRFS_INODE_EXTREF_KEY))
1248 return -ENOENT;
1249
1250 ret = iterate_inode_ref(root, p, &found_key, true, __copy_first_ref, path);
1251 if (ret < 0)
1252 return ret;
1253 return 0;
1254 }
1255
1256 struct backref_ctx {
1257 struct send_ctx *sctx;
1258
1259 /* number of total found references */
1260 u64 found;
1261
1262 /*
1263 * used for clones found in send_root. clones found behind cur_objectid
1264 * and cur_offset are not considered as allowed clones.
1265 */
1266 u64 cur_objectid;
1267 u64 cur_offset;
1268
1269 /* may be truncated in case it's the last extent in a file */
1270 u64 extent_len;
1271
1272 /* The bytenr the file extent item we are processing refers to. */
1273 u64 bytenr;
1274 /* The owner (root id) of the data backref for the current extent. */
1275 u64 backref_owner;
1276 /* The offset of the data backref for the current extent. */
1277 u64 backref_offset;
1278 };
1279
__clone_root_cmp_bsearch(const void * key,const void * elt)1280 static int __clone_root_cmp_bsearch(const void *key, const void *elt)
1281 {
1282 u64 root = (u64)(uintptr_t)key;
1283 const struct clone_root *cr = elt;
1284
1285 if (root < btrfs_root_id(cr->root))
1286 return -1;
1287 if (root > btrfs_root_id(cr->root))
1288 return 1;
1289 return 0;
1290 }
1291
__clone_root_cmp_sort(const void * e1,const void * e2)1292 static int __clone_root_cmp_sort(const void *e1, const void *e2)
1293 {
1294 const struct clone_root *cr1 = e1;
1295 const struct clone_root *cr2 = e2;
1296
1297 if (btrfs_root_id(cr1->root) < btrfs_root_id(cr2->root))
1298 return -1;
1299 if (btrfs_root_id(cr1->root) > btrfs_root_id(cr2->root))
1300 return 1;
1301 return 0;
1302 }
1303
1304 /*
1305 * Called for every backref that is found for the current extent.
1306 * Results are collected in sctx->clone_roots->ino/offset.
1307 */
iterate_backrefs(u64 ino,u64 offset,u64 num_bytes,u64 root_id,void * ctx_)1308 static int iterate_backrefs(u64 ino, u64 offset, u64 num_bytes, u64 root_id,
1309 void *ctx_)
1310 {
1311 struct backref_ctx *bctx = ctx_;
1312 struct clone_root *clone_root;
1313
1314 /* First check if the root is in the list of accepted clone sources */
1315 clone_root = bsearch((void *)(uintptr_t)root_id, bctx->sctx->clone_roots,
1316 bctx->sctx->clone_roots_cnt,
1317 sizeof(struct clone_root),
1318 __clone_root_cmp_bsearch);
1319 if (!clone_root)
1320 return 0;
1321
1322 /* This is our own reference, bail out as we can't clone from it. */
1323 if (clone_root->root == bctx->sctx->send_root &&
1324 ino == bctx->cur_objectid &&
1325 offset == bctx->cur_offset)
1326 return 0;
1327
1328 /*
1329 * Make sure we don't consider clones from send_root that are
1330 * behind the current inode/offset.
1331 */
1332 if (clone_root->root == bctx->sctx->send_root) {
1333 /*
1334 * If the source inode was not yet processed we can't issue a
1335 * clone operation, as the source extent does not exist yet at
1336 * the destination of the stream.
1337 */
1338 if (ino > bctx->cur_objectid)
1339 return 0;
1340 /*
1341 * We clone from the inode currently being sent as long as the
1342 * source extent is already processed, otherwise we could try
1343 * to clone from an extent that does not exist yet at the
1344 * destination of the stream.
1345 */
1346 if (ino == bctx->cur_objectid &&
1347 offset + bctx->extent_len >
1348 bctx->sctx->cur_inode_next_write_offset)
1349 return 0;
1350 }
1351
1352 bctx->found++;
1353 clone_root->found_ref = true;
1354
1355 /*
1356 * If the given backref refers to a file extent item with a larger
1357 * number of bytes than what we found before, use the new one so that
1358 * we clone more optimally and end up doing less writes and getting
1359 * less exclusive, non-shared extents at the destination.
1360 */
1361 if (num_bytes > clone_root->num_bytes) {
1362 clone_root->ino = ino;
1363 clone_root->offset = offset;
1364 clone_root->num_bytes = num_bytes;
1365
1366 /*
1367 * Found a perfect candidate, so there's no need to continue
1368 * backref walking.
1369 */
1370 if (num_bytes >= bctx->extent_len)
1371 return BTRFS_ITERATE_EXTENT_INODES_STOP;
1372 }
1373
1374 return 0;
1375 }
1376
lookup_backref_cache(u64 leaf_bytenr,void * ctx,const u64 ** root_ids_ret,int * root_count_ret)1377 static bool lookup_backref_cache(u64 leaf_bytenr, void *ctx,
1378 const u64 **root_ids_ret, int *root_count_ret)
1379 {
1380 struct backref_ctx *bctx = ctx;
1381 struct send_ctx *sctx = bctx->sctx;
1382 struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
1383 const u64 key = leaf_bytenr >> fs_info->nodesize_bits;
1384 struct btrfs_lru_cache_entry *raw_entry;
1385 struct backref_cache_entry *entry;
1386
1387 if (sctx->backref_cache.size == 0)
1388 return false;
1389
1390 /*
1391 * If relocation happened since we first filled the cache, then we must
1392 * empty the cache and can not use it, because even though we operate on
1393 * read-only roots, their leaves and nodes may have been reallocated and
1394 * now be used for different nodes/leaves of the same tree or some other
1395 * tree.
1396 *
1397 * We are called from iterate_extent_inodes() while either holding a
1398 * transaction handle or holding fs_info->commit_root_sem, so no need
1399 * to take any lock here.
1400 */
1401 if (fs_info->last_reloc_trans > sctx->backref_cache_last_reloc_trans) {
1402 btrfs_lru_cache_clear(&sctx->backref_cache);
1403 return false;
1404 }
1405
1406 raw_entry = btrfs_lru_cache_lookup(&sctx->backref_cache, key, 0);
1407 if (!raw_entry)
1408 return false;
1409
1410 entry = container_of(raw_entry, struct backref_cache_entry, entry);
1411 *root_ids_ret = entry->root_ids;
1412 *root_count_ret = entry->num_roots;
1413
1414 return true;
1415 }
1416
store_backref_cache(u64 leaf_bytenr,const struct ulist * root_ids,void * ctx)1417 static void store_backref_cache(u64 leaf_bytenr, const struct ulist *root_ids,
1418 void *ctx)
1419 {
1420 struct backref_ctx *bctx = ctx;
1421 struct send_ctx *sctx = bctx->sctx;
1422 struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
1423 struct backref_cache_entry *new_entry;
1424 struct ulist_iterator uiter;
1425 struct ulist_node *node;
1426 int ret;
1427
1428 /*
1429 * We're called while holding a transaction handle or while holding
1430 * fs_info->commit_root_sem (at iterate_extent_inodes()), so must do a
1431 * NOFS allocation.
1432 */
1433 new_entry = kmalloc(sizeof(struct backref_cache_entry), GFP_NOFS);
1434 /* No worries, cache is optional. */
1435 if (!new_entry)
1436 return;
1437
1438 new_entry->entry.key = leaf_bytenr >> fs_info->nodesize_bits;
1439 new_entry->entry.gen = 0;
1440 new_entry->num_roots = 0;
1441 ULIST_ITER_INIT(&uiter);
1442 while ((node = ulist_next(root_ids, &uiter)) != NULL) {
1443 const u64 root_id = node->val;
1444 struct clone_root *root;
1445
1446 root = bsearch((void *)(uintptr_t)root_id, sctx->clone_roots,
1447 sctx->clone_roots_cnt, sizeof(struct clone_root),
1448 __clone_root_cmp_bsearch);
1449 if (!root)
1450 continue;
1451
1452 /* Too many roots, just exit, no worries as caching is optional. */
1453 if (new_entry->num_roots >= SEND_MAX_BACKREF_CACHE_ROOTS) {
1454 kfree(new_entry);
1455 return;
1456 }
1457
1458 new_entry->root_ids[new_entry->num_roots] = root_id;
1459 new_entry->num_roots++;
1460 }
1461
1462 /*
1463 * We may have not added any roots to the new cache entry, which means
1464 * none of the roots is part of the list of roots from which we are
1465 * allowed to clone. Cache the new entry as it's still useful to avoid
1466 * backref walking to determine which roots have a path to the leaf.
1467 *
1468 * Also use GFP_NOFS because we're called while holding a transaction
1469 * handle or while holding fs_info->commit_root_sem.
1470 */
1471 ret = btrfs_lru_cache_store(&sctx->backref_cache, &new_entry->entry,
1472 GFP_NOFS);
1473 ASSERT(ret == 0 || ret == -ENOMEM);
1474 if (ret) {
1475 /* Caching is optional, no worries. */
1476 kfree(new_entry);
1477 return;
1478 }
1479
1480 /*
1481 * We are called from iterate_extent_inodes() while either holding a
1482 * transaction handle or holding fs_info->commit_root_sem, so no need
1483 * to take any lock here.
1484 */
1485 if (sctx->backref_cache.size == 1)
1486 sctx->backref_cache_last_reloc_trans = fs_info->last_reloc_trans;
1487 }
1488
check_extent_item(u64 bytenr,const struct btrfs_extent_item * ei,const struct extent_buffer * leaf,void * ctx)1489 static int check_extent_item(u64 bytenr, const struct btrfs_extent_item *ei,
1490 const struct extent_buffer *leaf, void *ctx)
1491 {
1492 const u64 refs = btrfs_extent_refs(leaf, ei);
1493 const struct backref_ctx *bctx = ctx;
1494 const struct send_ctx *sctx = bctx->sctx;
1495
1496 if (bytenr == bctx->bytenr) {
1497 const u64 flags = btrfs_extent_flags(leaf, ei);
1498
1499 if (WARN_ON(flags & BTRFS_EXTENT_FLAG_TREE_BLOCK))
1500 return -EUCLEAN;
1501
1502 /*
1503 * If we have only one reference and only the send root as a
1504 * clone source - meaning no clone roots were given in the
1505 * struct btrfs_ioctl_send_args passed to the send ioctl - then
1506 * it's our reference and there's no point in doing backref
1507 * walking which is expensive, so exit early.
1508 */
1509 if (refs == 1 && sctx->clone_roots_cnt == 1)
1510 return -ENOENT;
1511 }
1512
1513 /*
1514 * Backreference walking (iterate_extent_inodes() below) is currently
1515 * too expensive when an extent has a large number of references, both
1516 * in time spent and used memory. So for now just fallback to write
1517 * operations instead of clone operations when an extent has more than
1518 * a certain amount of references.
1519 */
1520 if (refs > SEND_MAX_EXTENT_REFS)
1521 return -ENOENT;
1522
1523 return 0;
1524 }
1525
skip_self_data_ref(u64 root,u64 ino,u64 offset,void * ctx)1526 static bool skip_self_data_ref(u64 root, u64 ino, u64 offset, void *ctx)
1527 {
1528 const struct backref_ctx *bctx = ctx;
1529
1530 if (ino == bctx->cur_objectid &&
1531 root == bctx->backref_owner &&
1532 offset == bctx->backref_offset)
1533 return true;
1534
1535 return false;
1536 }
1537
1538 /*
1539 * Given an inode, offset and extent item, it finds a good clone for a clone
1540 * instruction. Returns -ENOENT when none could be found. The function makes
1541 * sure that the returned clone is usable at the point where sending is at the
1542 * moment. This means, that no clones are accepted which lie behind the current
1543 * inode+offset.
1544 *
1545 * path must point to the extent item when called.
1546 */
find_extent_clone(struct send_ctx * sctx,struct btrfs_path * path,u64 ino,u64 data_offset,u64 ino_size,struct clone_root ** found)1547 static int find_extent_clone(struct send_ctx *sctx,
1548 struct btrfs_path *path,
1549 u64 ino, u64 data_offset,
1550 u64 ino_size,
1551 struct clone_root **found)
1552 {
1553 struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
1554 int ret;
1555 int extent_type;
1556 u64 disk_byte;
1557 u64 num_bytes;
1558 struct btrfs_file_extent_item *fi;
1559 struct extent_buffer *eb = path->nodes[0];
1560 struct backref_ctx backref_ctx = { 0 };
1561 struct btrfs_backref_walk_ctx backref_walk_ctx = { 0 };
1562 struct clone_root *cur_clone_root;
1563 int compressed;
1564 u32 i;
1565
1566 /*
1567 * With fallocate we can get prealloc extents beyond the inode's i_size,
1568 * so we don't do anything here because clone operations can not clone
1569 * to a range beyond i_size without increasing the i_size of the
1570 * destination inode.
1571 */
1572 if (data_offset >= ino_size)
1573 return 0;
1574
1575 fi = btrfs_item_ptr(eb, path->slots[0], struct btrfs_file_extent_item);
1576 extent_type = btrfs_file_extent_type(eb, fi);
1577 if (extent_type == BTRFS_FILE_EXTENT_INLINE)
1578 return -ENOENT;
1579
1580 disk_byte = btrfs_file_extent_disk_bytenr(eb, fi);
1581 if (disk_byte == 0)
1582 return -ENOENT;
1583
1584 compressed = btrfs_file_extent_compression(eb, fi);
1585 num_bytes = btrfs_file_extent_num_bytes(eb, fi);
1586
1587 /*
1588 * Setup the clone roots.
1589 */
1590 for (i = 0; i < sctx->clone_roots_cnt; i++) {
1591 cur_clone_root = sctx->clone_roots + i;
1592 cur_clone_root->ino = (u64)-1;
1593 cur_clone_root->offset = 0;
1594 cur_clone_root->num_bytes = 0;
1595 cur_clone_root->found_ref = false;
1596 }
1597
1598 backref_ctx.sctx = sctx;
1599 backref_ctx.cur_objectid = ino;
1600 backref_ctx.cur_offset = data_offset;
1601 backref_ctx.bytenr = disk_byte;
1602 /*
1603 * Use the header owner and not the send root's id, because in case of a
1604 * snapshot we can have shared subtrees.
1605 */
1606 backref_ctx.backref_owner = btrfs_header_owner(eb);
1607 backref_ctx.backref_offset = data_offset - btrfs_file_extent_offset(eb, fi);
1608
1609 /*
1610 * The last extent of a file may be too large due to page alignment.
1611 * We need to adjust extent_len in this case so that the checks in
1612 * iterate_backrefs() work.
1613 */
1614 if (data_offset + num_bytes >= ino_size)
1615 backref_ctx.extent_len = ino_size - data_offset;
1616 else
1617 backref_ctx.extent_len = num_bytes;
1618
1619 /*
1620 * Now collect all backrefs.
1621 */
1622 backref_walk_ctx.bytenr = disk_byte;
1623 if (compressed == BTRFS_COMPRESS_NONE)
1624 backref_walk_ctx.extent_item_pos = btrfs_file_extent_offset(eb, fi);
1625 backref_walk_ctx.fs_info = fs_info;
1626 backref_walk_ctx.cache_lookup = lookup_backref_cache;
1627 backref_walk_ctx.cache_store = store_backref_cache;
1628 backref_walk_ctx.indirect_ref_iterator = iterate_backrefs;
1629 backref_walk_ctx.check_extent_item = check_extent_item;
1630 backref_walk_ctx.user_ctx = &backref_ctx;
1631
1632 /*
1633 * If have a single clone root, then it's the send root and we can tell
1634 * the backref walking code to skip our own backref and not resolve it,
1635 * since we can not use it for cloning - the source and destination
1636 * ranges can't overlap and in case the leaf is shared through a subtree
1637 * due to snapshots, we can't use those other roots since they are not
1638 * in the list of clone roots.
1639 */
1640 if (sctx->clone_roots_cnt == 1)
1641 backref_walk_ctx.skip_data_ref = skip_self_data_ref;
1642
1643 ret = iterate_extent_inodes(&backref_walk_ctx, true, iterate_backrefs,
1644 &backref_ctx);
1645 if (ret < 0)
1646 return ret;
1647
1648 down_read(&fs_info->commit_root_sem);
1649 if (fs_info->last_reloc_trans > sctx->last_reloc_trans) {
1650 /*
1651 * A transaction commit for a transaction in which block group
1652 * relocation was done just happened.
1653 * The disk_bytenr of the file extent item we processed is
1654 * possibly stale, referring to the extent's location before
1655 * relocation. So act as if we haven't found any clone sources
1656 * and fallback to write commands, which will read the correct
1657 * data from the new extent location. Otherwise we will fail
1658 * below because we haven't found our own back reference or we
1659 * could be getting incorrect sources in case the old extent
1660 * was already reallocated after the relocation.
1661 */
1662 up_read(&fs_info->commit_root_sem);
1663 return -ENOENT;
1664 }
1665 up_read(&fs_info->commit_root_sem);
1666
1667 if (!backref_ctx.found)
1668 return -ENOENT;
1669
1670 cur_clone_root = NULL;
1671 for (i = 0; i < sctx->clone_roots_cnt; i++) {
1672 struct clone_root *clone_root = &sctx->clone_roots[i];
1673
1674 if (!clone_root->found_ref)
1675 continue;
1676
1677 /*
1678 * Choose the root from which we can clone more bytes, to
1679 * minimize write operations and therefore have more extent
1680 * sharing at the destination (the same as in the source).
1681 */
1682 if (!cur_clone_root ||
1683 clone_root->num_bytes > cur_clone_root->num_bytes) {
1684 cur_clone_root = clone_root;
1685
1686 /*
1687 * We found an optimal clone candidate (any inode from
1688 * any root is fine), so we're done.
1689 */
1690 if (clone_root->num_bytes >= backref_ctx.extent_len)
1691 break;
1692 }
1693 }
1694
1695 if (cur_clone_root) {
1696 *found = cur_clone_root;
1697 ret = 0;
1698 } else {
1699 ret = -ENOENT;
1700 }
1701
1702 return ret;
1703 }
1704
read_symlink(struct btrfs_root * root,u64 ino,struct fs_path * dest)1705 static int read_symlink(struct btrfs_root *root,
1706 u64 ino,
1707 struct fs_path *dest)
1708 {
1709 int ret;
1710 BTRFS_PATH_AUTO_FREE(path);
1711 struct btrfs_key key;
1712 struct btrfs_file_extent_item *ei;
1713 u8 type;
1714 u8 compression;
1715 unsigned long off;
1716 int len;
1717
1718 path = alloc_path_for_send();
1719 if (!path)
1720 return -ENOMEM;
1721
1722 key.objectid = ino;
1723 key.type = BTRFS_EXTENT_DATA_KEY;
1724 key.offset = 0;
1725 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
1726 if (ret < 0)
1727 return ret;
1728 if (unlikely(ret)) {
1729 /*
1730 * An empty symlink inode. Can happen in rare error paths when
1731 * creating a symlink (transaction committed before the inode
1732 * eviction handler removed the symlink inode items and a crash
1733 * happened in between or the subvol was snapshotted in between).
1734 * Print an informative message to dmesg/syslog so that the user
1735 * can delete the symlink.
1736 */
1737 btrfs_err(root->fs_info,
1738 "Found empty symlink inode %llu at root %llu",
1739 ino, btrfs_root_id(root));
1740 return -EIO;
1741 }
1742
1743 ei = btrfs_item_ptr(path->nodes[0], path->slots[0],
1744 struct btrfs_file_extent_item);
1745 type = btrfs_file_extent_type(path->nodes[0], ei);
1746 if (unlikely(type != BTRFS_FILE_EXTENT_INLINE)) {
1747 ret = -EUCLEAN;
1748 btrfs_crit(root->fs_info,
1749 "send: found symlink extent that is not inline, ino %llu root %llu extent type %d",
1750 ino, btrfs_root_id(root), type);
1751 return ret;
1752 }
1753 compression = btrfs_file_extent_compression(path->nodes[0], ei);
1754 if (unlikely(compression != BTRFS_COMPRESS_NONE)) {
1755 ret = -EUCLEAN;
1756 btrfs_crit(root->fs_info,
1757 "send: found symlink extent with compression, ino %llu root %llu compression type %d",
1758 ino, btrfs_root_id(root), compression);
1759 return ret;
1760 }
1761
1762 off = btrfs_file_extent_inline_start(ei);
1763 len = btrfs_file_extent_ram_bytes(path->nodes[0], ei);
1764
1765 return fs_path_add_from_extent_buffer(dest, path->nodes[0], off, len);
1766 }
1767
1768 /*
1769 * Helper function to generate a file name that is unique in the root of
1770 * send_root and parent_root. This is used to generate names for orphan inodes.
1771 */
gen_unique_name(struct send_ctx * sctx,u64 ino,u64 gen,struct fs_path * dest)1772 static int gen_unique_name(struct send_ctx *sctx,
1773 u64 ino, u64 gen,
1774 struct fs_path *dest)
1775 {
1776 BTRFS_PATH_AUTO_FREE(path);
1777 struct btrfs_dir_item *di;
1778 char tmp[64];
1779 int len;
1780 u64 idx = 0;
1781
1782 path = alloc_path_for_send();
1783 if (!path)
1784 return -ENOMEM;
1785
1786 while (1) {
1787 struct fscrypt_str tmp_name;
1788
1789 len = snprintf(tmp, sizeof(tmp), "o%llu-%llu-%llu",
1790 ino, gen, idx);
1791 ASSERT(len < sizeof(tmp));
1792 tmp_name.name = tmp;
1793 tmp_name.len = len;
1794
1795 di = btrfs_lookup_dir_item(NULL, sctx->send_root,
1796 path, BTRFS_FIRST_FREE_OBJECTID,
1797 &tmp_name, 0);
1798 btrfs_release_path(path);
1799 if (IS_ERR(di))
1800 return PTR_ERR(di);
1801
1802 if (di) {
1803 /* not unique, try again */
1804 idx++;
1805 continue;
1806 }
1807
1808 if (!sctx->parent_root) {
1809 /* unique */
1810 break;
1811 }
1812
1813 di = btrfs_lookup_dir_item(NULL, sctx->parent_root,
1814 path, BTRFS_FIRST_FREE_OBJECTID,
1815 &tmp_name, 0);
1816 btrfs_release_path(path);
1817 if (IS_ERR(di))
1818 return PTR_ERR(di);
1819
1820 if (di) {
1821 /* not unique, try again */
1822 idx++;
1823 continue;
1824 }
1825 /* unique */
1826 break;
1827 }
1828
1829 return fs_path_add(dest, tmp, len);
1830 }
1831
1832 enum inode_state {
1833 inode_state_no_change,
1834 inode_state_will_create,
1835 inode_state_did_create,
1836 inode_state_will_delete,
1837 inode_state_did_delete,
1838 };
1839
get_cur_inode_state(struct send_ctx * sctx,u64 ino,u64 gen,u64 * send_gen,u64 * parent_gen)1840 static int get_cur_inode_state(struct send_ctx *sctx, u64 ino, u64 gen,
1841 u64 *send_gen, u64 *parent_gen)
1842 {
1843 int ret;
1844 int left_ret;
1845 int right_ret;
1846 u64 left_gen;
1847 u64 right_gen = 0;
1848 struct btrfs_inode_info info;
1849
1850 ret = get_inode_info(sctx->send_root, ino, &info);
1851 if (ret < 0 && ret != -ENOENT)
1852 return ret;
1853 left_ret = (info.nlink == 0) ? -ENOENT : ret;
1854 left_gen = info.gen;
1855 if (send_gen)
1856 *send_gen = ((left_ret == -ENOENT) ? 0 : info.gen);
1857
1858 if (!sctx->parent_root) {
1859 right_ret = -ENOENT;
1860 } else {
1861 ret = get_inode_info(sctx->parent_root, ino, &info);
1862 if (ret < 0 && ret != -ENOENT)
1863 return ret;
1864 right_ret = (info.nlink == 0) ? -ENOENT : ret;
1865 right_gen = info.gen;
1866 if (parent_gen)
1867 *parent_gen = ((right_ret == -ENOENT) ? 0 : info.gen);
1868 }
1869
1870 if (!left_ret && !right_ret) {
1871 if (left_gen == gen && right_gen == gen) {
1872 ret = inode_state_no_change;
1873 } else if (left_gen == gen) {
1874 if (ino < sctx->send_progress)
1875 ret = inode_state_did_create;
1876 else
1877 ret = inode_state_will_create;
1878 } else if (right_gen == gen) {
1879 if (ino < sctx->send_progress)
1880 ret = inode_state_did_delete;
1881 else
1882 ret = inode_state_will_delete;
1883 } else {
1884 ret = -ENOENT;
1885 }
1886 } else if (!left_ret) {
1887 if (left_gen == gen) {
1888 if (ino < sctx->send_progress)
1889 ret = inode_state_did_create;
1890 else
1891 ret = inode_state_will_create;
1892 } else {
1893 ret = -ENOENT;
1894 }
1895 } else if (!right_ret) {
1896 if (right_gen == gen) {
1897 if (ino < sctx->send_progress)
1898 ret = inode_state_did_delete;
1899 else
1900 ret = inode_state_will_delete;
1901 } else {
1902 ret = -ENOENT;
1903 }
1904 } else {
1905 ret = -ENOENT;
1906 }
1907
1908 return ret;
1909 }
1910
is_inode_existent(struct send_ctx * sctx,u64 ino,u64 gen,u64 * send_gen,u64 * parent_gen)1911 static int is_inode_existent(struct send_ctx *sctx, u64 ino, u64 gen,
1912 u64 *send_gen, u64 *parent_gen)
1913 {
1914 int ret;
1915
1916 if (ino == BTRFS_FIRST_FREE_OBJECTID)
1917 return 1;
1918
1919 ret = get_cur_inode_state(sctx, ino, gen, send_gen, parent_gen);
1920 if (ret < 0)
1921 return ret;
1922
1923 if (ret == inode_state_no_change ||
1924 ret == inode_state_did_create ||
1925 ret == inode_state_will_delete)
1926 return 1;
1927
1928 return 0;
1929 }
1930
1931 /*
1932 * Helper function to lookup a dir item in a dir.
1933 */
lookup_dir_item_inode(struct btrfs_root * root,u64 dir,const char * name,int name_len,u64 * found_inode)1934 static int lookup_dir_item_inode(struct btrfs_root *root,
1935 u64 dir, const char *name, int name_len,
1936 u64 *found_inode)
1937 {
1938 int ret = 0;
1939 struct btrfs_dir_item *di;
1940 struct btrfs_key key;
1941 BTRFS_PATH_AUTO_FREE(path);
1942 struct fscrypt_str name_str = FSTR_INIT((char *)name, name_len);
1943
1944 path = alloc_path_for_send();
1945 if (!path)
1946 return -ENOMEM;
1947
1948 di = btrfs_lookup_dir_item(NULL, root, path, dir, &name_str, 0);
1949 if (IS_ERR_OR_NULL(di))
1950 return di ? PTR_ERR(di) : -ENOENT;
1951
1952 btrfs_dir_item_key_to_cpu(path->nodes[0], di, &key);
1953 if (key.type == BTRFS_ROOT_ITEM_KEY)
1954 return -ENOENT;
1955
1956 *found_inode = key.objectid;
1957
1958 return ret;
1959 }
1960
1961 /*
1962 * Looks up the first btrfs_inode_ref of a given ino. It returns the parent dir,
1963 * generation of the parent dir and the name of the dir entry.
1964 */
get_first_ref(struct btrfs_root * root,u64 ino,u64 * dir,u64 * dir_gen,struct fs_path * name)1965 static int get_first_ref(struct btrfs_root *root, u64 ino,
1966 u64 *dir, u64 *dir_gen, struct fs_path *name)
1967 {
1968 int ret;
1969 struct btrfs_key key;
1970 struct btrfs_key found_key;
1971 BTRFS_PATH_AUTO_FREE(path);
1972 int len;
1973 u64 parent_dir;
1974
1975 path = alloc_path_for_send();
1976 if (!path)
1977 return -ENOMEM;
1978
1979 key.objectid = ino;
1980 key.type = BTRFS_INODE_REF_KEY;
1981 key.offset = 0;
1982
1983 ret = btrfs_search_slot_for_read(root, &key, path, 1, 0);
1984 if (ret < 0)
1985 return ret;
1986 if (!ret)
1987 btrfs_item_key_to_cpu(path->nodes[0], &found_key,
1988 path->slots[0]);
1989 if (ret || found_key.objectid != ino ||
1990 (found_key.type != BTRFS_INODE_REF_KEY &&
1991 found_key.type != BTRFS_INODE_EXTREF_KEY))
1992 return -ENOENT;
1993
1994 if (found_key.type == BTRFS_INODE_REF_KEY) {
1995 struct btrfs_inode_ref *iref;
1996 iref = btrfs_item_ptr(path->nodes[0], path->slots[0],
1997 struct btrfs_inode_ref);
1998 len = btrfs_inode_ref_name_len(path->nodes[0], iref);
1999 ret = fs_path_add_from_extent_buffer(name, path->nodes[0],
2000 (unsigned long)(iref + 1),
2001 len);
2002 parent_dir = found_key.offset;
2003 } else {
2004 struct btrfs_inode_extref *extref;
2005 extref = btrfs_item_ptr(path->nodes[0], path->slots[0],
2006 struct btrfs_inode_extref);
2007 len = btrfs_inode_extref_name_len(path->nodes[0], extref);
2008 ret = fs_path_add_from_extent_buffer(name, path->nodes[0],
2009 (unsigned long)&extref->name, len);
2010 parent_dir = btrfs_inode_extref_parent(path->nodes[0], extref);
2011 }
2012 if (ret < 0)
2013 return ret;
2014 btrfs_release_path(path);
2015
2016 if (dir_gen) {
2017 ret = get_inode_gen(root, parent_dir, dir_gen);
2018 if (ret < 0)
2019 return ret;
2020 }
2021
2022 *dir = parent_dir;
2023
2024 return ret;
2025 }
2026
is_first_ref(struct btrfs_root * root,u64 ino,u64 dir,const char * name,int name_len)2027 static int is_first_ref(struct btrfs_root *root,
2028 u64 ino, u64 dir,
2029 const char *name, int name_len)
2030 {
2031 int ret;
2032 struct fs_path *tmp_name;
2033 u64 tmp_dir;
2034
2035 tmp_name = fs_path_alloc();
2036 if (!tmp_name)
2037 return -ENOMEM;
2038
2039 ret = get_first_ref(root, ino, &tmp_dir, NULL, tmp_name);
2040 if (ret < 0)
2041 goto out;
2042
2043 if (dir != tmp_dir || name_len != fs_path_len(tmp_name)) {
2044 ret = 0;
2045 goto out;
2046 }
2047
2048 ret = !memcmp(tmp_name->start, name, name_len);
2049
2050 out:
2051 fs_path_free(tmp_name);
2052 return ret;
2053 }
2054
2055 /*
2056 * Used by process_recorded_refs to determine if a new ref would overwrite an
2057 * already existing ref. In case it detects an overwrite, it returns the
2058 * inode/gen in who_ino/who_gen.
2059 * When an overwrite is detected, process_recorded_refs does proper orphanizing
2060 * to make sure later references to the overwritten inode are possible.
2061 * Orphanizing is however only required for the first ref of an inode.
2062 * process_recorded_refs does an additional is_first_ref check to see if
2063 * orphanizing is really required.
2064 */
will_overwrite_ref(struct send_ctx * sctx,u64 dir,u64 dir_gen,const char * name,int name_len,u64 * who_ino,u64 * who_gen,u64 * who_mode)2065 static int will_overwrite_ref(struct send_ctx *sctx, u64 dir, u64 dir_gen,
2066 const char *name, int name_len,
2067 u64 *who_ino, u64 *who_gen, u64 *who_mode)
2068 {
2069 int ret;
2070 u64 parent_root_dir_gen;
2071 u64 other_inode = 0;
2072 struct btrfs_inode_info info;
2073
2074 if (!sctx->parent_root)
2075 return 0;
2076
2077 ret = is_inode_existent(sctx, dir, dir_gen, NULL, &parent_root_dir_gen);
2078 if (ret <= 0)
2079 return 0;
2080
2081 /*
2082 * If we have a parent root we need to verify that the parent dir was
2083 * not deleted and then re-created, if it was then we have no overwrite
2084 * and we can just unlink this entry.
2085 *
2086 * @parent_root_dir_gen was set to 0 if the inode does not exist in the
2087 * parent root.
2088 */
2089 if (sctx->parent_root && dir != BTRFS_FIRST_FREE_OBJECTID &&
2090 parent_root_dir_gen != dir_gen)
2091 return 0;
2092
2093 ret = lookup_dir_item_inode(sctx->parent_root, dir, name, name_len,
2094 &other_inode);
2095 if (ret == -ENOENT)
2096 return 0;
2097 else if (ret < 0)
2098 return ret;
2099
2100 /*
2101 * Check if the overwritten ref was already processed. If yes, the ref
2102 * was already unlinked/moved, so we can safely assume that we will not
2103 * overwrite anything at this point in time.
2104 */
2105 if (other_inode > sctx->send_progress ||
2106 is_waiting_for_move(sctx, other_inode)) {
2107 ret = get_inode_info(sctx->parent_root, other_inode, &info);
2108 if (ret < 0)
2109 return ret;
2110
2111 *who_ino = other_inode;
2112 *who_gen = info.gen;
2113 *who_mode = info.mode;
2114 return 1;
2115 }
2116
2117 return 0;
2118 }
2119
2120 /*
2121 * Checks if the ref was overwritten by an already processed inode. This is
2122 * used by __get_cur_name_and_parent to find out if the ref was orphanized and
2123 * thus the orphan name needs be used.
2124 * process_recorded_refs also uses it to avoid unlinking of refs that were
2125 * overwritten.
2126 */
did_overwrite_ref(struct send_ctx * sctx,u64 dir,u64 dir_gen,u64 ino,u64 ino_gen,const char * name,int name_len)2127 static int did_overwrite_ref(struct send_ctx *sctx,
2128 u64 dir, u64 dir_gen,
2129 u64 ino, u64 ino_gen,
2130 const char *name, int name_len)
2131 {
2132 int ret;
2133 u64 ow_inode;
2134 u64 ow_gen = 0;
2135 u64 send_root_dir_gen;
2136
2137 if (!sctx->parent_root)
2138 return 0;
2139
2140 ret = is_inode_existent(sctx, dir, dir_gen, &send_root_dir_gen, NULL);
2141 if (ret <= 0)
2142 return ret;
2143
2144 /*
2145 * @send_root_dir_gen was set to 0 if the inode does not exist in the
2146 * send root.
2147 */
2148 if (dir != BTRFS_FIRST_FREE_OBJECTID && send_root_dir_gen != dir_gen)
2149 return 0;
2150
2151 /* check if the ref was overwritten by another ref */
2152 ret = lookup_dir_item_inode(sctx->send_root, dir, name, name_len,
2153 &ow_inode);
2154 if (ret == -ENOENT) {
2155 /* was never and will never be overwritten */
2156 return 0;
2157 } else if (ret < 0) {
2158 return ret;
2159 }
2160
2161 if (ow_inode == ino) {
2162 ret = get_inode_gen(sctx->send_root, ow_inode, &ow_gen);
2163 if (ret < 0)
2164 return ret;
2165
2166 /* It's the same inode, so no overwrite happened. */
2167 if (ow_gen == ino_gen)
2168 return 0;
2169 }
2170
2171 /*
2172 * We know that it is or will be overwritten. Check this now.
2173 * The current inode being processed might have been the one that caused
2174 * inode 'ino' to be orphanized, therefore check if ow_inode matches
2175 * the current inode being processed.
2176 */
2177 if (ow_inode < sctx->send_progress)
2178 return 1;
2179
2180 if (ino != sctx->cur_ino && ow_inode == sctx->cur_ino) {
2181 if (ow_gen == 0) {
2182 ret = get_inode_gen(sctx->send_root, ow_inode, &ow_gen);
2183 if (ret < 0)
2184 return ret;
2185 }
2186 if (ow_gen == sctx->cur_inode_gen)
2187 return 1;
2188 }
2189
2190 return 0;
2191 }
2192
2193 /*
2194 * Same as did_overwrite_ref, but also checks if it is the first ref of an inode
2195 * that got overwritten. This is used by process_recorded_refs to determine
2196 * if it has to use the path as returned by get_cur_path or the orphan name.
2197 */
did_overwrite_first_ref(struct send_ctx * sctx,u64 ino,u64 gen)2198 static int did_overwrite_first_ref(struct send_ctx *sctx, u64 ino, u64 gen)
2199 {
2200 int ret = 0;
2201 struct fs_path *name = NULL;
2202 u64 dir;
2203 u64 dir_gen;
2204
2205 if (!sctx->parent_root)
2206 goto out;
2207
2208 name = fs_path_alloc();
2209 if (!name)
2210 return -ENOMEM;
2211
2212 ret = get_first_ref(sctx->parent_root, ino, &dir, &dir_gen, name);
2213 if (ret < 0)
2214 goto out;
2215
2216 ret = did_overwrite_ref(sctx, dir, dir_gen, ino, gen,
2217 name->start, fs_path_len(name));
2218
2219 out:
2220 fs_path_free(name);
2221 return ret;
2222 }
2223
name_cache_search(struct send_ctx * sctx,u64 ino,u64 gen)2224 static inline struct name_cache_entry *name_cache_search(struct send_ctx *sctx,
2225 u64 ino, u64 gen)
2226 {
2227 struct btrfs_lru_cache_entry *entry;
2228
2229 entry = btrfs_lru_cache_lookup(&sctx->name_cache, ino, gen);
2230 if (!entry)
2231 return NULL;
2232
2233 return container_of(entry, struct name_cache_entry, entry);
2234 }
2235
2236 /*
2237 * Used by get_cur_path for each ref up to the root.
2238 * Returns 0 if it succeeded.
2239 * Returns 1 if the inode is not existent or got overwritten. In that case, the
2240 * name is an orphan name. This instructs get_cur_path to stop iterating. If 1
2241 * is returned, parent_ino/parent_gen are not guaranteed to be valid.
2242 * Returns <0 in case of error.
2243 */
__get_cur_name_and_parent(struct send_ctx * sctx,u64 ino,u64 gen,u64 * parent_ino,u64 * parent_gen,struct fs_path * dest)2244 static int __get_cur_name_and_parent(struct send_ctx *sctx,
2245 u64 ino, u64 gen,
2246 u64 *parent_ino,
2247 u64 *parent_gen,
2248 struct fs_path *dest)
2249 {
2250 int ret;
2251 int nce_ret;
2252 struct name_cache_entry *nce;
2253
2254 /*
2255 * First check if we already did a call to this function with the same
2256 * ino/gen. If yes, check if the cache entry is still up-to-date. If yes
2257 * return the cached result.
2258 */
2259 nce = name_cache_search(sctx, ino, gen);
2260 if (nce) {
2261 if (ino < sctx->send_progress && nce->need_later_update) {
2262 btrfs_lru_cache_remove(&sctx->name_cache, &nce->entry);
2263 nce = NULL;
2264 } else {
2265 *parent_ino = nce->parent_ino;
2266 *parent_gen = nce->parent_gen;
2267 ret = fs_path_add(dest, nce->name, nce->name_len);
2268 if (ret < 0)
2269 return ret;
2270 return nce->ret;
2271 }
2272 }
2273
2274 /*
2275 * If the inode is not existent yet, add the orphan name and return 1.
2276 * This should only happen for the parent dir that we determine in
2277 * record_new_ref_if_needed().
2278 */
2279 ret = is_inode_existent(sctx, ino, gen, NULL, NULL);
2280 if (ret < 0)
2281 return ret;
2282
2283 if (!ret) {
2284 ret = gen_unique_name(sctx, ino, gen, dest);
2285 if (ret < 0)
2286 return ret;
2287 ret = 1;
2288 goto out_cache;
2289 }
2290
2291 /*
2292 * Depending on whether the inode was already processed or not, use
2293 * send_root or parent_root for ref lookup.
2294 */
2295 if (ino < sctx->send_progress)
2296 ret = get_first_ref(sctx->send_root, ino,
2297 parent_ino, parent_gen, dest);
2298 else
2299 ret = get_first_ref(sctx->parent_root, ino,
2300 parent_ino, parent_gen, dest);
2301 if (ret < 0)
2302 return ret;
2303
2304 /*
2305 * Check if the ref was overwritten by an inode's ref that was processed
2306 * earlier. If yes, treat as orphan and return 1.
2307 */
2308 ret = did_overwrite_ref(sctx, *parent_ino, *parent_gen, ino, gen,
2309 dest->start, fs_path_len(dest));
2310 if (ret < 0)
2311 return ret;
2312 if (ret) {
2313 fs_path_reset(dest);
2314 ret = gen_unique_name(sctx, ino, gen, dest);
2315 if (ret < 0)
2316 return ret;
2317 ret = 1;
2318 }
2319
2320 out_cache:
2321 /*
2322 * Store the result of the lookup in the name cache.
2323 */
2324 nce = kmalloc(sizeof(*nce) + fs_path_len(dest), GFP_KERNEL);
2325 if (!nce)
2326 return -ENOMEM;
2327
2328 nce->entry.key = ino;
2329 nce->entry.gen = gen;
2330 nce->parent_ino = *parent_ino;
2331 nce->parent_gen = *parent_gen;
2332 nce->name_len = fs_path_len(dest);
2333 nce->ret = ret;
2334 memcpy(nce->name, dest->start, nce->name_len);
2335
2336 if (ino < sctx->send_progress)
2337 nce->need_later_update = 0;
2338 else
2339 nce->need_later_update = 1;
2340
2341 nce_ret = btrfs_lru_cache_store(&sctx->name_cache, &nce->entry, GFP_KERNEL);
2342 if (nce_ret < 0) {
2343 kfree(nce);
2344 return nce_ret;
2345 }
2346
2347 return ret;
2348 }
2349
2350 /*
2351 * Magic happens here. This function returns the first ref to an inode as it
2352 * would look like while receiving the stream at this point in time.
2353 * We walk the path up to the root. For every inode in between, we check if it
2354 * was already processed/sent. If yes, we continue with the parent as found
2355 * in send_root. If not, we continue with the parent as found in parent_root.
2356 * If we encounter an inode that was deleted at this point in time, we use the
2357 * inodes "orphan" name instead of the real name and stop. Same with new inodes
2358 * that were not created yet and overwritten inodes/refs.
2359 *
2360 * When do we have orphan inodes:
2361 * 1. When an inode is freshly created and thus no valid refs are available yet
2362 * 2. When a directory lost all it's refs (deleted) but still has dir items
2363 * inside which were not processed yet (pending for move/delete). If anyone
2364 * tried to get the path to the dir items, it would get a path inside that
2365 * orphan directory.
2366 * 3. When an inode is moved around or gets new links, it may overwrite the ref
2367 * of an unprocessed inode. If in that case the first ref would be
2368 * overwritten, the overwritten inode gets "orphanized". Later when we
2369 * process this overwritten inode, it is restored at a new place by moving
2370 * the orphan inode.
2371 *
2372 * sctx->send_progress tells this function at which point in time receiving
2373 * would be.
2374 */
get_cur_path(struct send_ctx * sctx,u64 ino,u64 gen,struct fs_path * dest)2375 static int get_cur_path(struct send_ctx *sctx, u64 ino, u64 gen,
2376 struct fs_path *dest)
2377 {
2378 int ret = 0;
2379 struct fs_path *name = NULL;
2380 u64 parent_inode = 0;
2381 u64 parent_gen = 0;
2382 int stop = 0;
2383 const bool is_cur_inode = (ino == sctx->cur_ino && gen == sctx->cur_inode_gen);
2384
2385 if (is_cur_inode && fs_path_len(&sctx->cur_inode_path) > 0) {
2386 if (dest != &sctx->cur_inode_path)
2387 return fs_path_copy(dest, &sctx->cur_inode_path);
2388
2389 return 0;
2390 }
2391
2392 name = fs_path_alloc();
2393 if (!name) {
2394 ret = -ENOMEM;
2395 goto out;
2396 }
2397
2398 dest->reversed = 1;
2399 fs_path_reset(dest);
2400
2401 while (!stop && ino != BTRFS_FIRST_FREE_OBJECTID) {
2402 struct waiting_dir_move *wdm;
2403
2404 fs_path_reset(name);
2405
2406 if (is_waiting_for_rm(sctx, ino, gen)) {
2407 ret = gen_unique_name(sctx, ino, gen, name);
2408 if (ret < 0)
2409 goto out;
2410 ret = fs_path_add_path(dest, name);
2411 break;
2412 }
2413
2414 wdm = get_waiting_dir_move(sctx, ino);
2415 if (wdm && wdm->orphanized) {
2416 ret = gen_unique_name(sctx, ino, gen, name);
2417 stop = 1;
2418 } else if (wdm) {
2419 ret = get_first_ref(sctx->parent_root, ino,
2420 &parent_inode, &parent_gen, name);
2421 } else {
2422 ret = __get_cur_name_and_parent(sctx, ino, gen,
2423 &parent_inode,
2424 &parent_gen, name);
2425 if (ret)
2426 stop = 1;
2427 }
2428
2429 if (ret < 0)
2430 goto out;
2431
2432 ret = fs_path_add_path(dest, name);
2433 if (ret < 0)
2434 goto out;
2435
2436 ino = parent_inode;
2437 gen = parent_gen;
2438 }
2439
2440 out:
2441 fs_path_free(name);
2442 if (!ret) {
2443 fs_path_unreverse(dest);
2444 if (is_cur_inode && dest != &sctx->cur_inode_path)
2445 ret = fs_path_copy(&sctx->cur_inode_path, dest);
2446 }
2447
2448 return ret;
2449 }
2450
2451 /*
2452 * Sends a BTRFS_SEND_C_SUBVOL command/item to userspace
2453 */
send_subvol_begin(struct send_ctx * sctx)2454 static int send_subvol_begin(struct send_ctx *sctx)
2455 {
2456 int ret;
2457 struct btrfs_root *send_root = sctx->send_root;
2458 struct btrfs_root *parent_root = sctx->parent_root;
2459 BTRFS_PATH_AUTO_FREE(path);
2460 struct btrfs_key key;
2461 struct btrfs_root_ref *ref;
2462 struct extent_buffer *leaf;
2463 char *name = NULL;
2464 int namelen;
2465
2466 path = btrfs_alloc_path();
2467 if (!path)
2468 return -ENOMEM;
2469
2470 name = kmalloc(BTRFS_PATH_NAME_MAX, GFP_KERNEL);
2471 if (!name)
2472 return -ENOMEM;
2473
2474 key.objectid = btrfs_root_id(send_root);
2475 key.type = BTRFS_ROOT_BACKREF_KEY;
2476 key.offset = 0;
2477
2478 ret = btrfs_search_slot_for_read(send_root->fs_info->tree_root,
2479 &key, path, 1, 0);
2480 if (ret < 0)
2481 goto out;
2482 if (ret) {
2483 ret = -ENOENT;
2484 goto out;
2485 }
2486
2487 leaf = path->nodes[0];
2488 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
2489 if (key.type != BTRFS_ROOT_BACKREF_KEY ||
2490 key.objectid != btrfs_root_id(send_root)) {
2491 ret = -ENOENT;
2492 goto out;
2493 }
2494 ref = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_root_ref);
2495 namelen = btrfs_root_ref_name_len(leaf, ref);
2496 read_extent_buffer(leaf, name, (unsigned long)(ref + 1), namelen);
2497 btrfs_release_path(path);
2498
2499 if (parent_root) {
2500 ret = begin_cmd(sctx, BTRFS_SEND_C_SNAPSHOT);
2501 if (ret < 0)
2502 goto out;
2503 } else {
2504 ret = begin_cmd(sctx, BTRFS_SEND_C_SUBVOL);
2505 if (ret < 0)
2506 goto out;
2507 }
2508
2509 TLV_PUT_STRING(sctx, BTRFS_SEND_A_PATH, name, namelen);
2510
2511 if (!btrfs_is_empty_uuid(sctx->send_root->root_item.received_uuid))
2512 TLV_PUT_UUID(sctx, BTRFS_SEND_A_UUID,
2513 sctx->send_root->root_item.received_uuid);
2514 else
2515 TLV_PUT_UUID(sctx, BTRFS_SEND_A_UUID,
2516 sctx->send_root->root_item.uuid);
2517
2518 TLV_PUT_U64(sctx, BTRFS_SEND_A_CTRANSID,
2519 btrfs_root_ctransid(&sctx->send_root->root_item));
2520 if (parent_root) {
2521 if (!btrfs_is_empty_uuid(parent_root->root_item.received_uuid))
2522 TLV_PUT_UUID(sctx, BTRFS_SEND_A_CLONE_UUID,
2523 parent_root->root_item.received_uuid);
2524 else
2525 TLV_PUT_UUID(sctx, BTRFS_SEND_A_CLONE_UUID,
2526 parent_root->root_item.uuid);
2527 TLV_PUT_U64(sctx, BTRFS_SEND_A_CLONE_CTRANSID,
2528 btrfs_root_ctransid(&sctx->parent_root->root_item));
2529 }
2530
2531 ret = send_cmd(sctx);
2532
2533 tlv_put_failure:
2534 out:
2535 kfree(name);
2536 return ret;
2537 }
2538
get_cur_inode_path(struct send_ctx * sctx)2539 static struct fs_path *get_cur_inode_path(struct send_ctx *sctx)
2540 {
2541 if (fs_path_len(&sctx->cur_inode_path) == 0) {
2542 int ret;
2543
2544 ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen,
2545 &sctx->cur_inode_path);
2546 if (ret < 0)
2547 return ERR_PTR(ret);
2548 }
2549
2550 return &sctx->cur_inode_path;
2551 }
2552
get_path_for_command(struct send_ctx * sctx,u64 ino,u64 gen)2553 static struct fs_path *get_path_for_command(struct send_ctx *sctx, u64 ino, u64 gen)
2554 {
2555 struct fs_path *path;
2556 int ret;
2557
2558 if (ino == sctx->cur_ino && gen == sctx->cur_inode_gen)
2559 return get_cur_inode_path(sctx);
2560
2561 path = fs_path_alloc();
2562 if (!path)
2563 return ERR_PTR(-ENOMEM);
2564
2565 ret = get_cur_path(sctx, ino, gen, path);
2566 if (ret < 0) {
2567 fs_path_free(path);
2568 return ERR_PTR(ret);
2569 }
2570
2571 return path;
2572 }
2573
free_path_for_command(const struct send_ctx * sctx,struct fs_path * path)2574 static void free_path_for_command(const struct send_ctx *sctx, struct fs_path *path)
2575 {
2576 if (path != &sctx->cur_inode_path)
2577 fs_path_free(path);
2578 }
2579
send_truncate(struct send_ctx * sctx,u64 ino,u64 gen,u64 size)2580 static int send_truncate(struct send_ctx *sctx, u64 ino, u64 gen, u64 size)
2581 {
2582 int ret = 0;
2583 struct fs_path *p;
2584
2585 p = get_path_for_command(sctx, ino, gen);
2586 if (IS_ERR(p))
2587 return PTR_ERR(p);
2588
2589 ret = begin_cmd(sctx, BTRFS_SEND_C_TRUNCATE);
2590 if (ret < 0)
2591 goto out;
2592
2593 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
2594 TLV_PUT_U64(sctx, BTRFS_SEND_A_SIZE, size);
2595
2596 ret = send_cmd(sctx);
2597
2598 tlv_put_failure:
2599 out:
2600 free_path_for_command(sctx, p);
2601 return ret;
2602 }
2603
send_chmod(struct send_ctx * sctx,u64 ino,u64 gen,u64 mode)2604 static int send_chmod(struct send_ctx *sctx, u64 ino, u64 gen, u64 mode)
2605 {
2606 int ret = 0;
2607 struct fs_path *p;
2608
2609 p = get_path_for_command(sctx, ino, gen);
2610 if (IS_ERR(p))
2611 return PTR_ERR(p);
2612
2613 ret = begin_cmd(sctx, BTRFS_SEND_C_CHMOD);
2614 if (ret < 0)
2615 goto out;
2616
2617 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
2618 TLV_PUT_U64(sctx, BTRFS_SEND_A_MODE, mode & 07777);
2619
2620 ret = send_cmd(sctx);
2621
2622 tlv_put_failure:
2623 out:
2624 free_path_for_command(sctx, p);
2625 return ret;
2626 }
2627
send_fileattr(struct send_ctx * sctx,u64 ino,u64 gen,u64 fileattr)2628 static int send_fileattr(struct send_ctx *sctx, u64 ino, u64 gen, u64 fileattr)
2629 {
2630 int ret = 0;
2631 struct fs_path *p;
2632
2633 if (sctx->proto < 2)
2634 return 0;
2635
2636 p = get_path_for_command(sctx, ino, gen);
2637 if (IS_ERR(p))
2638 return PTR_ERR(p);
2639
2640 ret = begin_cmd(sctx, BTRFS_SEND_C_FILEATTR);
2641 if (ret < 0)
2642 goto out;
2643
2644 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
2645 TLV_PUT_U64(sctx, BTRFS_SEND_A_FILEATTR, fileattr);
2646
2647 ret = send_cmd(sctx);
2648
2649 tlv_put_failure:
2650 out:
2651 free_path_for_command(sctx, p);
2652 return ret;
2653 }
2654
send_chown(struct send_ctx * sctx,u64 ino,u64 gen,u64 uid,u64 gid)2655 static int send_chown(struct send_ctx *sctx, u64 ino, u64 gen, u64 uid, u64 gid)
2656 {
2657 int ret = 0;
2658 struct fs_path *p;
2659
2660 p = get_path_for_command(sctx, ino, gen);
2661 if (IS_ERR(p))
2662 return PTR_ERR(p);
2663
2664 ret = begin_cmd(sctx, BTRFS_SEND_C_CHOWN);
2665 if (ret < 0)
2666 goto out;
2667
2668 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
2669 TLV_PUT_U64(sctx, BTRFS_SEND_A_UID, uid);
2670 TLV_PUT_U64(sctx, BTRFS_SEND_A_GID, gid);
2671
2672 ret = send_cmd(sctx);
2673
2674 tlv_put_failure:
2675 out:
2676 free_path_for_command(sctx, p);
2677 return ret;
2678 }
2679
send_utimes(struct send_ctx * sctx,u64 ino,u64 gen)2680 static int send_utimes(struct send_ctx *sctx, u64 ino, u64 gen)
2681 {
2682 int ret = 0;
2683 struct fs_path *p = NULL;
2684 struct btrfs_inode_item *ii;
2685 BTRFS_PATH_AUTO_FREE(path);
2686 struct extent_buffer *eb;
2687 struct btrfs_key key;
2688 int slot;
2689
2690 p = get_path_for_command(sctx, ino, gen);
2691 if (IS_ERR(p))
2692 return PTR_ERR(p);
2693
2694 path = alloc_path_for_send();
2695 if (!path) {
2696 ret = -ENOMEM;
2697 goto out;
2698 }
2699
2700 key.objectid = ino;
2701 key.type = BTRFS_INODE_ITEM_KEY;
2702 key.offset = 0;
2703 ret = btrfs_search_slot(NULL, sctx->send_root, &key, path, 0, 0);
2704 if (ret > 0)
2705 ret = -ENOENT;
2706 if (ret < 0)
2707 goto out;
2708
2709 eb = path->nodes[0];
2710 slot = path->slots[0];
2711 ii = btrfs_item_ptr(eb, slot, struct btrfs_inode_item);
2712
2713 ret = begin_cmd(sctx, BTRFS_SEND_C_UTIMES);
2714 if (ret < 0)
2715 goto out;
2716
2717 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
2718 TLV_PUT_BTRFS_TIMESPEC(sctx, BTRFS_SEND_A_ATIME, eb, &ii->atime);
2719 TLV_PUT_BTRFS_TIMESPEC(sctx, BTRFS_SEND_A_MTIME, eb, &ii->mtime);
2720 TLV_PUT_BTRFS_TIMESPEC(sctx, BTRFS_SEND_A_CTIME, eb, &ii->ctime);
2721 if (sctx->proto >= 2)
2722 TLV_PUT_BTRFS_TIMESPEC(sctx, BTRFS_SEND_A_OTIME, eb, &ii->otime);
2723
2724 ret = send_cmd(sctx);
2725
2726 tlv_put_failure:
2727 out:
2728 free_path_for_command(sctx, p);
2729 return ret;
2730 }
2731
2732 /*
2733 * If the cache is full, we can't remove entries from it and do a call to
2734 * send_utimes() for each respective inode, because we might be finishing
2735 * processing an inode that is a directory and it just got renamed, and existing
2736 * entries in the cache may refer to inodes that have the directory in their
2737 * full path - in which case we would generate outdated paths (pre-rename)
2738 * for the inodes that the cache entries point to. Instead of pruning the
2739 * cache when inserting, do it after we finish processing each inode at
2740 * finish_inode_if_needed().
2741 */
cache_dir_utimes(struct send_ctx * sctx,u64 dir,u64 gen)2742 static int cache_dir_utimes(struct send_ctx *sctx, u64 dir, u64 gen)
2743 {
2744 struct btrfs_lru_cache_entry *entry;
2745 int ret;
2746
2747 entry = btrfs_lru_cache_lookup(&sctx->dir_utimes_cache, dir, gen);
2748 if (entry != NULL)
2749 return 0;
2750
2751 /* Caching is optional, don't fail if we can't allocate memory. */
2752 entry = kmalloc(sizeof(*entry), GFP_KERNEL);
2753 if (!entry)
2754 return send_utimes(sctx, dir, gen);
2755
2756 entry->key = dir;
2757 entry->gen = gen;
2758
2759 ret = btrfs_lru_cache_store(&sctx->dir_utimes_cache, entry, GFP_KERNEL);
2760 ASSERT(ret != -EEXIST);
2761 if (ret) {
2762 kfree(entry);
2763 return send_utimes(sctx, dir, gen);
2764 }
2765
2766 return 0;
2767 }
2768
trim_dir_utimes_cache(struct send_ctx * sctx)2769 static int trim_dir_utimes_cache(struct send_ctx *sctx)
2770 {
2771 while (sctx->dir_utimes_cache.size > SEND_MAX_DIR_UTIMES_CACHE_SIZE) {
2772 struct btrfs_lru_cache_entry *lru;
2773 int ret;
2774
2775 lru = btrfs_lru_cache_lru_entry(&sctx->dir_utimes_cache);
2776 ASSERT(lru != NULL);
2777
2778 ret = send_utimes(sctx, lru->key, lru->gen);
2779 if (ret)
2780 return ret;
2781
2782 btrfs_lru_cache_remove(&sctx->dir_utimes_cache, lru);
2783 }
2784
2785 return 0;
2786 }
2787
2788 /*
2789 * Sends a BTRFS_SEND_C_MKXXX or SYMLINK command to user space. We don't have
2790 * a valid path yet because we did not process the refs yet. So, the inode
2791 * is created as orphan.
2792 */
send_create_inode(struct send_ctx * sctx,u64 ino)2793 static int send_create_inode(struct send_ctx *sctx, u64 ino)
2794 {
2795 int ret = 0;
2796 struct fs_path *p;
2797 int cmd;
2798 struct btrfs_inode_info info;
2799 u64 gen;
2800 u64 mode;
2801 u64 rdev;
2802
2803 p = fs_path_alloc();
2804 if (!p)
2805 return -ENOMEM;
2806
2807 if (ino != sctx->cur_ino) {
2808 ret = get_inode_info(sctx->send_root, ino, &info);
2809 if (ret < 0)
2810 goto out;
2811 gen = info.gen;
2812 mode = info.mode;
2813 rdev = info.rdev;
2814 } else {
2815 gen = sctx->cur_inode_gen;
2816 mode = sctx->cur_inode_mode;
2817 rdev = sctx->cur_inode_rdev;
2818 }
2819
2820 if (S_ISREG(mode)) {
2821 cmd = BTRFS_SEND_C_MKFILE;
2822 } else if (S_ISDIR(mode)) {
2823 cmd = BTRFS_SEND_C_MKDIR;
2824 } else if (S_ISLNK(mode)) {
2825 cmd = BTRFS_SEND_C_SYMLINK;
2826 } else if (S_ISCHR(mode) || S_ISBLK(mode)) {
2827 cmd = BTRFS_SEND_C_MKNOD;
2828 } else if (S_ISFIFO(mode)) {
2829 cmd = BTRFS_SEND_C_MKFIFO;
2830 } else if (S_ISSOCK(mode)) {
2831 cmd = BTRFS_SEND_C_MKSOCK;
2832 } else {
2833 btrfs_warn(sctx->send_root->fs_info, "unexpected inode type %o",
2834 (int)(mode & S_IFMT));
2835 ret = -EOPNOTSUPP;
2836 goto out;
2837 }
2838
2839 ret = begin_cmd(sctx, cmd);
2840 if (ret < 0)
2841 goto out;
2842
2843 ret = gen_unique_name(sctx, ino, gen, p);
2844 if (ret < 0)
2845 goto out;
2846
2847 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
2848 TLV_PUT_U64(sctx, BTRFS_SEND_A_INO, ino);
2849
2850 if (S_ISLNK(mode)) {
2851 fs_path_reset(p);
2852 ret = read_symlink(sctx->send_root, ino, p);
2853 if (ret < 0)
2854 goto out;
2855 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH_LINK, p);
2856 } else if (S_ISCHR(mode) || S_ISBLK(mode) ||
2857 S_ISFIFO(mode) || S_ISSOCK(mode)) {
2858 TLV_PUT_U64(sctx, BTRFS_SEND_A_RDEV, new_encode_dev(rdev));
2859 TLV_PUT_U64(sctx, BTRFS_SEND_A_MODE, mode);
2860 }
2861
2862 ret = send_cmd(sctx);
2863 if (ret < 0)
2864 goto out;
2865
2866
2867 tlv_put_failure:
2868 out:
2869 fs_path_free(p);
2870 return ret;
2871 }
2872
cache_dir_created(struct send_ctx * sctx,u64 dir)2873 static void cache_dir_created(struct send_ctx *sctx, u64 dir)
2874 {
2875 struct btrfs_lru_cache_entry *entry;
2876 int ret;
2877
2878 /* Caching is optional, ignore any failures. */
2879 entry = kmalloc(sizeof(*entry), GFP_KERNEL);
2880 if (!entry)
2881 return;
2882
2883 entry->key = dir;
2884 entry->gen = 0;
2885 ret = btrfs_lru_cache_store(&sctx->dir_created_cache, entry, GFP_KERNEL);
2886 if (ret < 0)
2887 kfree(entry);
2888 }
2889
2890 /*
2891 * We need some special handling for inodes that get processed before the parent
2892 * directory got created. See process_recorded_refs for details.
2893 * This function does the check if we already created the dir out of order.
2894 */
did_create_dir(struct send_ctx * sctx,u64 dir)2895 static int did_create_dir(struct send_ctx *sctx, u64 dir)
2896 {
2897 int ret = 0;
2898 int iter_ret = 0;
2899 BTRFS_PATH_AUTO_FREE(path);
2900 struct btrfs_key key;
2901 struct btrfs_key found_key;
2902 struct btrfs_key di_key;
2903 struct btrfs_dir_item *di;
2904
2905 if (btrfs_lru_cache_lookup(&sctx->dir_created_cache, dir, 0))
2906 return 1;
2907
2908 path = alloc_path_for_send();
2909 if (!path)
2910 return -ENOMEM;
2911
2912 key.objectid = dir;
2913 key.type = BTRFS_DIR_INDEX_KEY;
2914 key.offset = 0;
2915
2916 btrfs_for_each_slot(sctx->send_root, &key, &found_key, path, iter_ret) {
2917 struct extent_buffer *eb = path->nodes[0];
2918
2919 if (found_key.objectid != key.objectid ||
2920 found_key.type != key.type) {
2921 ret = 0;
2922 break;
2923 }
2924
2925 di = btrfs_item_ptr(eb, path->slots[0], struct btrfs_dir_item);
2926 btrfs_dir_item_key_to_cpu(eb, di, &di_key);
2927
2928 if (di_key.type != BTRFS_ROOT_ITEM_KEY &&
2929 di_key.objectid < sctx->send_progress) {
2930 ret = 1;
2931 cache_dir_created(sctx, dir);
2932 break;
2933 }
2934 }
2935 /* Catch error found during iteration */
2936 if (iter_ret < 0)
2937 ret = iter_ret;
2938
2939 return ret;
2940 }
2941
2942 /*
2943 * Only creates the inode if it is:
2944 * 1. Not a directory
2945 * 2. Or a directory which was not created already due to out of order
2946 * directories. See did_create_dir and process_recorded_refs for details.
2947 */
send_create_inode_if_needed(struct send_ctx * sctx)2948 static int send_create_inode_if_needed(struct send_ctx *sctx)
2949 {
2950 int ret;
2951
2952 if (S_ISDIR(sctx->cur_inode_mode)) {
2953 ret = did_create_dir(sctx, sctx->cur_ino);
2954 if (ret < 0)
2955 return ret;
2956 else if (ret > 0)
2957 return 0;
2958 }
2959
2960 ret = send_create_inode(sctx, sctx->cur_ino);
2961
2962 if (ret == 0 && S_ISDIR(sctx->cur_inode_mode))
2963 cache_dir_created(sctx, sctx->cur_ino);
2964
2965 return ret;
2966 }
2967
2968 struct recorded_ref {
2969 struct list_head list;
2970 char *name;
2971 struct fs_path *full_path;
2972 u64 dir;
2973 u64 dir_gen;
2974 int name_len;
2975 struct rb_node node;
2976 struct rb_root *root;
2977 };
2978
recorded_ref_alloc(void)2979 static struct recorded_ref *recorded_ref_alloc(void)
2980 {
2981 struct recorded_ref *ref;
2982
2983 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
2984 if (!ref)
2985 return NULL;
2986 RB_CLEAR_NODE(&ref->node);
2987 INIT_LIST_HEAD(&ref->list);
2988 return ref;
2989 }
2990
recorded_ref_free(struct recorded_ref * ref)2991 static void recorded_ref_free(struct recorded_ref *ref)
2992 {
2993 if (!ref)
2994 return;
2995 if (!RB_EMPTY_NODE(&ref->node))
2996 rb_erase(&ref->node, ref->root);
2997 list_del(&ref->list);
2998 fs_path_free(ref->full_path);
2999 kfree(ref);
3000 }
3001
set_ref_path(struct recorded_ref * ref,struct fs_path * path)3002 static void set_ref_path(struct recorded_ref *ref, struct fs_path *path)
3003 {
3004 ref->full_path = path;
3005 ref->name = (char *)kbasename(ref->full_path->start);
3006 ref->name_len = ref->full_path->end - ref->name;
3007 }
3008
dup_ref(struct recorded_ref * ref,struct list_head * list)3009 static int dup_ref(struct recorded_ref *ref, struct list_head *list)
3010 {
3011 struct recorded_ref *new;
3012
3013 new = recorded_ref_alloc();
3014 if (!new)
3015 return -ENOMEM;
3016
3017 new->dir = ref->dir;
3018 new->dir_gen = ref->dir_gen;
3019 list_add_tail(&new->list, list);
3020 return 0;
3021 }
3022
__free_recorded_refs(struct list_head * head)3023 static void __free_recorded_refs(struct list_head *head)
3024 {
3025 struct recorded_ref *cur;
3026
3027 while (!list_empty(head)) {
3028 cur = list_first_entry(head, struct recorded_ref, list);
3029 recorded_ref_free(cur);
3030 }
3031 }
3032
free_recorded_refs(struct send_ctx * sctx)3033 static void free_recorded_refs(struct send_ctx *sctx)
3034 {
3035 __free_recorded_refs(&sctx->new_refs);
3036 __free_recorded_refs(&sctx->deleted_refs);
3037 }
3038
3039 /*
3040 * Renames/moves a file/dir to its orphan name. Used when the first
3041 * ref of an unprocessed inode gets overwritten and for all non empty
3042 * directories.
3043 */
orphanize_inode(struct send_ctx * sctx,u64 ino,u64 gen,struct fs_path * path)3044 static int orphanize_inode(struct send_ctx *sctx, u64 ino, u64 gen,
3045 struct fs_path *path)
3046 {
3047 int ret;
3048 struct fs_path *orphan;
3049
3050 orphan = fs_path_alloc();
3051 if (!orphan)
3052 return -ENOMEM;
3053
3054 ret = gen_unique_name(sctx, ino, gen, orphan);
3055 if (ret < 0)
3056 goto out;
3057
3058 ret = send_rename(sctx, path, orphan);
3059 if (ret < 0)
3060 goto out;
3061
3062 if (ino == sctx->cur_ino && gen == sctx->cur_inode_gen)
3063 ret = fs_path_copy(&sctx->cur_inode_path, orphan);
3064
3065 out:
3066 fs_path_free(orphan);
3067 return ret;
3068 }
3069
add_orphan_dir_info(struct send_ctx * sctx,u64 dir_ino,u64 dir_gen)3070 static struct orphan_dir_info *add_orphan_dir_info(struct send_ctx *sctx,
3071 u64 dir_ino, u64 dir_gen)
3072 {
3073 struct rb_node **p = &sctx->orphan_dirs.rb_node;
3074 struct rb_node *parent = NULL;
3075 struct orphan_dir_info *entry, *odi;
3076
3077 while (*p) {
3078 parent = *p;
3079 entry = rb_entry(parent, struct orphan_dir_info, node);
3080 if (dir_ino < entry->ino)
3081 p = &(*p)->rb_left;
3082 else if (dir_ino > entry->ino)
3083 p = &(*p)->rb_right;
3084 else if (dir_gen < entry->gen)
3085 p = &(*p)->rb_left;
3086 else if (dir_gen > entry->gen)
3087 p = &(*p)->rb_right;
3088 else
3089 return entry;
3090 }
3091
3092 odi = kmalloc(sizeof(*odi), GFP_KERNEL);
3093 if (!odi)
3094 return ERR_PTR(-ENOMEM);
3095 odi->ino = dir_ino;
3096 odi->gen = dir_gen;
3097 odi->last_dir_index_offset = 0;
3098 odi->dir_high_seq_ino = 0;
3099
3100 rb_link_node(&odi->node, parent, p);
3101 rb_insert_color(&odi->node, &sctx->orphan_dirs);
3102 return odi;
3103 }
3104
get_orphan_dir_info(struct send_ctx * sctx,u64 dir_ino,u64 gen)3105 static struct orphan_dir_info *get_orphan_dir_info(struct send_ctx *sctx,
3106 u64 dir_ino, u64 gen)
3107 {
3108 struct rb_node *n = sctx->orphan_dirs.rb_node;
3109 struct orphan_dir_info *entry;
3110
3111 while (n) {
3112 entry = rb_entry(n, struct orphan_dir_info, node);
3113 if (dir_ino < entry->ino)
3114 n = n->rb_left;
3115 else if (dir_ino > entry->ino)
3116 n = n->rb_right;
3117 else if (gen < entry->gen)
3118 n = n->rb_left;
3119 else if (gen > entry->gen)
3120 n = n->rb_right;
3121 else
3122 return entry;
3123 }
3124 return NULL;
3125 }
3126
is_waiting_for_rm(struct send_ctx * sctx,u64 dir_ino,u64 gen)3127 static int is_waiting_for_rm(struct send_ctx *sctx, u64 dir_ino, u64 gen)
3128 {
3129 struct orphan_dir_info *odi = get_orphan_dir_info(sctx, dir_ino, gen);
3130
3131 return odi != NULL;
3132 }
3133
free_orphan_dir_info(struct send_ctx * sctx,struct orphan_dir_info * odi)3134 static void free_orphan_dir_info(struct send_ctx *sctx,
3135 struct orphan_dir_info *odi)
3136 {
3137 if (!odi)
3138 return;
3139 rb_erase(&odi->node, &sctx->orphan_dirs);
3140 kfree(odi);
3141 }
3142
3143 /*
3144 * Returns 1 if a directory can be removed at this point in time.
3145 * We check this by iterating all dir items and checking if the inode behind
3146 * the dir item was already processed.
3147 */
can_rmdir(struct send_ctx * sctx,u64 dir,u64 dir_gen)3148 static int can_rmdir(struct send_ctx *sctx, u64 dir, u64 dir_gen)
3149 {
3150 int ret = 0;
3151 int iter_ret = 0;
3152 struct btrfs_root *root = sctx->parent_root;
3153 struct btrfs_path *path;
3154 struct btrfs_key key;
3155 struct btrfs_key found_key;
3156 struct btrfs_key loc;
3157 struct btrfs_dir_item *di;
3158 struct orphan_dir_info *odi = NULL;
3159 u64 dir_high_seq_ino = 0;
3160 u64 last_dir_index_offset = 0;
3161
3162 /*
3163 * Don't try to rmdir the top/root subvolume dir.
3164 */
3165 if (dir == BTRFS_FIRST_FREE_OBJECTID)
3166 return 0;
3167
3168 odi = get_orphan_dir_info(sctx, dir, dir_gen);
3169 if (odi && sctx->cur_ino < odi->dir_high_seq_ino)
3170 return 0;
3171
3172 path = alloc_path_for_send();
3173 if (!path)
3174 return -ENOMEM;
3175
3176 if (!odi) {
3177 /*
3178 * Find the inode number associated with the last dir index
3179 * entry. This is very likely the inode with the highest number
3180 * of all inodes that have an entry in the directory. We can
3181 * then use it to avoid future calls to can_rmdir(), when
3182 * processing inodes with a lower number, from having to search
3183 * the parent root b+tree for dir index keys.
3184 */
3185 key.objectid = dir;
3186 key.type = BTRFS_DIR_INDEX_KEY;
3187 key.offset = (u64)-1;
3188
3189 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
3190 if (ret < 0) {
3191 goto out;
3192 } else if (ret > 0) {
3193 /* Can't happen, the root is never empty. */
3194 ASSERT(path->slots[0] > 0);
3195 if (WARN_ON(path->slots[0] == 0)) {
3196 ret = -EUCLEAN;
3197 goto out;
3198 }
3199 path->slots[0]--;
3200 }
3201
3202 btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
3203 if (key.objectid != dir || key.type != BTRFS_DIR_INDEX_KEY) {
3204 /* No index keys, dir can be removed. */
3205 ret = 1;
3206 goto out;
3207 }
3208
3209 di = btrfs_item_ptr(path->nodes[0], path->slots[0],
3210 struct btrfs_dir_item);
3211 btrfs_dir_item_key_to_cpu(path->nodes[0], di, &loc);
3212 dir_high_seq_ino = loc.objectid;
3213 if (sctx->cur_ino < dir_high_seq_ino) {
3214 ret = 0;
3215 goto out;
3216 }
3217
3218 btrfs_release_path(path);
3219 }
3220
3221 key.objectid = dir;
3222 key.type = BTRFS_DIR_INDEX_KEY;
3223 key.offset = (odi ? odi->last_dir_index_offset : 0);
3224
3225 btrfs_for_each_slot(root, &key, &found_key, path, iter_ret) {
3226 struct waiting_dir_move *dm;
3227
3228 if (found_key.objectid != key.objectid ||
3229 found_key.type != key.type)
3230 break;
3231
3232 di = btrfs_item_ptr(path->nodes[0], path->slots[0],
3233 struct btrfs_dir_item);
3234 btrfs_dir_item_key_to_cpu(path->nodes[0], di, &loc);
3235
3236 dir_high_seq_ino = max(dir_high_seq_ino, loc.objectid);
3237 last_dir_index_offset = found_key.offset;
3238
3239 dm = get_waiting_dir_move(sctx, loc.objectid);
3240 if (dm) {
3241 dm->rmdir_ino = dir;
3242 dm->rmdir_gen = dir_gen;
3243 ret = 0;
3244 goto out;
3245 }
3246
3247 if (loc.objectid > sctx->cur_ino) {
3248 ret = 0;
3249 goto out;
3250 }
3251 }
3252 if (iter_ret < 0) {
3253 ret = iter_ret;
3254 goto out;
3255 }
3256 free_orphan_dir_info(sctx, odi);
3257
3258 ret = 1;
3259
3260 out:
3261 btrfs_free_path(path);
3262
3263 if (ret)
3264 return ret;
3265
3266 if (!odi) {
3267 odi = add_orphan_dir_info(sctx, dir, dir_gen);
3268 if (IS_ERR(odi))
3269 return PTR_ERR(odi);
3270
3271 odi->gen = dir_gen;
3272 }
3273
3274 odi->last_dir_index_offset = last_dir_index_offset;
3275 odi->dir_high_seq_ino = max(odi->dir_high_seq_ino, dir_high_seq_ino);
3276
3277 return 0;
3278 }
3279
is_waiting_for_move(struct send_ctx * sctx,u64 ino)3280 static int is_waiting_for_move(struct send_ctx *sctx, u64 ino)
3281 {
3282 struct waiting_dir_move *entry = get_waiting_dir_move(sctx, ino);
3283
3284 return entry != NULL;
3285 }
3286
add_waiting_dir_move(struct send_ctx * sctx,u64 ino,bool orphanized)3287 static int add_waiting_dir_move(struct send_ctx *sctx, u64 ino, bool orphanized)
3288 {
3289 struct rb_node **p = &sctx->waiting_dir_moves.rb_node;
3290 struct rb_node *parent = NULL;
3291 struct waiting_dir_move *entry, *dm;
3292
3293 dm = kmalloc(sizeof(*dm), GFP_KERNEL);
3294 if (!dm)
3295 return -ENOMEM;
3296 dm->ino = ino;
3297 dm->rmdir_ino = 0;
3298 dm->rmdir_gen = 0;
3299 dm->orphanized = orphanized;
3300
3301 while (*p) {
3302 parent = *p;
3303 entry = rb_entry(parent, struct waiting_dir_move, node);
3304 if (ino < entry->ino) {
3305 p = &(*p)->rb_left;
3306 } else if (ino > entry->ino) {
3307 p = &(*p)->rb_right;
3308 } else {
3309 kfree(dm);
3310 return -EEXIST;
3311 }
3312 }
3313
3314 rb_link_node(&dm->node, parent, p);
3315 rb_insert_color(&dm->node, &sctx->waiting_dir_moves);
3316 return 0;
3317 }
3318
3319 static struct waiting_dir_move *
get_waiting_dir_move(struct send_ctx * sctx,u64 ino)3320 get_waiting_dir_move(struct send_ctx *sctx, u64 ino)
3321 {
3322 struct rb_node *n = sctx->waiting_dir_moves.rb_node;
3323 struct waiting_dir_move *entry;
3324
3325 while (n) {
3326 entry = rb_entry(n, struct waiting_dir_move, node);
3327 if (ino < entry->ino)
3328 n = n->rb_left;
3329 else if (ino > entry->ino)
3330 n = n->rb_right;
3331 else
3332 return entry;
3333 }
3334 return NULL;
3335 }
3336
free_waiting_dir_move(struct send_ctx * sctx,struct waiting_dir_move * dm)3337 static void free_waiting_dir_move(struct send_ctx *sctx,
3338 struct waiting_dir_move *dm)
3339 {
3340 if (!dm)
3341 return;
3342 rb_erase(&dm->node, &sctx->waiting_dir_moves);
3343 kfree(dm);
3344 }
3345
add_pending_dir_move(struct send_ctx * sctx,u64 ino,u64 ino_gen,u64 parent_ino,struct list_head * new_refs,struct list_head * deleted_refs,const bool is_orphan)3346 static int add_pending_dir_move(struct send_ctx *sctx,
3347 u64 ino,
3348 u64 ino_gen,
3349 u64 parent_ino,
3350 struct list_head *new_refs,
3351 struct list_head *deleted_refs,
3352 const bool is_orphan)
3353 {
3354 struct rb_node **p = &sctx->pending_dir_moves.rb_node;
3355 struct rb_node *parent = NULL;
3356 struct pending_dir_move *entry = NULL, *pm;
3357 struct recorded_ref *cur;
3358 int exists = 0;
3359 int ret;
3360
3361 pm = kmalloc(sizeof(*pm), GFP_KERNEL);
3362 if (!pm)
3363 return -ENOMEM;
3364 pm->parent_ino = parent_ino;
3365 pm->ino = ino;
3366 pm->gen = ino_gen;
3367 INIT_LIST_HEAD(&pm->list);
3368 INIT_LIST_HEAD(&pm->update_refs);
3369 RB_CLEAR_NODE(&pm->node);
3370
3371 while (*p) {
3372 parent = *p;
3373 entry = rb_entry(parent, struct pending_dir_move, node);
3374 if (parent_ino < entry->parent_ino) {
3375 p = &(*p)->rb_left;
3376 } else if (parent_ino > entry->parent_ino) {
3377 p = &(*p)->rb_right;
3378 } else {
3379 exists = 1;
3380 break;
3381 }
3382 }
3383
3384 list_for_each_entry(cur, deleted_refs, list) {
3385 ret = dup_ref(cur, &pm->update_refs);
3386 if (ret < 0)
3387 goto out;
3388 }
3389 list_for_each_entry(cur, new_refs, list) {
3390 ret = dup_ref(cur, &pm->update_refs);
3391 if (ret < 0)
3392 goto out;
3393 }
3394
3395 ret = add_waiting_dir_move(sctx, pm->ino, is_orphan);
3396 if (ret)
3397 goto out;
3398
3399 if (exists) {
3400 list_add_tail(&pm->list, &entry->list);
3401 } else {
3402 rb_link_node(&pm->node, parent, p);
3403 rb_insert_color(&pm->node, &sctx->pending_dir_moves);
3404 }
3405 ret = 0;
3406 out:
3407 if (ret) {
3408 __free_recorded_refs(&pm->update_refs);
3409 kfree(pm);
3410 }
3411 return ret;
3412 }
3413
get_pending_dir_moves(struct send_ctx * sctx,u64 parent_ino)3414 static struct pending_dir_move *get_pending_dir_moves(struct send_ctx *sctx,
3415 u64 parent_ino)
3416 {
3417 struct rb_node *n = sctx->pending_dir_moves.rb_node;
3418 struct pending_dir_move *entry;
3419
3420 while (n) {
3421 entry = rb_entry(n, struct pending_dir_move, node);
3422 if (parent_ino < entry->parent_ino)
3423 n = n->rb_left;
3424 else if (parent_ino > entry->parent_ino)
3425 n = n->rb_right;
3426 else
3427 return entry;
3428 }
3429 return NULL;
3430 }
3431
path_loop(struct send_ctx * sctx,struct fs_path * name,u64 ino,u64 gen,u64 * ancestor_ino)3432 static int path_loop(struct send_ctx *sctx, struct fs_path *name,
3433 u64 ino, u64 gen, u64 *ancestor_ino)
3434 {
3435 int ret = 0;
3436 u64 parent_inode = 0;
3437 u64 parent_gen = 0;
3438 u64 start_ino = ino;
3439
3440 *ancestor_ino = 0;
3441 while (ino != BTRFS_FIRST_FREE_OBJECTID) {
3442 fs_path_reset(name);
3443
3444 if (is_waiting_for_rm(sctx, ino, gen))
3445 break;
3446 if (is_waiting_for_move(sctx, ino)) {
3447 if (*ancestor_ino == 0)
3448 *ancestor_ino = ino;
3449 ret = get_first_ref(sctx->parent_root, ino,
3450 &parent_inode, &parent_gen, name);
3451 } else {
3452 ret = __get_cur_name_and_parent(sctx, ino, gen,
3453 &parent_inode,
3454 &parent_gen, name);
3455 if (ret > 0) {
3456 ret = 0;
3457 break;
3458 }
3459 }
3460 if (ret < 0)
3461 break;
3462 if (parent_inode == start_ino) {
3463 ret = 1;
3464 if (*ancestor_ino == 0)
3465 *ancestor_ino = ino;
3466 break;
3467 }
3468 ino = parent_inode;
3469 gen = parent_gen;
3470 }
3471 return ret;
3472 }
3473
apply_dir_move(struct send_ctx * sctx,struct pending_dir_move * pm)3474 static int apply_dir_move(struct send_ctx *sctx, struct pending_dir_move *pm)
3475 {
3476 struct fs_path *from_path = NULL;
3477 struct fs_path *to_path = NULL;
3478 struct fs_path *name = NULL;
3479 u64 orig_progress = sctx->send_progress;
3480 struct recorded_ref *cur;
3481 u64 parent_ino, parent_gen;
3482 struct waiting_dir_move *dm = NULL;
3483 u64 rmdir_ino = 0;
3484 u64 rmdir_gen;
3485 u64 ancestor;
3486 bool is_orphan;
3487 int ret;
3488
3489 name = fs_path_alloc();
3490 from_path = fs_path_alloc();
3491 if (!name || !from_path) {
3492 ret = -ENOMEM;
3493 goto out;
3494 }
3495
3496 dm = get_waiting_dir_move(sctx, pm->ino);
3497 ASSERT(dm);
3498 rmdir_ino = dm->rmdir_ino;
3499 rmdir_gen = dm->rmdir_gen;
3500 is_orphan = dm->orphanized;
3501 free_waiting_dir_move(sctx, dm);
3502
3503 if (is_orphan) {
3504 ret = gen_unique_name(sctx, pm->ino,
3505 pm->gen, from_path);
3506 } else {
3507 ret = get_first_ref(sctx->parent_root, pm->ino,
3508 &parent_ino, &parent_gen, name);
3509 if (ret < 0)
3510 goto out;
3511 ret = get_cur_path(sctx, parent_ino, parent_gen,
3512 from_path);
3513 if (ret < 0)
3514 goto out;
3515 ret = fs_path_add_path(from_path, name);
3516 }
3517 if (ret < 0)
3518 goto out;
3519
3520 sctx->send_progress = sctx->cur_ino + 1;
3521 ret = path_loop(sctx, name, pm->ino, pm->gen, &ancestor);
3522 if (ret < 0)
3523 goto out;
3524 if (ret) {
3525 LIST_HEAD(deleted_refs);
3526 ASSERT(ancestor > BTRFS_FIRST_FREE_OBJECTID);
3527 ret = add_pending_dir_move(sctx, pm->ino, pm->gen, ancestor,
3528 &pm->update_refs, &deleted_refs,
3529 is_orphan);
3530 if (ret < 0)
3531 goto out;
3532 if (rmdir_ino) {
3533 dm = get_waiting_dir_move(sctx, pm->ino);
3534 ASSERT(dm);
3535 dm->rmdir_ino = rmdir_ino;
3536 dm->rmdir_gen = rmdir_gen;
3537 }
3538 goto out;
3539 }
3540 fs_path_reset(name);
3541 to_path = name;
3542 name = NULL;
3543 ret = get_cur_path(sctx, pm->ino, pm->gen, to_path);
3544 if (ret < 0)
3545 goto out;
3546
3547 ret = send_rename(sctx, from_path, to_path);
3548 if (ret < 0)
3549 goto out;
3550
3551 if (rmdir_ino) {
3552 struct orphan_dir_info *odi;
3553 u64 gen;
3554
3555 odi = get_orphan_dir_info(sctx, rmdir_ino, rmdir_gen);
3556 if (!odi) {
3557 /* already deleted */
3558 goto finish;
3559 }
3560 gen = odi->gen;
3561
3562 ret = can_rmdir(sctx, rmdir_ino, gen);
3563 if (ret < 0)
3564 goto out;
3565 if (!ret)
3566 goto finish;
3567
3568 name = fs_path_alloc();
3569 if (!name) {
3570 ret = -ENOMEM;
3571 goto out;
3572 }
3573 ret = get_cur_path(sctx, rmdir_ino, gen, name);
3574 if (ret < 0)
3575 goto out;
3576 ret = send_rmdir(sctx, name);
3577 if (ret < 0)
3578 goto out;
3579 }
3580
3581 finish:
3582 ret = cache_dir_utimes(sctx, pm->ino, pm->gen);
3583 if (ret < 0)
3584 goto out;
3585
3586 /*
3587 * After rename/move, need to update the utimes of both new parent(s)
3588 * and old parent(s).
3589 */
3590 list_for_each_entry(cur, &pm->update_refs, list) {
3591 /*
3592 * The parent inode might have been deleted in the send snapshot
3593 */
3594 ret = get_inode_info(sctx->send_root, cur->dir, NULL);
3595 if (ret == -ENOENT) {
3596 ret = 0;
3597 continue;
3598 }
3599 if (ret < 0)
3600 goto out;
3601
3602 ret = cache_dir_utimes(sctx, cur->dir, cur->dir_gen);
3603 if (ret < 0)
3604 goto out;
3605 }
3606
3607 out:
3608 fs_path_free(name);
3609 fs_path_free(from_path);
3610 fs_path_free(to_path);
3611 sctx->send_progress = orig_progress;
3612
3613 return ret;
3614 }
3615
free_pending_move(struct send_ctx * sctx,struct pending_dir_move * m)3616 static void free_pending_move(struct send_ctx *sctx, struct pending_dir_move *m)
3617 {
3618 if (!list_empty(&m->list))
3619 list_del(&m->list);
3620 if (!RB_EMPTY_NODE(&m->node))
3621 rb_erase(&m->node, &sctx->pending_dir_moves);
3622 __free_recorded_refs(&m->update_refs);
3623 kfree(m);
3624 }
3625
tail_append_pending_moves(struct send_ctx * sctx,struct pending_dir_move * moves,struct list_head * stack)3626 static void tail_append_pending_moves(struct send_ctx *sctx,
3627 struct pending_dir_move *moves,
3628 struct list_head *stack)
3629 {
3630 if (list_empty(&moves->list)) {
3631 list_add_tail(&moves->list, stack);
3632 } else {
3633 LIST_HEAD(list);
3634 list_splice_init(&moves->list, &list);
3635 list_add_tail(&moves->list, stack);
3636 list_splice_tail(&list, stack);
3637 }
3638 if (!RB_EMPTY_NODE(&moves->node)) {
3639 rb_erase(&moves->node, &sctx->pending_dir_moves);
3640 RB_CLEAR_NODE(&moves->node);
3641 }
3642 }
3643
apply_children_dir_moves(struct send_ctx * sctx)3644 static int apply_children_dir_moves(struct send_ctx *sctx)
3645 {
3646 struct pending_dir_move *pm;
3647 LIST_HEAD(stack);
3648 u64 parent_ino = sctx->cur_ino;
3649 int ret = 0;
3650
3651 pm = get_pending_dir_moves(sctx, parent_ino);
3652 if (!pm)
3653 return 0;
3654
3655 tail_append_pending_moves(sctx, pm, &stack);
3656
3657 while (!list_empty(&stack)) {
3658 pm = list_first_entry(&stack, struct pending_dir_move, list);
3659 parent_ino = pm->ino;
3660 ret = apply_dir_move(sctx, pm);
3661 free_pending_move(sctx, pm);
3662 if (ret)
3663 goto out;
3664 pm = get_pending_dir_moves(sctx, parent_ino);
3665 if (pm)
3666 tail_append_pending_moves(sctx, pm, &stack);
3667 }
3668 return 0;
3669
3670 out:
3671 while (!list_empty(&stack)) {
3672 pm = list_first_entry(&stack, struct pending_dir_move, list);
3673 free_pending_move(sctx, pm);
3674 }
3675 return ret;
3676 }
3677
3678 /*
3679 * We might need to delay a directory rename even when no ancestor directory
3680 * (in the send root) with a higher inode number than ours (sctx->cur_ino) was
3681 * renamed. This happens when we rename a directory to the old name (the name
3682 * in the parent root) of some other unrelated directory that got its rename
3683 * delayed due to some ancestor with higher number that got renamed.
3684 *
3685 * Example:
3686 *
3687 * Parent snapshot:
3688 * . (ino 256)
3689 * |---- a/ (ino 257)
3690 * | |---- file (ino 260)
3691 * |
3692 * |---- b/ (ino 258)
3693 * |---- c/ (ino 259)
3694 *
3695 * Send snapshot:
3696 * . (ino 256)
3697 * |---- a/ (ino 258)
3698 * |---- x/ (ino 259)
3699 * |---- y/ (ino 257)
3700 * |----- file (ino 260)
3701 *
3702 * Here we can not rename 258 from 'b' to 'a' without the rename of inode 257
3703 * from 'a' to 'x/y' happening first, which in turn depends on the rename of
3704 * inode 259 from 'c' to 'x'. So the order of rename commands the send stream
3705 * must issue is:
3706 *
3707 * 1 - rename 259 from 'c' to 'x'
3708 * 2 - rename 257 from 'a' to 'x/y'
3709 * 3 - rename 258 from 'b' to 'a'
3710 *
3711 * Returns 1 if the rename of sctx->cur_ino needs to be delayed, 0 if it can
3712 * be done right away and < 0 on error.
3713 */
wait_for_dest_dir_move(struct send_ctx * sctx,struct recorded_ref * parent_ref,const bool is_orphan)3714 static int wait_for_dest_dir_move(struct send_ctx *sctx,
3715 struct recorded_ref *parent_ref,
3716 const bool is_orphan)
3717 {
3718 BTRFS_PATH_AUTO_FREE(path);
3719 struct btrfs_key key;
3720 struct btrfs_key di_key;
3721 struct btrfs_dir_item *di;
3722 u64 left_gen;
3723 u64 right_gen;
3724 int ret = 0;
3725 struct waiting_dir_move *wdm;
3726
3727 if (RB_EMPTY_ROOT(&sctx->waiting_dir_moves))
3728 return 0;
3729
3730 path = alloc_path_for_send();
3731 if (!path)
3732 return -ENOMEM;
3733
3734 key.objectid = parent_ref->dir;
3735 key.type = BTRFS_DIR_ITEM_KEY;
3736 key.offset = btrfs_name_hash(parent_ref->name, parent_ref->name_len);
3737
3738 ret = btrfs_search_slot(NULL, sctx->parent_root, &key, path, 0, 0);
3739 if (ret < 0)
3740 return ret;
3741 if (ret > 0)
3742 return 0;
3743
3744 di = btrfs_match_dir_item_name(path, parent_ref->name,
3745 parent_ref->name_len);
3746 if (!di)
3747 return 0;
3748 /*
3749 * di_key.objectid has the number of the inode that has a dentry in the
3750 * parent directory with the same name that sctx->cur_ino is being
3751 * renamed to. We need to check if that inode is in the send root as
3752 * well and if it is currently marked as an inode with a pending rename,
3753 * if it is, we need to delay the rename of sctx->cur_ino as well, so
3754 * that it happens after that other inode is renamed.
3755 */
3756 btrfs_dir_item_key_to_cpu(path->nodes[0], di, &di_key);
3757 if (di_key.type != BTRFS_INODE_ITEM_KEY)
3758 return 0;
3759
3760 ret = get_inode_gen(sctx->parent_root, di_key.objectid, &left_gen);
3761 if (ret < 0)
3762 return ret;
3763 ret = get_inode_gen(sctx->send_root, di_key.objectid, &right_gen);
3764 if (ret < 0) {
3765 if (ret == -ENOENT)
3766 ret = 0;
3767 return ret;
3768 }
3769
3770 /* Different inode, no need to delay the rename of sctx->cur_ino */
3771 if (right_gen != left_gen)
3772 return 0;
3773
3774 wdm = get_waiting_dir_move(sctx, di_key.objectid);
3775 if (wdm && !wdm->orphanized) {
3776 ret = add_pending_dir_move(sctx,
3777 sctx->cur_ino,
3778 sctx->cur_inode_gen,
3779 di_key.objectid,
3780 &sctx->new_refs,
3781 &sctx->deleted_refs,
3782 is_orphan);
3783 if (!ret)
3784 ret = 1;
3785 }
3786 return ret;
3787 }
3788
3789 /*
3790 * Check if inode ino2, or any of its ancestors, is inode ino1.
3791 * Return 1 if true, 0 if false and < 0 on error.
3792 */
check_ino_in_path(struct btrfs_root * root,const u64 ino1,const u64 ino1_gen,const u64 ino2,const u64 ino2_gen,struct fs_path * fs_path)3793 static int check_ino_in_path(struct btrfs_root *root,
3794 const u64 ino1,
3795 const u64 ino1_gen,
3796 const u64 ino2,
3797 const u64 ino2_gen,
3798 struct fs_path *fs_path)
3799 {
3800 u64 ino = ino2;
3801
3802 if (ino1 == ino2)
3803 return ino1_gen == ino2_gen;
3804
3805 while (ino > BTRFS_FIRST_FREE_OBJECTID) {
3806 u64 parent;
3807 u64 parent_gen;
3808 int ret;
3809
3810 fs_path_reset(fs_path);
3811 ret = get_first_ref(root, ino, &parent, &parent_gen, fs_path);
3812 if (ret < 0)
3813 return ret;
3814 if (parent == ino1)
3815 return parent_gen == ino1_gen;
3816 ino = parent;
3817 }
3818 return 0;
3819 }
3820
3821 /*
3822 * Check if inode ino1 is an ancestor of inode ino2 in the given root for any
3823 * possible path (in case ino2 is not a directory and has multiple hard links).
3824 * Return 1 if true, 0 if false and < 0 on error.
3825 */
is_ancestor(struct btrfs_root * root,const u64 ino1,const u64 ino1_gen,const u64 ino2,struct fs_path * fs_path)3826 static int is_ancestor(struct btrfs_root *root,
3827 const u64 ino1,
3828 const u64 ino1_gen,
3829 const u64 ino2,
3830 struct fs_path *fs_path)
3831 {
3832 bool free_fs_path = false;
3833 int ret = 0;
3834 int iter_ret = 0;
3835 BTRFS_PATH_AUTO_FREE(path);
3836 struct btrfs_key key;
3837
3838 if (!fs_path) {
3839 fs_path = fs_path_alloc();
3840 if (!fs_path)
3841 return -ENOMEM;
3842 free_fs_path = true;
3843 }
3844
3845 path = alloc_path_for_send();
3846 if (!path) {
3847 ret = -ENOMEM;
3848 goto out;
3849 }
3850
3851 key.objectid = ino2;
3852 key.type = BTRFS_INODE_REF_KEY;
3853 key.offset = 0;
3854
3855 btrfs_for_each_slot(root, &key, &key, path, iter_ret) {
3856 struct extent_buffer *leaf = path->nodes[0];
3857 int slot = path->slots[0];
3858 u32 cur_offset = 0;
3859 u32 item_size;
3860
3861 if (key.objectid != ino2)
3862 break;
3863 if (key.type != BTRFS_INODE_REF_KEY &&
3864 key.type != BTRFS_INODE_EXTREF_KEY)
3865 break;
3866
3867 item_size = btrfs_item_size(leaf, slot);
3868 while (cur_offset < item_size) {
3869 u64 parent;
3870 u64 parent_gen;
3871
3872 if (key.type == BTRFS_INODE_EXTREF_KEY) {
3873 unsigned long ptr;
3874 struct btrfs_inode_extref *extref;
3875
3876 ptr = btrfs_item_ptr_offset(leaf, slot);
3877 extref = (struct btrfs_inode_extref *)
3878 (ptr + cur_offset);
3879 parent = btrfs_inode_extref_parent(leaf,
3880 extref);
3881 cur_offset += sizeof(*extref);
3882 cur_offset += btrfs_inode_extref_name_len(leaf,
3883 extref);
3884 } else {
3885 parent = key.offset;
3886 cur_offset = item_size;
3887 }
3888
3889 ret = get_inode_gen(root, parent, &parent_gen);
3890 if (ret < 0)
3891 goto out;
3892 ret = check_ino_in_path(root, ino1, ino1_gen,
3893 parent, parent_gen, fs_path);
3894 if (ret)
3895 goto out;
3896 }
3897 }
3898 ret = 0;
3899 if (iter_ret < 0)
3900 ret = iter_ret;
3901
3902 out:
3903 if (free_fs_path)
3904 fs_path_free(fs_path);
3905 return ret;
3906 }
3907
wait_for_parent_move(struct send_ctx * sctx,struct recorded_ref * parent_ref,const bool is_orphan)3908 static int wait_for_parent_move(struct send_ctx *sctx,
3909 struct recorded_ref *parent_ref,
3910 const bool is_orphan)
3911 {
3912 int ret = 0;
3913 u64 ino = parent_ref->dir;
3914 u64 ino_gen = parent_ref->dir_gen;
3915 u64 parent_ino_before, parent_ino_after;
3916 struct fs_path *path_before = NULL;
3917 struct fs_path *path_after = NULL;
3918 int len1, len2;
3919
3920 path_after = fs_path_alloc();
3921 path_before = fs_path_alloc();
3922 if (!path_after || !path_before) {
3923 ret = -ENOMEM;
3924 goto out;
3925 }
3926
3927 /*
3928 * Our current directory inode may not yet be renamed/moved because some
3929 * ancestor (immediate or not) has to be renamed/moved first. So find if
3930 * such ancestor exists and make sure our own rename/move happens after
3931 * that ancestor is processed to avoid path build infinite loops (done
3932 * at get_cur_path()).
3933 */
3934 while (ino > BTRFS_FIRST_FREE_OBJECTID) {
3935 u64 parent_ino_after_gen;
3936
3937 if (is_waiting_for_move(sctx, ino)) {
3938 /*
3939 * If the current inode is an ancestor of ino in the
3940 * parent root, we need to delay the rename of the
3941 * current inode, otherwise don't delayed the rename
3942 * because we can end up with a circular dependency
3943 * of renames, resulting in some directories never
3944 * getting the respective rename operations issued in
3945 * the send stream or getting into infinite path build
3946 * loops.
3947 */
3948 ret = is_ancestor(sctx->parent_root,
3949 sctx->cur_ino, sctx->cur_inode_gen,
3950 ino, path_before);
3951 if (ret)
3952 break;
3953 }
3954
3955 fs_path_reset(path_before);
3956 fs_path_reset(path_after);
3957
3958 ret = get_first_ref(sctx->send_root, ino, &parent_ino_after,
3959 &parent_ino_after_gen, path_after);
3960 if (ret < 0)
3961 goto out;
3962 ret = get_first_ref(sctx->parent_root, ino, &parent_ino_before,
3963 NULL, path_before);
3964 if (ret < 0 && ret != -ENOENT) {
3965 goto out;
3966 } else if (ret == -ENOENT) {
3967 ret = 0;
3968 break;
3969 }
3970
3971 len1 = fs_path_len(path_before);
3972 len2 = fs_path_len(path_after);
3973 if (ino > sctx->cur_ino &&
3974 (parent_ino_before != parent_ino_after || len1 != len2 ||
3975 memcmp(path_before->start, path_after->start, len1))) {
3976 u64 parent_ino_gen;
3977
3978 ret = get_inode_gen(sctx->parent_root, ino, &parent_ino_gen);
3979 if (ret < 0)
3980 goto out;
3981 if (ino_gen == parent_ino_gen) {
3982 ret = 1;
3983 break;
3984 }
3985 }
3986 ino = parent_ino_after;
3987 ino_gen = parent_ino_after_gen;
3988 }
3989
3990 out:
3991 fs_path_free(path_before);
3992 fs_path_free(path_after);
3993
3994 if (ret == 1) {
3995 ret = add_pending_dir_move(sctx,
3996 sctx->cur_ino,
3997 sctx->cur_inode_gen,
3998 ino,
3999 &sctx->new_refs,
4000 &sctx->deleted_refs,
4001 is_orphan);
4002 if (!ret)
4003 ret = 1;
4004 }
4005
4006 return ret;
4007 }
4008
update_ref_path(struct send_ctx * sctx,struct recorded_ref * ref)4009 static int update_ref_path(struct send_ctx *sctx, struct recorded_ref *ref)
4010 {
4011 int ret;
4012 struct fs_path *new_path;
4013
4014 /*
4015 * Our reference's name member points to its full_path member string, so
4016 * we use here a new path.
4017 */
4018 new_path = fs_path_alloc();
4019 if (!new_path)
4020 return -ENOMEM;
4021
4022 ret = get_cur_path(sctx, ref->dir, ref->dir_gen, new_path);
4023 if (ret < 0) {
4024 fs_path_free(new_path);
4025 return ret;
4026 }
4027 ret = fs_path_add(new_path, ref->name, ref->name_len);
4028 if (ret < 0) {
4029 fs_path_free(new_path);
4030 return ret;
4031 }
4032
4033 fs_path_free(ref->full_path);
4034 set_ref_path(ref, new_path);
4035
4036 return 0;
4037 }
4038
4039 /*
4040 * When processing the new references for an inode we may orphanize an existing
4041 * directory inode because its old name conflicts with one of the new references
4042 * of the current inode. Later, when processing another new reference of our
4043 * inode, we might need to orphanize another inode, but the path we have in the
4044 * reference reflects the pre-orphanization name of the directory we previously
4045 * orphanized. For example:
4046 *
4047 * parent snapshot looks like:
4048 *
4049 * . (ino 256)
4050 * |----- f1 (ino 257)
4051 * |----- f2 (ino 258)
4052 * |----- d1/ (ino 259)
4053 * |----- d2/ (ino 260)
4054 *
4055 * send snapshot looks like:
4056 *
4057 * . (ino 256)
4058 * |----- d1 (ino 258)
4059 * |----- f2/ (ino 259)
4060 * |----- f2_link/ (ino 260)
4061 * | |----- f1 (ino 257)
4062 * |
4063 * |----- d2 (ino 258)
4064 *
4065 * When processing inode 257 we compute the name for inode 259 as "d1", and we
4066 * cache it in the name cache. Later when we start processing inode 258, when
4067 * collecting all its new references we set a full path of "d1/d2" for its new
4068 * reference with name "d2". When we start processing the new references we
4069 * start by processing the new reference with name "d1", and this results in
4070 * orphanizing inode 259, since its old reference causes a conflict. Then we
4071 * move on the next new reference, with name "d2", and we find out we must
4072 * orphanize inode 260, as its old reference conflicts with ours - but for the
4073 * orphanization we use a source path corresponding to the path we stored in the
4074 * new reference, which is "d1/d2" and not "o259-6-0/d2" - this makes the
4075 * receiver fail since the path component "d1/" no longer exists, it was renamed
4076 * to "o259-6-0/" when processing the previous new reference. So in this case we
4077 * must recompute the path in the new reference and use it for the new
4078 * orphanization operation.
4079 */
refresh_ref_path(struct send_ctx * sctx,struct recorded_ref * ref)4080 static int refresh_ref_path(struct send_ctx *sctx, struct recorded_ref *ref)
4081 {
4082 char *name;
4083 int ret;
4084
4085 name = kmemdup(ref->name, ref->name_len, GFP_KERNEL);
4086 if (!name)
4087 return -ENOMEM;
4088
4089 fs_path_reset(ref->full_path);
4090 ret = get_cur_path(sctx, ref->dir, ref->dir_gen, ref->full_path);
4091 if (ret < 0)
4092 goto out;
4093
4094 ret = fs_path_add(ref->full_path, name, ref->name_len);
4095 if (ret < 0)
4096 goto out;
4097
4098 /* Update the reference's base name pointer. */
4099 set_ref_path(ref, ref->full_path);
4100 out:
4101 kfree(name);
4102 return ret;
4103 }
4104
rename_current_inode(struct send_ctx * sctx,struct fs_path * current_path,struct fs_path * new_path)4105 static int rename_current_inode(struct send_ctx *sctx,
4106 struct fs_path *current_path,
4107 struct fs_path *new_path)
4108 {
4109 int ret;
4110
4111 ret = send_rename(sctx, current_path, new_path);
4112 if (ret < 0)
4113 return ret;
4114
4115 ret = fs_path_copy(&sctx->cur_inode_path, new_path);
4116 if (ret < 0)
4117 return ret;
4118
4119 return fs_path_copy(current_path, new_path);
4120 }
4121
4122 /*
4123 * This does all the move/link/unlink/rmdir magic.
4124 */
process_recorded_refs(struct send_ctx * sctx,int * pending_move)4125 static int process_recorded_refs(struct send_ctx *sctx, int *pending_move)
4126 {
4127 struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
4128 int ret = 0;
4129 struct recorded_ref *cur;
4130 struct recorded_ref *cur2;
4131 LIST_HEAD(check_dirs);
4132 struct fs_path *valid_path = NULL;
4133 u64 ow_inode = 0;
4134 u64 ow_gen;
4135 u64 ow_mode;
4136 u64 last_dir_ino_rm = 0;
4137 bool did_overwrite = false;
4138 bool is_orphan = false;
4139 bool can_rename = true;
4140 bool orphanized_dir = false;
4141 bool orphanized_ancestor = false;
4142
4143 /*
4144 * This should never happen as the root dir always has the same ref
4145 * which is always '..'
4146 */
4147 if (unlikely(sctx->cur_ino <= BTRFS_FIRST_FREE_OBJECTID)) {
4148 btrfs_err(fs_info,
4149 "send: unexpected inode %llu in process_recorded_refs()",
4150 sctx->cur_ino);
4151 ret = -EINVAL;
4152 goto out;
4153 }
4154
4155 valid_path = fs_path_alloc();
4156 if (!valid_path) {
4157 ret = -ENOMEM;
4158 goto out;
4159 }
4160
4161 /*
4162 * First, check if the first ref of the current inode was overwritten
4163 * before. If yes, we know that the current inode was already orphanized
4164 * and thus use the orphan name. If not, we can use get_cur_path to
4165 * get the path of the first ref as it would like while receiving at
4166 * this point in time.
4167 * New inodes are always orphan at the beginning, so force to use the
4168 * orphan name in this case.
4169 * The first ref is stored in valid_path and will be updated if it
4170 * gets moved around.
4171 */
4172 if (!sctx->cur_inode_new) {
4173 ret = did_overwrite_first_ref(sctx, sctx->cur_ino,
4174 sctx->cur_inode_gen);
4175 if (ret < 0)
4176 goto out;
4177 if (ret)
4178 did_overwrite = true;
4179 }
4180 if (sctx->cur_inode_new || did_overwrite) {
4181 ret = gen_unique_name(sctx, sctx->cur_ino,
4182 sctx->cur_inode_gen, valid_path);
4183 if (ret < 0)
4184 goto out;
4185 is_orphan = true;
4186 } else {
4187 ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen,
4188 valid_path);
4189 if (ret < 0)
4190 goto out;
4191 }
4192
4193 /*
4194 * Before doing any rename and link operations, do a first pass on the
4195 * new references to orphanize any unprocessed inodes that may have a
4196 * reference that conflicts with one of the new references of the current
4197 * inode. This needs to happen first because a new reference may conflict
4198 * with the old reference of a parent directory, so we must make sure
4199 * that the path used for link and rename commands don't use an
4200 * orphanized name when an ancestor was not yet orphanized.
4201 *
4202 * Example:
4203 *
4204 * Parent snapshot:
4205 *
4206 * . (ino 256)
4207 * |----- testdir/ (ino 259)
4208 * | |----- a (ino 257)
4209 * |
4210 * |----- b (ino 258)
4211 *
4212 * Send snapshot:
4213 *
4214 * . (ino 256)
4215 * |----- testdir_2/ (ino 259)
4216 * | |----- a (ino 260)
4217 * |
4218 * |----- testdir (ino 257)
4219 * |----- b (ino 257)
4220 * |----- b2 (ino 258)
4221 *
4222 * Processing the new reference for inode 257 with name "b" may happen
4223 * before processing the new reference with name "testdir". If so, we
4224 * must make sure that by the time we send a link command to create the
4225 * hard link "b", inode 259 was already orphanized, since the generated
4226 * path in "valid_path" already contains the orphanized name for 259.
4227 * We are processing inode 257, so only later when processing 259 we do
4228 * the rename operation to change its temporary (orphanized) name to
4229 * "testdir_2".
4230 */
4231 list_for_each_entry(cur, &sctx->new_refs, list) {
4232 ret = get_cur_inode_state(sctx, cur->dir, cur->dir_gen, NULL, NULL);
4233 if (ret < 0)
4234 goto out;
4235 if (ret == inode_state_will_create)
4236 continue;
4237
4238 /*
4239 * Check if this new ref would overwrite the first ref of another
4240 * unprocessed inode. If yes, orphanize the overwritten inode.
4241 * If we find an overwritten ref that is not the first ref,
4242 * simply unlink it.
4243 */
4244 ret = will_overwrite_ref(sctx, cur->dir, cur->dir_gen,
4245 cur->name, cur->name_len,
4246 &ow_inode, &ow_gen, &ow_mode);
4247 if (ret < 0)
4248 goto out;
4249 if (ret) {
4250 ret = is_first_ref(sctx->parent_root,
4251 ow_inode, cur->dir, cur->name,
4252 cur->name_len);
4253 if (ret < 0)
4254 goto out;
4255 if (ret) {
4256 struct name_cache_entry *nce;
4257 struct waiting_dir_move *wdm;
4258
4259 if (orphanized_dir) {
4260 ret = refresh_ref_path(sctx, cur);
4261 if (ret < 0)
4262 goto out;
4263 }
4264
4265 ret = orphanize_inode(sctx, ow_inode, ow_gen,
4266 cur->full_path);
4267 if (ret < 0)
4268 goto out;
4269 if (S_ISDIR(ow_mode))
4270 orphanized_dir = true;
4271
4272 /*
4273 * If ow_inode has its rename operation delayed
4274 * make sure that its orphanized name is used in
4275 * the source path when performing its rename
4276 * operation.
4277 */
4278 wdm = get_waiting_dir_move(sctx, ow_inode);
4279 if (wdm)
4280 wdm->orphanized = true;
4281
4282 /*
4283 * Make sure we clear our orphanized inode's
4284 * name from the name cache. This is because the
4285 * inode ow_inode might be an ancestor of some
4286 * other inode that will be orphanized as well
4287 * later and has an inode number greater than
4288 * sctx->send_progress. We need to prevent
4289 * future name lookups from using the old name
4290 * and get instead the orphan name.
4291 */
4292 nce = name_cache_search(sctx, ow_inode, ow_gen);
4293 if (nce)
4294 btrfs_lru_cache_remove(&sctx->name_cache,
4295 &nce->entry);
4296
4297 /*
4298 * ow_inode might currently be an ancestor of
4299 * cur_ino, therefore compute valid_path (the
4300 * current path of cur_ino) again because it
4301 * might contain the pre-orphanization name of
4302 * ow_inode, which is no longer valid.
4303 */
4304 ret = is_ancestor(sctx->parent_root,
4305 ow_inode, ow_gen,
4306 sctx->cur_ino, NULL);
4307 if (ret > 0) {
4308 orphanized_ancestor = true;
4309 fs_path_reset(valid_path);
4310 fs_path_reset(&sctx->cur_inode_path);
4311 ret = get_cur_path(sctx, sctx->cur_ino,
4312 sctx->cur_inode_gen,
4313 valid_path);
4314 }
4315 if (ret < 0)
4316 goto out;
4317 } else {
4318 /*
4319 * If we previously orphanized a directory that
4320 * collided with a new reference that we already
4321 * processed, recompute the current path because
4322 * that directory may be part of the path.
4323 */
4324 if (orphanized_dir) {
4325 ret = refresh_ref_path(sctx, cur);
4326 if (ret < 0)
4327 goto out;
4328 }
4329 ret = send_unlink(sctx, cur->full_path);
4330 if (ret < 0)
4331 goto out;
4332 }
4333 }
4334
4335 }
4336
4337 list_for_each_entry(cur, &sctx->new_refs, list) {
4338 /*
4339 * We may have refs where the parent directory does not exist
4340 * yet. This happens if the parent directories inum is higher
4341 * than the current inum. To handle this case, we create the
4342 * parent directory out of order. But we need to check if this
4343 * did already happen before due to other refs in the same dir.
4344 */
4345 ret = get_cur_inode_state(sctx, cur->dir, cur->dir_gen, NULL, NULL);
4346 if (ret < 0)
4347 goto out;
4348 if (ret == inode_state_will_create) {
4349 ret = 0;
4350 /*
4351 * First check if any of the current inodes refs did
4352 * already create the dir.
4353 */
4354 list_for_each_entry(cur2, &sctx->new_refs, list) {
4355 if (cur == cur2)
4356 break;
4357 if (cur2->dir == cur->dir) {
4358 ret = 1;
4359 break;
4360 }
4361 }
4362
4363 /*
4364 * If that did not happen, check if a previous inode
4365 * did already create the dir.
4366 */
4367 if (!ret)
4368 ret = did_create_dir(sctx, cur->dir);
4369 if (ret < 0)
4370 goto out;
4371 if (!ret) {
4372 ret = send_create_inode(sctx, cur->dir);
4373 if (ret < 0)
4374 goto out;
4375 cache_dir_created(sctx, cur->dir);
4376 }
4377 }
4378
4379 if (S_ISDIR(sctx->cur_inode_mode) && sctx->parent_root) {
4380 ret = wait_for_dest_dir_move(sctx, cur, is_orphan);
4381 if (ret < 0)
4382 goto out;
4383 if (ret == 1) {
4384 can_rename = false;
4385 *pending_move = 1;
4386 }
4387 }
4388
4389 if (S_ISDIR(sctx->cur_inode_mode) && sctx->parent_root &&
4390 can_rename) {
4391 ret = wait_for_parent_move(sctx, cur, is_orphan);
4392 if (ret < 0)
4393 goto out;
4394 if (ret == 1) {
4395 can_rename = false;
4396 *pending_move = 1;
4397 }
4398 }
4399
4400 /*
4401 * link/move the ref to the new place. If we have an orphan
4402 * inode, move it and update valid_path. If not, link or move
4403 * it depending on the inode mode.
4404 */
4405 if (is_orphan && can_rename) {
4406 ret = rename_current_inode(sctx, valid_path, cur->full_path);
4407 if (ret < 0)
4408 goto out;
4409 is_orphan = false;
4410 } else if (can_rename) {
4411 if (S_ISDIR(sctx->cur_inode_mode)) {
4412 /*
4413 * Dirs can't be linked, so move it. For moved
4414 * dirs, we always have one new and one deleted
4415 * ref. The deleted ref is ignored later.
4416 */
4417 ret = rename_current_inode(sctx, valid_path,
4418 cur->full_path);
4419 if (ret < 0)
4420 goto out;
4421 } else {
4422 /*
4423 * We might have previously orphanized an inode
4424 * which is an ancestor of our current inode,
4425 * so our reference's full path, which was
4426 * computed before any such orphanizations, must
4427 * be updated.
4428 */
4429 if (orphanized_dir) {
4430 ret = update_ref_path(sctx, cur);
4431 if (ret < 0)
4432 goto out;
4433 }
4434 ret = send_link(sctx, cur->full_path,
4435 valid_path);
4436 if (ret < 0)
4437 goto out;
4438 }
4439 }
4440 ret = dup_ref(cur, &check_dirs);
4441 if (ret < 0)
4442 goto out;
4443 }
4444
4445 if (S_ISDIR(sctx->cur_inode_mode) && sctx->cur_inode_deleted) {
4446 /*
4447 * Check if we can already rmdir the directory. If not,
4448 * orphanize it. For every dir item inside that gets deleted
4449 * later, we do this check again and rmdir it then if possible.
4450 * See the use of check_dirs for more details.
4451 */
4452 ret = can_rmdir(sctx, sctx->cur_ino, sctx->cur_inode_gen);
4453 if (ret < 0)
4454 goto out;
4455 if (ret) {
4456 ret = send_rmdir(sctx, valid_path);
4457 if (ret < 0)
4458 goto out;
4459 } else if (!is_orphan) {
4460 ret = orphanize_inode(sctx, sctx->cur_ino,
4461 sctx->cur_inode_gen, valid_path);
4462 if (ret < 0)
4463 goto out;
4464 is_orphan = true;
4465 }
4466
4467 list_for_each_entry(cur, &sctx->deleted_refs, list) {
4468 ret = dup_ref(cur, &check_dirs);
4469 if (ret < 0)
4470 goto out;
4471 }
4472 } else if (S_ISDIR(sctx->cur_inode_mode) &&
4473 !list_empty(&sctx->deleted_refs)) {
4474 /*
4475 * We have a moved dir. Add the old parent to check_dirs
4476 */
4477 cur = list_first_entry(&sctx->deleted_refs, struct recorded_ref, list);
4478 ret = dup_ref(cur, &check_dirs);
4479 if (ret < 0)
4480 goto out;
4481 } else if (!S_ISDIR(sctx->cur_inode_mode)) {
4482 /*
4483 * We have a non dir inode. Go through all deleted refs and
4484 * unlink them if they were not already overwritten by other
4485 * inodes.
4486 */
4487 list_for_each_entry(cur, &sctx->deleted_refs, list) {
4488 ret = did_overwrite_ref(sctx, cur->dir, cur->dir_gen,
4489 sctx->cur_ino, sctx->cur_inode_gen,
4490 cur->name, cur->name_len);
4491 if (ret < 0)
4492 goto out;
4493 if (!ret) {
4494 /*
4495 * If we orphanized any ancestor before, we need
4496 * to recompute the full path for deleted names,
4497 * since any such path was computed before we
4498 * processed any references and orphanized any
4499 * ancestor inode.
4500 */
4501 if (orphanized_ancestor) {
4502 ret = update_ref_path(sctx, cur);
4503 if (ret < 0)
4504 goto out;
4505 }
4506 ret = send_unlink(sctx, cur->full_path);
4507 if (ret < 0)
4508 goto out;
4509 if (is_current_inode_path(sctx, cur->full_path))
4510 fs_path_reset(&sctx->cur_inode_path);
4511 }
4512 ret = dup_ref(cur, &check_dirs);
4513 if (ret < 0)
4514 goto out;
4515 }
4516 /*
4517 * If the inode is still orphan, unlink the orphan. This may
4518 * happen when a previous inode did overwrite the first ref
4519 * of this inode and no new refs were added for the current
4520 * inode. Unlinking does not mean that the inode is deleted in
4521 * all cases. There may still be links to this inode in other
4522 * places.
4523 */
4524 if (is_orphan) {
4525 ret = send_unlink(sctx, valid_path);
4526 if (ret < 0)
4527 goto out;
4528 }
4529 }
4530
4531 /*
4532 * We did collect all parent dirs where cur_inode was once located. We
4533 * now go through all these dirs and check if they are pending for
4534 * deletion and if it's finally possible to perform the rmdir now.
4535 * We also update the inode stats of the parent dirs here.
4536 */
4537 list_for_each_entry(cur, &check_dirs, list) {
4538 /*
4539 * In case we had refs into dirs that were not processed yet,
4540 * we don't need to do the utime and rmdir logic for these dirs.
4541 * The dir will be processed later.
4542 */
4543 if (cur->dir > sctx->cur_ino)
4544 continue;
4545
4546 ret = get_cur_inode_state(sctx, cur->dir, cur->dir_gen, NULL, NULL);
4547 if (ret < 0)
4548 goto out;
4549
4550 if (ret == inode_state_did_create ||
4551 ret == inode_state_no_change) {
4552 ret = cache_dir_utimes(sctx, cur->dir, cur->dir_gen);
4553 if (ret < 0)
4554 goto out;
4555 } else if (ret == inode_state_did_delete &&
4556 cur->dir != last_dir_ino_rm) {
4557 ret = can_rmdir(sctx, cur->dir, cur->dir_gen);
4558 if (ret < 0)
4559 goto out;
4560 if (ret) {
4561 ret = get_cur_path(sctx, cur->dir,
4562 cur->dir_gen, valid_path);
4563 if (ret < 0)
4564 goto out;
4565 ret = send_rmdir(sctx, valid_path);
4566 if (ret < 0)
4567 goto out;
4568 last_dir_ino_rm = cur->dir;
4569 }
4570 }
4571 }
4572
4573 ret = 0;
4574
4575 out:
4576 __free_recorded_refs(&check_dirs);
4577 free_recorded_refs(sctx);
4578 fs_path_free(valid_path);
4579 return ret;
4580 }
4581
rbtree_ref_comp(const void * k,const struct rb_node * node)4582 static int rbtree_ref_comp(const void *k, const struct rb_node *node)
4583 {
4584 const struct recorded_ref *data = k;
4585 const struct recorded_ref *ref = rb_entry(node, struct recorded_ref, node);
4586
4587 if (data->dir > ref->dir)
4588 return 1;
4589 if (data->dir < ref->dir)
4590 return -1;
4591 if (data->dir_gen > ref->dir_gen)
4592 return 1;
4593 if (data->dir_gen < ref->dir_gen)
4594 return -1;
4595 if (data->name_len > ref->name_len)
4596 return 1;
4597 if (data->name_len < ref->name_len)
4598 return -1;
4599 return strcmp(data->name, ref->name);
4600 }
4601
rbtree_ref_less(struct rb_node * node,const struct rb_node * parent)4602 static bool rbtree_ref_less(struct rb_node *node, const struct rb_node *parent)
4603 {
4604 const struct recorded_ref *entry = rb_entry(node, struct recorded_ref, node);
4605
4606 return rbtree_ref_comp(entry, parent) < 0;
4607 }
4608
record_ref_in_tree(struct rb_root * root,struct list_head * refs,struct fs_path * name,u64 dir,u64 dir_gen,struct send_ctx * sctx)4609 static int record_ref_in_tree(struct rb_root *root, struct list_head *refs,
4610 struct fs_path *name, u64 dir, u64 dir_gen,
4611 struct send_ctx *sctx)
4612 {
4613 int ret = 0;
4614 struct fs_path *path = NULL;
4615 struct recorded_ref *ref = NULL;
4616
4617 path = fs_path_alloc();
4618 if (!path) {
4619 ret = -ENOMEM;
4620 goto out;
4621 }
4622
4623 ref = recorded_ref_alloc();
4624 if (!ref) {
4625 ret = -ENOMEM;
4626 goto out;
4627 }
4628
4629 ret = get_cur_path(sctx, dir, dir_gen, path);
4630 if (ret < 0)
4631 goto out;
4632 ret = fs_path_add_path(path, name);
4633 if (ret < 0)
4634 goto out;
4635
4636 ref->dir = dir;
4637 ref->dir_gen = dir_gen;
4638 set_ref_path(ref, path);
4639 list_add_tail(&ref->list, refs);
4640 rb_add(&ref->node, root, rbtree_ref_less);
4641 ref->root = root;
4642 out:
4643 if (ret) {
4644 if (path && (!ref || !ref->full_path))
4645 fs_path_free(path);
4646 recorded_ref_free(ref);
4647 }
4648 return ret;
4649 }
4650
record_new_ref_if_needed(u64 dir,struct fs_path * name,void * ctx)4651 static int record_new_ref_if_needed(u64 dir, struct fs_path *name, void *ctx)
4652 {
4653 int ret;
4654 struct send_ctx *sctx = ctx;
4655 struct rb_node *node = NULL;
4656 struct recorded_ref data;
4657 struct recorded_ref *ref;
4658 u64 dir_gen;
4659
4660 ret = get_inode_gen(sctx->send_root, dir, &dir_gen);
4661 if (ret < 0)
4662 return ret;
4663
4664 data.dir = dir;
4665 data.dir_gen = dir_gen;
4666 set_ref_path(&data, name);
4667 node = rb_find(&data, &sctx->rbtree_deleted_refs, rbtree_ref_comp);
4668 if (node) {
4669 ref = rb_entry(node, struct recorded_ref, node);
4670 recorded_ref_free(ref);
4671 } else {
4672 ret = record_ref_in_tree(&sctx->rbtree_new_refs,
4673 &sctx->new_refs, name, dir, dir_gen,
4674 sctx);
4675 }
4676
4677 return ret;
4678 }
4679
record_deleted_ref_if_needed(u64 dir,struct fs_path * name,void * ctx)4680 static int record_deleted_ref_if_needed(u64 dir, struct fs_path *name, void *ctx)
4681 {
4682 int ret;
4683 struct send_ctx *sctx = ctx;
4684 struct rb_node *node = NULL;
4685 struct recorded_ref data;
4686 struct recorded_ref *ref;
4687 u64 dir_gen;
4688
4689 ret = get_inode_gen(sctx->parent_root, dir, &dir_gen);
4690 if (ret < 0)
4691 return ret;
4692
4693 data.dir = dir;
4694 data.dir_gen = dir_gen;
4695 set_ref_path(&data, name);
4696 node = rb_find(&data, &sctx->rbtree_new_refs, rbtree_ref_comp);
4697 if (node) {
4698 ref = rb_entry(node, struct recorded_ref, node);
4699 recorded_ref_free(ref);
4700 } else {
4701 ret = record_ref_in_tree(&sctx->rbtree_deleted_refs,
4702 &sctx->deleted_refs, name, dir,
4703 dir_gen, sctx);
4704 }
4705
4706 return ret;
4707 }
4708
record_new_ref(struct send_ctx * sctx)4709 static int record_new_ref(struct send_ctx *sctx)
4710 {
4711 int ret;
4712
4713 ret = iterate_inode_ref(sctx->send_root, sctx->left_path, sctx->cmp_key,
4714 false, record_new_ref_if_needed, sctx);
4715 if (ret < 0)
4716 return ret;
4717
4718 return 0;
4719 }
4720
record_deleted_ref(struct send_ctx * sctx)4721 static int record_deleted_ref(struct send_ctx *sctx)
4722 {
4723 int ret;
4724
4725 ret = iterate_inode_ref(sctx->parent_root, sctx->right_path, sctx->cmp_key,
4726 false, record_deleted_ref_if_needed, sctx);
4727 if (ret < 0)
4728 return ret;
4729
4730 return 0;
4731 }
4732
record_changed_ref(struct send_ctx * sctx)4733 static int record_changed_ref(struct send_ctx *sctx)
4734 {
4735 int ret;
4736
4737 ret = iterate_inode_ref(sctx->send_root, sctx->left_path, sctx->cmp_key,
4738 false, record_new_ref_if_needed, sctx);
4739 if (ret < 0)
4740 return ret;
4741 ret = iterate_inode_ref(sctx->parent_root, sctx->right_path, sctx->cmp_key,
4742 false, record_deleted_ref_if_needed, sctx);
4743 if (ret < 0)
4744 return ret;
4745
4746 return 0;
4747 }
4748
4749 /*
4750 * Record and process all refs at once. Needed when an inode changes the
4751 * generation number, which means that it was deleted and recreated.
4752 */
process_all_refs(struct send_ctx * sctx,enum btrfs_compare_tree_result cmd)4753 static int process_all_refs(struct send_ctx *sctx,
4754 enum btrfs_compare_tree_result cmd)
4755 {
4756 int ret = 0;
4757 int iter_ret = 0;
4758 struct btrfs_root *root;
4759 BTRFS_PATH_AUTO_FREE(path);
4760 struct btrfs_key key;
4761 struct btrfs_key found_key;
4762 iterate_inode_ref_t cb;
4763 int pending_move = 0;
4764
4765 path = alloc_path_for_send();
4766 if (!path)
4767 return -ENOMEM;
4768
4769 if (cmd == BTRFS_COMPARE_TREE_NEW) {
4770 root = sctx->send_root;
4771 cb = record_new_ref_if_needed;
4772 } else if (cmd == BTRFS_COMPARE_TREE_DELETED) {
4773 root = sctx->parent_root;
4774 cb = record_deleted_ref_if_needed;
4775 } else {
4776 btrfs_err(sctx->send_root->fs_info,
4777 "Wrong command %d in process_all_refs", cmd);
4778 return -EINVAL;
4779 }
4780
4781 key.objectid = sctx->cmp_key->objectid;
4782 key.type = BTRFS_INODE_REF_KEY;
4783 key.offset = 0;
4784 btrfs_for_each_slot(root, &key, &found_key, path, iter_ret) {
4785 if (found_key.objectid != key.objectid ||
4786 (found_key.type != BTRFS_INODE_REF_KEY &&
4787 found_key.type != BTRFS_INODE_EXTREF_KEY))
4788 break;
4789
4790 ret = iterate_inode_ref(root, path, &found_key, false, cb, sctx);
4791 if (ret < 0)
4792 return ret;
4793 }
4794 /* Catch error found during iteration */
4795 if (iter_ret < 0)
4796 return iter_ret;
4797
4798 btrfs_release_path(path);
4799
4800 /*
4801 * We don't actually care about pending_move as we are simply
4802 * re-creating this inode and will be rename'ing it into place once we
4803 * rename the parent directory.
4804 */
4805 return process_recorded_refs(sctx, &pending_move);
4806 }
4807
send_set_xattr(struct send_ctx * sctx,const char * name,int name_len,const char * data,int data_len)4808 static int send_set_xattr(struct send_ctx *sctx,
4809 const char *name, int name_len,
4810 const char *data, int data_len)
4811 {
4812 struct fs_path *path;
4813 int ret;
4814
4815 path = get_cur_inode_path(sctx);
4816 if (IS_ERR(path))
4817 return PTR_ERR(path);
4818
4819 ret = begin_cmd(sctx, BTRFS_SEND_C_SET_XATTR);
4820 if (ret < 0)
4821 return ret;
4822
4823 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path);
4824 TLV_PUT_STRING(sctx, BTRFS_SEND_A_XATTR_NAME, name, name_len);
4825 TLV_PUT(sctx, BTRFS_SEND_A_XATTR_DATA, data, data_len);
4826
4827 ret = send_cmd(sctx);
4828
4829 tlv_put_failure:
4830 return ret;
4831 }
4832
send_remove_xattr(struct send_ctx * sctx,struct fs_path * path,const char * name,int name_len)4833 static int send_remove_xattr(struct send_ctx *sctx,
4834 struct fs_path *path,
4835 const char *name, int name_len)
4836 {
4837 int ret;
4838
4839 ret = begin_cmd(sctx, BTRFS_SEND_C_REMOVE_XATTR);
4840 if (ret < 0)
4841 return ret;
4842
4843 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path);
4844 TLV_PUT_STRING(sctx, BTRFS_SEND_A_XATTR_NAME, name, name_len);
4845
4846 ret = send_cmd(sctx);
4847
4848 tlv_put_failure:
4849 return ret;
4850 }
4851
__process_new_xattr(int num,struct btrfs_key * di_key,const char * name,int name_len,const char * data,int data_len,void * ctx)4852 static int __process_new_xattr(int num, struct btrfs_key *di_key,
4853 const char *name, int name_len, const char *data,
4854 int data_len, void *ctx)
4855 {
4856 struct send_ctx *sctx = ctx;
4857 struct posix_acl_xattr_header dummy_acl;
4858
4859 /* Capabilities are emitted by finish_inode_if_needed */
4860 if (!strncmp(name, XATTR_NAME_CAPS, name_len))
4861 return 0;
4862
4863 /*
4864 * This hack is needed because empty acls are stored as zero byte
4865 * data in xattrs. Problem with that is, that receiving these zero byte
4866 * acls will fail later. To fix this, we send a dummy acl list that
4867 * only contains the version number and no entries.
4868 */
4869 if (!strncmp(name, XATTR_NAME_POSIX_ACL_ACCESS, name_len) ||
4870 !strncmp(name, XATTR_NAME_POSIX_ACL_DEFAULT, name_len)) {
4871 if (data_len == 0) {
4872 dummy_acl.a_version =
4873 cpu_to_le32(POSIX_ACL_XATTR_VERSION);
4874 data = (char *)&dummy_acl;
4875 data_len = sizeof(dummy_acl);
4876 }
4877 }
4878
4879 return send_set_xattr(sctx, name, name_len, data, data_len);
4880 }
4881
__process_deleted_xattr(int num,struct btrfs_key * di_key,const char * name,int name_len,const char * data,int data_len,void * ctx)4882 static int __process_deleted_xattr(int num, struct btrfs_key *di_key,
4883 const char *name, int name_len,
4884 const char *data, int data_len, void *ctx)
4885 {
4886 struct send_ctx *sctx = ctx;
4887 struct fs_path *p;
4888
4889 p = get_cur_inode_path(sctx);
4890 if (IS_ERR(p))
4891 return PTR_ERR(p);
4892
4893 return send_remove_xattr(sctx, p, name, name_len);
4894 }
4895
process_new_xattr(struct send_ctx * sctx)4896 static int process_new_xattr(struct send_ctx *sctx)
4897 {
4898 return iterate_dir_item(sctx->send_root, sctx->left_path,
4899 __process_new_xattr, sctx);
4900 }
4901
process_deleted_xattr(struct send_ctx * sctx)4902 static int process_deleted_xattr(struct send_ctx *sctx)
4903 {
4904 return iterate_dir_item(sctx->parent_root, sctx->right_path,
4905 __process_deleted_xattr, sctx);
4906 }
4907
4908 struct find_xattr_ctx {
4909 const char *name;
4910 int name_len;
4911 int found_idx;
4912 char *found_data;
4913 int found_data_len;
4914 };
4915
__find_xattr(int num,struct btrfs_key * di_key,const char * name,int name_len,const char * data,int data_len,void * vctx)4916 static int __find_xattr(int num, struct btrfs_key *di_key, const char *name,
4917 int name_len, const char *data, int data_len, void *vctx)
4918 {
4919 struct find_xattr_ctx *ctx = vctx;
4920
4921 if (name_len == ctx->name_len &&
4922 strncmp(name, ctx->name, name_len) == 0) {
4923 ctx->found_idx = num;
4924 ctx->found_data_len = data_len;
4925 ctx->found_data = kmemdup(data, data_len, GFP_KERNEL);
4926 if (!ctx->found_data)
4927 return -ENOMEM;
4928 return 1;
4929 }
4930 return 0;
4931 }
4932
find_xattr(struct btrfs_root * root,struct btrfs_path * path,struct btrfs_key * key,const char * name,int name_len,char ** data,int * data_len)4933 static int find_xattr(struct btrfs_root *root,
4934 struct btrfs_path *path,
4935 struct btrfs_key *key,
4936 const char *name, int name_len,
4937 char **data, int *data_len)
4938 {
4939 int ret;
4940 struct find_xattr_ctx ctx;
4941
4942 ctx.name = name;
4943 ctx.name_len = name_len;
4944 ctx.found_idx = -1;
4945 ctx.found_data = NULL;
4946 ctx.found_data_len = 0;
4947
4948 ret = iterate_dir_item(root, path, __find_xattr, &ctx);
4949 if (ret < 0)
4950 return ret;
4951
4952 if (ctx.found_idx == -1)
4953 return -ENOENT;
4954 if (data) {
4955 *data = ctx.found_data;
4956 *data_len = ctx.found_data_len;
4957 } else {
4958 kfree(ctx.found_data);
4959 }
4960 return ctx.found_idx;
4961 }
4962
4963
__process_changed_new_xattr(int num,struct btrfs_key * di_key,const char * name,int name_len,const char * data,int data_len,void * ctx)4964 static int __process_changed_new_xattr(int num, struct btrfs_key *di_key,
4965 const char *name, int name_len,
4966 const char *data, int data_len,
4967 void *ctx)
4968 {
4969 int ret;
4970 struct send_ctx *sctx = ctx;
4971 char *found_data = NULL;
4972 int found_data_len = 0;
4973
4974 ret = find_xattr(sctx->parent_root, sctx->right_path,
4975 sctx->cmp_key, name, name_len, &found_data,
4976 &found_data_len);
4977 if (ret == -ENOENT) {
4978 ret = __process_new_xattr(num, di_key, name, name_len, data,
4979 data_len, ctx);
4980 } else if (ret >= 0) {
4981 if (data_len != found_data_len ||
4982 memcmp(data, found_data, data_len)) {
4983 ret = __process_new_xattr(num, di_key, name, name_len,
4984 data, data_len, ctx);
4985 } else {
4986 ret = 0;
4987 }
4988 }
4989
4990 kfree(found_data);
4991 return ret;
4992 }
4993
__process_changed_deleted_xattr(int num,struct btrfs_key * di_key,const char * name,int name_len,const char * data,int data_len,void * ctx)4994 static int __process_changed_deleted_xattr(int num, struct btrfs_key *di_key,
4995 const char *name, int name_len,
4996 const char *data, int data_len,
4997 void *ctx)
4998 {
4999 int ret;
5000 struct send_ctx *sctx = ctx;
5001
5002 ret = find_xattr(sctx->send_root, sctx->left_path, sctx->cmp_key,
5003 name, name_len, NULL, NULL);
5004 if (ret == -ENOENT)
5005 ret = __process_deleted_xattr(num, di_key, name, name_len, data,
5006 data_len, ctx);
5007 else if (ret >= 0)
5008 ret = 0;
5009
5010 return ret;
5011 }
5012
process_changed_xattr(struct send_ctx * sctx)5013 static int process_changed_xattr(struct send_ctx *sctx)
5014 {
5015 int ret;
5016
5017 ret = iterate_dir_item(sctx->send_root, sctx->left_path,
5018 __process_changed_new_xattr, sctx);
5019 if (ret < 0)
5020 return ret;
5021
5022 return iterate_dir_item(sctx->parent_root, sctx->right_path,
5023 __process_changed_deleted_xattr, sctx);
5024 }
5025
process_all_new_xattrs(struct send_ctx * sctx)5026 static int process_all_new_xattrs(struct send_ctx *sctx)
5027 {
5028 int ret = 0;
5029 int iter_ret = 0;
5030 struct btrfs_root *root;
5031 BTRFS_PATH_AUTO_FREE(path);
5032 struct btrfs_key key;
5033 struct btrfs_key found_key;
5034
5035 path = alloc_path_for_send();
5036 if (!path)
5037 return -ENOMEM;
5038
5039 root = sctx->send_root;
5040
5041 key.objectid = sctx->cmp_key->objectid;
5042 key.type = BTRFS_XATTR_ITEM_KEY;
5043 key.offset = 0;
5044 btrfs_for_each_slot(root, &key, &found_key, path, iter_ret) {
5045 if (found_key.objectid != key.objectid ||
5046 found_key.type != key.type) {
5047 ret = 0;
5048 break;
5049 }
5050
5051 ret = iterate_dir_item(root, path, __process_new_xattr, sctx);
5052 if (ret < 0)
5053 break;
5054 }
5055 /* Catch error found during iteration */
5056 if (iter_ret < 0)
5057 ret = iter_ret;
5058
5059 return ret;
5060 }
5061
send_verity(struct send_ctx * sctx,struct fs_path * path,struct fsverity_descriptor * desc)5062 static int send_verity(struct send_ctx *sctx, struct fs_path *path,
5063 struct fsverity_descriptor *desc)
5064 {
5065 int ret;
5066
5067 ret = begin_cmd(sctx, BTRFS_SEND_C_ENABLE_VERITY);
5068 if (ret < 0)
5069 return ret;
5070
5071 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path);
5072 TLV_PUT_U8(sctx, BTRFS_SEND_A_VERITY_ALGORITHM,
5073 le8_to_cpu(desc->hash_algorithm));
5074 TLV_PUT_U32(sctx, BTRFS_SEND_A_VERITY_BLOCK_SIZE,
5075 1U << le8_to_cpu(desc->log_blocksize));
5076 TLV_PUT(sctx, BTRFS_SEND_A_VERITY_SALT_DATA, desc->salt,
5077 le8_to_cpu(desc->salt_size));
5078 TLV_PUT(sctx, BTRFS_SEND_A_VERITY_SIG_DATA, desc->signature,
5079 le32_to_cpu(desc->sig_size));
5080
5081 ret = send_cmd(sctx);
5082
5083 tlv_put_failure:
5084 return ret;
5085 }
5086
process_verity(struct send_ctx * sctx)5087 static int process_verity(struct send_ctx *sctx)
5088 {
5089 int ret = 0;
5090 struct btrfs_inode *inode;
5091 struct fs_path *p;
5092
5093 inode = btrfs_iget(sctx->cur_ino, sctx->send_root);
5094 if (IS_ERR(inode))
5095 return PTR_ERR(inode);
5096
5097 ret = btrfs_get_verity_descriptor(&inode->vfs_inode, NULL, 0);
5098 if (ret < 0)
5099 goto iput;
5100
5101 if (ret > FS_VERITY_MAX_DESCRIPTOR_SIZE) {
5102 ret = -EMSGSIZE;
5103 goto iput;
5104 }
5105 if (!sctx->verity_descriptor) {
5106 sctx->verity_descriptor = kvmalloc(FS_VERITY_MAX_DESCRIPTOR_SIZE,
5107 GFP_KERNEL);
5108 if (!sctx->verity_descriptor) {
5109 ret = -ENOMEM;
5110 goto iput;
5111 }
5112 }
5113
5114 ret = btrfs_get_verity_descriptor(&inode->vfs_inode, sctx->verity_descriptor, ret);
5115 if (ret < 0)
5116 goto iput;
5117
5118 p = get_cur_inode_path(sctx);
5119 if (IS_ERR(p)) {
5120 ret = PTR_ERR(p);
5121 goto iput;
5122 }
5123
5124 ret = send_verity(sctx, p, sctx->verity_descriptor);
5125 iput:
5126 iput(&inode->vfs_inode);
5127 return ret;
5128 }
5129
max_send_read_size(const struct send_ctx * sctx)5130 static inline u64 max_send_read_size(const struct send_ctx *sctx)
5131 {
5132 return sctx->send_max_size - SZ_16K;
5133 }
5134
put_data_header(struct send_ctx * sctx,u32 len)5135 static int put_data_header(struct send_ctx *sctx, u32 len)
5136 {
5137 if (WARN_ON_ONCE(sctx->put_data))
5138 return -EINVAL;
5139 sctx->put_data = true;
5140 if (sctx->proto >= 2) {
5141 /*
5142 * Since v2, the data attribute header doesn't include a length,
5143 * it is implicitly to the end of the command.
5144 */
5145 if (sctx->send_max_size - sctx->send_size < sizeof(__le16) + len)
5146 return -EOVERFLOW;
5147 put_unaligned_le16(BTRFS_SEND_A_DATA, sctx->send_buf + sctx->send_size);
5148 sctx->send_size += sizeof(__le16);
5149 } else {
5150 struct btrfs_tlv_header *hdr;
5151
5152 if (sctx->send_max_size - sctx->send_size < sizeof(*hdr) + len)
5153 return -EOVERFLOW;
5154 hdr = (struct btrfs_tlv_header *)(sctx->send_buf + sctx->send_size);
5155 put_unaligned_le16(BTRFS_SEND_A_DATA, &hdr->tlv_type);
5156 put_unaligned_le16(len, &hdr->tlv_len);
5157 sctx->send_size += sizeof(*hdr);
5158 }
5159 return 0;
5160 }
5161
put_file_data(struct send_ctx * sctx,u64 offset,u32 len)5162 static int put_file_data(struct send_ctx *sctx, u64 offset, u32 len)
5163 {
5164 struct btrfs_root *root = sctx->send_root;
5165 struct btrfs_fs_info *fs_info = root->fs_info;
5166 u64 cur = offset;
5167 const u64 end = offset + len;
5168 const pgoff_t last_index = ((end - 1) >> PAGE_SHIFT);
5169 struct address_space *mapping = sctx->cur_inode->i_mapping;
5170 int ret;
5171
5172 ret = put_data_header(sctx, len);
5173 if (ret)
5174 return ret;
5175
5176 while (cur < end) {
5177 pgoff_t index = (cur >> PAGE_SHIFT);
5178 unsigned int cur_len;
5179 unsigned int pg_offset;
5180 struct folio *folio;
5181
5182 folio = filemap_lock_folio(mapping, index);
5183 if (IS_ERR(folio)) {
5184 page_cache_sync_readahead(mapping,
5185 &sctx->ra, NULL, index,
5186 last_index + 1 - index);
5187
5188 folio = filemap_grab_folio(mapping, index);
5189 if (IS_ERR(folio)) {
5190 ret = PTR_ERR(folio);
5191 break;
5192 }
5193 }
5194 pg_offset = offset_in_folio(folio, cur);
5195 cur_len = min_t(unsigned int, end - cur, folio_size(folio) - pg_offset);
5196
5197 if (folio_test_readahead(folio))
5198 page_cache_async_readahead(mapping, &sctx->ra, NULL, folio,
5199 last_index + 1 - index);
5200
5201 if (!folio_test_uptodate(folio)) {
5202 btrfs_read_folio(NULL, folio);
5203 folio_lock(folio);
5204 if (unlikely(!folio_test_uptodate(folio))) {
5205 folio_unlock(folio);
5206 btrfs_err(fs_info,
5207 "send: IO error at offset %llu for inode %llu root %llu",
5208 folio_pos(folio), sctx->cur_ino,
5209 btrfs_root_id(sctx->send_root));
5210 folio_put(folio);
5211 ret = -EIO;
5212 break;
5213 }
5214 if (folio->mapping != mapping) {
5215 folio_unlock(folio);
5216 folio_put(folio);
5217 continue;
5218 }
5219 }
5220
5221 memcpy_from_folio(sctx->send_buf + sctx->send_size, folio,
5222 pg_offset, cur_len);
5223 folio_unlock(folio);
5224 folio_put(folio);
5225 cur += cur_len;
5226 sctx->send_size += cur_len;
5227 }
5228
5229 return ret;
5230 }
5231
5232 /*
5233 * Read some bytes from the current inode/file and send a write command to
5234 * user space.
5235 */
send_write(struct send_ctx * sctx,u64 offset,u32 len)5236 static int send_write(struct send_ctx *sctx, u64 offset, u32 len)
5237 {
5238 int ret = 0;
5239 struct fs_path *p;
5240
5241 p = get_cur_inode_path(sctx);
5242 if (IS_ERR(p))
5243 return PTR_ERR(p);
5244
5245 ret = begin_cmd(sctx, BTRFS_SEND_C_WRITE);
5246 if (ret < 0)
5247 return ret;
5248
5249 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
5250 TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset);
5251 ret = put_file_data(sctx, offset, len);
5252 if (ret < 0)
5253 return ret;
5254
5255 ret = send_cmd(sctx);
5256
5257 tlv_put_failure:
5258 return ret;
5259 }
5260
5261 /*
5262 * Send a clone command to user space.
5263 */
send_clone(struct send_ctx * sctx,u64 offset,u32 len,struct clone_root * clone_root)5264 static int send_clone(struct send_ctx *sctx,
5265 u64 offset, u32 len,
5266 struct clone_root *clone_root)
5267 {
5268 int ret = 0;
5269 struct fs_path *p;
5270 struct fs_path *cur_inode_path;
5271 u64 gen;
5272
5273 cur_inode_path = get_cur_inode_path(sctx);
5274 if (IS_ERR(cur_inode_path))
5275 return PTR_ERR(cur_inode_path);
5276
5277 p = fs_path_alloc();
5278 if (!p)
5279 return -ENOMEM;
5280
5281 ret = begin_cmd(sctx, BTRFS_SEND_C_CLONE);
5282 if (ret < 0)
5283 goto out;
5284
5285 TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset);
5286 TLV_PUT_U64(sctx, BTRFS_SEND_A_CLONE_LEN, len);
5287 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, cur_inode_path);
5288
5289 if (clone_root->root == sctx->send_root) {
5290 ret = get_inode_gen(sctx->send_root, clone_root->ino, &gen);
5291 if (ret < 0)
5292 goto out;
5293 ret = get_cur_path(sctx, clone_root->ino, gen, p);
5294 } else {
5295 ret = get_inode_path(clone_root->root, clone_root->ino, p);
5296 }
5297 if (ret < 0)
5298 goto out;
5299
5300 /*
5301 * If the parent we're using has a received_uuid set then use that as
5302 * our clone source as that is what we will look for when doing a
5303 * receive.
5304 *
5305 * This covers the case that we create a snapshot off of a received
5306 * subvolume and then use that as the parent and try to receive on a
5307 * different host.
5308 */
5309 if (!btrfs_is_empty_uuid(clone_root->root->root_item.received_uuid))
5310 TLV_PUT_UUID(sctx, BTRFS_SEND_A_CLONE_UUID,
5311 clone_root->root->root_item.received_uuid);
5312 else
5313 TLV_PUT_UUID(sctx, BTRFS_SEND_A_CLONE_UUID,
5314 clone_root->root->root_item.uuid);
5315 TLV_PUT_U64(sctx, BTRFS_SEND_A_CLONE_CTRANSID,
5316 btrfs_root_ctransid(&clone_root->root->root_item));
5317 TLV_PUT_PATH(sctx, BTRFS_SEND_A_CLONE_PATH, p);
5318 TLV_PUT_U64(sctx, BTRFS_SEND_A_CLONE_OFFSET,
5319 clone_root->offset);
5320
5321 ret = send_cmd(sctx);
5322
5323 tlv_put_failure:
5324 out:
5325 fs_path_free(p);
5326 return ret;
5327 }
5328
5329 /*
5330 * Send an update extent command to user space.
5331 */
send_update_extent(struct send_ctx * sctx,u64 offset,u32 len)5332 static int send_update_extent(struct send_ctx *sctx,
5333 u64 offset, u32 len)
5334 {
5335 int ret = 0;
5336 struct fs_path *p;
5337
5338 p = get_cur_inode_path(sctx);
5339 if (IS_ERR(p))
5340 return PTR_ERR(p);
5341
5342 ret = begin_cmd(sctx, BTRFS_SEND_C_UPDATE_EXTENT);
5343 if (ret < 0)
5344 return ret;
5345
5346 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
5347 TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset);
5348 TLV_PUT_U64(sctx, BTRFS_SEND_A_SIZE, len);
5349
5350 ret = send_cmd(sctx);
5351
5352 tlv_put_failure:
5353 return ret;
5354 }
5355
send_fallocate(struct send_ctx * sctx,u32 mode,u64 offset,u64 len)5356 static int send_fallocate(struct send_ctx *sctx, u32 mode, u64 offset, u64 len)
5357 {
5358 struct fs_path *path;
5359 int ret;
5360
5361 path = get_cur_inode_path(sctx);
5362 if (IS_ERR(path))
5363 return PTR_ERR(path);
5364
5365 ret = begin_cmd(sctx, BTRFS_SEND_C_FALLOCATE);
5366 if (ret < 0)
5367 return ret;
5368
5369 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path);
5370 TLV_PUT_U32(sctx, BTRFS_SEND_A_FALLOCATE_MODE, mode);
5371 TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset);
5372 TLV_PUT_U64(sctx, BTRFS_SEND_A_SIZE, len);
5373
5374 ret = send_cmd(sctx);
5375
5376 tlv_put_failure:
5377 return ret;
5378 }
5379
send_hole(struct send_ctx * sctx,u64 end)5380 static int send_hole(struct send_ctx *sctx, u64 end)
5381 {
5382 struct fs_path *p = NULL;
5383 u64 read_size = max_send_read_size(sctx);
5384 u64 offset = sctx->cur_inode_last_extent;
5385 int ret = 0;
5386
5387 /*
5388 * Starting with send stream v2 we have fallocate and can use it to
5389 * punch holes instead of sending writes full of zeroes.
5390 */
5391 if (proto_cmd_ok(sctx, BTRFS_SEND_C_FALLOCATE))
5392 return send_fallocate(sctx, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
5393 offset, end - offset);
5394
5395 /*
5396 * A hole that starts at EOF or beyond it. Since we do not yet support
5397 * fallocate (for extent preallocation and hole punching), sending a
5398 * write of zeroes starting at EOF or beyond would later require issuing
5399 * a truncate operation which would undo the write and achieve nothing.
5400 */
5401 if (offset >= sctx->cur_inode_size)
5402 return 0;
5403
5404 /*
5405 * Don't go beyond the inode's i_size due to prealloc extents that start
5406 * after the i_size.
5407 */
5408 end = min_t(u64, end, sctx->cur_inode_size);
5409
5410 if (sctx->flags & BTRFS_SEND_FLAG_NO_FILE_DATA)
5411 return send_update_extent(sctx, offset, end - offset);
5412
5413 p = get_cur_inode_path(sctx);
5414 if (IS_ERR(p))
5415 return PTR_ERR(p);
5416
5417 while (offset < end) {
5418 u64 len = min(end - offset, read_size);
5419
5420 ret = begin_cmd(sctx, BTRFS_SEND_C_WRITE);
5421 if (ret < 0)
5422 break;
5423 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
5424 TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset);
5425 ret = put_data_header(sctx, len);
5426 if (ret < 0)
5427 break;
5428 memset(sctx->send_buf + sctx->send_size, 0, len);
5429 sctx->send_size += len;
5430 ret = send_cmd(sctx);
5431 if (ret < 0)
5432 break;
5433 offset += len;
5434 }
5435 sctx->cur_inode_next_write_offset = offset;
5436 tlv_put_failure:
5437 return ret;
5438 }
5439
send_encoded_inline_extent(struct send_ctx * sctx,struct btrfs_path * path,u64 offset,u64 len)5440 static int send_encoded_inline_extent(struct send_ctx *sctx,
5441 struct btrfs_path *path, u64 offset,
5442 u64 len)
5443 {
5444 struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
5445 struct fs_path *fspath;
5446 struct extent_buffer *leaf = path->nodes[0];
5447 struct btrfs_key key;
5448 struct btrfs_file_extent_item *ei;
5449 u64 ram_bytes;
5450 size_t inline_size;
5451 int ret;
5452
5453 fspath = get_cur_inode_path(sctx);
5454 if (IS_ERR(fspath))
5455 return PTR_ERR(fspath);
5456
5457 ret = begin_cmd(sctx, BTRFS_SEND_C_ENCODED_WRITE);
5458 if (ret < 0)
5459 return ret;
5460
5461 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
5462 ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item);
5463 ram_bytes = btrfs_file_extent_ram_bytes(leaf, ei);
5464 inline_size = btrfs_file_extent_inline_item_len(leaf, path->slots[0]);
5465
5466 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, fspath);
5467 TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset);
5468 TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_FILE_LEN,
5469 min(key.offset + ram_bytes - offset, len));
5470 TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_LEN, ram_bytes);
5471 TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_OFFSET, offset - key.offset);
5472 ret = btrfs_encoded_io_compression_from_extent(fs_info,
5473 btrfs_file_extent_compression(leaf, ei));
5474 if (ret < 0)
5475 return ret;
5476 TLV_PUT_U32(sctx, BTRFS_SEND_A_COMPRESSION, ret);
5477
5478 ret = put_data_header(sctx, inline_size);
5479 if (ret < 0)
5480 return ret;
5481 read_extent_buffer(leaf, sctx->send_buf + sctx->send_size,
5482 btrfs_file_extent_inline_start(ei), inline_size);
5483 sctx->send_size += inline_size;
5484
5485 ret = send_cmd(sctx);
5486
5487 tlv_put_failure:
5488 return ret;
5489 }
5490
send_encoded_extent(struct send_ctx * sctx,struct btrfs_path * path,u64 offset,u64 len)5491 static int send_encoded_extent(struct send_ctx *sctx, struct btrfs_path *path,
5492 u64 offset, u64 len)
5493 {
5494 struct btrfs_root *root = sctx->send_root;
5495 struct btrfs_fs_info *fs_info = root->fs_info;
5496 struct btrfs_inode *inode;
5497 struct fs_path *fspath;
5498 struct extent_buffer *leaf = path->nodes[0];
5499 struct btrfs_key key;
5500 struct btrfs_file_extent_item *ei;
5501 u64 disk_bytenr, disk_num_bytes;
5502 u32 data_offset;
5503 struct btrfs_cmd_header *hdr;
5504 u32 crc;
5505 int ret;
5506
5507 inode = btrfs_iget(sctx->cur_ino, root);
5508 if (IS_ERR(inode))
5509 return PTR_ERR(inode);
5510
5511 fspath = get_cur_inode_path(sctx);
5512 if (IS_ERR(fspath)) {
5513 ret = PTR_ERR(fspath);
5514 goto out;
5515 }
5516
5517 ret = begin_cmd(sctx, BTRFS_SEND_C_ENCODED_WRITE);
5518 if (ret < 0)
5519 goto out;
5520
5521 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
5522 ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item);
5523 disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, ei);
5524 disk_num_bytes = btrfs_file_extent_disk_num_bytes(leaf, ei);
5525
5526 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, fspath);
5527 TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset);
5528 TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_FILE_LEN,
5529 min(key.offset + btrfs_file_extent_num_bytes(leaf, ei) - offset,
5530 len));
5531 TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_LEN,
5532 btrfs_file_extent_ram_bytes(leaf, ei));
5533 TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_OFFSET,
5534 offset - key.offset + btrfs_file_extent_offset(leaf, ei));
5535 ret = btrfs_encoded_io_compression_from_extent(fs_info,
5536 btrfs_file_extent_compression(leaf, ei));
5537 if (ret < 0)
5538 goto out;
5539 TLV_PUT_U32(sctx, BTRFS_SEND_A_COMPRESSION, ret);
5540 TLV_PUT_U32(sctx, BTRFS_SEND_A_ENCRYPTION, 0);
5541
5542 ret = put_data_header(sctx, disk_num_bytes);
5543 if (ret < 0)
5544 goto out;
5545
5546 /*
5547 * We want to do I/O directly into the send buffer, so get the next page
5548 * boundary in the send buffer. This means that there may be a gap
5549 * between the beginning of the command and the file data.
5550 */
5551 data_offset = PAGE_ALIGN(sctx->send_size);
5552 if (data_offset > sctx->send_max_size ||
5553 sctx->send_max_size - data_offset < disk_num_bytes) {
5554 ret = -EOVERFLOW;
5555 goto out;
5556 }
5557
5558 /*
5559 * Note that send_buf is a mapping of send_buf_pages, so this is really
5560 * reading into send_buf.
5561 */
5562 ret = btrfs_encoded_read_regular_fill_pages(inode,
5563 disk_bytenr, disk_num_bytes,
5564 sctx->send_buf_pages +
5565 (data_offset >> PAGE_SHIFT),
5566 NULL);
5567 if (ret)
5568 goto out;
5569
5570 hdr = (struct btrfs_cmd_header *)sctx->send_buf;
5571 hdr->len = cpu_to_le32(sctx->send_size + disk_num_bytes - sizeof(*hdr));
5572 hdr->crc = 0;
5573 crc = crc32c(0, sctx->send_buf, sctx->send_size);
5574 crc = crc32c(crc, sctx->send_buf + data_offset, disk_num_bytes);
5575 hdr->crc = cpu_to_le32(crc);
5576
5577 ret = write_buf(sctx->send_filp, sctx->send_buf, sctx->send_size,
5578 &sctx->send_off);
5579 if (!ret) {
5580 ret = write_buf(sctx->send_filp, sctx->send_buf + data_offset,
5581 disk_num_bytes, &sctx->send_off);
5582 }
5583 sctx->send_size = 0;
5584 sctx->put_data = false;
5585
5586 tlv_put_failure:
5587 out:
5588 iput(&inode->vfs_inode);
5589 return ret;
5590 }
5591
send_extent_data(struct send_ctx * sctx,struct btrfs_path * path,const u64 offset,const u64 len)5592 static int send_extent_data(struct send_ctx *sctx, struct btrfs_path *path,
5593 const u64 offset, const u64 len)
5594 {
5595 const u64 end = offset + len;
5596 struct extent_buffer *leaf = path->nodes[0];
5597 struct btrfs_file_extent_item *ei;
5598 u64 read_size = max_send_read_size(sctx);
5599 u64 sent = 0;
5600
5601 if (sctx->flags & BTRFS_SEND_FLAG_NO_FILE_DATA)
5602 return send_update_extent(sctx, offset, len);
5603
5604 ei = btrfs_item_ptr(leaf, path->slots[0],
5605 struct btrfs_file_extent_item);
5606 /*
5607 * Do not go through encoded read for bs > ps cases.
5608 *
5609 * Encoded send is using vmallocated pages as buffer, which we can
5610 * not ensure every folio is large enough to contain a block.
5611 */
5612 if (sctx->send_root->fs_info->sectorsize <= PAGE_SIZE &&
5613 (sctx->flags & BTRFS_SEND_FLAG_COMPRESSED) &&
5614 btrfs_file_extent_compression(leaf, ei) != BTRFS_COMPRESS_NONE) {
5615 bool is_inline = (btrfs_file_extent_type(leaf, ei) ==
5616 BTRFS_FILE_EXTENT_INLINE);
5617
5618 /*
5619 * Send the compressed extent unless the compressed data is
5620 * larger than the decompressed data. This can happen if we're
5621 * not sending the entire extent, either because it has been
5622 * partially overwritten/truncated or because this is a part of
5623 * the extent that we couldn't clone in clone_range().
5624 */
5625 if (is_inline &&
5626 btrfs_file_extent_inline_item_len(leaf,
5627 path->slots[0]) <= len) {
5628 return send_encoded_inline_extent(sctx, path, offset,
5629 len);
5630 } else if (!is_inline &&
5631 btrfs_file_extent_disk_num_bytes(leaf, ei) <= len) {
5632 return send_encoded_extent(sctx, path, offset, len);
5633 }
5634 }
5635
5636 if (sctx->cur_inode == NULL) {
5637 struct btrfs_inode *btrfs_inode;
5638 struct btrfs_root *root = sctx->send_root;
5639
5640 btrfs_inode = btrfs_iget(sctx->cur_ino, root);
5641 if (IS_ERR(btrfs_inode))
5642 return PTR_ERR(btrfs_inode);
5643
5644 sctx->cur_inode = &btrfs_inode->vfs_inode;
5645 memset(&sctx->ra, 0, sizeof(struct file_ra_state));
5646 file_ra_state_init(&sctx->ra, sctx->cur_inode->i_mapping);
5647
5648 /*
5649 * It's very likely there are no pages from this inode in the page
5650 * cache, so after reading extents and sending their data, we clean
5651 * the page cache to avoid trashing the page cache (adding pressure
5652 * to the page cache and forcing eviction of other data more useful
5653 * for applications).
5654 *
5655 * We decide if we should clean the page cache simply by checking
5656 * if the inode's mapping nrpages is 0 when we first open it, and
5657 * not by using something like filemap_range_has_page() before
5658 * reading an extent because when we ask the readahead code to
5659 * read a given file range, it may (and almost always does) read
5660 * pages from beyond that range (see the documentation for
5661 * page_cache_sync_readahead()), so it would not be reliable,
5662 * because after reading the first extent future calls to
5663 * filemap_range_has_page() would return true because the readahead
5664 * on the previous extent resulted in reading pages of the current
5665 * extent as well.
5666 */
5667 sctx->clean_page_cache = (sctx->cur_inode->i_mapping->nrpages == 0);
5668 sctx->page_cache_clear_start = round_down(offset, PAGE_SIZE);
5669 }
5670
5671 while (sent < len) {
5672 u64 size = min(len - sent, read_size);
5673 int ret;
5674
5675 ret = send_write(sctx, offset + sent, size);
5676 if (ret < 0)
5677 return ret;
5678 sent += size;
5679 }
5680
5681 if (sctx->clean_page_cache && PAGE_ALIGNED(end)) {
5682 /*
5683 * Always operate only on ranges that are a multiple of the page
5684 * size. This is not only to prevent zeroing parts of a page in
5685 * the case of subpage sector size, but also to guarantee we evict
5686 * pages, as passing a range that is smaller than page size does
5687 * not evict the respective page (only zeroes part of its content).
5688 *
5689 * Always start from the end offset of the last range cleared.
5690 * This is because the readahead code may (and very often does)
5691 * reads pages beyond the range we request for readahead. So if
5692 * we have an extent layout like this:
5693 *
5694 * [ extent A ] [ extent B ] [ extent C ]
5695 *
5696 * When we ask page_cache_sync_readahead() to read extent A, it
5697 * may also trigger reads for pages of extent B. If we are doing
5698 * an incremental send and extent B has not changed between the
5699 * parent and send snapshots, some or all of its pages may end
5700 * up being read and placed in the page cache. So when truncating
5701 * the page cache we always start from the end offset of the
5702 * previously processed extent up to the end of the current
5703 * extent.
5704 */
5705 truncate_inode_pages_range(&sctx->cur_inode->i_data,
5706 sctx->page_cache_clear_start,
5707 end - 1);
5708 sctx->page_cache_clear_start = end;
5709 }
5710
5711 return 0;
5712 }
5713
5714 /*
5715 * Search for a capability xattr related to sctx->cur_ino. If the capability is
5716 * found, call send_set_xattr function to emit it.
5717 *
5718 * Return 0 if there isn't a capability, or when the capability was emitted
5719 * successfully, or < 0 if an error occurred.
5720 */
send_capabilities(struct send_ctx * sctx)5721 static int send_capabilities(struct send_ctx *sctx)
5722 {
5723 BTRFS_PATH_AUTO_FREE(path);
5724 struct btrfs_dir_item *di;
5725 struct extent_buffer *leaf;
5726 unsigned long data_ptr;
5727 char *buf = NULL;
5728 int buf_len;
5729 int ret = 0;
5730
5731 path = alloc_path_for_send();
5732 if (!path)
5733 return -ENOMEM;
5734
5735 di = btrfs_lookup_xattr(NULL, sctx->send_root, path, sctx->cur_ino,
5736 XATTR_NAME_CAPS, strlen(XATTR_NAME_CAPS), 0);
5737 if (!di) {
5738 /* There is no xattr for this inode */
5739 goto out;
5740 } else if (IS_ERR(di)) {
5741 ret = PTR_ERR(di);
5742 goto out;
5743 }
5744
5745 leaf = path->nodes[0];
5746 buf_len = btrfs_dir_data_len(leaf, di);
5747
5748 buf = kmalloc(buf_len, GFP_KERNEL);
5749 if (!buf) {
5750 ret = -ENOMEM;
5751 goto out;
5752 }
5753
5754 data_ptr = (unsigned long)(di + 1) + btrfs_dir_name_len(leaf, di);
5755 read_extent_buffer(leaf, buf, data_ptr, buf_len);
5756
5757 ret = send_set_xattr(sctx, XATTR_NAME_CAPS,
5758 strlen(XATTR_NAME_CAPS), buf, buf_len);
5759 out:
5760 kfree(buf);
5761 return ret;
5762 }
5763
clone_range(struct send_ctx * sctx,struct btrfs_path * dst_path,struct clone_root * clone_root,const u64 disk_byte,u64 data_offset,u64 offset,u64 len)5764 static int clone_range(struct send_ctx *sctx, struct btrfs_path *dst_path,
5765 struct clone_root *clone_root, const u64 disk_byte,
5766 u64 data_offset, u64 offset, u64 len)
5767 {
5768 BTRFS_PATH_AUTO_FREE(path);
5769 struct btrfs_key key;
5770 int ret;
5771 struct btrfs_inode_info info;
5772 u64 clone_src_i_size = 0;
5773
5774 /*
5775 * Prevent cloning from a zero offset with a length matching the sector
5776 * size because in some scenarios this will make the receiver fail.
5777 *
5778 * For example, if in the source filesystem the extent at offset 0
5779 * has a length of sectorsize and it was written using direct IO, then
5780 * it can never be an inline extent (even if compression is enabled).
5781 * Then this extent can be cloned in the original filesystem to a non
5782 * zero file offset, but it may not be possible to clone in the
5783 * destination filesystem because it can be inlined due to compression
5784 * on the destination filesystem (as the receiver's write operations are
5785 * always done using buffered IO). The same happens when the original
5786 * filesystem does not have compression enabled but the destination
5787 * filesystem has.
5788 */
5789 if (clone_root->offset == 0 &&
5790 len == sctx->send_root->fs_info->sectorsize)
5791 return send_extent_data(sctx, dst_path, offset, len);
5792
5793 path = alloc_path_for_send();
5794 if (!path)
5795 return -ENOMEM;
5796
5797 /*
5798 * There are inodes that have extents that lie behind its i_size. Don't
5799 * accept clones from these extents.
5800 */
5801 ret = get_inode_info(clone_root->root, clone_root->ino, &info);
5802 btrfs_release_path(path);
5803 if (ret < 0)
5804 return ret;
5805 clone_src_i_size = info.size;
5806
5807 /*
5808 * We can't send a clone operation for the entire range if we find
5809 * extent items in the respective range in the source file that
5810 * refer to different extents or if we find holes.
5811 * So check for that and do a mix of clone and regular write/copy
5812 * operations if needed.
5813 *
5814 * Example:
5815 *
5816 * mkfs.btrfs -f /dev/sda
5817 * mount /dev/sda /mnt
5818 * xfs_io -f -c "pwrite -S 0xaa 0K 100K" /mnt/foo
5819 * cp --reflink=always /mnt/foo /mnt/bar
5820 * xfs_io -c "pwrite -S 0xbb 50K 50K" /mnt/foo
5821 * btrfs subvolume snapshot -r /mnt /mnt/snap
5822 *
5823 * If when we send the snapshot and we are processing file bar (which
5824 * has a higher inode number than foo) we blindly send a clone operation
5825 * for the [0, 100K[ range from foo to bar, the receiver ends up getting
5826 * a file bar that matches the content of file foo - iow, doesn't match
5827 * the content from bar in the original filesystem.
5828 */
5829 key.objectid = clone_root->ino;
5830 key.type = BTRFS_EXTENT_DATA_KEY;
5831 key.offset = clone_root->offset;
5832 ret = btrfs_search_slot(NULL, clone_root->root, &key, path, 0, 0);
5833 if (ret < 0)
5834 return ret;
5835 if (ret > 0 && path->slots[0] > 0) {
5836 btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0] - 1);
5837 if (key.objectid == clone_root->ino &&
5838 key.type == BTRFS_EXTENT_DATA_KEY)
5839 path->slots[0]--;
5840 }
5841
5842 while (true) {
5843 struct extent_buffer *leaf = path->nodes[0];
5844 int slot = path->slots[0];
5845 struct btrfs_file_extent_item *ei;
5846 u8 type;
5847 u64 ext_len;
5848 u64 clone_len;
5849 u64 clone_data_offset;
5850 bool crossed_src_i_size = false;
5851
5852 if (slot >= btrfs_header_nritems(leaf)) {
5853 ret = btrfs_next_leaf(clone_root->root, path);
5854 if (ret < 0)
5855 return ret;
5856 else if (ret > 0)
5857 break;
5858 continue;
5859 }
5860
5861 btrfs_item_key_to_cpu(leaf, &key, slot);
5862
5863 /*
5864 * We might have an implicit trailing hole (NO_HOLES feature
5865 * enabled). We deal with it after leaving this loop.
5866 */
5867 if (key.objectid != clone_root->ino ||
5868 key.type != BTRFS_EXTENT_DATA_KEY)
5869 break;
5870
5871 ei = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
5872 type = btrfs_file_extent_type(leaf, ei);
5873 if (type == BTRFS_FILE_EXTENT_INLINE) {
5874 ext_len = btrfs_file_extent_ram_bytes(leaf, ei);
5875 ext_len = PAGE_ALIGN(ext_len);
5876 } else {
5877 ext_len = btrfs_file_extent_num_bytes(leaf, ei);
5878 }
5879
5880 if (key.offset + ext_len <= clone_root->offset)
5881 goto next;
5882
5883 if (key.offset > clone_root->offset) {
5884 /* Implicit hole, NO_HOLES feature enabled. */
5885 u64 hole_len = key.offset - clone_root->offset;
5886
5887 if (hole_len > len)
5888 hole_len = len;
5889 ret = send_extent_data(sctx, dst_path, offset,
5890 hole_len);
5891 if (ret < 0)
5892 return ret;
5893
5894 len -= hole_len;
5895 if (len == 0)
5896 break;
5897 offset += hole_len;
5898 clone_root->offset += hole_len;
5899 data_offset += hole_len;
5900 }
5901
5902 if (key.offset >= clone_root->offset + len)
5903 break;
5904
5905 if (key.offset >= clone_src_i_size)
5906 break;
5907
5908 if (key.offset + ext_len > clone_src_i_size) {
5909 ext_len = clone_src_i_size - key.offset;
5910 crossed_src_i_size = true;
5911 }
5912
5913 clone_data_offset = btrfs_file_extent_offset(leaf, ei);
5914 if (btrfs_file_extent_disk_bytenr(leaf, ei) == disk_byte) {
5915 clone_root->offset = key.offset;
5916 if (clone_data_offset < data_offset &&
5917 clone_data_offset + ext_len > data_offset) {
5918 u64 extent_offset;
5919
5920 extent_offset = data_offset - clone_data_offset;
5921 ext_len -= extent_offset;
5922 clone_data_offset += extent_offset;
5923 clone_root->offset += extent_offset;
5924 }
5925 }
5926
5927 clone_len = min_t(u64, ext_len, len);
5928
5929 if (btrfs_file_extent_disk_bytenr(leaf, ei) == disk_byte &&
5930 clone_data_offset == data_offset) {
5931 const u64 src_end = clone_root->offset + clone_len;
5932 const u64 sectorsize = SZ_64K;
5933
5934 /*
5935 * We can't clone the last block, when its size is not
5936 * sector size aligned, into the middle of a file. If we
5937 * do so, the receiver will get a failure (-EINVAL) when
5938 * trying to clone or will silently corrupt the data in
5939 * the destination file if it's on a kernel without the
5940 * fix introduced by commit ac765f83f1397646
5941 * ("Btrfs: fix data corruption due to cloning of eof
5942 * block).
5943 *
5944 * So issue a clone of the aligned down range plus a
5945 * regular write for the eof block, if we hit that case.
5946 *
5947 * Also, we use the maximum possible sector size, 64K,
5948 * because we don't know what's the sector size of the
5949 * filesystem that receives the stream, so we have to
5950 * assume the largest possible sector size.
5951 */
5952 if (src_end == clone_src_i_size &&
5953 !IS_ALIGNED(src_end, sectorsize) &&
5954 offset + clone_len < sctx->cur_inode_size) {
5955 u64 slen;
5956
5957 slen = ALIGN_DOWN(src_end - clone_root->offset,
5958 sectorsize);
5959 if (slen > 0) {
5960 ret = send_clone(sctx, offset, slen,
5961 clone_root);
5962 if (ret < 0)
5963 return ret;
5964 }
5965 ret = send_extent_data(sctx, dst_path,
5966 offset + slen,
5967 clone_len - slen);
5968 } else {
5969 ret = send_clone(sctx, offset, clone_len,
5970 clone_root);
5971 }
5972 } else if (crossed_src_i_size && clone_len < len) {
5973 /*
5974 * If we are at i_size of the clone source inode and we
5975 * can not clone from it, terminate the loop. This is
5976 * to avoid sending two write operations, one with a
5977 * length matching clone_len and the final one after
5978 * this loop with a length of len - clone_len.
5979 *
5980 * When using encoded writes (BTRFS_SEND_FLAG_COMPRESSED
5981 * was passed to the send ioctl), this helps avoid
5982 * sending an encoded write for an offset that is not
5983 * sector size aligned, in case the i_size of the source
5984 * inode is not sector size aligned. That will make the
5985 * receiver fallback to decompression of the data and
5986 * writing it using regular buffered IO, therefore while
5987 * not incorrect, it's not optimal due decompression and
5988 * possible re-compression at the receiver.
5989 */
5990 break;
5991 } else {
5992 ret = send_extent_data(sctx, dst_path, offset,
5993 clone_len);
5994 }
5995
5996 if (ret < 0)
5997 return ret;
5998
5999 len -= clone_len;
6000 if (len == 0)
6001 break;
6002 offset += clone_len;
6003 clone_root->offset += clone_len;
6004
6005 /*
6006 * If we are cloning from the file we are currently processing,
6007 * and using the send root as the clone root, we must stop once
6008 * the current clone offset reaches the current eof of the file
6009 * at the receiver, otherwise we would issue an invalid clone
6010 * operation (source range going beyond eof) and cause the
6011 * receiver to fail. So if we reach the current eof, bail out
6012 * and fallback to a regular write.
6013 */
6014 if (clone_root->root == sctx->send_root &&
6015 clone_root->ino == sctx->cur_ino &&
6016 clone_root->offset >= sctx->cur_inode_next_write_offset)
6017 break;
6018
6019 data_offset += clone_len;
6020 next:
6021 path->slots[0]++;
6022 }
6023
6024 if (len > 0)
6025 ret = send_extent_data(sctx, dst_path, offset, len);
6026 else
6027 ret = 0;
6028 return ret;
6029 }
6030
send_write_or_clone(struct send_ctx * sctx,struct btrfs_path * path,struct btrfs_key * key,struct clone_root * clone_root)6031 static int send_write_or_clone(struct send_ctx *sctx,
6032 struct btrfs_path *path,
6033 struct btrfs_key *key,
6034 struct clone_root *clone_root)
6035 {
6036 int ret = 0;
6037 u64 offset = key->offset;
6038 u64 end;
6039 u64 bs = sctx->send_root->fs_info->sectorsize;
6040 struct btrfs_file_extent_item *ei;
6041 u64 disk_byte;
6042 u64 data_offset;
6043 u64 num_bytes;
6044 struct btrfs_inode_info info = { 0 };
6045
6046 end = min_t(u64, btrfs_file_extent_end(path), sctx->cur_inode_size);
6047 if (offset >= end)
6048 return 0;
6049
6050 num_bytes = end - offset;
6051
6052 if (!clone_root)
6053 goto write_data;
6054
6055 if (IS_ALIGNED(end, bs))
6056 goto clone_data;
6057
6058 /*
6059 * If the extent end is not aligned, we can clone if the extent ends at
6060 * the i_size of the inode and the clone range ends at the i_size of the
6061 * source inode, otherwise the clone operation fails with -EINVAL.
6062 */
6063 if (end != sctx->cur_inode_size)
6064 goto write_data;
6065
6066 ret = get_inode_info(clone_root->root, clone_root->ino, &info);
6067 if (ret < 0)
6068 return ret;
6069
6070 if (clone_root->offset + num_bytes == info.size) {
6071 /*
6072 * The final size of our file matches the end offset, but it may
6073 * be that its current size is larger, so we have to truncate it
6074 * to any value between the start offset of the range and the
6075 * final i_size, otherwise the clone operation is invalid
6076 * because it's unaligned and it ends before the current EOF.
6077 * We do this truncate to the final i_size when we finish
6078 * processing the inode, but it's too late by then. And here we
6079 * truncate to the start offset of the range because it's always
6080 * sector size aligned while if it were the final i_size it
6081 * would result in dirtying part of a page, filling part of a
6082 * page with zeroes and then having the clone operation at the
6083 * receiver trigger IO and wait for it due to the dirty page.
6084 */
6085 if (sctx->parent_root != NULL) {
6086 ret = send_truncate(sctx, sctx->cur_ino,
6087 sctx->cur_inode_gen, offset);
6088 if (ret < 0)
6089 return ret;
6090 }
6091 goto clone_data;
6092 }
6093
6094 write_data:
6095 ret = send_extent_data(sctx, path, offset, num_bytes);
6096 sctx->cur_inode_next_write_offset = end;
6097 return ret;
6098
6099 clone_data:
6100 ei = btrfs_item_ptr(path->nodes[0], path->slots[0],
6101 struct btrfs_file_extent_item);
6102 disk_byte = btrfs_file_extent_disk_bytenr(path->nodes[0], ei);
6103 data_offset = btrfs_file_extent_offset(path->nodes[0], ei);
6104 ret = clone_range(sctx, path, clone_root, disk_byte, data_offset, offset,
6105 num_bytes);
6106 sctx->cur_inode_next_write_offset = end;
6107 return ret;
6108 }
6109
is_extent_unchanged(struct send_ctx * sctx,struct btrfs_path * left_path,struct btrfs_key * ekey)6110 static int is_extent_unchanged(struct send_ctx *sctx,
6111 struct btrfs_path *left_path,
6112 struct btrfs_key *ekey)
6113 {
6114 int ret = 0;
6115 struct btrfs_key key;
6116 BTRFS_PATH_AUTO_FREE(path);
6117 struct extent_buffer *eb;
6118 int slot;
6119 struct btrfs_key found_key;
6120 struct btrfs_file_extent_item *ei;
6121 u64 left_disknr;
6122 u64 right_disknr;
6123 u64 left_offset;
6124 u64 right_offset;
6125 u64 left_offset_fixed;
6126 u64 left_len;
6127 u64 right_len;
6128 u64 left_gen;
6129 u64 right_gen;
6130 u8 left_type;
6131 u8 right_type;
6132
6133 path = alloc_path_for_send();
6134 if (!path)
6135 return -ENOMEM;
6136
6137 eb = left_path->nodes[0];
6138 slot = left_path->slots[0];
6139 ei = btrfs_item_ptr(eb, slot, struct btrfs_file_extent_item);
6140 left_type = btrfs_file_extent_type(eb, ei);
6141
6142 if (left_type != BTRFS_FILE_EXTENT_REG)
6143 return 0;
6144
6145 left_disknr = btrfs_file_extent_disk_bytenr(eb, ei);
6146 left_len = btrfs_file_extent_num_bytes(eb, ei);
6147 left_offset = btrfs_file_extent_offset(eb, ei);
6148 left_gen = btrfs_file_extent_generation(eb, ei);
6149
6150 /*
6151 * Following comments will refer to these graphics. L is the left
6152 * extents which we are checking at the moment. 1-8 are the right
6153 * extents that we iterate.
6154 *
6155 * |-----L-----|
6156 * |-1-|-2a-|-3-|-4-|-5-|-6-|
6157 *
6158 * |-----L-----|
6159 * |--1--|-2b-|...(same as above)
6160 *
6161 * Alternative situation. Happens on files where extents got split.
6162 * |-----L-----|
6163 * |-----------7-----------|-6-|
6164 *
6165 * Alternative situation. Happens on files which got larger.
6166 * |-----L-----|
6167 * |-8-|
6168 * Nothing follows after 8.
6169 */
6170
6171 key.objectid = ekey->objectid;
6172 key.type = BTRFS_EXTENT_DATA_KEY;
6173 key.offset = ekey->offset;
6174 ret = btrfs_search_slot_for_read(sctx->parent_root, &key, path, 0, 0);
6175 if (ret < 0)
6176 return ret;
6177 if (ret)
6178 return 0;
6179
6180 /*
6181 * Handle special case where the right side has no extents at all.
6182 */
6183 eb = path->nodes[0];
6184 slot = path->slots[0];
6185 btrfs_item_key_to_cpu(eb, &found_key, slot);
6186 if (found_key.objectid != key.objectid ||
6187 found_key.type != key.type)
6188 /* If we're a hole then just pretend nothing changed */
6189 return (left_disknr ? 0 : 1);
6190
6191 /*
6192 * We're now on 2a, 2b or 7.
6193 */
6194 key = found_key;
6195 while (key.offset < ekey->offset + left_len) {
6196 ei = btrfs_item_ptr(eb, slot, struct btrfs_file_extent_item);
6197 right_type = btrfs_file_extent_type(eb, ei);
6198 if (right_type != BTRFS_FILE_EXTENT_REG &&
6199 right_type != BTRFS_FILE_EXTENT_INLINE)
6200 return 0;
6201
6202 if (right_type == BTRFS_FILE_EXTENT_INLINE) {
6203 right_len = btrfs_file_extent_ram_bytes(eb, ei);
6204 right_len = PAGE_ALIGN(right_len);
6205 } else {
6206 right_len = btrfs_file_extent_num_bytes(eb, ei);
6207 }
6208
6209 /*
6210 * Are we at extent 8? If yes, we know the extent is changed.
6211 * This may only happen on the first iteration.
6212 */
6213 if (found_key.offset + right_len <= ekey->offset)
6214 /* If we're a hole just pretend nothing changed */
6215 return (left_disknr ? 0 : 1);
6216
6217 /*
6218 * We just wanted to see if when we have an inline extent, what
6219 * follows it is a regular extent (wanted to check the above
6220 * condition for inline extents too). This should normally not
6221 * happen but it's possible for example when we have an inline
6222 * compressed extent representing data with a size matching
6223 * the page size (currently the same as sector size).
6224 */
6225 if (right_type == BTRFS_FILE_EXTENT_INLINE)
6226 return 0;
6227
6228 right_disknr = btrfs_file_extent_disk_bytenr(eb, ei);
6229 right_offset = btrfs_file_extent_offset(eb, ei);
6230 right_gen = btrfs_file_extent_generation(eb, ei);
6231
6232 left_offset_fixed = left_offset;
6233 if (key.offset < ekey->offset) {
6234 /* Fix the right offset for 2a and 7. */
6235 right_offset += ekey->offset - key.offset;
6236 } else {
6237 /* Fix the left offset for all behind 2a and 2b */
6238 left_offset_fixed += key.offset - ekey->offset;
6239 }
6240
6241 /*
6242 * Check if we have the same extent.
6243 */
6244 if (left_disknr != right_disknr ||
6245 left_offset_fixed != right_offset ||
6246 left_gen != right_gen)
6247 return 0;
6248
6249 /*
6250 * Go to the next extent.
6251 */
6252 ret = btrfs_next_item(sctx->parent_root, path);
6253 if (ret < 0)
6254 return ret;
6255 if (!ret) {
6256 eb = path->nodes[0];
6257 slot = path->slots[0];
6258 btrfs_item_key_to_cpu(eb, &found_key, slot);
6259 }
6260 if (ret || found_key.objectid != key.objectid ||
6261 found_key.type != key.type) {
6262 key.offset += right_len;
6263 break;
6264 }
6265 if (found_key.offset != key.offset + right_len)
6266 return 0;
6267
6268 key = found_key;
6269 }
6270
6271 /*
6272 * We're now behind the left extent (treat as unchanged) or at the end
6273 * of the right side (treat as changed).
6274 */
6275 if (key.offset >= ekey->offset + left_len)
6276 ret = 1;
6277 else
6278 ret = 0;
6279
6280 return ret;
6281 }
6282
get_last_extent(struct send_ctx * sctx,u64 offset)6283 static int get_last_extent(struct send_ctx *sctx, u64 offset)
6284 {
6285 BTRFS_PATH_AUTO_FREE(path);
6286 struct btrfs_root *root = sctx->send_root;
6287 struct btrfs_key key;
6288 int ret;
6289
6290 path = alloc_path_for_send();
6291 if (!path)
6292 return -ENOMEM;
6293
6294 sctx->cur_inode_last_extent = 0;
6295
6296 key.objectid = sctx->cur_ino;
6297 key.type = BTRFS_EXTENT_DATA_KEY;
6298 key.offset = offset;
6299 ret = btrfs_search_slot_for_read(root, &key, path, 0, 1);
6300 if (ret < 0)
6301 return ret;
6302 ret = 0;
6303 btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
6304 if (key.objectid != sctx->cur_ino || key.type != BTRFS_EXTENT_DATA_KEY)
6305 return ret;
6306
6307 sctx->cur_inode_last_extent = btrfs_file_extent_end(path);
6308 return ret;
6309 }
6310
range_is_hole_in_parent(struct send_ctx * sctx,const u64 start,const u64 end)6311 static int range_is_hole_in_parent(struct send_ctx *sctx,
6312 const u64 start,
6313 const u64 end)
6314 {
6315 BTRFS_PATH_AUTO_FREE(path);
6316 struct btrfs_key key;
6317 struct btrfs_root *root = sctx->parent_root;
6318 u64 search_start = start;
6319 int ret;
6320
6321 path = alloc_path_for_send();
6322 if (!path)
6323 return -ENOMEM;
6324
6325 key.objectid = sctx->cur_ino;
6326 key.type = BTRFS_EXTENT_DATA_KEY;
6327 key.offset = search_start;
6328 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
6329 if (ret < 0)
6330 return ret;
6331 if (ret > 0 && path->slots[0] > 0)
6332 path->slots[0]--;
6333
6334 while (search_start < end) {
6335 struct extent_buffer *leaf = path->nodes[0];
6336 int slot = path->slots[0];
6337 struct btrfs_file_extent_item *fi;
6338 u64 extent_end;
6339
6340 if (slot >= btrfs_header_nritems(leaf)) {
6341 ret = btrfs_next_leaf(root, path);
6342 if (ret < 0)
6343 return ret;
6344 if (ret > 0)
6345 break;
6346 continue;
6347 }
6348
6349 btrfs_item_key_to_cpu(leaf, &key, slot);
6350 if (key.objectid < sctx->cur_ino ||
6351 key.type < BTRFS_EXTENT_DATA_KEY)
6352 goto next;
6353 if (key.objectid > sctx->cur_ino ||
6354 key.type > BTRFS_EXTENT_DATA_KEY ||
6355 key.offset >= end)
6356 break;
6357
6358 fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
6359 extent_end = btrfs_file_extent_end(path);
6360 if (extent_end <= start)
6361 goto next;
6362 if (btrfs_file_extent_disk_bytenr(leaf, fi) == 0) {
6363 search_start = extent_end;
6364 goto next;
6365 }
6366 return 0;
6367 next:
6368 path->slots[0]++;
6369 }
6370 return 1;
6371 }
6372
maybe_send_hole(struct send_ctx * sctx,struct btrfs_path * path,struct btrfs_key * key)6373 static int maybe_send_hole(struct send_ctx *sctx, struct btrfs_path *path,
6374 struct btrfs_key *key)
6375 {
6376 int ret = 0;
6377
6378 if (sctx->cur_ino != key->objectid || !need_send_hole(sctx))
6379 return 0;
6380
6381 /*
6382 * Get last extent's end offset (exclusive) if we haven't determined it
6383 * yet (we're processing the first file extent item that is new), or if
6384 * we're at the first slot of a leaf and the last extent's end is less
6385 * than the current extent's offset, because we might have skipped
6386 * entire leaves that contained only file extent items for our current
6387 * inode. These leaves have a generation number smaller (older) than the
6388 * one in the current leaf and the leaf our last extent came from, and
6389 * are located between these 2 leaves.
6390 */
6391 if ((sctx->cur_inode_last_extent == (u64)-1) ||
6392 (path->slots[0] == 0 && sctx->cur_inode_last_extent < key->offset)) {
6393 ret = get_last_extent(sctx, key->offset - 1);
6394 if (ret)
6395 return ret;
6396 }
6397
6398 if (sctx->cur_inode_last_extent < key->offset) {
6399 ret = range_is_hole_in_parent(sctx,
6400 sctx->cur_inode_last_extent,
6401 key->offset);
6402 if (ret < 0)
6403 return ret;
6404 else if (ret == 0)
6405 ret = send_hole(sctx, key->offset);
6406 else
6407 ret = 0;
6408 }
6409 sctx->cur_inode_last_extent = btrfs_file_extent_end(path);
6410 return ret;
6411 }
6412
process_extent(struct send_ctx * sctx,struct btrfs_path * path,struct btrfs_key * key)6413 static int process_extent(struct send_ctx *sctx,
6414 struct btrfs_path *path,
6415 struct btrfs_key *key)
6416 {
6417 struct clone_root *found_clone = NULL;
6418 int ret = 0;
6419
6420 if (S_ISLNK(sctx->cur_inode_mode))
6421 return 0;
6422
6423 if (sctx->parent_root && !sctx->cur_inode_new) {
6424 ret = is_extent_unchanged(sctx, path, key);
6425 if (ret < 0)
6426 goto out;
6427 if (ret) {
6428 ret = 0;
6429 goto out_hole;
6430 }
6431 } else {
6432 struct btrfs_file_extent_item *ei;
6433 u8 type;
6434
6435 ei = btrfs_item_ptr(path->nodes[0], path->slots[0],
6436 struct btrfs_file_extent_item);
6437 type = btrfs_file_extent_type(path->nodes[0], ei);
6438 if (type == BTRFS_FILE_EXTENT_PREALLOC ||
6439 type == BTRFS_FILE_EXTENT_REG) {
6440 /*
6441 * The send spec does not have a prealloc command yet,
6442 * so just leave a hole for prealloc'ed extents until
6443 * we have enough commands queued up to justify rev'ing
6444 * the send spec.
6445 */
6446 if (type == BTRFS_FILE_EXTENT_PREALLOC) {
6447 ret = 0;
6448 goto out;
6449 }
6450
6451 /* Have a hole, just skip it. */
6452 if (btrfs_file_extent_disk_bytenr(path->nodes[0], ei) == 0) {
6453 ret = 0;
6454 goto out;
6455 }
6456 }
6457 }
6458
6459 ret = find_extent_clone(sctx, path, key->objectid, key->offset,
6460 sctx->cur_inode_size, &found_clone);
6461 if (ret != -ENOENT && ret < 0)
6462 goto out;
6463
6464 ret = send_write_or_clone(sctx, path, key, found_clone);
6465 if (ret)
6466 goto out;
6467 out_hole:
6468 ret = maybe_send_hole(sctx, path, key);
6469 out:
6470 return ret;
6471 }
6472
process_all_extents(struct send_ctx * sctx)6473 static int process_all_extents(struct send_ctx *sctx)
6474 {
6475 int ret = 0;
6476 int iter_ret = 0;
6477 struct btrfs_root *root;
6478 BTRFS_PATH_AUTO_FREE(path);
6479 struct btrfs_key key;
6480 struct btrfs_key found_key;
6481
6482 root = sctx->send_root;
6483 path = alloc_path_for_send();
6484 if (!path)
6485 return -ENOMEM;
6486
6487 key.objectid = sctx->cmp_key->objectid;
6488 key.type = BTRFS_EXTENT_DATA_KEY;
6489 key.offset = 0;
6490 btrfs_for_each_slot(root, &key, &found_key, path, iter_ret) {
6491 if (found_key.objectid != key.objectid ||
6492 found_key.type != key.type) {
6493 ret = 0;
6494 break;
6495 }
6496
6497 ret = process_extent(sctx, path, &found_key);
6498 if (ret < 0)
6499 break;
6500 }
6501 /* Catch error found during iteration */
6502 if (iter_ret < 0)
6503 ret = iter_ret;
6504
6505 return ret;
6506 }
6507
process_recorded_refs_if_needed(struct send_ctx * sctx,bool at_end,int * pending_move,int * refs_processed)6508 static int process_recorded_refs_if_needed(struct send_ctx *sctx, bool at_end,
6509 int *pending_move,
6510 int *refs_processed)
6511 {
6512 int ret = 0;
6513
6514 if (sctx->cur_ino == 0)
6515 goto out;
6516 if (!at_end && sctx->cur_ino == sctx->cmp_key->objectid &&
6517 sctx->cmp_key->type <= BTRFS_INODE_EXTREF_KEY)
6518 goto out;
6519 if (list_empty(&sctx->new_refs) && list_empty(&sctx->deleted_refs))
6520 goto out;
6521
6522 ret = process_recorded_refs(sctx, pending_move);
6523 if (ret < 0)
6524 goto out;
6525
6526 *refs_processed = 1;
6527 out:
6528 return ret;
6529 }
6530
finish_inode_if_needed(struct send_ctx * sctx,bool at_end)6531 static int finish_inode_if_needed(struct send_ctx *sctx, bool at_end)
6532 {
6533 int ret = 0;
6534 struct btrfs_inode_info info;
6535 u64 left_mode;
6536 u64 left_uid;
6537 u64 left_gid;
6538 u64 left_fileattr;
6539 u64 right_mode;
6540 u64 right_uid;
6541 u64 right_gid;
6542 u64 right_fileattr;
6543 int need_chmod = 0;
6544 int need_chown = 0;
6545 bool need_fileattr = false;
6546 int need_truncate = 1;
6547 int pending_move = 0;
6548 int refs_processed = 0;
6549
6550 if (sctx->ignore_cur_inode)
6551 return 0;
6552
6553 ret = process_recorded_refs_if_needed(sctx, at_end, &pending_move,
6554 &refs_processed);
6555 if (ret < 0)
6556 goto out;
6557
6558 /*
6559 * We have processed the refs and thus need to advance send_progress.
6560 * Now, calls to get_cur_xxx will take the updated refs of the current
6561 * inode into account.
6562 *
6563 * On the other hand, if our current inode is a directory and couldn't
6564 * be moved/renamed because its parent was renamed/moved too and it has
6565 * a higher inode number, we can only move/rename our current inode
6566 * after we moved/renamed its parent. Therefore in this case operate on
6567 * the old path (pre move/rename) of our current inode, and the
6568 * move/rename will be performed later.
6569 */
6570 if (refs_processed && !pending_move)
6571 sctx->send_progress = sctx->cur_ino + 1;
6572
6573 if (sctx->cur_ino == 0 || sctx->cur_inode_deleted)
6574 goto out;
6575 if (!at_end && sctx->cmp_key->objectid == sctx->cur_ino)
6576 goto out;
6577 ret = get_inode_info(sctx->send_root, sctx->cur_ino, &info);
6578 if (ret < 0)
6579 goto out;
6580 left_mode = info.mode;
6581 left_uid = info.uid;
6582 left_gid = info.gid;
6583 left_fileattr = info.fileattr;
6584
6585 if (!sctx->parent_root || sctx->cur_inode_new) {
6586 need_chown = 1;
6587 if (!S_ISLNK(sctx->cur_inode_mode))
6588 need_chmod = 1;
6589 if (sctx->cur_inode_next_write_offset == sctx->cur_inode_size)
6590 need_truncate = 0;
6591 } else {
6592 u64 old_size;
6593
6594 ret = get_inode_info(sctx->parent_root, sctx->cur_ino, &info);
6595 if (ret < 0)
6596 goto out;
6597 old_size = info.size;
6598 right_mode = info.mode;
6599 right_uid = info.uid;
6600 right_gid = info.gid;
6601 right_fileattr = info.fileattr;
6602
6603 if (left_uid != right_uid || left_gid != right_gid)
6604 need_chown = 1;
6605 if (!S_ISLNK(sctx->cur_inode_mode) && left_mode != right_mode)
6606 need_chmod = 1;
6607 if (!S_ISLNK(sctx->cur_inode_mode) && left_fileattr != right_fileattr)
6608 need_fileattr = true;
6609 if ((old_size == sctx->cur_inode_size) ||
6610 (sctx->cur_inode_size > old_size &&
6611 sctx->cur_inode_next_write_offset == sctx->cur_inode_size))
6612 need_truncate = 0;
6613 }
6614
6615 if (S_ISREG(sctx->cur_inode_mode)) {
6616 if (need_send_hole(sctx)) {
6617 if (sctx->cur_inode_last_extent == (u64)-1 ||
6618 sctx->cur_inode_last_extent <
6619 sctx->cur_inode_size) {
6620 ret = get_last_extent(sctx, (u64)-1);
6621 if (ret)
6622 goto out;
6623 }
6624 if (sctx->cur_inode_last_extent < sctx->cur_inode_size) {
6625 ret = range_is_hole_in_parent(sctx,
6626 sctx->cur_inode_last_extent,
6627 sctx->cur_inode_size);
6628 if (ret < 0) {
6629 goto out;
6630 } else if (ret == 0) {
6631 ret = send_hole(sctx, sctx->cur_inode_size);
6632 if (ret < 0)
6633 goto out;
6634 } else {
6635 /* Range is already a hole, skip. */
6636 ret = 0;
6637 }
6638 }
6639 }
6640 if (need_truncate) {
6641 ret = send_truncate(sctx, sctx->cur_ino,
6642 sctx->cur_inode_gen,
6643 sctx->cur_inode_size);
6644 if (ret < 0)
6645 goto out;
6646 }
6647 }
6648
6649 if (need_chown) {
6650 ret = send_chown(sctx, sctx->cur_ino, sctx->cur_inode_gen,
6651 left_uid, left_gid);
6652 if (ret < 0)
6653 goto out;
6654 }
6655 if (need_chmod) {
6656 ret = send_chmod(sctx, sctx->cur_ino, sctx->cur_inode_gen,
6657 left_mode);
6658 if (ret < 0)
6659 goto out;
6660 }
6661 if (need_fileattr) {
6662 ret = send_fileattr(sctx, sctx->cur_ino, sctx->cur_inode_gen,
6663 left_fileattr);
6664 if (ret < 0)
6665 goto out;
6666 }
6667
6668 if (proto_cmd_ok(sctx, BTRFS_SEND_C_ENABLE_VERITY)
6669 && sctx->cur_inode_needs_verity) {
6670 ret = process_verity(sctx);
6671 if (ret < 0)
6672 goto out;
6673 }
6674
6675 ret = send_capabilities(sctx);
6676 if (ret < 0)
6677 goto out;
6678
6679 /*
6680 * If other directory inodes depended on our current directory
6681 * inode's move/rename, now do their move/rename operations.
6682 */
6683 if (!is_waiting_for_move(sctx, sctx->cur_ino)) {
6684 ret = apply_children_dir_moves(sctx);
6685 if (ret)
6686 goto out;
6687 /*
6688 * Need to send that every time, no matter if it actually
6689 * changed between the two trees as we have done changes to
6690 * the inode before. If our inode is a directory and it's
6691 * waiting to be moved/renamed, we will send its utimes when
6692 * it's moved/renamed, therefore we don't need to do it here.
6693 */
6694 sctx->send_progress = sctx->cur_ino + 1;
6695
6696 /*
6697 * If the current inode is a non-empty directory, delay issuing
6698 * the utimes command for it, as it's very likely we have inodes
6699 * with an higher number inside it. We want to issue the utimes
6700 * command only after adding all dentries to it.
6701 */
6702 if (S_ISDIR(sctx->cur_inode_mode) && sctx->cur_inode_size > 0)
6703 ret = cache_dir_utimes(sctx, sctx->cur_ino, sctx->cur_inode_gen);
6704 else
6705 ret = send_utimes(sctx, sctx->cur_ino, sctx->cur_inode_gen);
6706
6707 if (ret < 0)
6708 goto out;
6709 }
6710
6711 out:
6712 if (!ret)
6713 ret = trim_dir_utimes_cache(sctx);
6714
6715 return ret;
6716 }
6717
close_current_inode(struct send_ctx * sctx)6718 static void close_current_inode(struct send_ctx *sctx)
6719 {
6720 u64 i_size;
6721
6722 if (sctx->cur_inode == NULL)
6723 return;
6724
6725 i_size = i_size_read(sctx->cur_inode);
6726
6727 /*
6728 * If we are doing an incremental send, we may have extents between the
6729 * last processed extent and the i_size that have not been processed
6730 * because they haven't changed but we may have read some of their pages
6731 * through readahead, see the comments at send_extent_data().
6732 */
6733 if (sctx->clean_page_cache && sctx->page_cache_clear_start < i_size)
6734 truncate_inode_pages_range(&sctx->cur_inode->i_data,
6735 sctx->page_cache_clear_start,
6736 round_up(i_size, PAGE_SIZE) - 1);
6737
6738 iput(sctx->cur_inode);
6739 sctx->cur_inode = NULL;
6740 }
6741
changed_inode(struct send_ctx * sctx,enum btrfs_compare_tree_result result)6742 static int changed_inode(struct send_ctx *sctx,
6743 enum btrfs_compare_tree_result result)
6744 {
6745 int ret = 0;
6746 struct btrfs_key *key = sctx->cmp_key;
6747 struct btrfs_inode_item *left_ii = NULL;
6748 struct btrfs_inode_item *right_ii = NULL;
6749 u64 left_gen = 0;
6750 u64 right_gen = 0;
6751
6752 close_current_inode(sctx);
6753
6754 sctx->cur_ino = key->objectid;
6755 sctx->cur_inode_new_gen = false;
6756 sctx->cur_inode_last_extent = (u64)-1;
6757 sctx->cur_inode_next_write_offset = 0;
6758 sctx->ignore_cur_inode = false;
6759 fs_path_reset(&sctx->cur_inode_path);
6760
6761 /*
6762 * Set send_progress to current inode. This will tell all get_cur_xxx
6763 * functions that the current inode's refs are not updated yet. Later,
6764 * when process_recorded_refs is finished, it is set to cur_ino + 1.
6765 */
6766 sctx->send_progress = sctx->cur_ino;
6767
6768 if (result == BTRFS_COMPARE_TREE_NEW ||
6769 result == BTRFS_COMPARE_TREE_CHANGED) {
6770 left_ii = btrfs_item_ptr(sctx->left_path->nodes[0],
6771 sctx->left_path->slots[0],
6772 struct btrfs_inode_item);
6773 left_gen = btrfs_inode_generation(sctx->left_path->nodes[0],
6774 left_ii);
6775 } else {
6776 right_ii = btrfs_item_ptr(sctx->right_path->nodes[0],
6777 sctx->right_path->slots[0],
6778 struct btrfs_inode_item);
6779 right_gen = btrfs_inode_generation(sctx->right_path->nodes[0],
6780 right_ii);
6781 }
6782 if (result == BTRFS_COMPARE_TREE_CHANGED) {
6783 right_ii = btrfs_item_ptr(sctx->right_path->nodes[0],
6784 sctx->right_path->slots[0],
6785 struct btrfs_inode_item);
6786
6787 right_gen = btrfs_inode_generation(sctx->right_path->nodes[0],
6788 right_ii);
6789
6790 /*
6791 * The cur_ino = root dir case is special here. We can't treat
6792 * the inode as deleted+reused because it would generate a
6793 * stream that tries to delete/mkdir the root dir.
6794 */
6795 if (left_gen != right_gen &&
6796 sctx->cur_ino != BTRFS_FIRST_FREE_OBJECTID)
6797 sctx->cur_inode_new_gen = true;
6798 }
6799
6800 /*
6801 * Normally we do not find inodes with a link count of zero (orphans)
6802 * because the most common case is to create a snapshot and use it
6803 * for a send operation. However other less common use cases involve
6804 * using a subvolume and send it after turning it to RO mode just
6805 * after deleting all hard links of a file while holding an open
6806 * file descriptor against it or turning a RO snapshot into RW mode,
6807 * keep an open file descriptor against a file, delete it and then
6808 * turn the snapshot back to RO mode before using it for a send
6809 * operation. The former is what the receiver operation does.
6810 * Therefore, if we want to send these snapshots soon after they're
6811 * received, we need to handle orphan inodes as well. Moreover, orphans
6812 * can appear not only in the send snapshot but also in the parent
6813 * snapshot. Here are several cases:
6814 *
6815 * Case 1: BTRFS_COMPARE_TREE_NEW
6816 * | send snapshot | action
6817 * --------------------------------
6818 * nlink | 0 | ignore
6819 *
6820 * Case 2: BTRFS_COMPARE_TREE_DELETED
6821 * | parent snapshot | action
6822 * ----------------------------------
6823 * nlink | 0 | as usual
6824 * Note: No unlinks will be sent because there're no paths for it.
6825 *
6826 * Case 3: BTRFS_COMPARE_TREE_CHANGED
6827 * | | parent snapshot | send snapshot | action
6828 * -----------------------------------------------------------------------
6829 * subcase 1 | nlink | 0 | 0 | ignore
6830 * subcase 2 | nlink | >0 | 0 | new_gen(deletion)
6831 * subcase 3 | nlink | 0 | >0 | new_gen(creation)
6832 *
6833 */
6834 if (result == BTRFS_COMPARE_TREE_NEW) {
6835 if (btrfs_inode_nlink(sctx->left_path->nodes[0], left_ii) == 0) {
6836 sctx->ignore_cur_inode = true;
6837 goto out;
6838 }
6839 sctx->cur_inode_gen = left_gen;
6840 sctx->cur_inode_new = true;
6841 sctx->cur_inode_deleted = false;
6842 sctx->cur_inode_size = btrfs_inode_size(
6843 sctx->left_path->nodes[0], left_ii);
6844 sctx->cur_inode_mode = btrfs_inode_mode(
6845 sctx->left_path->nodes[0], left_ii);
6846 sctx->cur_inode_rdev = btrfs_inode_rdev(
6847 sctx->left_path->nodes[0], left_ii);
6848 if (sctx->cur_ino != BTRFS_FIRST_FREE_OBJECTID)
6849 ret = send_create_inode_if_needed(sctx);
6850 } else if (result == BTRFS_COMPARE_TREE_DELETED) {
6851 sctx->cur_inode_gen = right_gen;
6852 sctx->cur_inode_new = false;
6853 sctx->cur_inode_deleted = true;
6854 sctx->cur_inode_size = btrfs_inode_size(
6855 sctx->right_path->nodes[0], right_ii);
6856 sctx->cur_inode_mode = btrfs_inode_mode(
6857 sctx->right_path->nodes[0], right_ii);
6858 } else if (result == BTRFS_COMPARE_TREE_CHANGED) {
6859 u32 new_nlinks, old_nlinks;
6860
6861 new_nlinks = btrfs_inode_nlink(sctx->left_path->nodes[0], left_ii);
6862 old_nlinks = btrfs_inode_nlink(sctx->right_path->nodes[0], right_ii);
6863 if (new_nlinks == 0 && old_nlinks == 0) {
6864 sctx->ignore_cur_inode = true;
6865 goto out;
6866 } else if (new_nlinks == 0 || old_nlinks == 0) {
6867 sctx->cur_inode_new_gen = 1;
6868 }
6869 /*
6870 * We need to do some special handling in case the inode was
6871 * reported as changed with a changed generation number. This
6872 * means that the original inode was deleted and new inode
6873 * reused the same inum. So we have to treat the old inode as
6874 * deleted and the new one as new.
6875 */
6876 if (sctx->cur_inode_new_gen) {
6877 /*
6878 * First, process the inode as if it was deleted.
6879 */
6880 if (old_nlinks > 0) {
6881 sctx->cur_inode_gen = right_gen;
6882 sctx->cur_inode_new = false;
6883 sctx->cur_inode_deleted = true;
6884 sctx->cur_inode_size = btrfs_inode_size(
6885 sctx->right_path->nodes[0], right_ii);
6886 sctx->cur_inode_mode = btrfs_inode_mode(
6887 sctx->right_path->nodes[0], right_ii);
6888 ret = process_all_refs(sctx,
6889 BTRFS_COMPARE_TREE_DELETED);
6890 if (ret < 0)
6891 goto out;
6892 }
6893
6894 /*
6895 * Now process the inode as if it was new.
6896 */
6897 if (new_nlinks > 0) {
6898 sctx->cur_inode_gen = left_gen;
6899 sctx->cur_inode_new = true;
6900 sctx->cur_inode_deleted = false;
6901 sctx->cur_inode_size = btrfs_inode_size(
6902 sctx->left_path->nodes[0],
6903 left_ii);
6904 sctx->cur_inode_mode = btrfs_inode_mode(
6905 sctx->left_path->nodes[0],
6906 left_ii);
6907 sctx->cur_inode_rdev = btrfs_inode_rdev(
6908 sctx->left_path->nodes[0],
6909 left_ii);
6910 ret = send_create_inode_if_needed(sctx);
6911 if (ret < 0)
6912 goto out;
6913
6914 ret = process_all_refs(sctx, BTRFS_COMPARE_TREE_NEW);
6915 if (ret < 0)
6916 goto out;
6917 /*
6918 * Advance send_progress now as we did not get
6919 * into process_recorded_refs_if_needed in the
6920 * new_gen case.
6921 */
6922 sctx->send_progress = sctx->cur_ino + 1;
6923
6924 /*
6925 * Now process all extents and xattrs of the
6926 * inode as if they were all new.
6927 */
6928 ret = process_all_extents(sctx);
6929 if (ret < 0)
6930 goto out;
6931 ret = process_all_new_xattrs(sctx);
6932 if (ret < 0)
6933 goto out;
6934 }
6935 } else {
6936 sctx->cur_inode_gen = left_gen;
6937 sctx->cur_inode_new = false;
6938 sctx->cur_inode_new_gen = false;
6939 sctx->cur_inode_deleted = false;
6940 sctx->cur_inode_size = btrfs_inode_size(
6941 sctx->left_path->nodes[0], left_ii);
6942 sctx->cur_inode_mode = btrfs_inode_mode(
6943 sctx->left_path->nodes[0], left_ii);
6944 }
6945 }
6946
6947 out:
6948 return ret;
6949 }
6950
6951 /*
6952 * We have to process new refs before deleted refs, but compare_trees gives us
6953 * the new and deleted refs mixed. To fix this, we record the new/deleted refs
6954 * first and later process them in process_recorded_refs.
6955 * For the cur_inode_new_gen case, we skip recording completely because
6956 * changed_inode did already initiate processing of refs. The reason for this is
6957 * that in this case, compare_tree actually compares the refs of 2 different
6958 * inodes. To fix this, process_all_refs is used in changed_inode to handle all
6959 * refs of the right tree as deleted and all refs of the left tree as new.
6960 */
changed_ref(struct send_ctx * sctx,enum btrfs_compare_tree_result result)6961 static int changed_ref(struct send_ctx *sctx,
6962 enum btrfs_compare_tree_result result)
6963 {
6964 int ret = 0;
6965
6966 if (unlikely(sctx->cur_ino != sctx->cmp_key->objectid)) {
6967 inconsistent_snapshot_error(sctx, result, "reference");
6968 return -EIO;
6969 }
6970
6971 if (!sctx->cur_inode_new_gen &&
6972 sctx->cur_ino != BTRFS_FIRST_FREE_OBJECTID) {
6973 if (result == BTRFS_COMPARE_TREE_NEW)
6974 ret = record_new_ref(sctx);
6975 else if (result == BTRFS_COMPARE_TREE_DELETED)
6976 ret = record_deleted_ref(sctx);
6977 else if (result == BTRFS_COMPARE_TREE_CHANGED)
6978 ret = record_changed_ref(sctx);
6979 }
6980
6981 return ret;
6982 }
6983
6984 /*
6985 * Process new/deleted/changed xattrs. We skip processing in the
6986 * cur_inode_new_gen case because changed_inode did already initiate processing
6987 * of xattrs. The reason is the same as in changed_ref
6988 */
changed_xattr(struct send_ctx * sctx,enum btrfs_compare_tree_result result)6989 static int changed_xattr(struct send_ctx *sctx,
6990 enum btrfs_compare_tree_result result)
6991 {
6992 int ret = 0;
6993
6994 if (unlikely(sctx->cur_ino != sctx->cmp_key->objectid)) {
6995 inconsistent_snapshot_error(sctx, result, "xattr");
6996 return -EIO;
6997 }
6998
6999 if (!sctx->cur_inode_new_gen && !sctx->cur_inode_deleted) {
7000 if (result == BTRFS_COMPARE_TREE_NEW)
7001 ret = process_new_xattr(sctx);
7002 else if (result == BTRFS_COMPARE_TREE_DELETED)
7003 ret = process_deleted_xattr(sctx);
7004 else if (result == BTRFS_COMPARE_TREE_CHANGED)
7005 ret = process_changed_xattr(sctx);
7006 }
7007
7008 return ret;
7009 }
7010
7011 /*
7012 * Process new/deleted/changed extents. We skip processing in the
7013 * cur_inode_new_gen case because changed_inode did already initiate processing
7014 * of extents. The reason is the same as in changed_ref
7015 */
changed_extent(struct send_ctx * sctx,enum btrfs_compare_tree_result result)7016 static int changed_extent(struct send_ctx *sctx,
7017 enum btrfs_compare_tree_result result)
7018 {
7019 int ret = 0;
7020
7021 /*
7022 * We have found an extent item that changed without the inode item
7023 * having changed. This can happen either after relocation (where the
7024 * disk_bytenr of an extent item is replaced at
7025 * relocation.c:replace_file_extents()) or after deduplication into a
7026 * file in both the parent and send snapshots (where an extent item can
7027 * get modified or replaced with a new one). Note that deduplication
7028 * updates the inode item, but it only changes the iversion (sequence
7029 * field in the inode item) of the inode, so if a file is deduplicated
7030 * the same amount of times in both the parent and send snapshots, its
7031 * iversion becomes the same in both snapshots, whence the inode item is
7032 * the same on both snapshots.
7033 */
7034 if (sctx->cur_ino != sctx->cmp_key->objectid)
7035 return 0;
7036
7037 if (!sctx->cur_inode_new_gen && !sctx->cur_inode_deleted) {
7038 if (result != BTRFS_COMPARE_TREE_DELETED)
7039 ret = process_extent(sctx, sctx->left_path,
7040 sctx->cmp_key);
7041 }
7042
7043 return ret;
7044 }
7045
changed_verity(struct send_ctx * sctx,enum btrfs_compare_tree_result result)7046 static int changed_verity(struct send_ctx *sctx, enum btrfs_compare_tree_result result)
7047 {
7048 if (!sctx->cur_inode_new_gen && !sctx->cur_inode_deleted) {
7049 if (result == BTRFS_COMPARE_TREE_NEW)
7050 sctx->cur_inode_needs_verity = true;
7051 }
7052 return 0;
7053 }
7054
dir_changed(struct send_ctx * sctx,u64 dir)7055 static int dir_changed(struct send_ctx *sctx, u64 dir)
7056 {
7057 u64 orig_gen, new_gen;
7058 int ret;
7059
7060 ret = get_inode_gen(sctx->send_root, dir, &new_gen);
7061 if (ret)
7062 return ret;
7063
7064 ret = get_inode_gen(sctx->parent_root, dir, &orig_gen);
7065 if (ret)
7066 return ret;
7067
7068 return (orig_gen != new_gen) ? 1 : 0;
7069 }
7070
compare_refs(struct send_ctx * sctx,struct btrfs_path * path,struct btrfs_key * key)7071 static int compare_refs(struct send_ctx *sctx, struct btrfs_path *path,
7072 struct btrfs_key *key)
7073 {
7074 struct btrfs_inode_extref *extref;
7075 struct extent_buffer *leaf;
7076 u64 dirid = 0, last_dirid = 0;
7077 unsigned long ptr;
7078 u32 item_size;
7079 u32 cur_offset = 0;
7080 int ref_name_len;
7081 int ret = 0;
7082
7083 /* Easy case, just check this one dirid */
7084 if (key->type == BTRFS_INODE_REF_KEY) {
7085 dirid = key->offset;
7086
7087 ret = dir_changed(sctx, dirid);
7088 goto out;
7089 }
7090
7091 leaf = path->nodes[0];
7092 item_size = btrfs_item_size(leaf, path->slots[0]);
7093 ptr = btrfs_item_ptr_offset(leaf, path->slots[0]);
7094 while (cur_offset < item_size) {
7095 extref = (struct btrfs_inode_extref *)(ptr +
7096 cur_offset);
7097 dirid = btrfs_inode_extref_parent(leaf, extref);
7098 ref_name_len = btrfs_inode_extref_name_len(leaf, extref);
7099 cur_offset += ref_name_len + sizeof(*extref);
7100 if (dirid == last_dirid)
7101 continue;
7102 ret = dir_changed(sctx, dirid);
7103 if (ret)
7104 break;
7105 last_dirid = dirid;
7106 }
7107 out:
7108 return ret;
7109 }
7110
7111 /*
7112 * Updates compare related fields in sctx and simply forwards to the actual
7113 * changed_xxx functions.
7114 */
changed_cb(struct btrfs_path * left_path,struct btrfs_path * right_path,struct btrfs_key * key,enum btrfs_compare_tree_result result,struct send_ctx * sctx)7115 static int changed_cb(struct btrfs_path *left_path,
7116 struct btrfs_path *right_path,
7117 struct btrfs_key *key,
7118 enum btrfs_compare_tree_result result,
7119 struct send_ctx *sctx)
7120 {
7121 int ret;
7122
7123 /*
7124 * We can not hold the commit root semaphore here. This is because in
7125 * the case of sending and receiving to the same filesystem, using a
7126 * pipe, could result in a deadlock:
7127 *
7128 * 1) The task running send blocks on the pipe because it's full;
7129 *
7130 * 2) The task running receive, which is the only consumer of the pipe,
7131 * is waiting for a transaction commit (for example due to a space
7132 * reservation when doing a write or triggering a transaction commit
7133 * when creating a subvolume);
7134 *
7135 * 3) The transaction is waiting to write lock the commit root semaphore,
7136 * but can not acquire it since it's being held at 1).
7137 *
7138 * Down this call chain we write to the pipe through kernel_write().
7139 * The same type of problem can also happen when sending to a file that
7140 * is stored in the same filesystem - when reserving space for a write
7141 * into the file, we can trigger a transaction commit.
7142 *
7143 * Our caller has supplied us with clones of leaves from the send and
7144 * parent roots, so we're safe here from a concurrent relocation and
7145 * further reallocation of metadata extents while we are here. Below we
7146 * also assert that the leaves are clones.
7147 */
7148 lockdep_assert_not_held(&sctx->send_root->fs_info->commit_root_sem);
7149
7150 /*
7151 * We always have a send root, so left_path is never NULL. We will not
7152 * have a leaf when we have reached the end of the send root but have
7153 * not yet reached the end of the parent root.
7154 */
7155 if (left_path->nodes[0])
7156 ASSERT(test_bit(EXTENT_BUFFER_UNMAPPED,
7157 &left_path->nodes[0]->bflags));
7158 /*
7159 * When doing a full send we don't have a parent root, so right_path is
7160 * NULL. When doing an incremental send, we may have reached the end of
7161 * the parent root already, so we don't have a leaf at right_path.
7162 */
7163 if (right_path && right_path->nodes[0])
7164 ASSERT(test_bit(EXTENT_BUFFER_UNMAPPED,
7165 &right_path->nodes[0]->bflags));
7166
7167 if (result == BTRFS_COMPARE_TREE_SAME) {
7168 if (key->type == BTRFS_INODE_REF_KEY ||
7169 key->type == BTRFS_INODE_EXTREF_KEY) {
7170 ret = compare_refs(sctx, left_path, key);
7171 if (!ret)
7172 return 0;
7173 if (ret < 0)
7174 return ret;
7175 } else if (key->type == BTRFS_EXTENT_DATA_KEY) {
7176 return maybe_send_hole(sctx, left_path, key);
7177 } else {
7178 return 0;
7179 }
7180 result = BTRFS_COMPARE_TREE_CHANGED;
7181 }
7182
7183 sctx->left_path = left_path;
7184 sctx->right_path = right_path;
7185 sctx->cmp_key = key;
7186
7187 ret = finish_inode_if_needed(sctx, 0);
7188 if (ret < 0)
7189 goto out;
7190
7191 /* Ignore non-FS objects */
7192 if (key->objectid == BTRFS_FREE_INO_OBJECTID ||
7193 key->objectid == BTRFS_FREE_SPACE_OBJECTID)
7194 goto out;
7195
7196 if (key->type == BTRFS_INODE_ITEM_KEY) {
7197 ret = changed_inode(sctx, result);
7198 } else if (!sctx->ignore_cur_inode) {
7199 if (key->type == BTRFS_INODE_REF_KEY ||
7200 key->type == BTRFS_INODE_EXTREF_KEY)
7201 ret = changed_ref(sctx, result);
7202 else if (key->type == BTRFS_XATTR_ITEM_KEY)
7203 ret = changed_xattr(sctx, result);
7204 else if (key->type == BTRFS_EXTENT_DATA_KEY)
7205 ret = changed_extent(sctx, result);
7206 else if (key->type == BTRFS_VERITY_DESC_ITEM_KEY &&
7207 key->offset == 0)
7208 ret = changed_verity(sctx, result);
7209 }
7210
7211 out:
7212 return ret;
7213 }
7214
search_key_again(const struct send_ctx * sctx,struct btrfs_root * root,struct btrfs_path * path,const struct btrfs_key * key)7215 static int search_key_again(const struct send_ctx *sctx,
7216 struct btrfs_root *root,
7217 struct btrfs_path *path,
7218 const struct btrfs_key *key)
7219 {
7220 int ret;
7221
7222 if (!path->need_commit_sem)
7223 lockdep_assert_held_read(&root->fs_info->commit_root_sem);
7224
7225 /*
7226 * Roots used for send operations are readonly and no one can add,
7227 * update or remove keys from them, so we should be able to find our
7228 * key again. The only exception is deduplication, which can operate on
7229 * readonly roots and add, update or remove keys to/from them - but at
7230 * the moment we don't allow it to run in parallel with send.
7231 */
7232 ret = btrfs_search_slot(NULL, root, key, path, 0, 0);
7233 ASSERT(ret <= 0);
7234 if (unlikely(ret > 0)) {
7235 btrfs_print_tree(path->nodes[path->lowest_level], false);
7236 btrfs_err(root->fs_info,
7237 "send: key (%llu %u %llu) not found in %s root %llu, lowest_level %d, slot %d",
7238 key->objectid, key->type, key->offset,
7239 (root == sctx->parent_root ? "parent" : "send"),
7240 btrfs_root_id(root), path->lowest_level,
7241 path->slots[path->lowest_level]);
7242 return -EUCLEAN;
7243 }
7244
7245 return ret;
7246 }
7247
full_send_tree(struct send_ctx * sctx)7248 static int full_send_tree(struct send_ctx *sctx)
7249 {
7250 int ret;
7251 struct btrfs_root *send_root = sctx->send_root;
7252 struct btrfs_key key;
7253 struct btrfs_fs_info *fs_info = send_root->fs_info;
7254 BTRFS_PATH_AUTO_FREE(path);
7255
7256 path = alloc_path_for_send();
7257 if (!path)
7258 return -ENOMEM;
7259 path->reada = READA_FORWARD_ALWAYS;
7260
7261 key.objectid = BTRFS_FIRST_FREE_OBJECTID;
7262 key.type = BTRFS_INODE_ITEM_KEY;
7263 key.offset = 0;
7264
7265 down_read(&fs_info->commit_root_sem);
7266 sctx->last_reloc_trans = fs_info->last_reloc_trans;
7267 up_read(&fs_info->commit_root_sem);
7268
7269 ret = btrfs_search_slot_for_read(send_root, &key, path, 1, 0);
7270 if (ret < 0)
7271 return ret;
7272 if (ret)
7273 goto out_finish;
7274
7275 while (1) {
7276 btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
7277
7278 ret = changed_cb(path, NULL, &key,
7279 BTRFS_COMPARE_TREE_NEW, sctx);
7280 if (ret < 0)
7281 return ret;
7282
7283 down_read(&fs_info->commit_root_sem);
7284 if (fs_info->last_reloc_trans > sctx->last_reloc_trans) {
7285 sctx->last_reloc_trans = fs_info->last_reloc_trans;
7286 up_read(&fs_info->commit_root_sem);
7287 /*
7288 * A transaction used for relocating a block group was
7289 * committed or is about to finish its commit. Release
7290 * our path (leaf) and restart the search, so that we
7291 * avoid operating on any file extent items that are
7292 * stale, with a disk_bytenr that reflects a pre
7293 * relocation value. This way we avoid as much as
7294 * possible to fallback to regular writes when checking
7295 * if we can clone file ranges.
7296 */
7297 btrfs_release_path(path);
7298 ret = search_key_again(sctx, send_root, path, &key);
7299 if (ret < 0)
7300 return ret;
7301 } else {
7302 up_read(&fs_info->commit_root_sem);
7303 }
7304
7305 ret = btrfs_next_item(send_root, path);
7306 if (ret < 0)
7307 return ret;
7308 if (ret) {
7309 ret = 0;
7310 break;
7311 }
7312 }
7313
7314 out_finish:
7315 return finish_inode_if_needed(sctx, 1);
7316 }
7317
replace_node_with_clone(struct btrfs_path * path,int level)7318 static int replace_node_with_clone(struct btrfs_path *path, int level)
7319 {
7320 struct extent_buffer *clone;
7321
7322 clone = btrfs_clone_extent_buffer(path->nodes[level]);
7323 if (!clone)
7324 return -ENOMEM;
7325
7326 free_extent_buffer(path->nodes[level]);
7327 path->nodes[level] = clone;
7328
7329 return 0;
7330 }
7331
tree_move_down(struct btrfs_path * path,int * level,u64 reada_min_gen)7332 static int tree_move_down(struct btrfs_path *path, int *level, u64 reada_min_gen)
7333 {
7334 struct extent_buffer *eb;
7335 struct extent_buffer *parent = path->nodes[*level];
7336 int slot = path->slots[*level];
7337 const int nritems = btrfs_header_nritems(parent);
7338 u64 reada_max;
7339 u64 reada_done = 0;
7340
7341 lockdep_assert_held_read(&parent->fs_info->commit_root_sem);
7342 ASSERT(*level != 0);
7343
7344 eb = btrfs_read_node_slot(parent, slot);
7345 if (IS_ERR(eb))
7346 return PTR_ERR(eb);
7347
7348 /*
7349 * Trigger readahead for the next leaves we will process, so that it is
7350 * very likely that when we need them they are already in memory and we
7351 * will not block on disk IO. For nodes we only do readahead for one,
7352 * since the time window between processing nodes is typically larger.
7353 */
7354 reada_max = (*level == 1 ? SZ_128K : eb->fs_info->nodesize);
7355
7356 for (slot++; slot < nritems && reada_done < reada_max; slot++) {
7357 if (btrfs_node_ptr_generation(parent, slot) > reada_min_gen) {
7358 btrfs_readahead_node_child(parent, slot);
7359 reada_done += eb->fs_info->nodesize;
7360 }
7361 }
7362
7363 path->nodes[*level - 1] = eb;
7364 path->slots[*level - 1] = 0;
7365 (*level)--;
7366
7367 if (*level == 0)
7368 return replace_node_with_clone(path, 0);
7369
7370 return 0;
7371 }
7372
tree_move_next_or_upnext(struct btrfs_path * path,int * level,int root_level)7373 static int tree_move_next_or_upnext(struct btrfs_path *path,
7374 int *level, int root_level)
7375 {
7376 int ret = 0;
7377 int nritems;
7378 nritems = btrfs_header_nritems(path->nodes[*level]);
7379
7380 path->slots[*level]++;
7381
7382 while (path->slots[*level] >= nritems) {
7383 if (*level == root_level) {
7384 path->slots[*level] = nritems - 1;
7385 return -1;
7386 }
7387
7388 /* move upnext */
7389 path->slots[*level] = 0;
7390 free_extent_buffer(path->nodes[*level]);
7391 path->nodes[*level] = NULL;
7392 (*level)++;
7393 path->slots[*level]++;
7394
7395 nritems = btrfs_header_nritems(path->nodes[*level]);
7396 ret = 1;
7397 }
7398 return ret;
7399 }
7400
7401 /*
7402 * Returns 1 if it had to move up and next. 0 is returned if it moved only next
7403 * or down.
7404 */
tree_advance(struct btrfs_path * path,int * level,int root_level,int allow_down,struct btrfs_key * key,u64 reada_min_gen)7405 static int tree_advance(struct btrfs_path *path,
7406 int *level, int root_level,
7407 int allow_down,
7408 struct btrfs_key *key,
7409 u64 reada_min_gen)
7410 {
7411 int ret;
7412
7413 if (*level == 0 || !allow_down) {
7414 ret = tree_move_next_or_upnext(path, level, root_level);
7415 } else {
7416 ret = tree_move_down(path, level, reada_min_gen);
7417 }
7418
7419 /*
7420 * Even if we have reached the end of a tree, ret is -1, update the key
7421 * anyway, so that in case we need to restart due to a block group
7422 * relocation, we can assert that the last key of the root node still
7423 * exists in the tree.
7424 */
7425 if (*level == 0)
7426 btrfs_item_key_to_cpu(path->nodes[*level], key,
7427 path->slots[*level]);
7428 else
7429 btrfs_node_key_to_cpu(path->nodes[*level], key,
7430 path->slots[*level]);
7431
7432 return ret;
7433 }
7434
tree_compare_item(struct btrfs_path * left_path,struct btrfs_path * right_path,char * tmp_buf)7435 static int tree_compare_item(struct btrfs_path *left_path,
7436 struct btrfs_path *right_path,
7437 char *tmp_buf)
7438 {
7439 int cmp;
7440 int len1, len2;
7441 unsigned long off1, off2;
7442
7443 len1 = btrfs_item_size(left_path->nodes[0], left_path->slots[0]);
7444 len2 = btrfs_item_size(right_path->nodes[0], right_path->slots[0]);
7445 if (len1 != len2)
7446 return 1;
7447
7448 off1 = btrfs_item_ptr_offset(left_path->nodes[0], left_path->slots[0]);
7449 off2 = btrfs_item_ptr_offset(right_path->nodes[0],
7450 right_path->slots[0]);
7451
7452 read_extent_buffer(left_path->nodes[0], tmp_buf, off1, len1);
7453
7454 cmp = memcmp_extent_buffer(right_path->nodes[0], tmp_buf, off2, len1);
7455 if (cmp)
7456 return 1;
7457 return 0;
7458 }
7459
7460 /*
7461 * A transaction used for relocating a block group was committed or is about to
7462 * finish its commit. Release our paths and restart the search, so that we are
7463 * not using stale extent buffers:
7464 *
7465 * 1) For levels > 0, we are only holding references of extent buffers, without
7466 * any locks on them, which does not prevent them from having been relocated
7467 * and reallocated after the last time we released the commit root semaphore.
7468 * The exception are the root nodes, for which we always have a clone, see
7469 * the comment at btrfs_compare_trees();
7470 *
7471 * 2) For leaves, level 0, we are holding copies (clones) of extent buffers, so
7472 * we are safe from the concurrent relocation and reallocation. However they
7473 * can have file extent items with a pre relocation disk_bytenr value, so we
7474 * restart the start from the current commit roots and clone the new leaves so
7475 * that we get the post relocation disk_bytenr values. Not doing so, could
7476 * make us clone the wrong data in case there are new extents using the old
7477 * disk_bytenr that happen to be shared.
7478 */
restart_after_relocation(struct btrfs_path * left_path,struct btrfs_path * right_path,const struct btrfs_key * left_key,const struct btrfs_key * right_key,int left_level,int right_level,const struct send_ctx * sctx)7479 static int restart_after_relocation(struct btrfs_path *left_path,
7480 struct btrfs_path *right_path,
7481 const struct btrfs_key *left_key,
7482 const struct btrfs_key *right_key,
7483 int left_level,
7484 int right_level,
7485 const struct send_ctx *sctx)
7486 {
7487 int root_level;
7488 int ret;
7489
7490 lockdep_assert_held_read(&sctx->send_root->fs_info->commit_root_sem);
7491
7492 btrfs_release_path(left_path);
7493 btrfs_release_path(right_path);
7494
7495 /*
7496 * Since keys can not be added or removed to/from our roots because they
7497 * are readonly and we do not allow deduplication to run in parallel
7498 * (which can add, remove or change keys), the layout of the trees should
7499 * not change.
7500 */
7501 left_path->lowest_level = left_level;
7502 ret = search_key_again(sctx, sctx->send_root, left_path, left_key);
7503 if (ret < 0)
7504 return ret;
7505
7506 right_path->lowest_level = right_level;
7507 ret = search_key_again(sctx, sctx->parent_root, right_path, right_key);
7508 if (ret < 0)
7509 return ret;
7510
7511 /*
7512 * If the lowest level nodes are leaves, clone them so that they can be
7513 * safely used by changed_cb() while not under the protection of the
7514 * commit root semaphore, even if relocation and reallocation happens in
7515 * parallel.
7516 */
7517 if (left_level == 0) {
7518 ret = replace_node_with_clone(left_path, 0);
7519 if (ret < 0)
7520 return ret;
7521 }
7522
7523 if (right_level == 0) {
7524 ret = replace_node_with_clone(right_path, 0);
7525 if (ret < 0)
7526 return ret;
7527 }
7528
7529 /*
7530 * Now clone the root nodes (unless they happen to be the leaves we have
7531 * already cloned). This is to protect against concurrent snapshotting of
7532 * the send and parent roots (see the comment at btrfs_compare_trees()).
7533 */
7534 root_level = btrfs_header_level(sctx->send_root->commit_root);
7535 if (root_level > 0) {
7536 ret = replace_node_with_clone(left_path, root_level);
7537 if (ret < 0)
7538 return ret;
7539 }
7540
7541 root_level = btrfs_header_level(sctx->parent_root->commit_root);
7542 if (root_level > 0) {
7543 ret = replace_node_with_clone(right_path, root_level);
7544 if (ret < 0)
7545 return ret;
7546 }
7547
7548 return 0;
7549 }
7550
7551 /*
7552 * This function compares two trees and calls the provided callback for
7553 * every changed/new/deleted item it finds.
7554 * If shared tree blocks are encountered, whole subtrees are skipped, making
7555 * the compare pretty fast on snapshotted subvolumes.
7556 *
7557 * This currently works on commit roots only. As commit roots are read only,
7558 * we don't do any locking. The commit roots are protected with transactions.
7559 * Transactions are ended and rejoined when a commit is tried in between.
7560 *
7561 * This function checks for modifications done to the trees while comparing.
7562 * If it detects a change, it aborts immediately.
7563 */
btrfs_compare_trees(struct btrfs_root * left_root,struct btrfs_root * right_root,struct send_ctx * sctx)7564 static int btrfs_compare_trees(struct btrfs_root *left_root,
7565 struct btrfs_root *right_root, struct send_ctx *sctx)
7566 {
7567 struct btrfs_fs_info *fs_info = left_root->fs_info;
7568 int ret;
7569 int cmp;
7570 BTRFS_PATH_AUTO_FREE(left_path);
7571 BTRFS_PATH_AUTO_FREE(right_path);
7572 struct btrfs_key left_key;
7573 struct btrfs_key right_key;
7574 char *tmp_buf = NULL;
7575 int left_root_level;
7576 int right_root_level;
7577 int left_level;
7578 int right_level;
7579 int left_end_reached = 0;
7580 int right_end_reached = 0;
7581 int advance_left = 0;
7582 int advance_right = 0;
7583 u64 left_blockptr;
7584 u64 right_blockptr;
7585 u64 left_gen;
7586 u64 right_gen;
7587 u64 reada_min_gen;
7588
7589 left_path = btrfs_alloc_path();
7590 if (!left_path) {
7591 ret = -ENOMEM;
7592 goto out;
7593 }
7594 right_path = btrfs_alloc_path();
7595 if (!right_path) {
7596 ret = -ENOMEM;
7597 goto out;
7598 }
7599
7600 tmp_buf = kvmalloc(fs_info->nodesize, GFP_KERNEL);
7601 if (!tmp_buf) {
7602 ret = -ENOMEM;
7603 goto out;
7604 }
7605
7606 left_path->search_commit_root = 1;
7607 left_path->skip_locking = 1;
7608 right_path->search_commit_root = 1;
7609 right_path->skip_locking = 1;
7610
7611 /*
7612 * Strategy: Go to the first items of both trees. Then do
7613 *
7614 * If both trees are at level 0
7615 * Compare keys of current items
7616 * If left < right treat left item as new, advance left tree
7617 * and repeat
7618 * If left > right treat right item as deleted, advance right tree
7619 * and repeat
7620 * If left == right do deep compare of items, treat as changed if
7621 * needed, advance both trees and repeat
7622 * If both trees are at the same level but not at level 0
7623 * Compare keys of current nodes/leafs
7624 * If left < right advance left tree and repeat
7625 * If left > right advance right tree and repeat
7626 * If left == right compare blockptrs of the next nodes/leafs
7627 * If they match advance both trees but stay at the same level
7628 * and repeat
7629 * If they don't match advance both trees while allowing to go
7630 * deeper and repeat
7631 * If tree levels are different
7632 * Advance the tree that needs it and repeat
7633 *
7634 * Advancing a tree means:
7635 * If we are at level 0, try to go to the next slot. If that's not
7636 * possible, go one level up and repeat. Stop when we found a level
7637 * where we could go to the next slot. We may at this point be on a
7638 * node or a leaf.
7639 *
7640 * If we are not at level 0 and not on shared tree blocks, go one
7641 * level deeper.
7642 *
7643 * If we are not at level 0 and on shared tree blocks, go one slot to
7644 * the right if possible or go up and right.
7645 */
7646
7647 down_read(&fs_info->commit_root_sem);
7648 left_level = btrfs_header_level(left_root->commit_root);
7649 left_root_level = left_level;
7650 /*
7651 * We clone the root node of the send and parent roots to prevent races
7652 * with snapshot creation of these roots. Snapshot creation COWs the
7653 * root node of a tree, so after the transaction is committed the old
7654 * extent can be reallocated while this send operation is still ongoing.
7655 * So we clone them, under the commit root semaphore, to be race free.
7656 */
7657 left_path->nodes[left_level] =
7658 btrfs_clone_extent_buffer(left_root->commit_root);
7659 if (!left_path->nodes[left_level]) {
7660 ret = -ENOMEM;
7661 goto out_unlock;
7662 }
7663
7664 right_level = btrfs_header_level(right_root->commit_root);
7665 right_root_level = right_level;
7666 right_path->nodes[right_level] =
7667 btrfs_clone_extent_buffer(right_root->commit_root);
7668 if (!right_path->nodes[right_level]) {
7669 ret = -ENOMEM;
7670 goto out_unlock;
7671 }
7672 /*
7673 * Our right root is the parent root, while the left root is the "send"
7674 * root. We know that all new nodes/leaves in the left root must have
7675 * a generation greater than the right root's generation, so we trigger
7676 * readahead for those nodes and leaves of the left root, as we know we
7677 * will need to read them at some point.
7678 */
7679 reada_min_gen = btrfs_header_generation(right_root->commit_root);
7680
7681 if (left_level == 0)
7682 btrfs_item_key_to_cpu(left_path->nodes[left_level],
7683 &left_key, left_path->slots[left_level]);
7684 else
7685 btrfs_node_key_to_cpu(left_path->nodes[left_level],
7686 &left_key, left_path->slots[left_level]);
7687 if (right_level == 0)
7688 btrfs_item_key_to_cpu(right_path->nodes[right_level],
7689 &right_key, right_path->slots[right_level]);
7690 else
7691 btrfs_node_key_to_cpu(right_path->nodes[right_level],
7692 &right_key, right_path->slots[right_level]);
7693
7694 sctx->last_reloc_trans = fs_info->last_reloc_trans;
7695
7696 while (1) {
7697 if (need_resched() ||
7698 rwsem_is_contended(&fs_info->commit_root_sem)) {
7699 up_read(&fs_info->commit_root_sem);
7700 cond_resched();
7701 down_read(&fs_info->commit_root_sem);
7702 }
7703
7704 if (fs_info->last_reloc_trans > sctx->last_reloc_trans) {
7705 ret = restart_after_relocation(left_path, right_path,
7706 &left_key, &right_key,
7707 left_level, right_level,
7708 sctx);
7709 if (ret < 0)
7710 goto out_unlock;
7711 sctx->last_reloc_trans = fs_info->last_reloc_trans;
7712 }
7713
7714 if (advance_left && !left_end_reached) {
7715 ret = tree_advance(left_path, &left_level,
7716 left_root_level,
7717 advance_left != ADVANCE_ONLY_NEXT,
7718 &left_key, reada_min_gen);
7719 if (ret == -1)
7720 left_end_reached = ADVANCE;
7721 else if (ret < 0)
7722 goto out_unlock;
7723 advance_left = 0;
7724 }
7725 if (advance_right && !right_end_reached) {
7726 ret = tree_advance(right_path, &right_level,
7727 right_root_level,
7728 advance_right != ADVANCE_ONLY_NEXT,
7729 &right_key, reada_min_gen);
7730 if (ret == -1)
7731 right_end_reached = ADVANCE;
7732 else if (ret < 0)
7733 goto out_unlock;
7734 advance_right = 0;
7735 }
7736
7737 if (left_end_reached && right_end_reached) {
7738 ret = 0;
7739 goto out_unlock;
7740 } else if (left_end_reached) {
7741 if (right_level == 0) {
7742 up_read(&fs_info->commit_root_sem);
7743 ret = changed_cb(left_path, right_path,
7744 &right_key,
7745 BTRFS_COMPARE_TREE_DELETED,
7746 sctx);
7747 if (ret < 0)
7748 goto out;
7749 down_read(&fs_info->commit_root_sem);
7750 }
7751 advance_right = ADVANCE;
7752 continue;
7753 } else if (right_end_reached) {
7754 if (left_level == 0) {
7755 up_read(&fs_info->commit_root_sem);
7756 ret = changed_cb(left_path, right_path,
7757 &left_key,
7758 BTRFS_COMPARE_TREE_NEW,
7759 sctx);
7760 if (ret < 0)
7761 goto out;
7762 down_read(&fs_info->commit_root_sem);
7763 }
7764 advance_left = ADVANCE;
7765 continue;
7766 }
7767
7768 if (left_level == 0 && right_level == 0) {
7769 up_read(&fs_info->commit_root_sem);
7770 cmp = btrfs_comp_cpu_keys(&left_key, &right_key);
7771 if (cmp < 0) {
7772 ret = changed_cb(left_path, right_path,
7773 &left_key,
7774 BTRFS_COMPARE_TREE_NEW,
7775 sctx);
7776 advance_left = ADVANCE;
7777 } else if (cmp > 0) {
7778 ret = changed_cb(left_path, right_path,
7779 &right_key,
7780 BTRFS_COMPARE_TREE_DELETED,
7781 sctx);
7782 advance_right = ADVANCE;
7783 } else {
7784 enum btrfs_compare_tree_result result;
7785
7786 WARN_ON(!extent_buffer_uptodate(left_path->nodes[0]));
7787 ret = tree_compare_item(left_path, right_path,
7788 tmp_buf);
7789 if (ret)
7790 result = BTRFS_COMPARE_TREE_CHANGED;
7791 else
7792 result = BTRFS_COMPARE_TREE_SAME;
7793 ret = changed_cb(left_path, right_path,
7794 &left_key, result, sctx);
7795 advance_left = ADVANCE;
7796 advance_right = ADVANCE;
7797 }
7798
7799 if (ret < 0)
7800 goto out;
7801 down_read(&fs_info->commit_root_sem);
7802 } else if (left_level == right_level) {
7803 cmp = btrfs_comp_cpu_keys(&left_key, &right_key);
7804 if (cmp < 0) {
7805 advance_left = ADVANCE;
7806 } else if (cmp > 0) {
7807 advance_right = ADVANCE;
7808 } else {
7809 left_blockptr = btrfs_node_blockptr(
7810 left_path->nodes[left_level],
7811 left_path->slots[left_level]);
7812 right_blockptr = btrfs_node_blockptr(
7813 right_path->nodes[right_level],
7814 right_path->slots[right_level]);
7815 left_gen = btrfs_node_ptr_generation(
7816 left_path->nodes[left_level],
7817 left_path->slots[left_level]);
7818 right_gen = btrfs_node_ptr_generation(
7819 right_path->nodes[right_level],
7820 right_path->slots[right_level]);
7821 if (left_blockptr == right_blockptr &&
7822 left_gen == right_gen) {
7823 /*
7824 * As we're on a shared block, don't
7825 * allow to go deeper.
7826 */
7827 advance_left = ADVANCE_ONLY_NEXT;
7828 advance_right = ADVANCE_ONLY_NEXT;
7829 } else {
7830 advance_left = ADVANCE;
7831 advance_right = ADVANCE;
7832 }
7833 }
7834 } else if (left_level < right_level) {
7835 advance_right = ADVANCE;
7836 } else {
7837 advance_left = ADVANCE;
7838 }
7839 }
7840
7841 out_unlock:
7842 up_read(&fs_info->commit_root_sem);
7843 out:
7844 kvfree(tmp_buf);
7845 return ret;
7846 }
7847
send_subvol(struct send_ctx * sctx)7848 static int send_subvol(struct send_ctx *sctx)
7849 {
7850 int ret;
7851
7852 if (!(sctx->flags & BTRFS_SEND_FLAG_OMIT_STREAM_HEADER)) {
7853 ret = send_header(sctx);
7854 if (ret < 0)
7855 goto out;
7856 }
7857
7858 ret = send_subvol_begin(sctx);
7859 if (ret < 0)
7860 goto out;
7861
7862 if (sctx->parent_root) {
7863 ret = btrfs_compare_trees(sctx->send_root, sctx->parent_root, sctx);
7864 if (ret < 0)
7865 goto out;
7866 ret = finish_inode_if_needed(sctx, 1);
7867 if (ret < 0)
7868 goto out;
7869 } else {
7870 ret = full_send_tree(sctx);
7871 if (ret < 0)
7872 goto out;
7873 }
7874
7875 out:
7876 free_recorded_refs(sctx);
7877 return ret;
7878 }
7879
7880 /*
7881 * If orphan cleanup did remove any orphans from a root, it means the tree
7882 * was modified and therefore the commit root is not the same as the current
7883 * root anymore. This is a problem, because send uses the commit root and
7884 * therefore can see inode items that don't exist in the current root anymore,
7885 * and for example make calls to btrfs_iget, which will do tree lookups based
7886 * on the current root and not on the commit root. Those lookups will fail,
7887 * returning a -ESTALE error, and making send fail with that error. So make
7888 * sure a send does not see any orphans we have just removed, and that it will
7889 * see the same inodes regardless of whether a transaction commit happened
7890 * before it started (meaning that the commit root will be the same as the
7891 * current root) or not.
7892 */
ensure_commit_roots_uptodate(struct send_ctx * sctx)7893 static int ensure_commit_roots_uptodate(struct send_ctx *sctx)
7894 {
7895 struct btrfs_root *root = sctx->parent_root;
7896
7897 if (root && root->node != root->commit_root)
7898 return btrfs_commit_current_transaction(root);
7899
7900 for (int i = 0; i < sctx->clone_roots_cnt; i++) {
7901 root = sctx->clone_roots[i].root;
7902 if (root->node != root->commit_root)
7903 return btrfs_commit_current_transaction(root);
7904 }
7905
7906 return 0;
7907 }
7908
7909 /*
7910 * Make sure any existing delalloc is flushed for any root used by a send
7911 * operation so that we do not miss any data and we do not race with writeback
7912 * finishing and changing a tree while send is using the tree. This could
7913 * happen if a subvolume is in RW mode, has delalloc, is turned to RO mode and
7914 * a send operation then uses the subvolume.
7915 * After flushing delalloc ensure_commit_roots_uptodate() must be called.
7916 */
flush_delalloc_roots(struct send_ctx * sctx)7917 static int flush_delalloc_roots(struct send_ctx *sctx)
7918 {
7919 struct btrfs_root *root = sctx->parent_root;
7920 int ret;
7921 int i;
7922
7923 if (root) {
7924 ret = btrfs_start_delalloc_snapshot(root, false);
7925 if (ret)
7926 return ret;
7927 btrfs_wait_ordered_extents(root, U64_MAX, NULL);
7928 }
7929
7930 for (i = 0; i < sctx->clone_roots_cnt; i++) {
7931 root = sctx->clone_roots[i].root;
7932 ret = btrfs_start_delalloc_snapshot(root, false);
7933 if (ret)
7934 return ret;
7935 btrfs_wait_ordered_extents(root, U64_MAX, NULL);
7936 }
7937
7938 return 0;
7939 }
7940
btrfs_root_dec_send_in_progress(struct btrfs_root * root)7941 static void btrfs_root_dec_send_in_progress(struct btrfs_root* root)
7942 {
7943 spin_lock(&root->root_item_lock);
7944 root->send_in_progress--;
7945 /*
7946 * Not much left to do, we don't know why it's unbalanced and
7947 * can't blindly reset it to 0.
7948 */
7949 if (root->send_in_progress < 0)
7950 btrfs_err(root->fs_info,
7951 "send_in_progress unbalanced %d root %llu",
7952 root->send_in_progress, btrfs_root_id(root));
7953 spin_unlock(&root->root_item_lock);
7954 }
7955
dedupe_in_progress_warn(const struct btrfs_root * root)7956 static void dedupe_in_progress_warn(const struct btrfs_root *root)
7957 {
7958 btrfs_warn_rl(root->fs_info,
7959 "cannot use root %llu for send while deduplications on it are in progress (%d in progress)",
7960 btrfs_root_id(root), root->dedupe_in_progress);
7961 }
7962
btrfs_ioctl_send(struct btrfs_root * send_root,const struct btrfs_ioctl_send_args * arg)7963 long btrfs_ioctl_send(struct btrfs_root *send_root, const struct btrfs_ioctl_send_args *arg)
7964 {
7965 int ret = 0;
7966 struct btrfs_fs_info *fs_info = send_root->fs_info;
7967 struct btrfs_root *clone_root;
7968 struct send_ctx *sctx = NULL;
7969 u32 i;
7970 u64 *clone_sources_tmp = NULL;
7971 int clone_sources_to_rollback = 0;
7972 size_t alloc_size;
7973 int sort_clone_roots = 0;
7974 struct btrfs_lru_cache_entry *entry;
7975 struct btrfs_lru_cache_entry *tmp;
7976
7977 if (!capable(CAP_SYS_ADMIN))
7978 return -EPERM;
7979
7980 /*
7981 * The subvolume must remain read-only during send, protect against
7982 * making it RW. This also protects against deletion.
7983 */
7984 spin_lock(&send_root->root_item_lock);
7985 /*
7986 * Unlikely but possible, if the subvolume is marked for deletion but
7987 * is slow to remove the directory entry, send can still be started.
7988 */
7989 if (btrfs_root_dead(send_root)) {
7990 spin_unlock(&send_root->root_item_lock);
7991 return -EPERM;
7992 }
7993 /* Userspace tools do the checks and warn the user if it's not RO. */
7994 if (!btrfs_root_readonly(send_root)) {
7995 spin_unlock(&send_root->root_item_lock);
7996 return -EPERM;
7997 }
7998 if (send_root->dedupe_in_progress) {
7999 dedupe_in_progress_warn(send_root);
8000 spin_unlock(&send_root->root_item_lock);
8001 return -EAGAIN;
8002 }
8003 send_root->send_in_progress++;
8004 spin_unlock(&send_root->root_item_lock);
8005
8006 /*
8007 * Check that we don't overflow at later allocations, we request
8008 * clone_sources_count + 1 items, and compare to unsigned long inside
8009 * access_ok. Also set an upper limit for allocation size so this can't
8010 * easily exhaust memory. Max number of clone sources is about 200K.
8011 */
8012 if (arg->clone_sources_count > SZ_8M / sizeof(struct clone_root)) {
8013 ret = -EINVAL;
8014 goto out;
8015 }
8016
8017 if (arg->flags & ~BTRFS_SEND_FLAG_MASK) {
8018 ret = -EOPNOTSUPP;
8019 goto out;
8020 }
8021
8022 sctx = kzalloc(sizeof(struct send_ctx), GFP_KERNEL);
8023 if (!sctx) {
8024 ret = -ENOMEM;
8025 goto out;
8026 }
8027
8028 init_path(&sctx->cur_inode_path);
8029 INIT_LIST_HEAD(&sctx->new_refs);
8030 INIT_LIST_HEAD(&sctx->deleted_refs);
8031
8032 btrfs_lru_cache_init(&sctx->name_cache, SEND_MAX_NAME_CACHE_SIZE);
8033 btrfs_lru_cache_init(&sctx->backref_cache, SEND_MAX_BACKREF_CACHE_SIZE);
8034 btrfs_lru_cache_init(&sctx->dir_created_cache,
8035 SEND_MAX_DIR_CREATED_CACHE_SIZE);
8036 /*
8037 * This cache is periodically trimmed to a fixed size elsewhere, see
8038 * cache_dir_utimes() and trim_dir_utimes_cache().
8039 */
8040 btrfs_lru_cache_init(&sctx->dir_utimes_cache, 0);
8041
8042 sctx->pending_dir_moves = RB_ROOT;
8043 sctx->waiting_dir_moves = RB_ROOT;
8044 sctx->orphan_dirs = RB_ROOT;
8045 sctx->rbtree_new_refs = RB_ROOT;
8046 sctx->rbtree_deleted_refs = RB_ROOT;
8047
8048 sctx->flags = arg->flags;
8049
8050 if (arg->flags & BTRFS_SEND_FLAG_VERSION) {
8051 if (arg->version > BTRFS_SEND_STREAM_VERSION) {
8052 ret = -EPROTO;
8053 goto out;
8054 }
8055 /* Zero means "use the highest version" */
8056 sctx->proto = arg->version ?: BTRFS_SEND_STREAM_VERSION;
8057 } else {
8058 sctx->proto = 1;
8059 }
8060 if ((arg->flags & BTRFS_SEND_FLAG_COMPRESSED) && sctx->proto < 2) {
8061 ret = -EINVAL;
8062 goto out;
8063 }
8064
8065 sctx->send_filp = fget(arg->send_fd);
8066 if (!sctx->send_filp || !(sctx->send_filp->f_mode & FMODE_WRITE)) {
8067 ret = -EBADF;
8068 goto out;
8069 }
8070
8071 sctx->send_root = send_root;
8072 sctx->clone_roots_cnt = arg->clone_sources_count;
8073
8074 if (sctx->proto >= 2) {
8075 u32 send_buf_num_pages;
8076
8077 sctx->send_max_size = BTRFS_SEND_BUF_SIZE_V2;
8078 sctx->send_buf = vmalloc(sctx->send_max_size);
8079 if (!sctx->send_buf) {
8080 ret = -ENOMEM;
8081 goto out;
8082 }
8083 send_buf_num_pages = sctx->send_max_size >> PAGE_SHIFT;
8084 sctx->send_buf_pages = kcalloc(send_buf_num_pages,
8085 sizeof(*sctx->send_buf_pages),
8086 GFP_KERNEL);
8087 if (!sctx->send_buf_pages) {
8088 ret = -ENOMEM;
8089 goto out;
8090 }
8091 for (i = 0; i < send_buf_num_pages; i++) {
8092 sctx->send_buf_pages[i] =
8093 vmalloc_to_page(sctx->send_buf + (i << PAGE_SHIFT));
8094 }
8095 } else {
8096 sctx->send_max_size = BTRFS_SEND_BUF_SIZE_V1;
8097 sctx->send_buf = kvmalloc(sctx->send_max_size, GFP_KERNEL);
8098 }
8099 if (!sctx->send_buf) {
8100 ret = -ENOMEM;
8101 goto out;
8102 }
8103
8104 sctx->clone_roots = kvcalloc(arg->clone_sources_count + 1,
8105 sizeof(*sctx->clone_roots),
8106 GFP_KERNEL);
8107 if (!sctx->clone_roots) {
8108 ret = -ENOMEM;
8109 goto out;
8110 }
8111
8112 alloc_size = array_size(sizeof(*arg->clone_sources),
8113 arg->clone_sources_count);
8114
8115 if (arg->clone_sources_count) {
8116 clone_sources_tmp = kvmalloc(alloc_size, GFP_KERNEL);
8117 if (!clone_sources_tmp) {
8118 ret = -ENOMEM;
8119 goto out;
8120 }
8121
8122 ret = copy_from_user(clone_sources_tmp, arg->clone_sources,
8123 alloc_size);
8124 if (ret) {
8125 ret = -EFAULT;
8126 goto out;
8127 }
8128
8129 for (i = 0; i < arg->clone_sources_count; i++) {
8130 clone_root = btrfs_get_fs_root(fs_info,
8131 clone_sources_tmp[i], true);
8132 if (IS_ERR(clone_root)) {
8133 ret = PTR_ERR(clone_root);
8134 goto out;
8135 }
8136 spin_lock(&clone_root->root_item_lock);
8137 if (!btrfs_root_readonly(clone_root) ||
8138 btrfs_root_dead(clone_root)) {
8139 spin_unlock(&clone_root->root_item_lock);
8140 btrfs_put_root(clone_root);
8141 ret = -EPERM;
8142 goto out;
8143 }
8144 if (clone_root->dedupe_in_progress) {
8145 dedupe_in_progress_warn(clone_root);
8146 spin_unlock(&clone_root->root_item_lock);
8147 btrfs_put_root(clone_root);
8148 ret = -EAGAIN;
8149 goto out;
8150 }
8151 clone_root->send_in_progress++;
8152 spin_unlock(&clone_root->root_item_lock);
8153
8154 sctx->clone_roots[i].root = clone_root;
8155 clone_sources_to_rollback = i + 1;
8156 }
8157 kvfree(clone_sources_tmp);
8158 clone_sources_tmp = NULL;
8159 }
8160
8161 if (arg->parent_root) {
8162 sctx->parent_root = btrfs_get_fs_root(fs_info, arg->parent_root,
8163 true);
8164 if (IS_ERR(sctx->parent_root)) {
8165 ret = PTR_ERR(sctx->parent_root);
8166 goto out;
8167 }
8168
8169 spin_lock(&sctx->parent_root->root_item_lock);
8170 sctx->parent_root->send_in_progress++;
8171 if (!btrfs_root_readonly(sctx->parent_root) ||
8172 btrfs_root_dead(sctx->parent_root)) {
8173 spin_unlock(&sctx->parent_root->root_item_lock);
8174 ret = -EPERM;
8175 goto out;
8176 }
8177 if (sctx->parent_root->dedupe_in_progress) {
8178 dedupe_in_progress_warn(sctx->parent_root);
8179 spin_unlock(&sctx->parent_root->root_item_lock);
8180 ret = -EAGAIN;
8181 goto out;
8182 }
8183 spin_unlock(&sctx->parent_root->root_item_lock);
8184 }
8185
8186 /*
8187 * Clones from send_root are allowed, but only if the clone source
8188 * is behind the current send position. This is checked while searching
8189 * for possible clone sources.
8190 */
8191 sctx->clone_roots[sctx->clone_roots_cnt++].root =
8192 btrfs_grab_root(sctx->send_root);
8193
8194 /* We do a bsearch later */
8195 sort(sctx->clone_roots, sctx->clone_roots_cnt,
8196 sizeof(*sctx->clone_roots), __clone_root_cmp_sort,
8197 NULL);
8198 sort_clone_roots = 1;
8199
8200 ret = flush_delalloc_roots(sctx);
8201 if (ret)
8202 goto out;
8203
8204 ret = ensure_commit_roots_uptodate(sctx);
8205 if (ret)
8206 goto out;
8207
8208 ret = send_subvol(sctx);
8209 if (ret < 0)
8210 goto out;
8211
8212 btrfs_lru_cache_for_each_entry_safe(&sctx->dir_utimes_cache, entry, tmp) {
8213 ret = send_utimes(sctx, entry->key, entry->gen);
8214 if (ret < 0)
8215 goto out;
8216 btrfs_lru_cache_remove(&sctx->dir_utimes_cache, entry);
8217 }
8218
8219 if (!(sctx->flags & BTRFS_SEND_FLAG_OMIT_END_CMD)) {
8220 ret = begin_cmd(sctx, BTRFS_SEND_C_END);
8221 if (ret < 0)
8222 goto out;
8223 ret = send_cmd(sctx);
8224 if (ret < 0)
8225 goto out;
8226 }
8227
8228 out:
8229 WARN_ON(sctx && !ret && !RB_EMPTY_ROOT(&sctx->pending_dir_moves));
8230 while (sctx && !RB_EMPTY_ROOT(&sctx->pending_dir_moves)) {
8231 struct rb_node *n;
8232 struct pending_dir_move *pm;
8233
8234 n = rb_first(&sctx->pending_dir_moves);
8235 pm = rb_entry(n, struct pending_dir_move, node);
8236 while (!list_empty(&pm->list)) {
8237 struct pending_dir_move *pm2;
8238
8239 pm2 = list_first_entry(&pm->list,
8240 struct pending_dir_move, list);
8241 free_pending_move(sctx, pm2);
8242 }
8243 free_pending_move(sctx, pm);
8244 }
8245
8246 WARN_ON(sctx && !ret && !RB_EMPTY_ROOT(&sctx->waiting_dir_moves));
8247 while (sctx && !RB_EMPTY_ROOT(&sctx->waiting_dir_moves)) {
8248 struct rb_node *n;
8249 struct waiting_dir_move *dm;
8250
8251 n = rb_first(&sctx->waiting_dir_moves);
8252 dm = rb_entry(n, struct waiting_dir_move, node);
8253 rb_erase(&dm->node, &sctx->waiting_dir_moves);
8254 kfree(dm);
8255 }
8256
8257 WARN_ON(sctx && !ret && !RB_EMPTY_ROOT(&sctx->orphan_dirs));
8258 while (sctx && !RB_EMPTY_ROOT(&sctx->orphan_dirs)) {
8259 struct rb_node *n;
8260 struct orphan_dir_info *odi;
8261
8262 n = rb_first(&sctx->orphan_dirs);
8263 odi = rb_entry(n, struct orphan_dir_info, node);
8264 free_orphan_dir_info(sctx, odi);
8265 }
8266
8267 if (sort_clone_roots) {
8268 for (i = 0; i < sctx->clone_roots_cnt; i++) {
8269 btrfs_root_dec_send_in_progress(
8270 sctx->clone_roots[i].root);
8271 btrfs_put_root(sctx->clone_roots[i].root);
8272 }
8273 } else {
8274 for (i = 0; sctx && i < clone_sources_to_rollback; i++) {
8275 btrfs_root_dec_send_in_progress(
8276 sctx->clone_roots[i].root);
8277 btrfs_put_root(sctx->clone_roots[i].root);
8278 }
8279
8280 btrfs_root_dec_send_in_progress(send_root);
8281 }
8282 if (sctx && !IS_ERR_OR_NULL(sctx->parent_root)) {
8283 btrfs_root_dec_send_in_progress(sctx->parent_root);
8284 btrfs_put_root(sctx->parent_root);
8285 }
8286
8287 kvfree(clone_sources_tmp);
8288
8289 if (sctx) {
8290 if (sctx->send_filp)
8291 fput(sctx->send_filp);
8292
8293 kvfree(sctx->clone_roots);
8294 kfree(sctx->send_buf_pages);
8295 kvfree(sctx->send_buf);
8296 kvfree(sctx->verity_descriptor);
8297
8298 close_current_inode(sctx);
8299
8300 btrfs_lru_cache_clear(&sctx->name_cache);
8301 btrfs_lru_cache_clear(&sctx->backref_cache);
8302 btrfs_lru_cache_clear(&sctx->dir_created_cache);
8303 btrfs_lru_cache_clear(&sctx->dir_utimes_cache);
8304
8305 if (sctx->cur_inode_path.buf != sctx->cur_inode_path.inline_buf)
8306 kfree(sctx->cur_inode_path.buf);
8307
8308 kfree(sctx);
8309 }
8310
8311 return ret;
8312 }
8313