xref: /linux/fs/btrfs/send.c (revision 9f388a653c8a481cbdbdedca081a1f9f3ba204a2)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) 2012 Alexander Block.  All rights reserved.
4  */
5 
6 #include <linux/bsearch.h>
7 #include <linux/falloc.h>
8 #include <linux/fs.h>
9 #include <linux/file.h>
10 #include <linux/sort.h>
11 #include <linux/mount.h>
12 #include <linux/xattr.h>
13 #include <linux/posix_acl_xattr.h>
14 #include <linux/radix-tree.h>
15 #include <linux/vmalloc.h>
16 #include <linux/string.h>
17 #include <linux/compat.h>
18 #include <linux/crc32c.h>
19 #include <linux/fsverity.h>
20 #include "send.h"
21 #include "ctree.h"
22 #include "backref.h"
23 #include "locking.h"
24 #include "disk-io.h"
25 #include "btrfs_inode.h"
26 #include "transaction.h"
27 #include "compression.h"
28 #include "print-tree.h"
29 #include "accessors.h"
30 #include "dir-item.h"
31 #include "file-item.h"
32 #include "ioctl.h"
33 #include "verity.h"
34 #include "lru_cache.h"
35 
36 /*
37  * Maximum number of references an extent can have in order for us to attempt to
38  * issue clone operations instead of write operations. This currently exists to
39  * avoid hitting limitations of the backreference walking code (taking a lot of
40  * time and using too much memory for extents with large number of references).
41  */
42 #define SEND_MAX_EXTENT_REFS	1024
43 
44 /*
45  * A fs_path is a helper to dynamically build path names with unknown size.
46  * It reallocates the internal buffer on demand.
47  * It allows fast adding of path elements on the right side (normal path) and
48  * fast adding to the left side (reversed path). A reversed path can also be
49  * unreversed if needed.
50  */
51 struct fs_path {
52 	union {
53 		struct {
54 			char *start;
55 			char *end;
56 
57 			char *buf;
58 			unsigned short buf_len:15;
59 			unsigned short reversed:1;
60 			char inline_buf[];
61 		};
62 		/*
63 		 * Average path length does not exceed 200 bytes, we'll have
64 		 * better packing in the slab and higher chance to satisfy
65 		 * an allocation later during send.
66 		 */
67 		char pad[256];
68 	};
69 };
70 #define FS_PATH_INLINE_SIZE \
71 	(sizeof(struct fs_path) - offsetof(struct fs_path, inline_buf))
72 
73 
74 /* reused for each extent */
75 struct clone_root {
76 	struct btrfs_root *root;
77 	u64 ino;
78 	u64 offset;
79 	u64 num_bytes;
80 	bool found_ref;
81 };
82 
83 #define SEND_MAX_NAME_CACHE_SIZE			256
84 
85 /*
86  * Limit the root_ids array of struct backref_cache_entry to 17 elements.
87  * This makes the size of a cache entry to be exactly 192 bytes on x86_64, which
88  * can be satisfied from the kmalloc-192 slab, without wasting any space.
89  * The most common case is to have a single root for cloning, which corresponds
90  * to the send root. Having the user specify more than 16 clone roots is not
91  * common, and in such rare cases we simply don't use caching if the number of
92  * cloning roots that lead down to a leaf is more than 17.
93  */
94 #define SEND_MAX_BACKREF_CACHE_ROOTS			17
95 
96 /*
97  * Max number of entries in the cache.
98  * With SEND_MAX_BACKREF_CACHE_ROOTS as 17, the size in bytes, excluding
99  * maple tree's internal nodes, is 24K.
100  */
101 #define SEND_MAX_BACKREF_CACHE_SIZE 128
102 
103 /*
104  * A backref cache entry maps a leaf to a list of IDs of roots from which the
105  * leaf is accessible and we can use for clone operations.
106  * With SEND_MAX_BACKREF_CACHE_ROOTS as 12, each cache entry is 128 bytes (on
107  * x86_64).
108  */
109 struct backref_cache_entry {
110 	struct btrfs_lru_cache_entry entry;
111 	u64 root_ids[SEND_MAX_BACKREF_CACHE_ROOTS];
112 	/* Number of valid elements in the root_ids array. */
113 	int num_roots;
114 };
115 
116 /* See the comment at lru_cache.h about struct btrfs_lru_cache_entry. */
117 static_assert(offsetof(struct backref_cache_entry, entry) == 0);
118 
119 /*
120  * Max number of entries in the cache that stores directories that were already
121  * created. The cache uses raw struct btrfs_lru_cache_entry entries, so it uses
122  * at most 4096 bytes - sizeof(struct btrfs_lru_cache_entry) is 48 bytes, but
123  * the kmalloc-64 slab is used, so we get 4096 bytes (64 bytes * 64).
124  */
125 #define SEND_MAX_DIR_CREATED_CACHE_SIZE			64
126 
127 /*
128  * Max number of entries in the cache that stores directories that were already
129  * created. The cache uses raw struct btrfs_lru_cache_entry entries, so it uses
130  * at most 4096 bytes - sizeof(struct btrfs_lru_cache_entry) is 48 bytes, but
131  * the kmalloc-64 slab is used, so we get 4096 bytes (64 bytes * 64).
132  */
133 #define SEND_MAX_DIR_UTIMES_CACHE_SIZE			64
134 
135 struct send_ctx {
136 	struct file *send_filp;
137 	loff_t send_off;
138 	char *send_buf;
139 	u32 send_size;
140 	u32 send_max_size;
141 	/*
142 	 * Whether BTRFS_SEND_A_DATA attribute was already added to current
143 	 * command (since protocol v2, data must be the last attribute).
144 	 */
145 	bool put_data;
146 	struct page **send_buf_pages;
147 	u64 flags;	/* 'flags' member of btrfs_ioctl_send_args is u64 */
148 	/* Protocol version compatibility requested */
149 	u32 proto;
150 
151 	struct btrfs_root *send_root;
152 	struct btrfs_root *parent_root;
153 	struct clone_root *clone_roots;
154 	int clone_roots_cnt;
155 
156 	/* current state of the compare_tree call */
157 	struct btrfs_path *left_path;
158 	struct btrfs_path *right_path;
159 	struct btrfs_key *cmp_key;
160 
161 	/*
162 	 * Keep track of the generation of the last transaction that was used
163 	 * for relocating a block group. This is periodically checked in order
164 	 * to detect if a relocation happened since the last check, so that we
165 	 * don't operate on stale extent buffers for nodes (level >= 1) or on
166 	 * stale disk_bytenr values of file extent items.
167 	 */
168 	u64 last_reloc_trans;
169 
170 	/*
171 	 * infos of the currently processed inode. In case of deleted inodes,
172 	 * these are the values from the deleted inode.
173 	 */
174 	u64 cur_ino;
175 	u64 cur_inode_gen;
176 	u64 cur_inode_size;
177 	u64 cur_inode_mode;
178 	u64 cur_inode_rdev;
179 	u64 cur_inode_last_extent;
180 	u64 cur_inode_next_write_offset;
181 	bool cur_inode_new;
182 	bool cur_inode_new_gen;
183 	bool cur_inode_deleted;
184 	bool ignore_cur_inode;
185 	bool cur_inode_needs_verity;
186 	void *verity_descriptor;
187 
188 	u64 send_progress;
189 
190 	struct list_head new_refs;
191 	struct list_head deleted_refs;
192 
193 	struct btrfs_lru_cache name_cache;
194 
195 	/*
196 	 * The inode we are currently processing. It's not NULL only when we
197 	 * need to issue write commands for data extents from this inode.
198 	 */
199 	struct inode *cur_inode;
200 	struct file_ra_state ra;
201 	u64 page_cache_clear_start;
202 	bool clean_page_cache;
203 
204 	/*
205 	 * We process inodes by their increasing order, so if before an
206 	 * incremental send we reverse the parent/child relationship of
207 	 * directories such that a directory with a lower inode number was
208 	 * the parent of a directory with a higher inode number, and the one
209 	 * becoming the new parent got renamed too, we can't rename/move the
210 	 * directory with lower inode number when we finish processing it - we
211 	 * must process the directory with higher inode number first, then
212 	 * rename/move it and then rename/move the directory with lower inode
213 	 * number. Example follows.
214 	 *
215 	 * Tree state when the first send was performed:
216 	 *
217 	 * .
218 	 * |-- a                   (ino 257)
219 	 *     |-- b               (ino 258)
220 	 *         |
221 	 *         |
222 	 *         |-- c           (ino 259)
223 	 *         |   |-- d       (ino 260)
224 	 *         |
225 	 *         |-- c2          (ino 261)
226 	 *
227 	 * Tree state when the second (incremental) send is performed:
228 	 *
229 	 * .
230 	 * |-- a                   (ino 257)
231 	 *     |-- b               (ino 258)
232 	 *         |-- c2          (ino 261)
233 	 *             |-- d2      (ino 260)
234 	 *                 |-- cc  (ino 259)
235 	 *
236 	 * The sequence of steps that lead to the second state was:
237 	 *
238 	 * mv /a/b/c/d /a/b/c2/d2
239 	 * mv /a/b/c /a/b/c2/d2/cc
240 	 *
241 	 * "c" has lower inode number, but we can't move it (2nd mv operation)
242 	 * before we move "d", which has higher inode number.
243 	 *
244 	 * So we just memorize which move/rename operations must be performed
245 	 * later when their respective parent is processed and moved/renamed.
246 	 */
247 
248 	/* Indexed by parent directory inode number. */
249 	struct rb_root pending_dir_moves;
250 
251 	/*
252 	 * Reverse index, indexed by the inode number of a directory that
253 	 * is waiting for the move/rename of its immediate parent before its
254 	 * own move/rename can be performed.
255 	 */
256 	struct rb_root waiting_dir_moves;
257 
258 	/*
259 	 * A directory that is going to be rm'ed might have a child directory
260 	 * which is in the pending directory moves index above. In this case,
261 	 * the directory can only be removed after the move/rename of its child
262 	 * is performed. Example:
263 	 *
264 	 * Parent snapshot:
265 	 *
266 	 * .                        (ino 256)
267 	 * |-- a/                   (ino 257)
268 	 *     |-- b/               (ino 258)
269 	 *         |-- c/           (ino 259)
270 	 *         |   |-- x/       (ino 260)
271 	 *         |
272 	 *         |-- y/           (ino 261)
273 	 *
274 	 * Send snapshot:
275 	 *
276 	 * .                        (ino 256)
277 	 * |-- a/                   (ino 257)
278 	 *     |-- b/               (ino 258)
279 	 *         |-- YY/          (ino 261)
280 	 *              |-- x/      (ino 260)
281 	 *
282 	 * Sequence of steps that lead to the send snapshot:
283 	 * rm -f /a/b/c/foo.txt
284 	 * mv /a/b/y /a/b/YY
285 	 * mv /a/b/c/x /a/b/YY
286 	 * rmdir /a/b/c
287 	 *
288 	 * When the child is processed, its move/rename is delayed until its
289 	 * parent is processed (as explained above), but all other operations
290 	 * like update utimes, chown, chgrp, etc, are performed and the paths
291 	 * that it uses for those operations must use the orphanized name of
292 	 * its parent (the directory we're going to rm later), so we need to
293 	 * memorize that name.
294 	 *
295 	 * Indexed by the inode number of the directory to be deleted.
296 	 */
297 	struct rb_root orphan_dirs;
298 
299 	struct rb_root rbtree_new_refs;
300 	struct rb_root rbtree_deleted_refs;
301 
302 	struct btrfs_lru_cache backref_cache;
303 	u64 backref_cache_last_reloc_trans;
304 
305 	struct btrfs_lru_cache dir_created_cache;
306 	struct btrfs_lru_cache dir_utimes_cache;
307 
308 	/* Must be last as it ends in a flexible-array member. */
309 	struct fs_path cur_inode_path;
310 };
311 
312 struct pending_dir_move {
313 	struct rb_node node;
314 	struct list_head list;
315 	u64 parent_ino;
316 	u64 ino;
317 	u64 gen;
318 	struct list_head update_refs;
319 };
320 
321 struct waiting_dir_move {
322 	struct rb_node node;
323 	u64 ino;
324 	/*
325 	 * There might be some directory that could not be removed because it
326 	 * was waiting for this directory inode to be moved first. Therefore
327 	 * after this directory is moved, we can try to rmdir the ino rmdir_ino.
328 	 */
329 	u64 rmdir_ino;
330 	u64 rmdir_gen;
331 	bool orphanized;
332 };
333 
334 struct orphan_dir_info {
335 	struct rb_node node;
336 	u64 ino;
337 	u64 gen;
338 	u64 last_dir_index_offset;
339 	u64 dir_high_seq_ino;
340 };
341 
342 struct name_cache_entry {
343 	/*
344 	 * The key in the entry is an inode number, and the generation matches
345 	 * the inode's generation.
346 	 */
347 	struct btrfs_lru_cache_entry entry;
348 	u64 parent_ino;
349 	u64 parent_gen;
350 	int ret;
351 	int need_later_update;
352 	/* Name length without NUL terminator. */
353 	int name_len;
354 	/* Not NUL terminated. */
355 	char name[] __counted_by(name_len) __nonstring;
356 };
357 
358 /* See the comment at lru_cache.h about struct btrfs_lru_cache_entry. */
359 static_assert(offsetof(struct name_cache_entry, entry) == 0);
360 
361 #define ADVANCE							1
362 #define ADVANCE_ONLY_NEXT					-1
363 
364 enum btrfs_compare_tree_result {
365 	BTRFS_COMPARE_TREE_NEW,
366 	BTRFS_COMPARE_TREE_DELETED,
367 	BTRFS_COMPARE_TREE_CHANGED,
368 	BTRFS_COMPARE_TREE_SAME,
369 };
370 
371 __cold
inconsistent_snapshot_error(struct send_ctx * sctx,enum btrfs_compare_tree_result result,const char * what)372 static void inconsistent_snapshot_error(struct send_ctx *sctx,
373 					enum btrfs_compare_tree_result result,
374 					const char *what)
375 {
376 	const char *result_string;
377 
378 	switch (result) {
379 	case BTRFS_COMPARE_TREE_NEW:
380 		result_string = "new";
381 		break;
382 	case BTRFS_COMPARE_TREE_DELETED:
383 		result_string = "deleted";
384 		break;
385 	case BTRFS_COMPARE_TREE_CHANGED:
386 		result_string = "updated";
387 		break;
388 	case BTRFS_COMPARE_TREE_SAME:
389 		DEBUG_WARN("no change between trees");
390 		result_string = "unchanged";
391 		break;
392 	default:
393 		DEBUG_WARN("unexpected comparison result %d", result);
394 		result_string = "unexpected";
395 	}
396 
397 	btrfs_err(sctx->send_root->fs_info,
398 		  "Send: inconsistent snapshot, found %s %s for inode %llu without updated inode item, send root is %llu, parent root is %llu",
399 		  result_string, what, sctx->cmp_key->objectid,
400 		  btrfs_root_id(sctx->send_root),
401 		  (sctx->parent_root ?  btrfs_root_id(sctx->parent_root) : 0));
402 }
403 
404 __maybe_unused
proto_cmd_ok(const struct send_ctx * sctx,int cmd)405 static bool proto_cmd_ok(const struct send_ctx *sctx, int cmd)
406 {
407 	switch (sctx->proto) {
408 	case 1:	 return cmd <= BTRFS_SEND_C_MAX_V1;
409 	case 2:	 return cmd <= BTRFS_SEND_C_MAX_V2;
410 	case 3:	 return cmd <= BTRFS_SEND_C_MAX_V3;
411 	default: return false;
412 	}
413 }
414 
415 static int is_waiting_for_move(struct send_ctx *sctx, u64 ino);
416 
417 static struct waiting_dir_move *
418 get_waiting_dir_move(struct send_ctx *sctx, u64 ino);
419 
420 static int is_waiting_for_rm(struct send_ctx *sctx, u64 dir_ino, u64 gen);
421 
need_send_hole(struct send_ctx * sctx)422 static int need_send_hole(struct send_ctx *sctx)
423 {
424 	return (sctx->parent_root && !sctx->cur_inode_new &&
425 		!sctx->cur_inode_new_gen && !sctx->cur_inode_deleted &&
426 		S_ISREG(sctx->cur_inode_mode));
427 }
428 
fs_path_reset(struct fs_path * p)429 static void fs_path_reset(struct fs_path *p)
430 {
431 	if (p->reversed)
432 		p->start = p->buf + p->buf_len - 1;
433 	else
434 		p->start = p->buf;
435 
436 	p->end = p->start;
437 	*p->start = 0;
438 }
439 
init_path(struct fs_path * p)440 static void init_path(struct fs_path *p)
441 {
442 	p->reversed = 0;
443 	p->buf = p->inline_buf;
444 	p->buf_len = FS_PATH_INLINE_SIZE;
445 	fs_path_reset(p);
446 }
447 
fs_path_alloc(void)448 static struct fs_path *fs_path_alloc(void)
449 {
450 	struct fs_path *p;
451 
452 	p = kmalloc(sizeof(*p), GFP_KERNEL);
453 	if (!p)
454 		return NULL;
455 	init_path(p);
456 	return p;
457 }
458 
fs_path_alloc_reversed(void)459 static struct fs_path *fs_path_alloc_reversed(void)
460 {
461 	struct fs_path *p;
462 
463 	p = fs_path_alloc();
464 	if (!p)
465 		return NULL;
466 	p->reversed = 1;
467 	fs_path_reset(p);
468 	return p;
469 }
470 
fs_path_free(struct fs_path * p)471 static void fs_path_free(struct fs_path *p)
472 {
473 	if (!p)
474 		return;
475 	if (p->buf != p->inline_buf)
476 		kfree(p->buf);
477 	kfree(p);
478 }
479 
fs_path_len(const struct fs_path * p)480 static inline int fs_path_len(const struct fs_path *p)
481 {
482 	return p->end - p->start;
483 }
484 
fs_path_ensure_buf(struct fs_path * p,int len)485 static int fs_path_ensure_buf(struct fs_path *p, int len)
486 {
487 	char *tmp_buf;
488 	int path_len;
489 	int old_buf_len;
490 
491 	len++;
492 
493 	if (p->buf_len >= len)
494 		return 0;
495 
496 	if (WARN_ON(len > PATH_MAX))
497 		return -ENAMETOOLONG;
498 
499 	path_len = fs_path_len(p);
500 	old_buf_len = p->buf_len;
501 
502 	/*
503 	 * Allocate to the next largest kmalloc bucket size, to let
504 	 * the fast path happen most of the time.
505 	 */
506 	len = kmalloc_size_roundup(len);
507 	/*
508 	 * First time the inline_buf does not suffice
509 	 */
510 	if (p->buf == p->inline_buf) {
511 		tmp_buf = kmalloc(len, GFP_KERNEL);
512 		if (tmp_buf)
513 			memcpy(tmp_buf, p->buf, old_buf_len);
514 	} else {
515 		tmp_buf = krealloc(p->buf, len, GFP_KERNEL);
516 	}
517 	if (!tmp_buf)
518 		return -ENOMEM;
519 	p->buf = tmp_buf;
520 	p->buf_len = len;
521 
522 	if (p->reversed) {
523 		tmp_buf = p->buf + old_buf_len - path_len - 1;
524 		p->end = p->buf + p->buf_len - 1;
525 		p->start = p->end - path_len;
526 		memmove(p->start, tmp_buf, path_len + 1);
527 	} else {
528 		p->start = p->buf;
529 		p->end = p->start + path_len;
530 	}
531 	return 0;
532 }
533 
fs_path_prepare_for_add(struct fs_path * p,int name_len,char ** prepared)534 static int fs_path_prepare_for_add(struct fs_path *p, int name_len,
535 				   char **prepared)
536 {
537 	int ret;
538 	int new_len;
539 
540 	new_len = fs_path_len(p) + name_len;
541 	if (p->start != p->end)
542 		new_len++;
543 	ret = fs_path_ensure_buf(p, new_len);
544 	if (ret < 0)
545 		return ret;
546 
547 	if (p->reversed) {
548 		if (p->start != p->end)
549 			*--p->start = '/';
550 		p->start -= name_len;
551 		*prepared = p->start;
552 	} else {
553 		if (p->start != p->end)
554 			*p->end++ = '/';
555 		*prepared = p->end;
556 		p->end += name_len;
557 		*p->end = 0;
558 	}
559 
560 	return 0;
561 }
562 
fs_path_add(struct fs_path * p,const char * name,int name_len)563 static int fs_path_add(struct fs_path *p, const char *name, int name_len)
564 {
565 	int ret;
566 	char *prepared;
567 
568 	ret = fs_path_prepare_for_add(p, name_len, &prepared);
569 	if (ret < 0)
570 		return ret;
571 	memcpy(prepared, name, name_len);
572 
573 	return 0;
574 }
575 
fs_path_add_path(struct fs_path * p,const struct fs_path * p2)576 static inline int fs_path_add_path(struct fs_path *p, const struct fs_path *p2)
577 {
578 	return fs_path_add(p, p2->start, fs_path_len(p2));
579 }
580 
fs_path_add_from_extent_buffer(struct fs_path * p,struct extent_buffer * eb,unsigned long off,int len)581 static int fs_path_add_from_extent_buffer(struct fs_path *p,
582 					  struct extent_buffer *eb,
583 					  unsigned long off, int len)
584 {
585 	int ret;
586 	char *prepared;
587 
588 	ret = fs_path_prepare_for_add(p, len, &prepared);
589 	if (ret < 0)
590 		return ret;
591 
592 	read_extent_buffer(eb, prepared, off, len);
593 
594 	return 0;
595 }
596 
fs_path_copy(struct fs_path * p,struct fs_path * from)597 static int fs_path_copy(struct fs_path *p, struct fs_path *from)
598 {
599 	p->reversed = from->reversed;
600 	fs_path_reset(p);
601 
602 	return fs_path_add_path(p, from);
603 }
604 
fs_path_unreverse(struct fs_path * p)605 static void fs_path_unreverse(struct fs_path *p)
606 {
607 	char *tmp;
608 	int len;
609 
610 	if (!p->reversed)
611 		return;
612 
613 	tmp = p->start;
614 	len = fs_path_len(p);
615 	p->start = p->buf;
616 	p->end = p->start + len;
617 	memmove(p->start, tmp, len + 1);
618 	p->reversed = 0;
619 }
620 
is_current_inode_path(const struct send_ctx * sctx,const struct fs_path * path)621 static inline bool is_current_inode_path(const struct send_ctx *sctx,
622 					 const struct fs_path *path)
623 {
624 	const struct fs_path *cur = &sctx->cur_inode_path;
625 
626 	return (strncmp(path->start, cur->start, fs_path_len(cur)) == 0);
627 }
628 
alloc_path_for_send(void)629 static struct btrfs_path *alloc_path_for_send(void)
630 {
631 	struct btrfs_path *path;
632 
633 	path = btrfs_alloc_path();
634 	if (!path)
635 		return NULL;
636 	path->search_commit_root = 1;
637 	path->skip_locking = 1;
638 	path->need_commit_sem = 1;
639 	return path;
640 }
641 
write_buf(struct file * filp,const void * buf,u32 len,loff_t * off)642 static int write_buf(struct file *filp, const void *buf, u32 len, loff_t *off)
643 {
644 	int ret;
645 	u32 pos = 0;
646 
647 	while (pos < len) {
648 		ret = kernel_write(filp, buf + pos, len - pos, off);
649 		if (ret < 0)
650 			return ret;
651 		if (unlikely(ret == 0))
652 			return -EIO;
653 		pos += ret;
654 	}
655 
656 	return 0;
657 }
658 
tlv_put(struct send_ctx * sctx,u16 attr,const void * data,int len)659 static int tlv_put(struct send_ctx *sctx, u16 attr, const void *data, int len)
660 {
661 	struct btrfs_tlv_header *hdr;
662 	int total_len = sizeof(*hdr) + len;
663 	int left = sctx->send_max_size - sctx->send_size;
664 
665 	if (WARN_ON_ONCE(sctx->put_data))
666 		return -EINVAL;
667 
668 	if (unlikely(left < total_len))
669 		return -EOVERFLOW;
670 
671 	hdr = (struct btrfs_tlv_header *) (sctx->send_buf + sctx->send_size);
672 	put_unaligned_le16(attr, &hdr->tlv_type);
673 	put_unaligned_le16(len, &hdr->tlv_len);
674 	memcpy(hdr + 1, data, len);
675 	sctx->send_size += total_len;
676 
677 	return 0;
678 }
679 
680 #define TLV_PUT_DEFINE_INT(bits) \
681 	static int tlv_put_u##bits(struct send_ctx *sctx,	 	\
682 			u##bits attr, u##bits value)			\
683 	{								\
684 		__le##bits __tmp = cpu_to_le##bits(value);		\
685 		return tlv_put(sctx, attr, &__tmp, sizeof(__tmp));	\
686 	}
687 
688 TLV_PUT_DEFINE_INT(8)
689 TLV_PUT_DEFINE_INT(32)
690 TLV_PUT_DEFINE_INT(64)
691 
tlv_put_string(struct send_ctx * sctx,u16 attr,const char * str,int len)692 static int tlv_put_string(struct send_ctx *sctx, u16 attr,
693 			  const char *str, int len)
694 {
695 	if (len == -1)
696 		len = strlen(str);
697 	return tlv_put(sctx, attr, str, len);
698 }
699 
tlv_put_uuid(struct send_ctx * sctx,u16 attr,const u8 * uuid)700 static int tlv_put_uuid(struct send_ctx *sctx, u16 attr,
701 			const u8 *uuid)
702 {
703 	return tlv_put(sctx, attr, uuid, BTRFS_UUID_SIZE);
704 }
705 
tlv_put_btrfs_timespec(struct send_ctx * sctx,u16 attr,struct extent_buffer * eb,struct btrfs_timespec * ts)706 static int tlv_put_btrfs_timespec(struct send_ctx *sctx, u16 attr,
707 				  struct extent_buffer *eb,
708 				  struct btrfs_timespec *ts)
709 {
710 	struct btrfs_timespec bts;
711 	read_extent_buffer(eb, &bts, (unsigned long)ts, sizeof(bts));
712 	return tlv_put(sctx, attr, &bts, sizeof(bts));
713 }
714 
715 
716 #define TLV_PUT(sctx, attrtype, data, attrlen) \
717 	do { \
718 		ret = tlv_put(sctx, attrtype, data, attrlen); \
719 		if (ret < 0) \
720 			goto tlv_put_failure; \
721 	} while (0)
722 
723 #define TLV_PUT_INT(sctx, attrtype, bits, value) \
724 	do { \
725 		ret = tlv_put_u##bits(sctx, attrtype, value); \
726 		if (ret < 0) \
727 			goto tlv_put_failure; \
728 	} while (0)
729 
730 #define TLV_PUT_U8(sctx, attrtype, data) TLV_PUT_INT(sctx, attrtype, 8, data)
731 #define TLV_PUT_U16(sctx, attrtype, data) TLV_PUT_INT(sctx, attrtype, 16, data)
732 #define TLV_PUT_U32(sctx, attrtype, data) TLV_PUT_INT(sctx, attrtype, 32, data)
733 #define TLV_PUT_U64(sctx, attrtype, data) TLV_PUT_INT(sctx, attrtype, 64, data)
734 #define TLV_PUT_STRING(sctx, attrtype, str, len) \
735 	do { \
736 		ret = tlv_put_string(sctx, attrtype, str, len); \
737 		if (ret < 0) \
738 			goto tlv_put_failure; \
739 	} while (0)
740 #define TLV_PUT_PATH(sctx, attrtype, p) \
741 	do { \
742 		ret = tlv_put_string(sctx, attrtype, p->start, \
743 				     fs_path_len((p)));	       \
744 		if (ret < 0) \
745 			goto tlv_put_failure; \
746 	} while(0)
747 #define TLV_PUT_UUID(sctx, attrtype, uuid) \
748 	do { \
749 		ret = tlv_put_uuid(sctx, attrtype, uuid); \
750 		if (ret < 0) \
751 			goto tlv_put_failure; \
752 	} while (0)
753 #define TLV_PUT_BTRFS_TIMESPEC(sctx, attrtype, eb, ts) \
754 	do { \
755 		ret = tlv_put_btrfs_timespec(sctx, attrtype, eb, ts); \
756 		if (ret < 0) \
757 			goto tlv_put_failure; \
758 	} while (0)
759 
send_header(struct send_ctx * sctx)760 static int send_header(struct send_ctx *sctx)
761 {
762 	struct btrfs_stream_header hdr;
763 
764 	strscpy(hdr.magic, BTRFS_SEND_STREAM_MAGIC);
765 	hdr.version = cpu_to_le32(sctx->proto);
766 	return write_buf(sctx->send_filp, &hdr, sizeof(hdr),
767 					&sctx->send_off);
768 }
769 
770 /*
771  * For each command/item we want to send to userspace, we call this function.
772  */
begin_cmd(struct send_ctx * sctx,int cmd)773 static int begin_cmd(struct send_ctx *sctx, int cmd)
774 {
775 	struct btrfs_cmd_header *hdr;
776 
777 	if (WARN_ON(!sctx->send_buf))
778 		return -EINVAL;
779 
780 	if (unlikely(sctx->send_size != 0)) {
781 		btrfs_err(sctx->send_root->fs_info,
782 			  "send: command header buffer not empty cmd %d offset %llu",
783 			  cmd, sctx->send_off);
784 		return -EINVAL;
785 	}
786 
787 	sctx->send_size += sizeof(*hdr);
788 	hdr = (struct btrfs_cmd_header *)sctx->send_buf;
789 	put_unaligned_le16(cmd, &hdr->cmd);
790 
791 	return 0;
792 }
793 
send_cmd(struct send_ctx * sctx)794 static int send_cmd(struct send_ctx *sctx)
795 {
796 	int ret;
797 	struct btrfs_cmd_header *hdr;
798 	u32 crc;
799 
800 	hdr = (struct btrfs_cmd_header *)sctx->send_buf;
801 	put_unaligned_le32(sctx->send_size - sizeof(*hdr), &hdr->len);
802 	put_unaligned_le32(0, &hdr->crc);
803 
804 	crc = crc32c(0, (unsigned char *)sctx->send_buf, sctx->send_size);
805 	put_unaligned_le32(crc, &hdr->crc);
806 
807 	ret = write_buf(sctx->send_filp, sctx->send_buf, sctx->send_size,
808 					&sctx->send_off);
809 
810 	sctx->send_size = 0;
811 	sctx->put_data = false;
812 
813 	return ret;
814 }
815 
816 /*
817  * Sends a move instruction to user space
818  */
send_rename(struct send_ctx * sctx,struct fs_path * from,struct fs_path * to)819 static int send_rename(struct send_ctx *sctx,
820 		     struct fs_path *from, struct fs_path *to)
821 {
822 	int ret;
823 
824 	ret = begin_cmd(sctx, BTRFS_SEND_C_RENAME);
825 	if (ret < 0)
826 		return ret;
827 
828 	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, from);
829 	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH_TO, to);
830 
831 	ret = send_cmd(sctx);
832 
833 tlv_put_failure:
834 	return ret;
835 }
836 
837 /*
838  * Sends a link instruction to user space
839  */
send_link(struct send_ctx * sctx,struct fs_path * path,struct fs_path * lnk)840 static int send_link(struct send_ctx *sctx,
841 		     struct fs_path *path, struct fs_path *lnk)
842 {
843 	int ret;
844 
845 	ret = begin_cmd(sctx, BTRFS_SEND_C_LINK);
846 	if (ret < 0)
847 		return ret;
848 
849 	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path);
850 	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH_LINK, lnk);
851 
852 	ret = send_cmd(sctx);
853 
854 tlv_put_failure:
855 	return ret;
856 }
857 
858 /*
859  * Sends an unlink instruction to user space
860  */
send_unlink(struct send_ctx * sctx,struct fs_path * path)861 static int send_unlink(struct send_ctx *sctx, struct fs_path *path)
862 {
863 	int ret;
864 
865 	ret = begin_cmd(sctx, BTRFS_SEND_C_UNLINK);
866 	if (ret < 0)
867 		return ret;
868 
869 	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path);
870 
871 	ret = send_cmd(sctx);
872 
873 tlv_put_failure:
874 	return ret;
875 }
876 
877 /*
878  * Sends a rmdir instruction to user space
879  */
send_rmdir(struct send_ctx * sctx,struct fs_path * path)880 static int send_rmdir(struct send_ctx *sctx, struct fs_path *path)
881 {
882 	int ret;
883 
884 	ret = begin_cmd(sctx, BTRFS_SEND_C_RMDIR);
885 	if (ret < 0)
886 		return ret;
887 
888 	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path);
889 
890 	ret = send_cmd(sctx);
891 
892 tlv_put_failure:
893 	return ret;
894 }
895 
896 struct btrfs_inode_info {
897 	u64 size;
898 	u64 gen;
899 	u64 mode;
900 	u64 uid;
901 	u64 gid;
902 	u64 rdev;
903 	u64 fileattr;
904 	u64 nlink;
905 };
906 
907 /*
908  * Helper function to retrieve some fields from an inode item.
909  */
get_inode_info(struct btrfs_root * root,u64 ino,struct btrfs_inode_info * info)910 static int get_inode_info(struct btrfs_root *root, u64 ino,
911 			  struct btrfs_inode_info *info)
912 {
913 	int ret;
914 	BTRFS_PATH_AUTO_FREE(path);
915 	struct btrfs_inode_item *ii;
916 	struct btrfs_key key;
917 
918 	path = alloc_path_for_send();
919 	if (!path)
920 		return -ENOMEM;
921 
922 	key.objectid = ino;
923 	key.type = BTRFS_INODE_ITEM_KEY;
924 	key.offset = 0;
925 	ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
926 	if (ret) {
927 		if (ret > 0)
928 			ret = -ENOENT;
929 		return ret;
930 	}
931 
932 	if (!info)
933 		return 0;
934 
935 	ii = btrfs_item_ptr(path->nodes[0], path->slots[0],
936 			struct btrfs_inode_item);
937 	info->size = btrfs_inode_size(path->nodes[0], ii);
938 	info->gen = btrfs_inode_generation(path->nodes[0], ii);
939 	info->mode = btrfs_inode_mode(path->nodes[0], ii);
940 	info->uid = btrfs_inode_uid(path->nodes[0], ii);
941 	info->gid = btrfs_inode_gid(path->nodes[0], ii);
942 	info->rdev = btrfs_inode_rdev(path->nodes[0], ii);
943 	info->nlink = btrfs_inode_nlink(path->nodes[0], ii);
944 	/*
945 	 * Transfer the unchanged u64 value of btrfs_inode_item::flags, that's
946 	 * otherwise logically split to 32/32 parts.
947 	 */
948 	info->fileattr = btrfs_inode_flags(path->nodes[0], ii);
949 
950 	return 0;
951 }
952 
get_inode_gen(struct btrfs_root * root,u64 ino,u64 * gen)953 static int get_inode_gen(struct btrfs_root *root, u64 ino, u64 *gen)
954 {
955 	int ret;
956 	struct btrfs_inode_info info = { 0 };
957 
958 	ASSERT(gen);
959 
960 	ret = get_inode_info(root, ino, &info);
961 	*gen = info.gen;
962 	return ret;
963 }
964 
965 typedef int (*iterate_inode_ref_t)(u64 dir, struct fs_path *p, void *ctx);
966 
967 /*
968  * Helper function to iterate the entries in ONE btrfs_inode_ref or
969  * btrfs_inode_extref.
970  * The iterate callback may return a non zero value to stop iteration. This can
971  * be a negative value for error codes or 1 to simply stop it.
972  *
973  * path must point to the INODE_REF or INODE_EXTREF when called.
974  */
iterate_inode_ref(struct btrfs_root * root,struct btrfs_path * path,struct btrfs_key * found_key,bool resolve,iterate_inode_ref_t iterate,void * ctx)975 static int iterate_inode_ref(struct btrfs_root *root, struct btrfs_path *path,
976 			     struct btrfs_key *found_key, bool resolve,
977 			     iterate_inode_ref_t iterate, void *ctx)
978 {
979 	struct extent_buffer *eb = path->nodes[0];
980 	struct btrfs_inode_ref *iref;
981 	struct btrfs_inode_extref *extref;
982 	BTRFS_PATH_AUTO_FREE(tmp_path);
983 	struct fs_path *p;
984 	u32 cur = 0;
985 	u32 total;
986 	int slot = path->slots[0];
987 	u32 name_len;
988 	char *start;
989 	int ret = 0;
990 	u64 dir;
991 	unsigned long name_off;
992 	unsigned long elem_size;
993 	unsigned long ptr;
994 
995 	p = fs_path_alloc_reversed();
996 	if (!p)
997 		return -ENOMEM;
998 
999 	tmp_path = alloc_path_for_send();
1000 	if (!tmp_path) {
1001 		fs_path_free(p);
1002 		return -ENOMEM;
1003 	}
1004 
1005 
1006 	if (found_key->type == BTRFS_INODE_REF_KEY) {
1007 		ptr = (unsigned long)btrfs_item_ptr(eb, slot,
1008 						    struct btrfs_inode_ref);
1009 		total = btrfs_item_size(eb, slot);
1010 		elem_size = sizeof(*iref);
1011 	} else {
1012 		ptr = btrfs_item_ptr_offset(eb, slot);
1013 		total = btrfs_item_size(eb, slot);
1014 		elem_size = sizeof(*extref);
1015 	}
1016 
1017 	while (cur < total) {
1018 		fs_path_reset(p);
1019 
1020 		if (found_key->type == BTRFS_INODE_REF_KEY) {
1021 			iref = (struct btrfs_inode_ref *)(ptr + cur);
1022 			name_len = btrfs_inode_ref_name_len(eb, iref);
1023 			name_off = (unsigned long)(iref + 1);
1024 			dir = found_key->offset;
1025 		} else {
1026 			extref = (struct btrfs_inode_extref *)(ptr + cur);
1027 			name_len = btrfs_inode_extref_name_len(eb, extref);
1028 			name_off = (unsigned long)&extref->name;
1029 			dir = btrfs_inode_extref_parent(eb, extref);
1030 		}
1031 
1032 		if (resolve) {
1033 			start = btrfs_ref_to_path(root, tmp_path, name_len,
1034 						  name_off, eb, dir,
1035 						  p->buf, p->buf_len);
1036 			if (IS_ERR(start)) {
1037 				ret = PTR_ERR(start);
1038 				goto out;
1039 			}
1040 			if (start < p->buf) {
1041 				/* overflow , try again with larger buffer */
1042 				ret = fs_path_ensure_buf(p,
1043 						p->buf_len + p->buf - start);
1044 				if (ret < 0)
1045 					goto out;
1046 				start = btrfs_ref_to_path(root, tmp_path,
1047 							  name_len, name_off,
1048 							  eb, dir,
1049 							  p->buf, p->buf_len);
1050 				if (IS_ERR(start)) {
1051 					ret = PTR_ERR(start);
1052 					goto out;
1053 				}
1054 				if (unlikely(start < p->buf)) {
1055 					btrfs_err(root->fs_info,
1056 			"send: path ref buffer underflow for key (%llu %u %llu)",
1057 						  found_key->objectid,
1058 						  found_key->type,
1059 						  found_key->offset);
1060 					ret = -EINVAL;
1061 					goto out;
1062 				}
1063 			}
1064 			p->start = start;
1065 		} else {
1066 			ret = fs_path_add_from_extent_buffer(p, eb, name_off,
1067 							     name_len);
1068 			if (ret < 0)
1069 				goto out;
1070 		}
1071 
1072 		cur += elem_size + name_len;
1073 		ret = iterate(dir, p, ctx);
1074 		if (ret)
1075 			goto out;
1076 	}
1077 
1078 out:
1079 	fs_path_free(p);
1080 	return ret;
1081 }
1082 
1083 typedef int (*iterate_dir_item_t)(int num, struct btrfs_key *di_key,
1084 				  const char *name, int name_len,
1085 				  const char *data, int data_len,
1086 				  void *ctx);
1087 
1088 /*
1089  * Helper function to iterate the entries in ONE btrfs_dir_item.
1090  * The iterate callback may return a non zero value to stop iteration. This can
1091  * be a negative value for error codes or 1 to simply stop it.
1092  *
1093  * path must point to the dir item when called.
1094  */
iterate_dir_item(struct btrfs_root * root,struct btrfs_path * path,iterate_dir_item_t iterate,void * ctx)1095 static int iterate_dir_item(struct btrfs_root *root, struct btrfs_path *path,
1096 			    iterate_dir_item_t iterate, void *ctx)
1097 {
1098 	int ret = 0;
1099 	struct extent_buffer *eb;
1100 	struct btrfs_dir_item *di;
1101 	struct btrfs_key di_key;
1102 	char *buf = NULL;
1103 	int buf_len;
1104 	u32 name_len;
1105 	u32 data_len;
1106 	u32 cur;
1107 	u32 len;
1108 	u32 total;
1109 	int slot;
1110 	int num;
1111 
1112 	/*
1113 	 * Start with a small buffer (1 page). If later we end up needing more
1114 	 * space, which can happen for xattrs on a fs with a leaf size greater
1115 	 * than the page size, attempt to increase the buffer. Typically xattr
1116 	 * values are small.
1117 	 */
1118 	buf_len = PATH_MAX;
1119 	buf = kmalloc(buf_len, GFP_KERNEL);
1120 	if (!buf) {
1121 		ret = -ENOMEM;
1122 		goto out;
1123 	}
1124 
1125 	eb = path->nodes[0];
1126 	slot = path->slots[0];
1127 	di = btrfs_item_ptr(eb, slot, struct btrfs_dir_item);
1128 	cur = 0;
1129 	len = 0;
1130 	total = btrfs_item_size(eb, slot);
1131 
1132 	num = 0;
1133 	while (cur < total) {
1134 		name_len = btrfs_dir_name_len(eb, di);
1135 		data_len = btrfs_dir_data_len(eb, di);
1136 		btrfs_dir_item_key_to_cpu(eb, di, &di_key);
1137 
1138 		if (btrfs_dir_ftype(eb, di) == BTRFS_FT_XATTR) {
1139 			if (name_len > XATTR_NAME_MAX) {
1140 				ret = -ENAMETOOLONG;
1141 				goto out;
1142 			}
1143 			if (name_len + data_len >
1144 					BTRFS_MAX_XATTR_SIZE(root->fs_info)) {
1145 				ret = -E2BIG;
1146 				goto out;
1147 			}
1148 		} else {
1149 			/*
1150 			 * Path too long
1151 			 */
1152 			if (name_len + data_len > PATH_MAX) {
1153 				ret = -ENAMETOOLONG;
1154 				goto out;
1155 			}
1156 		}
1157 
1158 		if (name_len + data_len > buf_len) {
1159 			buf_len = name_len + data_len;
1160 			if (is_vmalloc_addr(buf)) {
1161 				vfree(buf);
1162 				buf = NULL;
1163 			} else {
1164 				char *tmp = krealloc(buf, buf_len,
1165 						GFP_KERNEL | __GFP_NOWARN);
1166 
1167 				if (!tmp)
1168 					kfree(buf);
1169 				buf = tmp;
1170 			}
1171 			if (!buf) {
1172 				buf = kvmalloc(buf_len, GFP_KERNEL);
1173 				if (!buf) {
1174 					ret = -ENOMEM;
1175 					goto out;
1176 				}
1177 			}
1178 		}
1179 
1180 		read_extent_buffer(eb, buf, (unsigned long)(di + 1),
1181 				name_len + data_len);
1182 
1183 		len = sizeof(*di) + name_len + data_len;
1184 		di = (struct btrfs_dir_item *)((char *)di + len);
1185 		cur += len;
1186 
1187 		ret = iterate(num, &di_key, buf, name_len, buf + name_len,
1188 			      data_len, ctx);
1189 		if (ret < 0)
1190 			goto out;
1191 		if (ret) {
1192 			ret = 0;
1193 			goto out;
1194 		}
1195 
1196 		num++;
1197 	}
1198 
1199 out:
1200 	kvfree(buf);
1201 	return ret;
1202 }
1203 
__copy_first_ref(u64 dir,struct fs_path * p,void * ctx)1204 static int __copy_first_ref(u64 dir, struct fs_path *p, void *ctx)
1205 {
1206 	int ret;
1207 	struct fs_path *pt = ctx;
1208 
1209 	ret = fs_path_copy(pt, p);
1210 	if (ret < 0)
1211 		return ret;
1212 
1213 	/* we want the first only */
1214 	return 1;
1215 }
1216 
1217 /*
1218  * Retrieve the first path of an inode. If an inode has more then one
1219  * ref/hardlink, this is ignored.
1220  */
get_inode_path(struct btrfs_root * root,u64 ino,struct fs_path * path)1221 static int get_inode_path(struct btrfs_root *root,
1222 			  u64 ino, struct fs_path *path)
1223 {
1224 	int ret;
1225 	struct btrfs_key key, found_key;
1226 	BTRFS_PATH_AUTO_FREE(p);
1227 
1228 	p = alloc_path_for_send();
1229 	if (!p)
1230 		return -ENOMEM;
1231 
1232 	fs_path_reset(path);
1233 
1234 	key.objectid = ino;
1235 	key.type = BTRFS_INODE_REF_KEY;
1236 	key.offset = 0;
1237 
1238 	ret = btrfs_search_slot_for_read(root, &key, p, 1, 0);
1239 	if (ret < 0)
1240 		return ret;
1241 	if (ret)
1242 		return 1;
1243 
1244 	btrfs_item_key_to_cpu(p->nodes[0], &found_key, p->slots[0]);
1245 	if (found_key.objectid != ino ||
1246 	    (found_key.type != BTRFS_INODE_REF_KEY &&
1247 	     found_key.type != BTRFS_INODE_EXTREF_KEY))
1248 		return -ENOENT;
1249 
1250 	ret = iterate_inode_ref(root, p, &found_key, true, __copy_first_ref, path);
1251 	if (ret < 0)
1252 		return ret;
1253 	return 0;
1254 }
1255 
1256 struct backref_ctx {
1257 	struct send_ctx *sctx;
1258 
1259 	/* number of total found references */
1260 	u64 found;
1261 
1262 	/*
1263 	 * used for clones found in send_root. clones found behind cur_objectid
1264 	 * and cur_offset are not considered as allowed clones.
1265 	 */
1266 	u64 cur_objectid;
1267 	u64 cur_offset;
1268 
1269 	/* may be truncated in case it's the last extent in a file */
1270 	u64 extent_len;
1271 
1272 	/* The bytenr the file extent item we are processing refers to. */
1273 	u64 bytenr;
1274 	/* The owner (root id) of the data backref for the current extent. */
1275 	u64 backref_owner;
1276 	/* The offset of the data backref for the current extent. */
1277 	u64 backref_offset;
1278 };
1279 
__clone_root_cmp_bsearch(const void * key,const void * elt)1280 static int __clone_root_cmp_bsearch(const void *key, const void *elt)
1281 {
1282 	u64 root = (u64)(uintptr_t)key;
1283 	const struct clone_root *cr = elt;
1284 
1285 	if (root < btrfs_root_id(cr->root))
1286 		return -1;
1287 	if (root > btrfs_root_id(cr->root))
1288 		return 1;
1289 	return 0;
1290 }
1291 
__clone_root_cmp_sort(const void * e1,const void * e2)1292 static int __clone_root_cmp_sort(const void *e1, const void *e2)
1293 {
1294 	const struct clone_root *cr1 = e1;
1295 	const struct clone_root *cr2 = e2;
1296 
1297 	if (btrfs_root_id(cr1->root) < btrfs_root_id(cr2->root))
1298 		return -1;
1299 	if (btrfs_root_id(cr1->root) > btrfs_root_id(cr2->root))
1300 		return 1;
1301 	return 0;
1302 }
1303 
1304 /*
1305  * Called for every backref that is found for the current extent.
1306  * Results are collected in sctx->clone_roots->ino/offset.
1307  */
iterate_backrefs(u64 ino,u64 offset,u64 num_bytes,u64 root_id,void * ctx_)1308 static int iterate_backrefs(u64 ino, u64 offset, u64 num_bytes, u64 root_id,
1309 			    void *ctx_)
1310 {
1311 	struct backref_ctx *bctx = ctx_;
1312 	struct clone_root *clone_root;
1313 
1314 	/* First check if the root is in the list of accepted clone sources */
1315 	clone_root = bsearch((void *)(uintptr_t)root_id, bctx->sctx->clone_roots,
1316 			     bctx->sctx->clone_roots_cnt,
1317 			     sizeof(struct clone_root),
1318 			     __clone_root_cmp_bsearch);
1319 	if (!clone_root)
1320 		return 0;
1321 
1322 	/* This is our own reference, bail out as we can't clone from it. */
1323 	if (clone_root->root == bctx->sctx->send_root &&
1324 	    ino == bctx->cur_objectid &&
1325 	    offset == bctx->cur_offset)
1326 		return 0;
1327 
1328 	/*
1329 	 * Make sure we don't consider clones from send_root that are
1330 	 * behind the current inode/offset.
1331 	 */
1332 	if (clone_root->root == bctx->sctx->send_root) {
1333 		/*
1334 		 * If the source inode was not yet processed we can't issue a
1335 		 * clone operation, as the source extent does not exist yet at
1336 		 * the destination of the stream.
1337 		 */
1338 		if (ino > bctx->cur_objectid)
1339 			return 0;
1340 		/*
1341 		 * We clone from the inode currently being sent as long as the
1342 		 * source extent is already processed, otherwise we could try
1343 		 * to clone from an extent that does not exist yet at the
1344 		 * destination of the stream.
1345 		 */
1346 		if (ino == bctx->cur_objectid &&
1347 		    offset + bctx->extent_len >
1348 		    bctx->sctx->cur_inode_next_write_offset)
1349 			return 0;
1350 	}
1351 
1352 	bctx->found++;
1353 	clone_root->found_ref = true;
1354 
1355 	/*
1356 	 * If the given backref refers to a file extent item with a larger
1357 	 * number of bytes than what we found before, use the new one so that
1358 	 * we clone more optimally and end up doing less writes and getting
1359 	 * less exclusive, non-shared extents at the destination.
1360 	 */
1361 	if (num_bytes > clone_root->num_bytes) {
1362 		clone_root->ino = ino;
1363 		clone_root->offset = offset;
1364 		clone_root->num_bytes = num_bytes;
1365 
1366 		/*
1367 		 * Found a perfect candidate, so there's no need to continue
1368 		 * backref walking.
1369 		 */
1370 		if (num_bytes >= bctx->extent_len)
1371 			return BTRFS_ITERATE_EXTENT_INODES_STOP;
1372 	}
1373 
1374 	return 0;
1375 }
1376 
lookup_backref_cache(u64 leaf_bytenr,void * ctx,const u64 ** root_ids_ret,int * root_count_ret)1377 static bool lookup_backref_cache(u64 leaf_bytenr, void *ctx,
1378 				 const u64 **root_ids_ret, int *root_count_ret)
1379 {
1380 	struct backref_ctx *bctx = ctx;
1381 	struct send_ctx *sctx = bctx->sctx;
1382 	struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
1383 	const u64 key = leaf_bytenr >> fs_info->nodesize_bits;
1384 	struct btrfs_lru_cache_entry *raw_entry;
1385 	struct backref_cache_entry *entry;
1386 
1387 	if (sctx->backref_cache.size == 0)
1388 		return false;
1389 
1390 	/*
1391 	 * If relocation happened since we first filled the cache, then we must
1392 	 * empty the cache and can not use it, because even though we operate on
1393 	 * read-only roots, their leaves and nodes may have been reallocated and
1394 	 * now be used for different nodes/leaves of the same tree or some other
1395 	 * tree.
1396 	 *
1397 	 * We are called from iterate_extent_inodes() while either holding a
1398 	 * transaction handle or holding fs_info->commit_root_sem, so no need
1399 	 * to take any lock here.
1400 	 */
1401 	if (fs_info->last_reloc_trans > sctx->backref_cache_last_reloc_trans) {
1402 		btrfs_lru_cache_clear(&sctx->backref_cache);
1403 		return false;
1404 	}
1405 
1406 	raw_entry = btrfs_lru_cache_lookup(&sctx->backref_cache, key, 0);
1407 	if (!raw_entry)
1408 		return false;
1409 
1410 	entry = container_of(raw_entry, struct backref_cache_entry, entry);
1411 	*root_ids_ret = entry->root_ids;
1412 	*root_count_ret = entry->num_roots;
1413 
1414 	return true;
1415 }
1416 
store_backref_cache(u64 leaf_bytenr,const struct ulist * root_ids,void * ctx)1417 static void store_backref_cache(u64 leaf_bytenr, const struct ulist *root_ids,
1418 				void *ctx)
1419 {
1420 	struct backref_ctx *bctx = ctx;
1421 	struct send_ctx *sctx = bctx->sctx;
1422 	struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
1423 	struct backref_cache_entry *new_entry;
1424 	struct ulist_iterator uiter;
1425 	struct ulist_node *node;
1426 	int ret;
1427 
1428 	/*
1429 	 * We're called while holding a transaction handle or while holding
1430 	 * fs_info->commit_root_sem (at iterate_extent_inodes()), so must do a
1431 	 * NOFS allocation.
1432 	 */
1433 	new_entry = kmalloc(sizeof(struct backref_cache_entry), GFP_NOFS);
1434 	/* No worries, cache is optional. */
1435 	if (!new_entry)
1436 		return;
1437 
1438 	new_entry->entry.key = leaf_bytenr >> fs_info->nodesize_bits;
1439 	new_entry->entry.gen = 0;
1440 	new_entry->num_roots = 0;
1441 	ULIST_ITER_INIT(&uiter);
1442 	while ((node = ulist_next(root_ids, &uiter)) != NULL) {
1443 		const u64 root_id = node->val;
1444 		struct clone_root *root;
1445 
1446 		root = bsearch((void *)(uintptr_t)root_id, sctx->clone_roots,
1447 			       sctx->clone_roots_cnt, sizeof(struct clone_root),
1448 			       __clone_root_cmp_bsearch);
1449 		if (!root)
1450 			continue;
1451 
1452 		/* Too many roots, just exit, no worries as caching is optional. */
1453 		if (new_entry->num_roots >= SEND_MAX_BACKREF_CACHE_ROOTS) {
1454 			kfree(new_entry);
1455 			return;
1456 		}
1457 
1458 		new_entry->root_ids[new_entry->num_roots] = root_id;
1459 		new_entry->num_roots++;
1460 	}
1461 
1462 	/*
1463 	 * We may have not added any roots to the new cache entry, which means
1464 	 * none of the roots is part of the list of roots from which we are
1465 	 * allowed to clone. Cache the new entry as it's still useful to avoid
1466 	 * backref walking to determine which roots have a path to the leaf.
1467 	 *
1468 	 * Also use GFP_NOFS because we're called while holding a transaction
1469 	 * handle or while holding fs_info->commit_root_sem.
1470 	 */
1471 	ret = btrfs_lru_cache_store(&sctx->backref_cache, &new_entry->entry,
1472 				    GFP_NOFS);
1473 	ASSERT(ret == 0 || ret == -ENOMEM);
1474 	if (ret) {
1475 		/* Caching is optional, no worries. */
1476 		kfree(new_entry);
1477 		return;
1478 	}
1479 
1480 	/*
1481 	 * We are called from iterate_extent_inodes() while either holding a
1482 	 * transaction handle or holding fs_info->commit_root_sem, so no need
1483 	 * to take any lock here.
1484 	 */
1485 	if (sctx->backref_cache.size == 1)
1486 		sctx->backref_cache_last_reloc_trans = fs_info->last_reloc_trans;
1487 }
1488 
check_extent_item(u64 bytenr,const struct btrfs_extent_item * ei,const struct extent_buffer * leaf,void * ctx)1489 static int check_extent_item(u64 bytenr, const struct btrfs_extent_item *ei,
1490 			     const struct extent_buffer *leaf, void *ctx)
1491 {
1492 	const u64 refs = btrfs_extent_refs(leaf, ei);
1493 	const struct backref_ctx *bctx = ctx;
1494 	const struct send_ctx *sctx = bctx->sctx;
1495 
1496 	if (bytenr == bctx->bytenr) {
1497 		const u64 flags = btrfs_extent_flags(leaf, ei);
1498 
1499 		if (WARN_ON(flags & BTRFS_EXTENT_FLAG_TREE_BLOCK))
1500 			return -EUCLEAN;
1501 
1502 		/*
1503 		 * If we have only one reference and only the send root as a
1504 		 * clone source - meaning no clone roots were given in the
1505 		 * struct btrfs_ioctl_send_args passed to the send ioctl - then
1506 		 * it's our reference and there's no point in doing backref
1507 		 * walking which is expensive, so exit early.
1508 		 */
1509 		if (refs == 1 && sctx->clone_roots_cnt == 1)
1510 			return -ENOENT;
1511 	}
1512 
1513 	/*
1514 	 * Backreference walking (iterate_extent_inodes() below) is currently
1515 	 * too expensive when an extent has a large number of references, both
1516 	 * in time spent and used memory. So for now just fallback to write
1517 	 * operations instead of clone operations when an extent has more than
1518 	 * a certain amount of references.
1519 	 */
1520 	if (refs > SEND_MAX_EXTENT_REFS)
1521 		return -ENOENT;
1522 
1523 	return 0;
1524 }
1525 
skip_self_data_ref(u64 root,u64 ino,u64 offset,void * ctx)1526 static bool skip_self_data_ref(u64 root, u64 ino, u64 offset, void *ctx)
1527 {
1528 	const struct backref_ctx *bctx = ctx;
1529 
1530 	if (ino == bctx->cur_objectid &&
1531 	    root == bctx->backref_owner &&
1532 	    offset == bctx->backref_offset)
1533 		return true;
1534 
1535 	return false;
1536 }
1537 
1538 /*
1539  * Given an inode, offset and extent item, it finds a good clone for a clone
1540  * instruction. Returns -ENOENT when none could be found. The function makes
1541  * sure that the returned clone is usable at the point where sending is at the
1542  * moment. This means, that no clones are accepted which lie behind the current
1543  * inode+offset.
1544  *
1545  * path must point to the extent item when called.
1546  */
find_extent_clone(struct send_ctx * sctx,struct btrfs_path * path,u64 ino,u64 data_offset,u64 ino_size,struct clone_root ** found)1547 static int find_extent_clone(struct send_ctx *sctx,
1548 			     struct btrfs_path *path,
1549 			     u64 ino, u64 data_offset,
1550 			     u64 ino_size,
1551 			     struct clone_root **found)
1552 {
1553 	struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
1554 	int ret;
1555 	int extent_type;
1556 	u64 disk_byte;
1557 	u64 num_bytes;
1558 	struct btrfs_file_extent_item *fi;
1559 	struct extent_buffer *eb = path->nodes[0];
1560 	struct backref_ctx backref_ctx = { 0 };
1561 	struct btrfs_backref_walk_ctx backref_walk_ctx = { 0 };
1562 	struct clone_root *cur_clone_root;
1563 	int compressed;
1564 	u32 i;
1565 
1566 	/*
1567 	 * With fallocate we can get prealloc extents beyond the inode's i_size,
1568 	 * so we don't do anything here because clone operations can not clone
1569 	 * to a range beyond i_size without increasing the i_size of the
1570 	 * destination inode.
1571 	 */
1572 	if (data_offset >= ino_size)
1573 		return 0;
1574 
1575 	fi = btrfs_item_ptr(eb, path->slots[0], struct btrfs_file_extent_item);
1576 	extent_type = btrfs_file_extent_type(eb, fi);
1577 	if (extent_type == BTRFS_FILE_EXTENT_INLINE)
1578 		return -ENOENT;
1579 
1580 	disk_byte = btrfs_file_extent_disk_bytenr(eb, fi);
1581 	if (disk_byte == 0)
1582 		return -ENOENT;
1583 
1584 	compressed = btrfs_file_extent_compression(eb, fi);
1585 	num_bytes = btrfs_file_extent_num_bytes(eb, fi);
1586 
1587 	/*
1588 	 * Setup the clone roots.
1589 	 */
1590 	for (i = 0; i < sctx->clone_roots_cnt; i++) {
1591 		cur_clone_root = sctx->clone_roots + i;
1592 		cur_clone_root->ino = (u64)-1;
1593 		cur_clone_root->offset = 0;
1594 		cur_clone_root->num_bytes = 0;
1595 		cur_clone_root->found_ref = false;
1596 	}
1597 
1598 	backref_ctx.sctx = sctx;
1599 	backref_ctx.cur_objectid = ino;
1600 	backref_ctx.cur_offset = data_offset;
1601 	backref_ctx.bytenr = disk_byte;
1602 	/*
1603 	 * Use the header owner and not the send root's id, because in case of a
1604 	 * snapshot we can have shared subtrees.
1605 	 */
1606 	backref_ctx.backref_owner = btrfs_header_owner(eb);
1607 	backref_ctx.backref_offset = data_offset - btrfs_file_extent_offset(eb, fi);
1608 
1609 	/*
1610 	 * The last extent of a file may be too large due to page alignment.
1611 	 * We need to adjust extent_len in this case so that the checks in
1612 	 * iterate_backrefs() work.
1613 	 */
1614 	if (data_offset + num_bytes >= ino_size)
1615 		backref_ctx.extent_len = ino_size - data_offset;
1616 	else
1617 		backref_ctx.extent_len = num_bytes;
1618 
1619 	/*
1620 	 * Now collect all backrefs.
1621 	 */
1622 	backref_walk_ctx.bytenr = disk_byte;
1623 	if (compressed == BTRFS_COMPRESS_NONE)
1624 		backref_walk_ctx.extent_item_pos = btrfs_file_extent_offset(eb, fi);
1625 	backref_walk_ctx.fs_info = fs_info;
1626 	backref_walk_ctx.cache_lookup = lookup_backref_cache;
1627 	backref_walk_ctx.cache_store = store_backref_cache;
1628 	backref_walk_ctx.indirect_ref_iterator = iterate_backrefs;
1629 	backref_walk_ctx.check_extent_item = check_extent_item;
1630 	backref_walk_ctx.user_ctx = &backref_ctx;
1631 
1632 	/*
1633 	 * If have a single clone root, then it's the send root and we can tell
1634 	 * the backref walking code to skip our own backref and not resolve it,
1635 	 * since we can not use it for cloning - the source and destination
1636 	 * ranges can't overlap and in case the leaf is shared through a subtree
1637 	 * due to snapshots, we can't use those other roots since they are not
1638 	 * in the list of clone roots.
1639 	 */
1640 	if (sctx->clone_roots_cnt == 1)
1641 		backref_walk_ctx.skip_data_ref = skip_self_data_ref;
1642 
1643 	ret = iterate_extent_inodes(&backref_walk_ctx, true, iterate_backrefs,
1644 				    &backref_ctx);
1645 	if (ret < 0)
1646 		return ret;
1647 
1648 	down_read(&fs_info->commit_root_sem);
1649 	if (fs_info->last_reloc_trans > sctx->last_reloc_trans) {
1650 		/*
1651 		 * A transaction commit for a transaction in which block group
1652 		 * relocation was done just happened.
1653 		 * The disk_bytenr of the file extent item we processed is
1654 		 * possibly stale, referring to the extent's location before
1655 		 * relocation. So act as if we haven't found any clone sources
1656 		 * and fallback to write commands, which will read the correct
1657 		 * data from the new extent location. Otherwise we will fail
1658 		 * below because we haven't found our own back reference or we
1659 		 * could be getting incorrect sources in case the old extent
1660 		 * was already reallocated after the relocation.
1661 		 */
1662 		up_read(&fs_info->commit_root_sem);
1663 		return -ENOENT;
1664 	}
1665 	up_read(&fs_info->commit_root_sem);
1666 
1667 	if (!backref_ctx.found)
1668 		return -ENOENT;
1669 
1670 	cur_clone_root = NULL;
1671 	for (i = 0; i < sctx->clone_roots_cnt; i++) {
1672 		struct clone_root *clone_root = &sctx->clone_roots[i];
1673 
1674 		if (!clone_root->found_ref)
1675 			continue;
1676 
1677 		/*
1678 		 * Choose the root from which we can clone more bytes, to
1679 		 * minimize write operations and therefore have more extent
1680 		 * sharing at the destination (the same as in the source).
1681 		 */
1682 		if (!cur_clone_root ||
1683 		    clone_root->num_bytes > cur_clone_root->num_bytes) {
1684 			cur_clone_root = clone_root;
1685 
1686 			/*
1687 			 * We found an optimal clone candidate (any inode from
1688 			 * any root is fine), so we're done.
1689 			 */
1690 			if (clone_root->num_bytes >= backref_ctx.extent_len)
1691 				break;
1692 		}
1693 	}
1694 
1695 	if (cur_clone_root) {
1696 		*found = cur_clone_root;
1697 		ret = 0;
1698 	} else {
1699 		ret = -ENOENT;
1700 	}
1701 
1702 	return ret;
1703 }
1704 
read_symlink(struct btrfs_root * root,u64 ino,struct fs_path * dest)1705 static int read_symlink(struct btrfs_root *root,
1706 			u64 ino,
1707 			struct fs_path *dest)
1708 {
1709 	int ret;
1710 	BTRFS_PATH_AUTO_FREE(path);
1711 	struct btrfs_key key;
1712 	struct btrfs_file_extent_item *ei;
1713 	u8 type;
1714 	u8 compression;
1715 	unsigned long off;
1716 	int len;
1717 
1718 	path = alloc_path_for_send();
1719 	if (!path)
1720 		return -ENOMEM;
1721 
1722 	key.objectid = ino;
1723 	key.type = BTRFS_EXTENT_DATA_KEY;
1724 	key.offset = 0;
1725 	ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
1726 	if (ret < 0)
1727 		return ret;
1728 	if (unlikely(ret)) {
1729 		/*
1730 		 * An empty symlink inode. Can happen in rare error paths when
1731 		 * creating a symlink (transaction committed before the inode
1732 		 * eviction handler removed the symlink inode items and a crash
1733 		 * happened in between or the subvol was snapshotted in between).
1734 		 * Print an informative message to dmesg/syslog so that the user
1735 		 * can delete the symlink.
1736 		 */
1737 		btrfs_err(root->fs_info,
1738 			  "Found empty symlink inode %llu at root %llu",
1739 			  ino, btrfs_root_id(root));
1740 		return -EIO;
1741 	}
1742 
1743 	ei = btrfs_item_ptr(path->nodes[0], path->slots[0],
1744 			struct btrfs_file_extent_item);
1745 	type = btrfs_file_extent_type(path->nodes[0], ei);
1746 	if (unlikely(type != BTRFS_FILE_EXTENT_INLINE)) {
1747 		ret = -EUCLEAN;
1748 		btrfs_crit(root->fs_info,
1749 "send: found symlink extent that is not inline, ino %llu root %llu extent type %d",
1750 			   ino, btrfs_root_id(root), type);
1751 		return ret;
1752 	}
1753 	compression = btrfs_file_extent_compression(path->nodes[0], ei);
1754 	if (unlikely(compression != BTRFS_COMPRESS_NONE)) {
1755 		ret = -EUCLEAN;
1756 		btrfs_crit(root->fs_info,
1757 "send: found symlink extent with compression, ino %llu root %llu compression type %d",
1758 			   ino, btrfs_root_id(root), compression);
1759 		return ret;
1760 	}
1761 
1762 	off = btrfs_file_extent_inline_start(ei);
1763 	len = btrfs_file_extent_ram_bytes(path->nodes[0], ei);
1764 
1765 	return fs_path_add_from_extent_buffer(dest, path->nodes[0], off, len);
1766 }
1767 
1768 /*
1769  * Helper function to generate a file name that is unique in the root of
1770  * send_root and parent_root. This is used to generate names for orphan inodes.
1771  */
gen_unique_name(struct send_ctx * sctx,u64 ino,u64 gen,struct fs_path * dest)1772 static int gen_unique_name(struct send_ctx *sctx,
1773 			   u64 ino, u64 gen,
1774 			   struct fs_path *dest)
1775 {
1776 	BTRFS_PATH_AUTO_FREE(path);
1777 	struct btrfs_dir_item *di;
1778 	char tmp[64];
1779 	int len;
1780 	u64 idx = 0;
1781 
1782 	path = alloc_path_for_send();
1783 	if (!path)
1784 		return -ENOMEM;
1785 
1786 	while (1) {
1787 		struct fscrypt_str tmp_name;
1788 
1789 		len = snprintf(tmp, sizeof(tmp), "o%llu-%llu-%llu",
1790 				ino, gen, idx);
1791 		ASSERT(len < sizeof(tmp));
1792 		tmp_name.name = tmp;
1793 		tmp_name.len = len;
1794 
1795 		di = btrfs_lookup_dir_item(NULL, sctx->send_root,
1796 				path, BTRFS_FIRST_FREE_OBJECTID,
1797 				&tmp_name, 0);
1798 		btrfs_release_path(path);
1799 		if (IS_ERR(di))
1800 			return PTR_ERR(di);
1801 
1802 		if (di) {
1803 			/* not unique, try again */
1804 			idx++;
1805 			continue;
1806 		}
1807 
1808 		if (!sctx->parent_root) {
1809 			/* unique */
1810 			break;
1811 		}
1812 
1813 		di = btrfs_lookup_dir_item(NULL, sctx->parent_root,
1814 				path, BTRFS_FIRST_FREE_OBJECTID,
1815 				&tmp_name, 0);
1816 		btrfs_release_path(path);
1817 		if (IS_ERR(di))
1818 			return PTR_ERR(di);
1819 
1820 		if (di) {
1821 			/* not unique, try again */
1822 			idx++;
1823 			continue;
1824 		}
1825 		/* unique */
1826 		break;
1827 	}
1828 
1829 	return fs_path_add(dest, tmp, len);
1830 }
1831 
1832 enum inode_state {
1833 	inode_state_no_change,
1834 	inode_state_will_create,
1835 	inode_state_did_create,
1836 	inode_state_will_delete,
1837 	inode_state_did_delete,
1838 };
1839 
get_cur_inode_state(struct send_ctx * sctx,u64 ino,u64 gen,u64 * send_gen,u64 * parent_gen)1840 static int get_cur_inode_state(struct send_ctx *sctx, u64 ino, u64 gen,
1841 			       u64 *send_gen, u64 *parent_gen)
1842 {
1843 	int ret;
1844 	int left_ret;
1845 	int right_ret;
1846 	u64 left_gen;
1847 	u64 right_gen = 0;
1848 	struct btrfs_inode_info info;
1849 
1850 	ret = get_inode_info(sctx->send_root, ino, &info);
1851 	if (ret < 0 && ret != -ENOENT)
1852 		return ret;
1853 	left_ret = (info.nlink == 0) ? -ENOENT : ret;
1854 	left_gen = info.gen;
1855 	if (send_gen)
1856 		*send_gen = ((left_ret == -ENOENT) ? 0 : info.gen);
1857 
1858 	if (!sctx->parent_root) {
1859 		right_ret = -ENOENT;
1860 	} else {
1861 		ret = get_inode_info(sctx->parent_root, ino, &info);
1862 		if (ret < 0 && ret != -ENOENT)
1863 			return ret;
1864 		right_ret = (info.nlink == 0) ? -ENOENT : ret;
1865 		right_gen = info.gen;
1866 		if (parent_gen)
1867 			*parent_gen = ((right_ret == -ENOENT) ? 0 : info.gen);
1868 	}
1869 
1870 	if (!left_ret && !right_ret) {
1871 		if (left_gen == gen && right_gen == gen) {
1872 			ret = inode_state_no_change;
1873 		} else if (left_gen == gen) {
1874 			if (ino < sctx->send_progress)
1875 				ret = inode_state_did_create;
1876 			else
1877 				ret = inode_state_will_create;
1878 		} else if (right_gen == gen) {
1879 			if (ino < sctx->send_progress)
1880 				ret = inode_state_did_delete;
1881 			else
1882 				ret = inode_state_will_delete;
1883 		} else  {
1884 			ret = -ENOENT;
1885 		}
1886 	} else if (!left_ret) {
1887 		if (left_gen == gen) {
1888 			if (ino < sctx->send_progress)
1889 				ret = inode_state_did_create;
1890 			else
1891 				ret = inode_state_will_create;
1892 		} else {
1893 			ret = -ENOENT;
1894 		}
1895 	} else if (!right_ret) {
1896 		if (right_gen == gen) {
1897 			if (ino < sctx->send_progress)
1898 				ret = inode_state_did_delete;
1899 			else
1900 				ret = inode_state_will_delete;
1901 		} else {
1902 			ret = -ENOENT;
1903 		}
1904 	} else {
1905 		ret = -ENOENT;
1906 	}
1907 
1908 	return ret;
1909 }
1910 
is_inode_existent(struct send_ctx * sctx,u64 ino,u64 gen,u64 * send_gen,u64 * parent_gen)1911 static int is_inode_existent(struct send_ctx *sctx, u64 ino, u64 gen,
1912 			     u64 *send_gen, u64 *parent_gen)
1913 {
1914 	int ret;
1915 
1916 	if (ino == BTRFS_FIRST_FREE_OBJECTID)
1917 		return 1;
1918 
1919 	ret = get_cur_inode_state(sctx, ino, gen, send_gen, parent_gen);
1920 	if (ret < 0)
1921 		return ret;
1922 
1923 	if (ret == inode_state_no_change ||
1924 	    ret == inode_state_did_create ||
1925 	    ret == inode_state_will_delete)
1926 		return 1;
1927 
1928 	return 0;
1929 }
1930 
1931 /*
1932  * Helper function to lookup a dir item in a dir.
1933  */
lookup_dir_item_inode(struct btrfs_root * root,u64 dir,const char * name,int name_len,u64 * found_inode)1934 static int lookup_dir_item_inode(struct btrfs_root *root,
1935 				 u64 dir, const char *name, int name_len,
1936 				 u64 *found_inode)
1937 {
1938 	int ret = 0;
1939 	struct btrfs_dir_item *di;
1940 	struct btrfs_key key;
1941 	BTRFS_PATH_AUTO_FREE(path);
1942 	struct fscrypt_str name_str = FSTR_INIT((char *)name, name_len);
1943 
1944 	path = alloc_path_for_send();
1945 	if (!path)
1946 		return -ENOMEM;
1947 
1948 	di = btrfs_lookup_dir_item(NULL, root, path, dir, &name_str, 0);
1949 	if (IS_ERR_OR_NULL(di))
1950 		return di ? PTR_ERR(di) : -ENOENT;
1951 
1952 	btrfs_dir_item_key_to_cpu(path->nodes[0], di, &key);
1953 	if (key.type == BTRFS_ROOT_ITEM_KEY)
1954 		return -ENOENT;
1955 
1956 	*found_inode = key.objectid;
1957 
1958 	return ret;
1959 }
1960 
1961 /*
1962  * Looks up the first btrfs_inode_ref of a given ino. It returns the parent dir,
1963  * generation of the parent dir and the name of the dir entry.
1964  */
get_first_ref(struct btrfs_root * root,u64 ino,u64 * dir,u64 * dir_gen,struct fs_path * name)1965 static int get_first_ref(struct btrfs_root *root, u64 ino,
1966 			 u64 *dir, u64 *dir_gen, struct fs_path *name)
1967 {
1968 	int ret;
1969 	struct btrfs_key key;
1970 	struct btrfs_key found_key;
1971 	BTRFS_PATH_AUTO_FREE(path);
1972 	int len;
1973 	u64 parent_dir;
1974 
1975 	path = alloc_path_for_send();
1976 	if (!path)
1977 		return -ENOMEM;
1978 
1979 	key.objectid = ino;
1980 	key.type = BTRFS_INODE_REF_KEY;
1981 	key.offset = 0;
1982 
1983 	ret = btrfs_search_slot_for_read(root, &key, path, 1, 0);
1984 	if (ret < 0)
1985 		return ret;
1986 	if (!ret)
1987 		btrfs_item_key_to_cpu(path->nodes[0], &found_key,
1988 				path->slots[0]);
1989 	if (ret || found_key.objectid != ino ||
1990 	    (found_key.type != BTRFS_INODE_REF_KEY &&
1991 	     found_key.type != BTRFS_INODE_EXTREF_KEY))
1992 		return -ENOENT;
1993 
1994 	if (found_key.type == BTRFS_INODE_REF_KEY) {
1995 		struct btrfs_inode_ref *iref;
1996 		iref = btrfs_item_ptr(path->nodes[0], path->slots[0],
1997 				      struct btrfs_inode_ref);
1998 		len = btrfs_inode_ref_name_len(path->nodes[0], iref);
1999 		ret = fs_path_add_from_extent_buffer(name, path->nodes[0],
2000 						     (unsigned long)(iref + 1),
2001 						     len);
2002 		parent_dir = found_key.offset;
2003 	} else {
2004 		struct btrfs_inode_extref *extref;
2005 		extref = btrfs_item_ptr(path->nodes[0], path->slots[0],
2006 					struct btrfs_inode_extref);
2007 		len = btrfs_inode_extref_name_len(path->nodes[0], extref);
2008 		ret = fs_path_add_from_extent_buffer(name, path->nodes[0],
2009 					(unsigned long)&extref->name, len);
2010 		parent_dir = btrfs_inode_extref_parent(path->nodes[0], extref);
2011 	}
2012 	if (ret < 0)
2013 		return ret;
2014 	btrfs_release_path(path);
2015 
2016 	if (dir_gen) {
2017 		ret = get_inode_gen(root, parent_dir, dir_gen);
2018 		if (ret < 0)
2019 			return ret;
2020 	}
2021 
2022 	*dir = parent_dir;
2023 
2024 	return ret;
2025 }
2026 
is_first_ref(struct btrfs_root * root,u64 ino,u64 dir,const char * name,int name_len)2027 static int is_first_ref(struct btrfs_root *root,
2028 			u64 ino, u64 dir,
2029 			const char *name, int name_len)
2030 {
2031 	int ret;
2032 	struct fs_path *tmp_name;
2033 	u64 tmp_dir;
2034 
2035 	tmp_name = fs_path_alloc();
2036 	if (!tmp_name)
2037 		return -ENOMEM;
2038 
2039 	ret = get_first_ref(root, ino, &tmp_dir, NULL, tmp_name);
2040 	if (ret < 0)
2041 		goto out;
2042 
2043 	if (dir != tmp_dir || name_len != fs_path_len(tmp_name)) {
2044 		ret = 0;
2045 		goto out;
2046 	}
2047 
2048 	ret = !memcmp(tmp_name->start, name, name_len);
2049 
2050 out:
2051 	fs_path_free(tmp_name);
2052 	return ret;
2053 }
2054 
2055 /*
2056  * Used by process_recorded_refs to determine if a new ref would overwrite an
2057  * already existing ref. In case it detects an overwrite, it returns the
2058  * inode/gen in who_ino/who_gen.
2059  * When an overwrite is detected, process_recorded_refs does proper orphanizing
2060  * to make sure later references to the overwritten inode are possible.
2061  * Orphanizing is however only required for the first ref of an inode.
2062  * process_recorded_refs does an additional is_first_ref check to see if
2063  * orphanizing is really required.
2064  */
will_overwrite_ref(struct send_ctx * sctx,u64 dir,u64 dir_gen,const char * name,int name_len,u64 * who_ino,u64 * who_gen,u64 * who_mode)2065 static int will_overwrite_ref(struct send_ctx *sctx, u64 dir, u64 dir_gen,
2066 			      const char *name, int name_len,
2067 			      u64 *who_ino, u64 *who_gen, u64 *who_mode)
2068 {
2069 	int ret;
2070 	u64 parent_root_dir_gen;
2071 	u64 other_inode = 0;
2072 	struct btrfs_inode_info info;
2073 
2074 	if (!sctx->parent_root)
2075 		return 0;
2076 
2077 	ret = is_inode_existent(sctx, dir, dir_gen, NULL, &parent_root_dir_gen);
2078 	if (ret <= 0)
2079 		return 0;
2080 
2081 	/*
2082 	 * If we have a parent root we need to verify that the parent dir was
2083 	 * not deleted and then re-created, if it was then we have no overwrite
2084 	 * and we can just unlink this entry.
2085 	 *
2086 	 * @parent_root_dir_gen was set to 0 if the inode does not exist in the
2087 	 * parent root.
2088 	 */
2089 	if (sctx->parent_root && dir != BTRFS_FIRST_FREE_OBJECTID &&
2090 	    parent_root_dir_gen != dir_gen)
2091 		return 0;
2092 
2093 	ret = lookup_dir_item_inode(sctx->parent_root, dir, name, name_len,
2094 				    &other_inode);
2095 	if (ret == -ENOENT)
2096 		return 0;
2097 	else if (ret < 0)
2098 		return ret;
2099 
2100 	/*
2101 	 * Check if the overwritten ref was already processed. If yes, the ref
2102 	 * was already unlinked/moved, so we can safely assume that we will not
2103 	 * overwrite anything at this point in time.
2104 	 */
2105 	if (other_inode > sctx->send_progress ||
2106 	    is_waiting_for_move(sctx, other_inode)) {
2107 		ret = get_inode_info(sctx->parent_root, other_inode, &info);
2108 		if (ret < 0)
2109 			return ret;
2110 
2111 		*who_ino = other_inode;
2112 		*who_gen = info.gen;
2113 		*who_mode = info.mode;
2114 		return 1;
2115 	}
2116 
2117 	return 0;
2118 }
2119 
2120 /*
2121  * Checks if the ref was overwritten by an already processed inode. This is
2122  * used by __get_cur_name_and_parent to find out if the ref was orphanized and
2123  * thus the orphan name needs be used.
2124  * process_recorded_refs also uses it to avoid unlinking of refs that were
2125  * overwritten.
2126  */
did_overwrite_ref(struct send_ctx * sctx,u64 dir,u64 dir_gen,u64 ino,u64 ino_gen,const char * name,int name_len)2127 static int did_overwrite_ref(struct send_ctx *sctx,
2128 			    u64 dir, u64 dir_gen,
2129 			    u64 ino, u64 ino_gen,
2130 			    const char *name, int name_len)
2131 {
2132 	int ret;
2133 	u64 ow_inode;
2134 	u64 ow_gen = 0;
2135 	u64 send_root_dir_gen;
2136 
2137 	if (!sctx->parent_root)
2138 		return 0;
2139 
2140 	ret = is_inode_existent(sctx, dir, dir_gen, &send_root_dir_gen, NULL);
2141 	if (ret <= 0)
2142 		return ret;
2143 
2144 	/*
2145 	 * @send_root_dir_gen was set to 0 if the inode does not exist in the
2146 	 * send root.
2147 	 */
2148 	if (dir != BTRFS_FIRST_FREE_OBJECTID && send_root_dir_gen != dir_gen)
2149 		return 0;
2150 
2151 	/* check if the ref was overwritten by another ref */
2152 	ret = lookup_dir_item_inode(sctx->send_root, dir, name, name_len,
2153 				    &ow_inode);
2154 	if (ret == -ENOENT) {
2155 		/* was never and will never be overwritten */
2156 		return 0;
2157 	} else if (ret < 0) {
2158 		return ret;
2159 	}
2160 
2161 	if (ow_inode == ino) {
2162 		ret = get_inode_gen(sctx->send_root, ow_inode, &ow_gen);
2163 		if (ret < 0)
2164 			return ret;
2165 
2166 		/* It's the same inode, so no overwrite happened. */
2167 		if (ow_gen == ino_gen)
2168 			return 0;
2169 	}
2170 
2171 	/*
2172 	 * We know that it is or will be overwritten. Check this now.
2173 	 * The current inode being processed might have been the one that caused
2174 	 * inode 'ino' to be orphanized, therefore check if ow_inode matches
2175 	 * the current inode being processed.
2176 	 */
2177 	if (ow_inode < sctx->send_progress)
2178 		return 1;
2179 
2180 	if (ino != sctx->cur_ino && ow_inode == sctx->cur_ino) {
2181 		if (ow_gen == 0) {
2182 			ret = get_inode_gen(sctx->send_root, ow_inode, &ow_gen);
2183 			if (ret < 0)
2184 				return ret;
2185 		}
2186 		if (ow_gen == sctx->cur_inode_gen)
2187 			return 1;
2188 	}
2189 
2190 	return 0;
2191 }
2192 
2193 /*
2194  * Same as did_overwrite_ref, but also checks if it is the first ref of an inode
2195  * that got overwritten. This is used by process_recorded_refs to determine
2196  * if it has to use the path as returned by get_cur_path or the orphan name.
2197  */
did_overwrite_first_ref(struct send_ctx * sctx,u64 ino,u64 gen)2198 static int did_overwrite_first_ref(struct send_ctx *sctx, u64 ino, u64 gen)
2199 {
2200 	int ret = 0;
2201 	struct fs_path *name = NULL;
2202 	u64 dir;
2203 	u64 dir_gen;
2204 
2205 	if (!sctx->parent_root)
2206 		goto out;
2207 
2208 	name = fs_path_alloc();
2209 	if (!name)
2210 		return -ENOMEM;
2211 
2212 	ret = get_first_ref(sctx->parent_root, ino, &dir, &dir_gen, name);
2213 	if (ret < 0)
2214 		goto out;
2215 
2216 	ret = did_overwrite_ref(sctx, dir, dir_gen, ino, gen,
2217 			name->start, fs_path_len(name));
2218 
2219 out:
2220 	fs_path_free(name);
2221 	return ret;
2222 }
2223 
name_cache_search(struct send_ctx * sctx,u64 ino,u64 gen)2224 static inline struct name_cache_entry *name_cache_search(struct send_ctx *sctx,
2225 							 u64 ino, u64 gen)
2226 {
2227 	struct btrfs_lru_cache_entry *entry;
2228 
2229 	entry = btrfs_lru_cache_lookup(&sctx->name_cache, ino, gen);
2230 	if (!entry)
2231 		return NULL;
2232 
2233 	return container_of(entry, struct name_cache_entry, entry);
2234 }
2235 
2236 /*
2237  * Used by get_cur_path for each ref up to the root.
2238  * Returns 0 if it succeeded.
2239  * Returns 1 if the inode is not existent or got overwritten. In that case, the
2240  * name is an orphan name. This instructs get_cur_path to stop iterating. If 1
2241  * is returned, parent_ino/parent_gen are not guaranteed to be valid.
2242  * Returns <0 in case of error.
2243  */
__get_cur_name_and_parent(struct send_ctx * sctx,u64 ino,u64 gen,u64 * parent_ino,u64 * parent_gen,struct fs_path * dest)2244 static int __get_cur_name_and_parent(struct send_ctx *sctx,
2245 				     u64 ino, u64 gen,
2246 				     u64 *parent_ino,
2247 				     u64 *parent_gen,
2248 				     struct fs_path *dest)
2249 {
2250 	int ret;
2251 	int nce_ret;
2252 	struct name_cache_entry *nce;
2253 
2254 	/*
2255 	 * First check if we already did a call to this function with the same
2256 	 * ino/gen. If yes, check if the cache entry is still up-to-date. If yes
2257 	 * return the cached result.
2258 	 */
2259 	nce = name_cache_search(sctx, ino, gen);
2260 	if (nce) {
2261 		if (ino < sctx->send_progress && nce->need_later_update) {
2262 			btrfs_lru_cache_remove(&sctx->name_cache, &nce->entry);
2263 			nce = NULL;
2264 		} else {
2265 			*parent_ino = nce->parent_ino;
2266 			*parent_gen = nce->parent_gen;
2267 			ret = fs_path_add(dest, nce->name, nce->name_len);
2268 			if (ret < 0)
2269 				return ret;
2270 			return nce->ret;
2271 		}
2272 	}
2273 
2274 	/*
2275 	 * If the inode is not existent yet, add the orphan name and return 1.
2276 	 * This should only happen for the parent dir that we determine in
2277 	 * record_new_ref_if_needed().
2278 	 */
2279 	ret = is_inode_existent(sctx, ino, gen, NULL, NULL);
2280 	if (ret < 0)
2281 		return ret;
2282 
2283 	if (!ret) {
2284 		ret = gen_unique_name(sctx, ino, gen, dest);
2285 		if (ret < 0)
2286 			return ret;
2287 		ret = 1;
2288 		goto out_cache;
2289 	}
2290 
2291 	/*
2292 	 * Depending on whether the inode was already processed or not, use
2293 	 * send_root or parent_root for ref lookup.
2294 	 */
2295 	if (ino < sctx->send_progress)
2296 		ret = get_first_ref(sctx->send_root, ino,
2297 				    parent_ino, parent_gen, dest);
2298 	else
2299 		ret = get_first_ref(sctx->parent_root, ino,
2300 				    parent_ino, parent_gen, dest);
2301 	if (ret < 0)
2302 		return ret;
2303 
2304 	/*
2305 	 * Check if the ref was overwritten by an inode's ref that was processed
2306 	 * earlier. If yes, treat as orphan and return 1.
2307 	 */
2308 	ret = did_overwrite_ref(sctx, *parent_ino, *parent_gen, ino, gen,
2309 				dest->start, fs_path_len(dest));
2310 	if (ret < 0)
2311 		return ret;
2312 	if (ret) {
2313 		fs_path_reset(dest);
2314 		ret = gen_unique_name(sctx, ino, gen, dest);
2315 		if (ret < 0)
2316 			return ret;
2317 		ret = 1;
2318 	}
2319 
2320 out_cache:
2321 	/*
2322 	 * Store the result of the lookup in the name cache.
2323 	 */
2324 	nce = kmalloc(sizeof(*nce) + fs_path_len(dest), GFP_KERNEL);
2325 	if (!nce)
2326 		return -ENOMEM;
2327 
2328 	nce->entry.key = ino;
2329 	nce->entry.gen = gen;
2330 	nce->parent_ino = *parent_ino;
2331 	nce->parent_gen = *parent_gen;
2332 	nce->name_len = fs_path_len(dest);
2333 	nce->ret = ret;
2334 	memcpy(nce->name, dest->start, nce->name_len);
2335 
2336 	if (ino < sctx->send_progress)
2337 		nce->need_later_update = 0;
2338 	else
2339 		nce->need_later_update = 1;
2340 
2341 	nce_ret = btrfs_lru_cache_store(&sctx->name_cache, &nce->entry, GFP_KERNEL);
2342 	if (nce_ret < 0) {
2343 		kfree(nce);
2344 		return nce_ret;
2345 	}
2346 
2347 	return ret;
2348 }
2349 
2350 /*
2351  * Magic happens here. This function returns the first ref to an inode as it
2352  * would look like while receiving the stream at this point in time.
2353  * We walk the path up to the root. For every inode in between, we check if it
2354  * was already processed/sent. If yes, we continue with the parent as found
2355  * in send_root. If not, we continue with the parent as found in parent_root.
2356  * If we encounter an inode that was deleted at this point in time, we use the
2357  * inodes "orphan" name instead of the real name and stop. Same with new inodes
2358  * that were not created yet and overwritten inodes/refs.
2359  *
2360  * When do we have orphan inodes:
2361  * 1. When an inode is freshly created and thus no valid refs are available yet
2362  * 2. When a directory lost all it's refs (deleted) but still has dir items
2363  *    inside which were not processed yet (pending for move/delete). If anyone
2364  *    tried to get the path to the dir items, it would get a path inside that
2365  *    orphan directory.
2366  * 3. When an inode is moved around or gets new links, it may overwrite the ref
2367  *    of an unprocessed inode. If in that case the first ref would be
2368  *    overwritten, the overwritten inode gets "orphanized". Later when we
2369  *    process this overwritten inode, it is restored at a new place by moving
2370  *    the orphan inode.
2371  *
2372  * sctx->send_progress tells this function at which point in time receiving
2373  * would be.
2374  */
get_cur_path(struct send_ctx * sctx,u64 ino,u64 gen,struct fs_path * dest)2375 static int get_cur_path(struct send_ctx *sctx, u64 ino, u64 gen,
2376 			struct fs_path *dest)
2377 {
2378 	int ret = 0;
2379 	struct fs_path *name = NULL;
2380 	u64 parent_inode = 0;
2381 	u64 parent_gen = 0;
2382 	int stop = 0;
2383 	const bool is_cur_inode = (ino == sctx->cur_ino && gen == sctx->cur_inode_gen);
2384 
2385 	if (is_cur_inode && fs_path_len(&sctx->cur_inode_path) > 0) {
2386 		if (dest != &sctx->cur_inode_path)
2387 			return fs_path_copy(dest, &sctx->cur_inode_path);
2388 
2389 		return 0;
2390 	}
2391 
2392 	name = fs_path_alloc();
2393 	if (!name) {
2394 		ret = -ENOMEM;
2395 		goto out;
2396 	}
2397 
2398 	dest->reversed = 1;
2399 	fs_path_reset(dest);
2400 
2401 	while (!stop && ino != BTRFS_FIRST_FREE_OBJECTID) {
2402 		struct waiting_dir_move *wdm;
2403 
2404 		fs_path_reset(name);
2405 
2406 		if (is_waiting_for_rm(sctx, ino, gen)) {
2407 			ret = gen_unique_name(sctx, ino, gen, name);
2408 			if (ret < 0)
2409 				goto out;
2410 			ret = fs_path_add_path(dest, name);
2411 			break;
2412 		}
2413 
2414 		wdm = get_waiting_dir_move(sctx, ino);
2415 		if (wdm && wdm->orphanized) {
2416 			ret = gen_unique_name(sctx, ino, gen, name);
2417 			stop = 1;
2418 		} else if (wdm) {
2419 			ret = get_first_ref(sctx->parent_root, ino,
2420 					    &parent_inode, &parent_gen, name);
2421 		} else {
2422 			ret = __get_cur_name_and_parent(sctx, ino, gen,
2423 							&parent_inode,
2424 							&parent_gen, name);
2425 			if (ret)
2426 				stop = 1;
2427 		}
2428 
2429 		if (ret < 0)
2430 			goto out;
2431 
2432 		ret = fs_path_add_path(dest, name);
2433 		if (ret < 0)
2434 			goto out;
2435 
2436 		ino = parent_inode;
2437 		gen = parent_gen;
2438 	}
2439 
2440 out:
2441 	fs_path_free(name);
2442 	if (!ret) {
2443 		fs_path_unreverse(dest);
2444 		if (is_cur_inode && dest != &sctx->cur_inode_path)
2445 			ret = fs_path_copy(&sctx->cur_inode_path, dest);
2446 	}
2447 
2448 	return ret;
2449 }
2450 
2451 /*
2452  * Sends a BTRFS_SEND_C_SUBVOL command/item to userspace
2453  */
send_subvol_begin(struct send_ctx * sctx)2454 static int send_subvol_begin(struct send_ctx *sctx)
2455 {
2456 	int ret;
2457 	struct btrfs_root *send_root = sctx->send_root;
2458 	struct btrfs_root *parent_root = sctx->parent_root;
2459 	BTRFS_PATH_AUTO_FREE(path);
2460 	struct btrfs_key key;
2461 	struct btrfs_root_ref *ref;
2462 	struct extent_buffer *leaf;
2463 	char *name = NULL;
2464 	int namelen;
2465 
2466 	path = btrfs_alloc_path();
2467 	if (!path)
2468 		return -ENOMEM;
2469 
2470 	name = kmalloc(BTRFS_PATH_NAME_MAX, GFP_KERNEL);
2471 	if (!name)
2472 		return -ENOMEM;
2473 
2474 	key.objectid = btrfs_root_id(send_root);
2475 	key.type = BTRFS_ROOT_BACKREF_KEY;
2476 	key.offset = 0;
2477 
2478 	ret = btrfs_search_slot_for_read(send_root->fs_info->tree_root,
2479 				&key, path, 1, 0);
2480 	if (ret < 0)
2481 		goto out;
2482 	if (ret) {
2483 		ret = -ENOENT;
2484 		goto out;
2485 	}
2486 
2487 	leaf = path->nodes[0];
2488 	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
2489 	if (key.type != BTRFS_ROOT_BACKREF_KEY ||
2490 	    key.objectid != btrfs_root_id(send_root)) {
2491 		ret = -ENOENT;
2492 		goto out;
2493 	}
2494 	ref = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_root_ref);
2495 	namelen = btrfs_root_ref_name_len(leaf, ref);
2496 	read_extent_buffer(leaf, name, (unsigned long)(ref + 1), namelen);
2497 	btrfs_release_path(path);
2498 
2499 	if (parent_root) {
2500 		ret = begin_cmd(sctx, BTRFS_SEND_C_SNAPSHOT);
2501 		if (ret < 0)
2502 			goto out;
2503 	} else {
2504 		ret = begin_cmd(sctx, BTRFS_SEND_C_SUBVOL);
2505 		if (ret < 0)
2506 			goto out;
2507 	}
2508 
2509 	TLV_PUT_STRING(sctx, BTRFS_SEND_A_PATH, name, namelen);
2510 
2511 	if (!btrfs_is_empty_uuid(sctx->send_root->root_item.received_uuid))
2512 		TLV_PUT_UUID(sctx, BTRFS_SEND_A_UUID,
2513 			    sctx->send_root->root_item.received_uuid);
2514 	else
2515 		TLV_PUT_UUID(sctx, BTRFS_SEND_A_UUID,
2516 			    sctx->send_root->root_item.uuid);
2517 
2518 	TLV_PUT_U64(sctx, BTRFS_SEND_A_CTRANSID,
2519 		    btrfs_root_ctransid(&sctx->send_root->root_item));
2520 	if (parent_root) {
2521 		if (!btrfs_is_empty_uuid(parent_root->root_item.received_uuid))
2522 			TLV_PUT_UUID(sctx, BTRFS_SEND_A_CLONE_UUID,
2523 				     parent_root->root_item.received_uuid);
2524 		else
2525 			TLV_PUT_UUID(sctx, BTRFS_SEND_A_CLONE_UUID,
2526 				     parent_root->root_item.uuid);
2527 		TLV_PUT_U64(sctx, BTRFS_SEND_A_CLONE_CTRANSID,
2528 			    btrfs_root_ctransid(&sctx->parent_root->root_item));
2529 	}
2530 
2531 	ret = send_cmd(sctx);
2532 
2533 tlv_put_failure:
2534 out:
2535 	kfree(name);
2536 	return ret;
2537 }
2538 
get_cur_inode_path(struct send_ctx * sctx)2539 static struct fs_path *get_cur_inode_path(struct send_ctx *sctx)
2540 {
2541 	if (fs_path_len(&sctx->cur_inode_path) == 0) {
2542 		int ret;
2543 
2544 		ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen,
2545 				   &sctx->cur_inode_path);
2546 		if (ret < 0)
2547 			return ERR_PTR(ret);
2548 	}
2549 
2550 	return &sctx->cur_inode_path;
2551 }
2552 
get_path_for_command(struct send_ctx * sctx,u64 ino,u64 gen)2553 static struct fs_path *get_path_for_command(struct send_ctx *sctx, u64 ino, u64 gen)
2554 {
2555 	struct fs_path *path;
2556 	int ret;
2557 
2558 	if (ino == sctx->cur_ino && gen == sctx->cur_inode_gen)
2559 		return get_cur_inode_path(sctx);
2560 
2561 	path = fs_path_alloc();
2562 	if (!path)
2563 		return ERR_PTR(-ENOMEM);
2564 
2565 	ret = get_cur_path(sctx, ino, gen, path);
2566 	if (ret < 0) {
2567 		fs_path_free(path);
2568 		return ERR_PTR(ret);
2569 	}
2570 
2571 	return path;
2572 }
2573 
free_path_for_command(const struct send_ctx * sctx,struct fs_path * path)2574 static void free_path_for_command(const struct send_ctx *sctx, struct fs_path *path)
2575 {
2576 	if (path != &sctx->cur_inode_path)
2577 		fs_path_free(path);
2578 }
2579 
send_truncate(struct send_ctx * sctx,u64 ino,u64 gen,u64 size)2580 static int send_truncate(struct send_ctx *sctx, u64 ino, u64 gen, u64 size)
2581 {
2582 	int ret = 0;
2583 	struct fs_path *p;
2584 
2585 	p = get_path_for_command(sctx, ino, gen);
2586 	if (IS_ERR(p))
2587 		return PTR_ERR(p);
2588 
2589 	ret = begin_cmd(sctx, BTRFS_SEND_C_TRUNCATE);
2590 	if (ret < 0)
2591 		goto out;
2592 
2593 	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
2594 	TLV_PUT_U64(sctx, BTRFS_SEND_A_SIZE, size);
2595 
2596 	ret = send_cmd(sctx);
2597 
2598 tlv_put_failure:
2599 out:
2600 	free_path_for_command(sctx, p);
2601 	return ret;
2602 }
2603 
send_chmod(struct send_ctx * sctx,u64 ino,u64 gen,u64 mode)2604 static int send_chmod(struct send_ctx *sctx, u64 ino, u64 gen, u64 mode)
2605 {
2606 	int ret = 0;
2607 	struct fs_path *p;
2608 
2609 	p = get_path_for_command(sctx, ino, gen);
2610 	if (IS_ERR(p))
2611 		return PTR_ERR(p);
2612 
2613 	ret = begin_cmd(sctx, BTRFS_SEND_C_CHMOD);
2614 	if (ret < 0)
2615 		goto out;
2616 
2617 	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
2618 	TLV_PUT_U64(sctx, BTRFS_SEND_A_MODE, mode & 07777);
2619 
2620 	ret = send_cmd(sctx);
2621 
2622 tlv_put_failure:
2623 out:
2624 	free_path_for_command(sctx, p);
2625 	return ret;
2626 }
2627 
send_fileattr(struct send_ctx * sctx,u64 ino,u64 gen,u64 fileattr)2628 static int send_fileattr(struct send_ctx *sctx, u64 ino, u64 gen, u64 fileattr)
2629 {
2630 	int ret = 0;
2631 	struct fs_path *p;
2632 
2633 	if (sctx->proto < 2)
2634 		return 0;
2635 
2636 	p = get_path_for_command(sctx, ino, gen);
2637 	if (IS_ERR(p))
2638 		return PTR_ERR(p);
2639 
2640 	ret = begin_cmd(sctx, BTRFS_SEND_C_FILEATTR);
2641 	if (ret < 0)
2642 		goto out;
2643 
2644 	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
2645 	TLV_PUT_U64(sctx, BTRFS_SEND_A_FILEATTR, fileattr);
2646 
2647 	ret = send_cmd(sctx);
2648 
2649 tlv_put_failure:
2650 out:
2651 	free_path_for_command(sctx, p);
2652 	return ret;
2653 }
2654 
send_chown(struct send_ctx * sctx,u64 ino,u64 gen,u64 uid,u64 gid)2655 static int send_chown(struct send_ctx *sctx, u64 ino, u64 gen, u64 uid, u64 gid)
2656 {
2657 	int ret = 0;
2658 	struct fs_path *p;
2659 
2660 	p = get_path_for_command(sctx, ino, gen);
2661 	if (IS_ERR(p))
2662 		return PTR_ERR(p);
2663 
2664 	ret = begin_cmd(sctx, BTRFS_SEND_C_CHOWN);
2665 	if (ret < 0)
2666 		goto out;
2667 
2668 	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
2669 	TLV_PUT_U64(sctx, BTRFS_SEND_A_UID, uid);
2670 	TLV_PUT_U64(sctx, BTRFS_SEND_A_GID, gid);
2671 
2672 	ret = send_cmd(sctx);
2673 
2674 tlv_put_failure:
2675 out:
2676 	free_path_for_command(sctx, p);
2677 	return ret;
2678 }
2679 
send_utimes(struct send_ctx * sctx,u64 ino,u64 gen)2680 static int send_utimes(struct send_ctx *sctx, u64 ino, u64 gen)
2681 {
2682 	int ret = 0;
2683 	struct fs_path *p = NULL;
2684 	struct btrfs_inode_item *ii;
2685 	BTRFS_PATH_AUTO_FREE(path);
2686 	struct extent_buffer *eb;
2687 	struct btrfs_key key;
2688 	int slot;
2689 
2690 	p = get_path_for_command(sctx, ino, gen);
2691 	if (IS_ERR(p))
2692 		return PTR_ERR(p);
2693 
2694 	path = alloc_path_for_send();
2695 	if (!path) {
2696 		ret = -ENOMEM;
2697 		goto out;
2698 	}
2699 
2700 	key.objectid = ino;
2701 	key.type = BTRFS_INODE_ITEM_KEY;
2702 	key.offset = 0;
2703 	ret = btrfs_search_slot(NULL, sctx->send_root, &key, path, 0, 0);
2704 	if (ret > 0)
2705 		ret = -ENOENT;
2706 	if (ret < 0)
2707 		goto out;
2708 
2709 	eb = path->nodes[0];
2710 	slot = path->slots[0];
2711 	ii = btrfs_item_ptr(eb, slot, struct btrfs_inode_item);
2712 
2713 	ret = begin_cmd(sctx, BTRFS_SEND_C_UTIMES);
2714 	if (ret < 0)
2715 		goto out;
2716 
2717 	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
2718 	TLV_PUT_BTRFS_TIMESPEC(sctx, BTRFS_SEND_A_ATIME, eb, &ii->atime);
2719 	TLV_PUT_BTRFS_TIMESPEC(sctx, BTRFS_SEND_A_MTIME, eb, &ii->mtime);
2720 	TLV_PUT_BTRFS_TIMESPEC(sctx, BTRFS_SEND_A_CTIME, eb, &ii->ctime);
2721 	if (sctx->proto >= 2)
2722 		TLV_PUT_BTRFS_TIMESPEC(sctx, BTRFS_SEND_A_OTIME, eb, &ii->otime);
2723 
2724 	ret = send_cmd(sctx);
2725 
2726 tlv_put_failure:
2727 out:
2728 	free_path_for_command(sctx, p);
2729 	return ret;
2730 }
2731 
2732 /*
2733  * If the cache is full, we can't remove entries from it and do a call to
2734  * send_utimes() for each respective inode, because we might be finishing
2735  * processing an inode that is a directory and it just got renamed, and existing
2736  * entries in the cache may refer to inodes that have the directory in their
2737  * full path - in which case we would generate outdated paths (pre-rename)
2738  * for the inodes that the cache entries point to. Instead of pruning the
2739  * cache when inserting, do it after we finish processing each inode at
2740  * finish_inode_if_needed().
2741  */
cache_dir_utimes(struct send_ctx * sctx,u64 dir,u64 gen)2742 static int cache_dir_utimes(struct send_ctx *sctx, u64 dir, u64 gen)
2743 {
2744 	struct btrfs_lru_cache_entry *entry;
2745 	int ret;
2746 
2747 	entry = btrfs_lru_cache_lookup(&sctx->dir_utimes_cache, dir, gen);
2748 	if (entry != NULL)
2749 		return 0;
2750 
2751 	/* Caching is optional, don't fail if we can't allocate memory. */
2752 	entry = kmalloc(sizeof(*entry), GFP_KERNEL);
2753 	if (!entry)
2754 		return send_utimes(sctx, dir, gen);
2755 
2756 	entry->key = dir;
2757 	entry->gen = gen;
2758 
2759 	ret = btrfs_lru_cache_store(&sctx->dir_utimes_cache, entry, GFP_KERNEL);
2760 	ASSERT(ret != -EEXIST);
2761 	if (ret) {
2762 		kfree(entry);
2763 		return send_utimes(sctx, dir, gen);
2764 	}
2765 
2766 	return 0;
2767 }
2768 
trim_dir_utimes_cache(struct send_ctx * sctx)2769 static int trim_dir_utimes_cache(struct send_ctx *sctx)
2770 {
2771 	while (sctx->dir_utimes_cache.size > SEND_MAX_DIR_UTIMES_CACHE_SIZE) {
2772 		struct btrfs_lru_cache_entry *lru;
2773 		int ret;
2774 
2775 		lru = btrfs_lru_cache_lru_entry(&sctx->dir_utimes_cache);
2776 		ASSERT(lru != NULL);
2777 
2778 		ret = send_utimes(sctx, lru->key, lru->gen);
2779 		if (ret)
2780 			return ret;
2781 
2782 		btrfs_lru_cache_remove(&sctx->dir_utimes_cache, lru);
2783 	}
2784 
2785 	return 0;
2786 }
2787 
2788 /*
2789  * Sends a BTRFS_SEND_C_MKXXX or SYMLINK command to user space. We don't have
2790  * a valid path yet because we did not process the refs yet. So, the inode
2791  * is created as orphan.
2792  */
send_create_inode(struct send_ctx * sctx,u64 ino)2793 static int send_create_inode(struct send_ctx *sctx, u64 ino)
2794 {
2795 	int ret = 0;
2796 	struct fs_path *p;
2797 	int cmd;
2798 	struct btrfs_inode_info info;
2799 	u64 gen;
2800 	u64 mode;
2801 	u64 rdev;
2802 
2803 	p = fs_path_alloc();
2804 	if (!p)
2805 		return -ENOMEM;
2806 
2807 	if (ino != sctx->cur_ino) {
2808 		ret = get_inode_info(sctx->send_root, ino, &info);
2809 		if (ret < 0)
2810 			goto out;
2811 		gen = info.gen;
2812 		mode = info.mode;
2813 		rdev = info.rdev;
2814 	} else {
2815 		gen = sctx->cur_inode_gen;
2816 		mode = sctx->cur_inode_mode;
2817 		rdev = sctx->cur_inode_rdev;
2818 	}
2819 
2820 	if (S_ISREG(mode)) {
2821 		cmd = BTRFS_SEND_C_MKFILE;
2822 	} else if (S_ISDIR(mode)) {
2823 		cmd = BTRFS_SEND_C_MKDIR;
2824 	} else if (S_ISLNK(mode)) {
2825 		cmd = BTRFS_SEND_C_SYMLINK;
2826 	} else if (S_ISCHR(mode) || S_ISBLK(mode)) {
2827 		cmd = BTRFS_SEND_C_MKNOD;
2828 	} else if (S_ISFIFO(mode)) {
2829 		cmd = BTRFS_SEND_C_MKFIFO;
2830 	} else if (S_ISSOCK(mode)) {
2831 		cmd = BTRFS_SEND_C_MKSOCK;
2832 	} else {
2833 		btrfs_warn(sctx->send_root->fs_info, "unexpected inode type %o",
2834 				(int)(mode & S_IFMT));
2835 		ret = -EOPNOTSUPP;
2836 		goto out;
2837 	}
2838 
2839 	ret = begin_cmd(sctx, cmd);
2840 	if (ret < 0)
2841 		goto out;
2842 
2843 	ret = gen_unique_name(sctx, ino, gen, p);
2844 	if (ret < 0)
2845 		goto out;
2846 
2847 	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
2848 	TLV_PUT_U64(sctx, BTRFS_SEND_A_INO, ino);
2849 
2850 	if (S_ISLNK(mode)) {
2851 		fs_path_reset(p);
2852 		ret = read_symlink(sctx->send_root, ino, p);
2853 		if (ret < 0)
2854 			goto out;
2855 		TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH_LINK, p);
2856 	} else if (S_ISCHR(mode) || S_ISBLK(mode) ||
2857 		   S_ISFIFO(mode) || S_ISSOCK(mode)) {
2858 		TLV_PUT_U64(sctx, BTRFS_SEND_A_RDEV, new_encode_dev(rdev));
2859 		TLV_PUT_U64(sctx, BTRFS_SEND_A_MODE, mode);
2860 	}
2861 
2862 	ret = send_cmd(sctx);
2863 	if (ret < 0)
2864 		goto out;
2865 
2866 
2867 tlv_put_failure:
2868 out:
2869 	fs_path_free(p);
2870 	return ret;
2871 }
2872 
cache_dir_created(struct send_ctx * sctx,u64 dir)2873 static void cache_dir_created(struct send_ctx *sctx, u64 dir)
2874 {
2875 	struct btrfs_lru_cache_entry *entry;
2876 	int ret;
2877 
2878 	/* Caching is optional, ignore any failures. */
2879 	entry = kmalloc(sizeof(*entry), GFP_KERNEL);
2880 	if (!entry)
2881 		return;
2882 
2883 	entry->key = dir;
2884 	entry->gen = 0;
2885 	ret = btrfs_lru_cache_store(&sctx->dir_created_cache, entry, GFP_KERNEL);
2886 	if (ret < 0)
2887 		kfree(entry);
2888 }
2889 
2890 /*
2891  * We need some special handling for inodes that get processed before the parent
2892  * directory got created. See process_recorded_refs for details.
2893  * This function does the check if we already created the dir out of order.
2894  */
did_create_dir(struct send_ctx * sctx,u64 dir)2895 static int did_create_dir(struct send_ctx *sctx, u64 dir)
2896 {
2897 	int ret = 0;
2898 	int iter_ret = 0;
2899 	BTRFS_PATH_AUTO_FREE(path);
2900 	struct btrfs_key key;
2901 	struct btrfs_key found_key;
2902 	struct btrfs_key di_key;
2903 	struct btrfs_dir_item *di;
2904 
2905 	if (btrfs_lru_cache_lookup(&sctx->dir_created_cache, dir, 0))
2906 		return 1;
2907 
2908 	path = alloc_path_for_send();
2909 	if (!path)
2910 		return -ENOMEM;
2911 
2912 	key.objectid = dir;
2913 	key.type = BTRFS_DIR_INDEX_KEY;
2914 	key.offset = 0;
2915 
2916 	btrfs_for_each_slot(sctx->send_root, &key, &found_key, path, iter_ret) {
2917 		struct extent_buffer *eb = path->nodes[0];
2918 
2919 		if (found_key.objectid != key.objectid ||
2920 		    found_key.type != key.type) {
2921 			ret = 0;
2922 			break;
2923 		}
2924 
2925 		di = btrfs_item_ptr(eb, path->slots[0], struct btrfs_dir_item);
2926 		btrfs_dir_item_key_to_cpu(eb, di, &di_key);
2927 
2928 		if (di_key.type != BTRFS_ROOT_ITEM_KEY &&
2929 		    di_key.objectid < sctx->send_progress) {
2930 			ret = 1;
2931 			cache_dir_created(sctx, dir);
2932 			break;
2933 		}
2934 	}
2935 	/* Catch error found during iteration */
2936 	if (iter_ret < 0)
2937 		ret = iter_ret;
2938 
2939 	return ret;
2940 }
2941 
2942 /*
2943  * Only creates the inode if it is:
2944  * 1. Not a directory
2945  * 2. Or a directory which was not created already due to out of order
2946  *    directories. See did_create_dir and process_recorded_refs for details.
2947  */
send_create_inode_if_needed(struct send_ctx * sctx)2948 static int send_create_inode_if_needed(struct send_ctx *sctx)
2949 {
2950 	int ret;
2951 
2952 	if (S_ISDIR(sctx->cur_inode_mode)) {
2953 		ret = did_create_dir(sctx, sctx->cur_ino);
2954 		if (ret < 0)
2955 			return ret;
2956 		else if (ret > 0)
2957 			return 0;
2958 	}
2959 
2960 	ret = send_create_inode(sctx, sctx->cur_ino);
2961 
2962 	if (ret == 0 && S_ISDIR(sctx->cur_inode_mode))
2963 		cache_dir_created(sctx, sctx->cur_ino);
2964 
2965 	return ret;
2966 }
2967 
2968 struct recorded_ref {
2969 	struct list_head list;
2970 	char *name;
2971 	struct fs_path *full_path;
2972 	u64 dir;
2973 	u64 dir_gen;
2974 	int name_len;
2975 	struct rb_node node;
2976 	struct rb_root *root;
2977 };
2978 
recorded_ref_alloc(void)2979 static struct recorded_ref *recorded_ref_alloc(void)
2980 {
2981 	struct recorded_ref *ref;
2982 
2983 	ref = kzalloc(sizeof(*ref), GFP_KERNEL);
2984 	if (!ref)
2985 		return NULL;
2986 	RB_CLEAR_NODE(&ref->node);
2987 	INIT_LIST_HEAD(&ref->list);
2988 	return ref;
2989 }
2990 
recorded_ref_free(struct recorded_ref * ref)2991 static void recorded_ref_free(struct recorded_ref *ref)
2992 {
2993 	if (!ref)
2994 		return;
2995 	if (!RB_EMPTY_NODE(&ref->node))
2996 		rb_erase(&ref->node, ref->root);
2997 	list_del(&ref->list);
2998 	fs_path_free(ref->full_path);
2999 	kfree(ref);
3000 }
3001 
set_ref_path(struct recorded_ref * ref,struct fs_path * path)3002 static void set_ref_path(struct recorded_ref *ref, struct fs_path *path)
3003 {
3004 	ref->full_path = path;
3005 	ref->name = (char *)kbasename(ref->full_path->start);
3006 	ref->name_len = ref->full_path->end - ref->name;
3007 }
3008 
dup_ref(struct recorded_ref * ref,struct list_head * list)3009 static int dup_ref(struct recorded_ref *ref, struct list_head *list)
3010 {
3011 	struct recorded_ref *new;
3012 
3013 	new = recorded_ref_alloc();
3014 	if (!new)
3015 		return -ENOMEM;
3016 
3017 	new->dir = ref->dir;
3018 	new->dir_gen = ref->dir_gen;
3019 	list_add_tail(&new->list, list);
3020 	return 0;
3021 }
3022 
__free_recorded_refs(struct list_head * head)3023 static void __free_recorded_refs(struct list_head *head)
3024 {
3025 	struct recorded_ref *cur;
3026 
3027 	while (!list_empty(head)) {
3028 		cur = list_first_entry(head, struct recorded_ref, list);
3029 		recorded_ref_free(cur);
3030 	}
3031 }
3032 
free_recorded_refs(struct send_ctx * sctx)3033 static void free_recorded_refs(struct send_ctx *sctx)
3034 {
3035 	__free_recorded_refs(&sctx->new_refs);
3036 	__free_recorded_refs(&sctx->deleted_refs);
3037 }
3038 
3039 /*
3040  * Renames/moves a file/dir to its orphan name. Used when the first
3041  * ref of an unprocessed inode gets overwritten and for all non empty
3042  * directories.
3043  */
orphanize_inode(struct send_ctx * sctx,u64 ino,u64 gen,struct fs_path * path)3044 static int orphanize_inode(struct send_ctx *sctx, u64 ino, u64 gen,
3045 			  struct fs_path *path)
3046 {
3047 	int ret;
3048 	struct fs_path *orphan;
3049 
3050 	orphan = fs_path_alloc();
3051 	if (!orphan)
3052 		return -ENOMEM;
3053 
3054 	ret = gen_unique_name(sctx, ino, gen, orphan);
3055 	if (ret < 0)
3056 		goto out;
3057 
3058 	ret = send_rename(sctx, path, orphan);
3059 	if (ret < 0)
3060 		goto out;
3061 
3062 	if (ino == sctx->cur_ino && gen == sctx->cur_inode_gen)
3063 		ret = fs_path_copy(&sctx->cur_inode_path, orphan);
3064 
3065 out:
3066 	fs_path_free(orphan);
3067 	return ret;
3068 }
3069 
add_orphan_dir_info(struct send_ctx * sctx,u64 dir_ino,u64 dir_gen)3070 static struct orphan_dir_info *add_orphan_dir_info(struct send_ctx *sctx,
3071 						   u64 dir_ino, u64 dir_gen)
3072 {
3073 	struct rb_node **p = &sctx->orphan_dirs.rb_node;
3074 	struct rb_node *parent = NULL;
3075 	struct orphan_dir_info *entry, *odi;
3076 
3077 	while (*p) {
3078 		parent = *p;
3079 		entry = rb_entry(parent, struct orphan_dir_info, node);
3080 		if (dir_ino < entry->ino)
3081 			p = &(*p)->rb_left;
3082 		else if (dir_ino > entry->ino)
3083 			p = &(*p)->rb_right;
3084 		else if (dir_gen < entry->gen)
3085 			p = &(*p)->rb_left;
3086 		else if (dir_gen > entry->gen)
3087 			p = &(*p)->rb_right;
3088 		else
3089 			return entry;
3090 	}
3091 
3092 	odi = kmalloc(sizeof(*odi), GFP_KERNEL);
3093 	if (!odi)
3094 		return ERR_PTR(-ENOMEM);
3095 	odi->ino = dir_ino;
3096 	odi->gen = dir_gen;
3097 	odi->last_dir_index_offset = 0;
3098 	odi->dir_high_seq_ino = 0;
3099 
3100 	rb_link_node(&odi->node, parent, p);
3101 	rb_insert_color(&odi->node, &sctx->orphan_dirs);
3102 	return odi;
3103 }
3104 
get_orphan_dir_info(struct send_ctx * sctx,u64 dir_ino,u64 gen)3105 static struct orphan_dir_info *get_orphan_dir_info(struct send_ctx *sctx,
3106 						   u64 dir_ino, u64 gen)
3107 {
3108 	struct rb_node *n = sctx->orphan_dirs.rb_node;
3109 	struct orphan_dir_info *entry;
3110 
3111 	while (n) {
3112 		entry = rb_entry(n, struct orphan_dir_info, node);
3113 		if (dir_ino < entry->ino)
3114 			n = n->rb_left;
3115 		else if (dir_ino > entry->ino)
3116 			n = n->rb_right;
3117 		else if (gen < entry->gen)
3118 			n = n->rb_left;
3119 		else if (gen > entry->gen)
3120 			n = n->rb_right;
3121 		else
3122 			return entry;
3123 	}
3124 	return NULL;
3125 }
3126 
is_waiting_for_rm(struct send_ctx * sctx,u64 dir_ino,u64 gen)3127 static int is_waiting_for_rm(struct send_ctx *sctx, u64 dir_ino, u64 gen)
3128 {
3129 	struct orphan_dir_info *odi = get_orphan_dir_info(sctx, dir_ino, gen);
3130 
3131 	return odi != NULL;
3132 }
3133 
free_orphan_dir_info(struct send_ctx * sctx,struct orphan_dir_info * odi)3134 static void free_orphan_dir_info(struct send_ctx *sctx,
3135 				 struct orphan_dir_info *odi)
3136 {
3137 	if (!odi)
3138 		return;
3139 	rb_erase(&odi->node, &sctx->orphan_dirs);
3140 	kfree(odi);
3141 }
3142 
3143 /*
3144  * Returns 1 if a directory can be removed at this point in time.
3145  * We check this by iterating all dir items and checking if the inode behind
3146  * the dir item was already processed.
3147  */
can_rmdir(struct send_ctx * sctx,u64 dir,u64 dir_gen)3148 static int can_rmdir(struct send_ctx *sctx, u64 dir, u64 dir_gen)
3149 {
3150 	int ret = 0;
3151 	int iter_ret = 0;
3152 	struct btrfs_root *root = sctx->parent_root;
3153 	struct btrfs_path *path;
3154 	struct btrfs_key key;
3155 	struct btrfs_key found_key;
3156 	struct btrfs_key loc;
3157 	struct btrfs_dir_item *di;
3158 	struct orphan_dir_info *odi = NULL;
3159 	u64 dir_high_seq_ino = 0;
3160 	u64 last_dir_index_offset = 0;
3161 
3162 	/*
3163 	 * Don't try to rmdir the top/root subvolume dir.
3164 	 */
3165 	if (dir == BTRFS_FIRST_FREE_OBJECTID)
3166 		return 0;
3167 
3168 	odi = get_orphan_dir_info(sctx, dir, dir_gen);
3169 	if (odi && sctx->cur_ino < odi->dir_high_seq_ino)
3170 		return 0;
3171 
3172 	path = alloc_path_for_send();
3173 	if (!path)
3174 		return -ENOMEM;
3175 
3176 	if (!odi) {
3177 		/*
3178 		 * Find the inode number associated with the last dir index
3179 		 * entry. This is very likely the inode with the highest number
3180 		 * of all inodes that have an entry in the directory. We can
3181 		 * then use it to avoid future calls to can_rmdir(), when
3182 		 * processing inodes with a lower number, from having to search
3183 		 * the parent root b+tree for dir index keys.
3184 		 */
3185 		key.objectid = dir;
3186 		key.type = BTRFS_DIR_INDEX_KEY;
3187 		key.offset = (u64)-1;
3188 
3189 		ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
3190 		if (ret < 0) {
3191 			goto out;
3192 		} else if (ret > 0) {
3193 			/* Can't happen, the root is never empty. */
3194 			ASSERT(path->slots[0] > 0);
3195 			if (WARN_ON(path->slots[0] == 0)) {
3196 				ret = -EUCLEAN;
3197 				goto out;
3198 			}
3199 			path->slots[0]--;
3200 		}
3201 
3202 		btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
3203 		if (key.objectid != dir || key.type != BTRFS_DIR_INDEX_KEY) {
3204 			/* No index keys, dir can be removed. */
3205 			ret = 1;
3206 			goto out;
3207 		}
3208 
3209 		di = btrfs_item_ptr(path->nodes[0], path->slots[0],
3210 				    struct btrfs_dir_item);
3211 		btrfs_dir_item_key_to_cpu(path->nodes[0], di, &loc);
3212 		dir_high_seq_ino = loc.objectid;
3213 		if (sctx->cur_ino < dir_high_seq_ino) {
3214 			ret = 0;
3215 			goto out;
3216 		}
3217 
3218 		btrfs_release_path(path);
3219 	}
3220 
3221 	key.objectid = dir;
3222 	key.type = BTRFS_DIR_INDEX_KEY;
3223 	key.offset = (odi ? odi->last_dir_index_offset : 0);
3224 
3225 	btrfs_for_each_slot(root, &key, &found_key, path, iter_ret) {
3226 		struct waiting_dir_move *dm;
3227 
3228 		if (found_key.objectid != key.objectid ||
3229 		    found_key.type != key.type)
3230 			break;
3231 
3232 		di = btrfs_item_ptr(path->nodes[0], path->slots[0],
3233 				struct btrfs_dir_item);
3234 		btrfs_dir_item_key_to_cpu(path->nodes[0], di, &loc);
3235 
3236 		dir_high_seq_ino = max(dir_high_seq_ino, loc.objectid);
3237 		last_dir_index_offset = found_key.offset;
3238 
3239 		dm = get_waiting_dir_move(sctx, loc.objectid);
3240 		if (dm) {
3241 			dm->rmdir_ino = dir;
3242 			dm->rmdir_gen = dir_gen;
3243 			ret = 0;
3244 			goto out;
3245 		}
3246 
3247 		if (loc.objectid > sctx->cur_ino) {
3248 			ret = 0;
3249 			goto out;
3250 		}
3251 	}
3252 	if (iter_ret < 0) {
3253 		ret = iter_ret;
3254 		goto out;
3255 	}
3256 	free_orphan_dir_info(sctx, odi);
3257 
3258 	ret = 1;
3259 
3260 out:
3261 	btrfs_free_path(path);
3262 
3263 	if (ret)
3264 		return ret;
3265 
3266 	if (!odi) {
3267 		odi = add_orphan_dir_info(sctx, dir, dir_gen);
3268 		if (IS_ERR(odi))
3269 			return PTR_ERR(odi);
3270 
3271 		odi->gen = dir_gen;
3272 	}
3273 
3274 	odi->last_dir_index_offset = last_dir_index_offset;
3275 	odi->dir_high_seq_ino = max(odi->dir_high_seq_ino, dir_high_seq_ino);
3276 
3277 	return 0;
3278 }
3279 
is_waiting_for_move(struct send_ctx * sctx,u64 ino)3280 static int is_waiting_for_move(struct send_ctx *sctx, u64 ino)
3281 {
3282 	struct waiting_dir_move *entry = get_waiting_dir_move(sctx, ino);
3283 
3284 	return entry != NULL;
3285 }
3286 
add_waiting_dir_move(struct send_ctx * sctx,u64 ino,bool orphanized)3287 static int add_waiting_dir_move(struct send_ctx *sctx, u64 ino, bool orphanized)
3288 {
3289 	struct rb_node **p = &sctx->waiting_dir_moves.rb_node;
3290 	struct rb_node *parent = NULL;
3291 	struct waiting_dir_move *entry, *dm;
3292 
3293 	dm = kmalloc(sizeof(*dm), GFP_KERNEL);
3294 	if (!dm)
3295 		return -ENOMEM;
3296 	dm->ino = ino;
3297 	dm->rmdir_ino = 0;
3298 	dm->rmdir_gen = 0;
3299 	dm->orphanized = orphanized;
3300 
3301 	while (*p) {
3302 		parent = *p;
3303 		entry = rb_entry(parent, struct waiting_dir_move, node);
3304 		if (ino < entry->ino) {
3305 			p = &(*p)->rb_left;
3306 		} else if (ino > entry->ino) {
3307 			p = &(*p)->rb_right;
3308 		} else {
3309 			kfree(dm);
3310 			return -EEXIST;
3311 		}
3312 	}
3313 
3314 	rb_link_node(&dm->node, parent, p);
3315 	rb_insert_color(&dm->node, &sctx->waiting_dir_moves);
3316 	return 0;
3317 }
3318 
3319 static struct waiting_dir_move *
get_waiting_dir_move(struct send_ctx * sctx,u64 ino)3320 get_waiting_dir_move(struct send_ctx *sctx, u64 ino)
3321 {
3322 	struct rb_node *n = sctx->waiting_dir_moves.rb_node;
3323 	struct waiting_dir_move *entry;
3324 
3325 	while (n) {
3326 		entry = rb_entry(n, struct waiting_dir_move, node);
3327 		if (ino < entry->ino)
3328 			n = n->rb_left;
3329 		else if (ino > entry->ino)
3330 			n = n->rb_right;
3331 		else
3332 			return entry;
3333 	}
3334 	return NULL;
3335 }
3336 
free_waiting_dir_move(struct send_ctx * sctx,struct waiting_dir_move * dm)3337 static void free_waiting_dir_move(struct send_ctx *sctx,
3338 				  struct waiting_dir_move *dm)
3339 {
3340 	if (!dm)
3341 		return;
3342 	rb_erase(&dm->node, &sctx->waiting_dir_moves);
3343 	kfree(dm);
3344 }
3345 
add_pending_dir_move(struct send_ctx * sctx,u64 ino,u64 ino_gen,u64 parent_ino,struct list_head * new_refs,struct list_head * deleted_refs,const bool is_orphan)3346 static int add_pending_dir_move(struct send_ctx *sctx,
3347 				u64 ino,
3348 				u64 ino_gen,
3349 				u64 parent_ino,
3350 				struct list_head *new_refs,
3351 				struct list_head *deleted_refs,
3352 				const bool is_orphan)
3353 {
3354 	struct rb_node **p = &sctx->pending_dir_moves.rb_node;
3355 	struct rb_node *parent = NULL;
3356 	struct pending_dir_move *entry = NULL, *pm;
3357 	struct recorded_ref *cur;
3358 	int exists = 0;
3359 	int ret;
3360 
3361 	pm = kmalloc(sizeof(*pm), GFP_KERNEL);
3362 	if (!pm)
3363 		return -ENOMEM;
3364 	pm->parent_ino = parent_ino;
3365 	pm->ino = ino;
3366 	pm->gen = ino_gen;
3367 	INIT_LIST_HEAD(&pm->list);
3368 	INIT_LIST_HEAD(&pm->update_refs);
3369 	RB_CLEAR_NODE(&pm->node);
3370 
3371 	while (*p) {
3372 		parent = *p;
3373 		entry = rb_entry(parent, struct pending_dir_move, node);
3374 		if (parent_ino < entry->parent_ino) {
3375 			p = &(*p)->rb_left;
3376 		} else if (parent_ino > entry->parent_ino) {
3377 			p = &(*p)->rb_right;
3378 		} else {
3379 			exists = 1;
3380 			break;
3381 		}
3382 	}
3383 
3384 	list_for_each_entry(cur, deleted_refs, list) {
3385 		ret = dup_ref(cur, &pm->update_refs);
3386 		if (ret < 0)
3387 			goto out;
3388 	}
3389 	list_for_each_entry(cur, new_refs, list) {
3390 		ret = dup_ref(cur, &pm->update_refs);
3391 		if (ret < 0)
3392 			goto out;
3393 	}
3394 
3395 	ret = add_waiting_dir_move(sctx, pm->ino, is_orphan);
3396 	if (ret)
3397 		goto out;
3398 
3399 	if (exists) {
3400 		list_add_tail(&pm->list, &entry->list);
3401 	} else {
3402 		rb_link_node(&pm->node, parent, p);
3403 		rb_insert_color(&pm->node, &sctx->pending_dir_moves);
3404 	}
3405 	ret = 0;
3406 out:
3407 	if (ret) {
3408 		__free_recorded_refs(&pm->update_refs);
3409 		kfree(pm);
3410 	}
3411 	return ret;
3412 }
3413 
get_pending_dir_moves(struct send_ctx * sctx,u64 parent_ino)3414 static struct pending_dir_move *get_pending_dir_moves(struct send_ctx *sctx,
3415 						      u64 parent_ino)
3416 {
3417 	struct rb_node *n = sctx->pending_dir_moves.rb_node;
3418 	struct pending_dir_move *entry;
3419 
3420 	while (n) {
3421 		entry = rb_entry(n, struct pending_dir_move, node);
3422 		if (parent_ino < entry->parent_ino)
3423 			n = n->rb_left;
3424 		else if (parent_ino > entry->parent_ino)
3425 			n = n->rb_right;
3426 		else
3427 			return entry;
3428 	}
3429 	return NULL;
3430 }
3431 
path_loop(struct send_ctx * sctx,struct fs_path * name,u64 ino,u64 gen,u64 * ancestor_ino)3432 static int path_loop(struct send_ctx *sctx, struct fs_path *name,
3433 		     u64 ino, u64 gen, u64 *ancestor_ino)
3434 {
3435 	int ret = 0;
3436 	u64 parent_inode = 0;
3437 	u64 parent_gen = 0;
3438 	u64 start_ino = ino;
3439 
3440 	*ancestor_ino = 0;
3441 	while (ino != BTRFS_FIRST_FREE_OBJECTID) {
3442 		fs_path_reset(name);
3443 
3444 		if (is_waiting_for_rm(sctx, ino, gen))
3445 			break;
3446 		if (is_waiting_for_move(sctx, ino)) {
3447 			if (*ancestor_ino == 0)
3448 				*ancestor_ino = ino;
3449 			ret = get_first_ref(sctx->parent_root, ino,
3450 					    &parent_inode, &parent_gen, name);
3451 		} else {
3452 			ret = __get_cur_name_and_parent(sctx, ino, gen,
3453 							&parent_inode,
3454 							&parent_gen, name);
3455 			if (ret > 0) {
3456 				ret = 0;
3457 				break;
3458 			}
3459 		}
3460 		if (ret < 0)
3461 			break;
3462 		if (parent_inode == start_ino) {
3463 			ret = 1;
3464 			if (*ancestor_ino == 0)
3465 				*ancestor_ino = ino;
3466 			break;
3467 		}
3468 		ino = parent_inode;
3469 		gen = parent_gen;
3470 	}
3471 	return ret;
3472 }
3473 
apply_dir_move(struct send_ctx * sctx,struct pending_dir_move * pm)3474 static int apply_dir_move(struct send_ctx *sctx, struct pending_dir_move *pm)
3475 {
3476 	struct fs_path *from_path = NULL;
3477 	struct fs_path *to_path = NULL;
3478 	struct fs_path *name = NULL;
3479 	u64 orig_progress = sctx->send_progress;
3480 	struct recorded_ref *cur;
3481 	u64 parent_ino, parent_gen;
3482 	struct waiting_dir_move *dm = NULL;
3483 	u64 rmdir_ino = 0;
3484 	u64 rmdir_gen;
3485 	u64 ancestor;
3486 	bool is_orphan;
3487 	int ret;
3488 
3489 	name = fs_path_alloc();
3490 	from_path = fs_path_alloc();
3491 	if (!name || !from_path) {
3492 		ret = -ENOMEM;
3493 		goto out;
3494 	}
3495 
3496 	dm = get_waiting_dir_move(sctx, pm->ino);
3497 	ASSERT(dm);
3498 	rmdir_ino = dm->rmdir_ino;
3499 	rmdir_gen = dm->rmdir_gen;
3500 	is_orphan = dm->orphanized;
3501 	free_waiting_dir_move(sctx, dm);
3502 
3503 	if (is_orphan) {
3504 		ret = gen_unique_name(sctx, pm->ino,
3505 				      pm->gen, from_path);
3506 	} else {
3507 		ret = get_first_ref(sctx->parent_root, pm->ino,
3508 				    &parent_ino, &parent_gen, name);
3509 		if (ret < 0)
3510 			goto out;
3511 		ret = get_cur_path(sctx, parent_ino, parent_gen,
3512 				   from_path);
3513 		if (ret < 0)
3514 			goto out;
3515 		ret = fs_path_add_path(from_path, name);
3516 	}
3517 	if (ret < 0)
3518 		goto out;
3519 
3520 	sctx->send_progress = sctx->cur_ino + 1;
3521 	ret = path_loop(sctx, name, pm->ino, pm->gen, &ancestor);
3522 	if (ret < 0)
3523 		goto out;
3524 	if (ret) {
3525 		LIST_HEAD(deleted_refs);
3526 		ASSERT(ancestor > BTRFS_FIRST_FREE_OBJECTID);
3527 		ret = add_pending_dir_move(sctx, pm->ino, pm->gen, ancestor,
3528 					   &pm->update_refs, &deleted_refs,
3529 					   is_orphan);
3530 		if (ret < 0)
3531 			goto out;
3532 		if (rmdir_ino) {
3533 			dm = get_waiting_dir_move(sctx, pm->ino);
3534 			ASSERT(dm);
3535 			dm->rmdir_ino = rmdir_ino;
3536 			dm->rmdir_gen = rmdir_gen;
3537 		}
3538 		goto out;
3539 	}
3540 	fs_path_reset(name);
3541 	to_path = name;
3542 	name = NULL;
3543 	ret = get_cur_path(sctx, pm->ino, pm->gen, to_path);
3544 	if (ret < 0)
3545 		goto out;
3546 
3547 	ret = send_rename(sctx, from_path, to_path);
3548 	if (ret < 0)
3549 		goto out;
3550 
3551 	if (rmdir_ino) {
3552 		struct orphan_dir_info *odi;
3553 		u64 gen;
3554 
3555 		odi = get_orphan_dir_info(sctx, rmdir_ino, rmdir_gen);
3556 		if (!odi) {
3557 			/* already deleted */
3558 			goto finish;
3559 		}
3560 		gen = odi->gen;
3561 
3562 		ret = can_rmdir(sctx, rmdir_ino, gen);
3563 		if (ret < 0)
3564 			goto out;
3565 		if (!ret)
3566 			goto finish;
3567 
3568 		name = fs_path_alloc();
3569 		if (!name) {
3570 			ret = -ENOMEM;
3571 			goto out;
3572 		}
3573 		ret = get_cur_path(sctx, rmdir_ino, gen, name);
3574 		if (ret < 0)
3575 			goto out;
3576 		ret = send_rmdir(sctx, name);
3577 		if (ret < 0)
3578 			goto out;
3579 	}
3580 
3581 finish:
3582 	ret = cache_dir_utimes(sctx, pm->ino, pm->gen);
3583 	if (ret < 0)
3584 		goto out;
3585 
3586 	/*
3587 	 * After rename/move, need to update the utimes of both new parent(s)
3588 	 * and old parent(s).
3589 	 */
3590 	list_for_each_entry(cur, &pm->update_refs, list) {
3591 		/*
3592 		 * The parent inode might have been deleted in the send snapshot
3593 		 */
3594 		ret = get_inode_info(sctx->send_root, cur->dir, NULL);
3595 		if (ret == -ENOENT) {
3596 			ret = 0;
3597 			continue;
3598 		}
3599 		if (ret < 0)
3600 			goto out;
3601 
3602 		ret = cache_dir_utimes(sctx, cur->dir, cur->dir_gen);
3603 		if (ret < 0)
3604 			goto out;
3605 	}
3606 
3607 out:
3608 	fs_path_free(name);
3609 	fs_path_free(from_path);
3610 	fs_path_free(to_path);
3611 	sctx->send_progress = orig_progress;
3612 
3613 	return ret;
3614 }
3615 
free_pending_move(struct send_ctx * sctx,struct pending_dir_move * m)3616 static void free_pending_move(struct send_ctx *sctx, struct pending_dir_move *m)
3617 {
3618 	if (!list_empty(&m->list))
3619 		list_del(&m->list);
3620 	if (!RB_EMPTY_NODE(&m->node))
3621 		rb_erase(&m->node, &sctx->pending_dir_moves);
3622 	__free_recorded_refs(&m->update_refs);
3623 	kfree(m);
3624 }
3625 
tail_append_pending_moves(struct send_ctx * sctx,struct pending_dir_move * moves,struct list_head * stack)3626 static void tail_append_pending_moves(struct send_ctx *sctx,
3627 				      struct pending_dir_move *moves,
3628 				      struct list_head *stack)
3629 {
3630 	if (list_empty(&moves->list)) {
3631 		list_add_tail(&moves->list, stack);
3632 	} else {
3633 		LIST_HEAD(list);
3634 		list_splice_init(&moves->list, &list);
3635 		list_add_tail(&moves->list, stack);
3636 		list_splice_tail(&list, stack);
3637 	}
3638 	if (!RB_EMPTY_NODE(&moves->node)) {
3639 		rb_erase(&moves->node, &sctx->pending_dir_moves);
3640 		RB_CLEAR_NODE(&moves->node);
3641 	}
3642 }
3643 
apply_children_dir_moves(struct send_ctx * sctx)3644 static int apply_children_dir_moves(struct send_ctx *sctx)
3645 {
3646 	struct pending_dir_move *pm;
3647 	LIST_HEAD(stack);
3648 	u64 parent_ino = sctx->cur_ino;
3649 	int ret = 0;
3650 
3651 	pm = get_pending_dir_moves(sctx, parent_ino);
3652 	if (!pm)
3653 		return 0;
3654 
3655 	tail_append_pending_moves(sctx, pm, &stack);
3656 
3657 	while (!list_empty(&stack)) {
3658 		pm = list_first_entry(&stack, struct pending_dir_move, list);
3659 		parent_ino = pm->ino;
3660 		ret = apply_dir_move(sctx, pm);
3661 		free_pending_move(sctx, pm);
3662 		if (ret)
3663 			goto out;
3664 		pm = get_pending_dir_moves(sctx, parent_ino);
3665 		if (pm)
3666 			tail_append_pending_moves(sctx, pm, &stack);
3667 	}
3668 	return 0;
3669 
3670 out:
3671 	while (!list_empty(&stack)) {
3672 		pm = list_first_entry(&stack, struct pending_dir_move, list);
3673 		free_pending_move(sctx, pm);
3674 	}
3675 	return ret;
3676 }
3677 
3678 /*
3679  * We might need to delay a directory rename even when no ancestor directory
3680  * (in the send root) with a higher inode number than ours (sctx->cur_ino) was
3681  * renamed. This happens when we rename a directory to the old name (the name
3682  * in the parent root) of some other unrelated directory that got its rename
3683  * delayed due to some ancestor with higher number that got renamed.
3684  *
3685  * Example:
3686  *
3687  * Parent snapshot:
3688  * .                                       (ino 256)
3689  * |---- a/                                (ino 257)
3690  * |     |---- file                        (ino 260)
3691  * |
3692  * |---- b/                                (ino 258)
3693  * |---- c/                                (ino 259)
3694  *
3695  * Send snapshot:
3696  * .                                       (ino 256)
3697  * |---- a/                                (ino 258)
3698  * |---- x/                                (ino 259)
3699  *       |---- y/                          (ino 257)
3700  *             |----- file                 (ino 260)
3701  *
3702  * Here we can not rename 258 from 'b' to 'a' without the rename of inode 257
3703  * from 'a' to 'x/y' happening first, which in turn depends on the rename of
3704  * inode 259 from 'c' to 'x'. So the order of rename commands the send stream
3705  * must issue is:
3706  *
3707  * 1 - rename 259 from 'c' to 'x'
3708  * 2 - rename 257 from 'a' to 'x/y'
3709  * 3 - rename 258 from 'b' to 'a'
3710  *
3711  * Returns 1 if the rename of sctx->cur_ino needs to be delayed, 0 if it can
3712  * be done right away and < 0 on error.
3713  */
wait_for_dest_dir_move(struct send_ctx * sctx,struct recorded_ref * parent_ref,const bool is_orphan)3714 static int wait_for_dest_dir_move(struct send_ctx *sctx,
3715 				  struct recorded_ref *parent_ref,
3716 				  const bool is_orphan)
3717 {
3718 	BTRFS_PATH_AUTO_FREE(path);
3719 	struct btrfs_key key;
3720 	struct btrfs_key di_key;
3721 	struct btrfs_dir_item *di;
3722 	u64 left_gen;
3723 	u64 right_gen;
3724 	int ret = 0;
3725 	struct waiting_dir_move *wdm;
3726 
3727 	if (RB_EMPTY_ROOT(&sctx->waiting_dir_moves))
3728 		return 0;
3729 
3730 	path = alloc_path_for_send();
3731 	if (!path)
3732 		return -ENOMEM;
3733 
3734 	key.objectid = parent_ref->dir;
3735 	key.type = BTRFS_DIR_ITEM_KEY;
3736 	key.offset = btrfs_name_hash(parent_ref->name, parent_ref->name_len);
3737 
3738 	ret = btrfs_search_slot(NULL, sctx->parent_root, &key, path, 0, 0);
3739 	if (ret < 0)
3740 		return ret;
3741 	if (ret > 0)
3742 		return 0;
3743 
3744 	di = btrfs_match_dir_item_name(path, parent_ref->name,
3745 				       parent_ref->name_len);
3746 	if (!di)
3747 		return 0;
3748 	/*
3749 	 * di_key.objectid has the number of the inode that has a dentry in the
3750 	 * parent directory with the same name that sctx->cur_ino is being
3751 	 * renamed to. We need to check if that inode is in the send root as
3752 	 * well and if it is currently marked as an inode with a pending rename,
3753 	 * if it is, we need to delay the rename of sctx->cur_ino as well, so
3754 	 * that it happens after that other inode is renamed.
3755 	 */
3756 	btrfs_dir_item_key_to_cpu(path->nodes[0], di, &di_key);
3757 	if (di_key.type != BTRFS_INODE_ITEM_KEY)
3758 		return 0;
3759 
3760 	ret = get_inode_gen(sctx->parent_root, di_key.objectid, &left_gen);
3761 	if (ret < 0)
3762 		return ret;
3763 	ret = get_inode_gen(sctx->send_root, di_key.objectid, &right_gen);
3764 	if (ret < 0) {
3765 		if (ret == -ENOENT)
3766 			ret = 0;
3767 		return ret;
3768 	}
3769 
3770 	/* Different inode, no need to delay the rename of sctx->cur_ino */
3771 	if (right_gen != left_gen)
3772 		return 0;
3773 
3774 	wdm = get_waiting_dir_move(sctx, di_key.objectid);
3775 	if (wdm && !wdm->orphanized) {
3776 		ret = add_pending_dir_move(sctx,
3777 					   sctx->cur_ino,
3778 					   sctx->cur_inode_gen,
3779 					   di_key.objectid,
3780 					   &sctx->new_refs,
3781 					   &sctx->deleted_refs,
3782 					   is_orphan);
3783 		if (!ret)
3784 			ret = 1;
3785 	}
3786 	return ret;
3787 }
3788 
3789 /*
3790  * Check if inode ino2, or any of its ancestors, is inode ino1.
3791  * Return 1 if true, 0 if false and < 0 on error.
3792  */
check_ino_in_path(struct btrfs_root * root,const u64 ino1,const u64 ino1_gen,const u64 ino2,const u64 ino2_gen,struct fs_path * fs_path)3793 static int check_ino_in_path(struct btrfs_root *root,
3794 			     const u64 ino1,
3795 			     const u64 ino1_gen,
3796 			     const u64 ino2,
3797 			     const u64 ino2_gen,
3798 			     struct fs_path *fs_path)
3799 {
3800 	u64 ino = ino2;
3801 
3802 	if (ino1 == ino2)
3803 		return ino1_gen == ino2_gen;
3804 
3805 	while (ino > BTRFS_FIRST_FREE_OBJECTID) {
3806 		u64 parent;
3807 		u64 parent_gen;
3808 		int ret;
3809 
3810 		fs_path_reset(fs_path);
3811 		ret = get_first_ref(root, ino, &parent, &parent_gen, fs_path);
3812 		if (ret < 0)
3813 			return ret;
3814 		if (parent == ino1)
3815 			return parent_gen == ino1_gen;
3816 		ino = parent;
3817 	}
3818 	return 0;
3819 }
3820 
3821 /*
3822  * Check if inode ino1 is an ancestor of inode ino2 in the given root for any
3823  * possible path (in case ino2 is not a directory and has multiple hard links).
3824  * Return 1 if true, 0 if false and < 0 on error.
3825  */
is_ancestor(struct btrfs_root * root,const u64 ino1,const u64 ino1_gen,const u64 ino2,struct fs_path * fs_path)3826 static int is_ancestor(struct btrfs_root *root,
3827 		       const u64 ino1,
3828 		       const u64 ino1_gen,
3829 		       const u64 ino2,
3830 		       struct fs_path *fs_path)
3831 {
3832 	bool free_fs_path = false;
3833 	int ret = 0;
3834 	int iter_ret = 0;
3835 	BTRFS_PATH_AUTO_FREE(path);
3836 	struct btrfs_key key;
3837 
3838 	if (!fs_path) {
3839 		fs_path = fs_path_alloc();
3840 		if (!fs_path)
3841 			return -ENOMEM;
3842 		free_fs_path = true;
3843 	}
3844 
3845 	path = alloc_path_for_send();
3846 	if (!path) {
3847 		ret = -ENOMEM;
3848 		goto out;
3849 	}
3850 
3851 	key.objectid = ino2;
3852 	key.type = BTRFS_INODE_REF_KEY;
3853 	key.offset = 0;
3854 
3855 	btrfs_for_each_slot(root, &key, &key, path, iter_ret) {
3856 		struct extent_buffer *leaf = path->nodes[0];
3857 		int slot = path->slots[0];
3858 		u32 cur_offset = 0;
3859 		u32 item_size;
3860 
3861 		if (key.objectid != ino2)
3862 			break;
3863 		if (key.type != BTRFS_INODE_REF_KEY &&
3864 		    key.type != BTRFS_INODE_EXTREF_KEY)
3865 			break;
3866 
3867 		item_size = btrfs_item_size(leaf, slot);
3868 		while (cur_offset < item_size) {
3869 			u64 parent;
3870 			u64 parent_gen;
3871 
3872 			if (key.type == BTRFS_INODE_EXTREF_KEY) {
3873 				unsigned long ptr;
3874 				struct btrfs_inode_extref *extref;
3875 
3876 				ptr = btrfs_item_ptr_offset(leaf, slot);
3877 				extref = (struct btrfs_inode_extref *)
3878 					(ptr + cur_offset);
3879 				parent = btrfs_inode_extref_parent(leaf,
3880 								   extref);
3881 				cur_offset += sizeof(*extref);
3882 				cur_offset += btrfs_inode_extref_name_len(leaf,
3883 								  extref);
3884 			} else {
3885 				parent = key.offset;
3886 				cur_offset = item_size;
3887 			}
3888 
3889 			ret = get_inode_gen(root, parent, &parent_gen);
3890 			if (ret < 0)
3891 				goto out;
3892 			ret = check_ino_in_path(root, ino1, ino1_gen,
3893 						parent, parent_gen, fs_path);
3894 			if (ret)
3895 				goto out;
3896 		}
3897 	}
3898 	ret = 0;
3899 	if (iter_ret < 0)
3900 		ret = iter_ret;
3901 
3902 out:
3903 	if (free_fs_path)
3904 		fs_path_free(fs_path);
3905 	return ret;
3906 }
3907 
wait_for_parent_move(struct send_ctx * sctx,struct recorded_ref * parent_ref,const bool is_orphan)3908 static int wait_for_parent_move(struct send_ctx *sctx,
3909 				struct recorded_ref *parent_ref,
3910 				const bool is_orphan)
3911 {
3912 	int ret = 0;
3913 	u64 ino = parent_ref->dir;
3914 	u64 ino_gen = parent_ref->dir_gen;
3915 	u64 parent_ino_before, parent_ino_after;
3916 	struct fs_path *path_before = NULL;
3917 	struct fs_path *path_after = NULL;
3918 	int len1, len2;
3919 
3920 	path_after = fs_path_alloc();
3921 	path_before = fs_path_alloc();
3922 	if (!path_after || !path_before) {
3923 		ret = -ENOMEM;
3924 		goto out;
3925 	}
3926 
3927 	/*
3928 	 * Our current directory inode may not yet be renamed/moved because some
3929 	 * ancestor (immediate or not) has to be renamed/moved first. So find if
3930 	 * such ancestor exists and make sure our own rename/move happens after
3931 	 * that ancestor is processed to avoid path build infinite loops (done
3932 	 * at get_cur_path()).
3933 	 */
3934 	while (ino > BTRFS_FIRST_FREE_OBJECTID) {
3935 		u64 parent_ino_after_gen;
3936 
3937 		if (is_waiting_for_move(sctx, ino)) {
3938 			/*
3939 			 * If the current inode is an ancestor of ino in the
3940 			 * parent root, we need to delay the rename of the
3941 			 * current inode, otherwise don't delayed the rename
3942 			 * because we can end up with a circular dependency
3943 			 * of renames, resulting in some directories never
3944 			 * getting the respective rename operations issued in
3945 			 * the send stream or getting into infinite path build
3946 			 * loops.
3947 			 */
3948 			ret = is_ancestor(sctx->parent_root,
3949 					  sctx->cur_ino, sctx->cur_inode_gen,
3950 					  ino, path_before);
3951 			if (ret)
3952 				break;
3953 		}
3954 
3955 		fs_path_reset(path_before);
3956 		fs_path_reset(path_after);
3957 
3958 		ret = get_first_ref(sctx->send_root, ino, &parent_ino_after,
3959 				    &parent_ino_after_gen, path_after);
3960 		if (ret < 0)
3961 			goto out;
3962 		ret = get_first_ref(sctx->parent_root, ino, &parent_ino_before,
3963 				    NULL, path_before);
3964 		if (ret < 0 && ret != -ENOENT) {
3965 			goto out;
3966 		} else if (ret == -ENOENT) {
3967 			ret = 0;
3968 			break;
3969 		}
3970 
3971 		len1 = fs_path_len(path_before);
3972 		len2 = fs_path_len(path_after);
3973 		if (ino > sctx->cur_ino &&
3974 		    (parent_ino_before != parent_ino_after || len1 != len2 ||
3975 		     memcmp(path_before->start, path_after->start, len1))) {
3976 			u64 parent_ino_gen;
3977 
3978 			ret = get_inode_gen(sctx->parent_root, ino, &parent_ino_gen);
3979 			if (ret < 0)
3980 				goto out;
3981 			if (ino_gen == parent_ino_gen) {
3982 				ret = 1;
3983 				break;
3984 			}
3985 		}
3986 		ino = parent_ino_after;
3987 		ino_gen = parent_ino_after_gen;
3988 	}
3989 
3990 out:
3991 	fs_path_free(path_before);
3992 	fs_path_free(path_after);
3993 
3994 	if (ret == 1) {
3995 		ret = add_pending_dir_move(sctx,
3996 					   sctx->cur_ino,
3997 					   sctx->cur_inode_gen,
3998 					   ino,
3999 					   &sctx->new_refs,
4000 					   &sctx->deleted_refs,
4001 					   is_orphan);
4002 		if (!ret)
4003 			ret = 1;
4004 	}
4005 
4006 	return ret;
4007 }
4008 
update_ref_path(struct send_ctx * sctx,struct recorded_ref * ref)4009 static int update_ref_path(struct send_ctx *sctx, struct recorded_ref *ref)
4010 {
4011 	int ret;
4012 	struct fs_path *new_path;
4013 
4014 	/*
4015 	 * Our reference's name member points to its full_path member string, so
4016 	 * we use here a new path.
4017 	 */
4018 	new_path = fs_path_alloc();
4019 	if (!new_path)
4020 		return -ENOMEM;
4021 
4022 	ret = get_cur_path(sctx, ref->dir, ref->dir_gen, new_path);
4023 	if (ret < 0) {
4024 		fs_path_free(new_path);
4025 		return ret;
4026 	}
4027 	ret = fs_path_add(new_path, ref->name, ref->name_len);
4028 	if (ret < 0) {
4029 		fs_path_free(new_path);
4030 		return ret;
4031 	}
4032 
4033 	fs_path_free(ref->full_path);
4034 	set_ref_path(ref, new_path);
4035 
4036 	return 0;
4037 }
4038 
4039 /*
4040  * When processing the new references for an inode we may orphanize an existing
4041  * directory inode because its old name conflicts with one of the new references
4042  * of the current inode. Later, when processing another new reference of our
4043  * inode, we might need to orphanize another inode, but the path we have in the
4044  * reference reflects the pre-orphanization name of the directory we previously
4045  * orphanized. For example:
4046  *
4047  * parent snapshot looks like:
4048  *
4049  * .                                     (ino 256)
4050  * |----- f1                             (ino 257)
4051  * |----- f2                             (ino 258)
4052  * |----- d1/                            (ino 259)
4053  *        |----- d2/                     (ino 260)
4054  *
4055  * send snapshot looks like:
4056  *
4057  * .                                     (ino 256)
4058  * |----- d1                             (ino 258)
4059  * |----- f2/                            (ino 259)
4060  *        |----- f2_link/                (ino 260)
4061  *        |       |----- f1              (ino 257)
4062  *        |
4063  *        |----- d2                      (ino 258)
4064  *
4065  * When processing inode 257 we compute the name for inode 259 as "d1", and we
4066  * cache it in the name cache. Later when we start processing inode 258, when
4067  * collecting all its new references we set a full path of "d1/d2" for its new
4068  * reference with name "d2". When we start processing the new references we
4069  * start by processing the new reference with name "d1", and this results in
4070  * orphanizing inode 259, since its old reference causes a conflict. Then we
4071  * move on the next new reference, with name "d2", and we find out we must
4072  * orphanize inode 260, as its old reference conflicts with ours - but for the
4073  * orphanization we use a source path corresponding to the path we stored in the
4074  * new reference, which is "d1/d2" and not "o259-6-0/d2" - this makes the
4075  * receiver fail since the path component "d1/" no longer exists, it was renamed
4076  * to "o259-6-0/" when processing the previous new reference. So in this case we
4077  * must recompute the path in the new reference and use it for the new
4078  * orphanization operation.
4079  */
refresh_ref_path(struct send_ctx * sctx,struct recorded_ref * ref)4080 static int refresh_ref_path(struct send_ctx *sctx, struct recorded_ref *ref)
4081 {
4082 	char *name;
4083 	int ret;
4084 
4085 	name = kmemdup(ref->name, ref->name_len, GFP_KERNEL);
4086 	if (!name)
4087 		return -ENOMEM;
4088 
4089 	fs_path_reset(ref->full_path);
4090 	ret = get_cur_path(sctx, ref->dir, ref->dir_gen, ref->full_path);
4091 	if (ret < 0)
4092 		goto out;
4093 
4094 	ret = fs_path_add(ref->full_path, name, ref->name_len);
4095 	if (ret < 0)
4096 		goto out;
4097 
4098 	/* Update the reference's base name pointer. */
4099 	set_ref_path(ref, ref->full_path);
4100 out:
4101 	kfree(name);
4102 	return ret;
4103 }
4104 
rename_current_inode(struct send_ctx * sctx,struct fs_path * current_path,struct fs_path * new_path)4105 static int rename_current_inode(struct send_ctx *sctx,
4106 				struct fs_path *current_path,
4107 				struct fs_path *new_path)
4108 {
4109 	int ret;
4110 
4111 	ret = send_rename(sctx, current_path, new_path);
4112 	if (ret < 0)
4113 		return ret;
4114 
4115 	ret = fs_path_copy(&sctx->cur_inode_path, new_path);
4116 	if (ret < 0)
4117 		return ret;
4118 
4119 	return fs_path_copy(current_path, new_path);
4120 }
4121 
4122 /*
4123  * This does all the move/link/unlink/rmdir magic.
4124  */
process_recorded_refs(struct send_ctx * sctx,int * pending_move)4125 static int process_recorded_refs(struct send_ctx *sctx, int *pending_move)
4126 {
4127 	struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
4128 	int ret = 0;
4129 	struct recorded_ref *cur;
4130 	struct recorded_ref *cur2;
4131 	LIST_HEAD(check_dirs);
4132 	struct fs_path *valid_path = NULL;
4133 	u64 ow_inode = 0;
4134 	u64 ow_gen;
4135 	u64 ow_mode;
4136 	u64 last_dir_ino_rm = 0;
4137 	bool did_overwrite = false;
4138 	bool is_orphan = false;
4139 	bool can_rename = true;
4140 	bool orphanized_dir = false;
4141 	bool orphanized_ancestor = false;
4142 
4143 	/*
4144 	 * This should never happen as the root dir always has the same ref
4145 	 * which is always '..'
4146 	 */
4147 	if (unlikely(sctx->cur_ino <= BTRFS_FIRST_FREE_OBJECTID)) {
4148 		btrfs_err(fs_info,
4149 			  "send: unexpected inode %llu in process_recorded_refs()",
4150 			  sctx->cur_ino);
4151 		ret = -EINVAL;
4152 		goto out;
4153 	}
4154 
4155 	valid_path = fs_path_alloc();
4156 	if (!valid_path) {
4157 		ret = -ENOMEM;
4158 		goto out;
4159 	}
4160 
4161 	/*
4162 	 * First, check if the first ref of the current inode was overwritten
4163 	 * before. If yes, we know that the current inode was already orphanized
4164 	 * and thus use the orphan name. If not, we can use get_cur_path to
4165 	 * get the path of the first ref as it would like while receiving at
4166 	 * this point in time.
4167 	 * New inodes are always orphan at the beginning, so force to use the
4168 	 * orphan name in this case.
4169 	 * The first ref is stored in valid_path and will be updated if it
4170 	 * gets moved around.
4171 	 */
4172 	if (!sctx->cur_inode_new) {
4173 		ret = did_overwrite_first_ref(sctx, sctx->cur_ino,
4174 				sctx->cur_inode_gen);
4175 		if (ret < 0)
4176 			goto out;
4177 		if (ret)
4178 			did_overwrite = true;
4179 	}
4180 	if (sctx->cur_inode_new || did_overwrite) {
4181 		ret = gen_unique_name(sctx, sctx->cur_ino,
4182 				sctx->cur_inode_gen, valid_path);
4183 		if (ret < 0)
4184 			goto out;
4185 		is_orphan = true;
4186 	} else {
4187 		ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen,
4188 				valid_path);
4189 		if (ret < 0)
4190 			goto out;
4191 	}
4192 
4193 	/*
4194 	 * Before doing any rename and link operations, do a first pass on the
4195 	 * new references to orphanize any unprocessed inodes that may have a
4196 	 * reference that conflicts with one of the new references of the current
4197 	 * inode. This needs to happen first because a new reference may conflict
4198 	 * with the old reference of a parent directory, so we must make sure
4199 	 * that the path used for link and rename commands don't use an
4200 	 * orphanized name when an ancestor was not yet orphanized.
4201 	 *
4202 	 * Example:
4203 	 *
4204 	 * Parent snapshot:
4205 	 *
4206 	 * .                                                      (ino 256)
4207 	 * |----- testdir/                                        (ino 259)
4208 	 * |          |----- a                                    (ino 257)
4209 	 * |
4210 	 * |----- b                                               (ino 258)
4211 	 *
4212 	 * Send snapshot:
4213 	 *
4214 	 * .                                                      (ino 256)
4215 	 * |----- testdir_2/                                      (ino 259)
4216 	 * |          |----- a                                    (ino 260)
4217 	 * |
4218 	 * |----- testdir                                         (ino 257)
4219 	 * |----- b                                               (ino 257)
4220 	 * |----- b2                                              (ino 258)
4221 	 *
4222 	 * Processing the new reference for inode 257 with name "b" may happen
4223 	 * before processing the new reference with name "testdir". If so, we
4224 	 * must make sure that by the time we send a link command to create the
4225 	 * hard link "b", inode 259 was already orphanized, since the generated
4226 	 * path in "valid_path" already contains the orphanized name for 259.
4227 	 * We are processing inode 257, so only later when processing 259 we do
4228 	 * the rename operation to change its temporary (orphanized) name to
4229 	 * "testdir_2".
4230 	 */
4231 	list_for_each_entry(cur, &sctx->new_refs, list) {
4232 		ret = get_cur_inode_state(sctx, cur->dir, cur->dir_gen, NULL, NULL);
4233 		if (ret < 0)
4234 			goto out;
4235 		if (ret == inode_state_will_create)
4236 			continue;
4237 
4238 		/*
4239 		 * Check if this new ref would overwrite the first ref of another
4240 		 * unprocessed inode. If yes, orphanize the overwritten inode.
4241 		 * If we find an overwritten ref that is not the first ref,
4242 		 * simply unlink it.
4243 		 */
4244 		ret = will_overwrite_ref(sctx, cur->dir, cur->dir_gen,
4245 				cur->name, cur->name_len,
4246 				&ow_inode, &ow_gen, &ow_mode);
4247 		if (ret < 0)
4248 			goto out;
4249 		if (ret) {
4250 			ret = is_first_ref(sctx->parent_root,
4251 					   ow_inode, cur->dir, cur->name,
4252 					   cur->name_len);
4253 			if (ret < 0)
4254 				goto out;
4255 			if (ret) {
4256 				struct name_cache_entry *nce;
4257 				struct waiting_dir_move *wdm;
4258 
4259 				if (orphanized_dir) {
4260 					ret = refresh_ref_path(sctx, cur);
4261 					if (ret < 0)
4262 						goto out;
4263 				}
4264 
4265 				ret = orphanize_inode(sctx, ow_inode, ow_gen,
4266 						cur->full_path);
4267 				if (ret < 0)
4268 					goto out;
4269 				if (S_ISDIR(ow_mode))
4270 					orphanized_dir = true;
4271 
4272 				/*
4273 				 * If ow_inode has its rename operation delayed
4274 				 * make sure that its orphanized name is used in
4275 				 * the source path when performing its rename
4276 				 * operation.
4277 				 */
4278 				wdm = get_waiting_dir_move(sctx, ow_inode);
4279 				if (wdm)
4280 					wdm->orphanized = true;
4281 
4282 				/*
4283 				 * Make sure we clear our orphanized inode's
4284 				 * name from the name cache. This is because the
4285 				 * inode ow_inode might be an ancestor of some
4286 				 * other inode that will be orphanized as well
4287 				 * later and has an inode number greater than
4288 				 * sctx->send_progress. We need to prevent
4289 				 * future name lookups from using the old name
4290 				 * and get instead the orphan name.
4291 				 */
4292 				nce = name_cache_search(sctx, ow_inode, ow_gen);
4293 				if (nce)
4294 					btrfs_lru_cache_remove(&sctx->name_cache,
4295 							       &nce->entry);
4296 
4297 				/*
4298 				 * ow_inode might currently be an ancestor of
4299 				 * cur_ino, therefore compute valid_path (the
4300 				 * current path of cur_ino) again because it
4301 				 * might contain the pre-orphanization name of
4302 				 * ow_inode, which is no longer valid.
4303 				 */
4304 				ret = is_ancestor(sctx->parent_root,
4305 						  ow_inode, ow_gen,
4306 						  sctx->cur_ino, NULL);
4307 				if (ret > 0) {
4308 					orphanized_ancestor = true;
4309 					fs_path_reset(valid_path);
4310 					fs_path_reset(&sctx->cur_inode_path);
4311 					ret = get_cur_path(sctx, sctx->cur_ino,
4312 							   sctx->cur_inode_gen,
4313 							   valid_path);
4314 				}
4315 				if (ret < 0)
4316 					goto out;
4317 			} else {
4318 				/*
4319 				 * If we previously orphanized a directory that
4320 				 * collided with a new reference that we already
4321 				 * processed, recompute the current path because
4322 				 * that directory may be part of the path.
4323 				 */
4324 				if (orphanized_dir) {
4325 					ret = refresh_ref_path(sctx, cur);
4326 					if (ret < 0)
4327 						goto out;
4328 				}
4329 				ret = send_unlink(sctx, cur->full_path);
4330 				if (ret < 0)
4331 					goto out;
4332 			}
4333 		}
4334 
4335 	}
4336 
4337 	list_for_each_entry(cur, &sctx->new_refs, list) {
4338 		/*
4339 		 * We may have refs where the parent directory does not exist
4340 		 * yet. This happens if the parent directories inum is higher
4341 		 * than the current inum. To handle this case, we create the
4342 		 * parent directory out of order. But we need to check if this
4343 		 * did already happen before due to other refs in the same dir.
4344 		 */
4345 		ret = get_cur_inode_state(sctx, cur->dir, cur->dir_gen, NULL, NULL);
4346 		if (ret < 0)
4347 			goto out;
4348 		if (ret == inode_state_will_create) {
4349 			ret = 0;
4350 			/*
4351 			 * First check if any of the current inodes refs did
4352 			 * already create the dir.
4353 			 */
4354 			list_for_each_entry(cur2, &sctx->new_refs, list) {
4355 				if (cur == cur2)
4356 					break;
4357 				if (cur2->dir == cur->dir) {
4358 					ret = 1;
4359 					break;
4360 				}
4361 			}
4362 
4363 			/*
4364 			 * If that did not happen, check if a previous inode
4365 			 * did already create the dir.
4366 			 */
4367 			if (!ret)
4368 				ret = did_create_dir(sctx, cur->dir);
4369 			if (ret < 0)
4370 				goto out;
4371 			if (!ret) {
4372 				ret = send_create_inode(sctx, cur->dir);
4373 				if (ret < 0)
4374 					goto out;
4375 				cache_dir_created(sctx, cur->dir);
4376 			}
4377 		}
4378 
4379 		if (S_ISDIR(sctx->cur_inode_mode) && sctx->parent_root) {
4380 			ret = wait_for_dest_dir_move(sctx, cur, is_orphan);
4381 			if (ret < 0)
4382 				goto out;
4383 			if (ret == 1) {
4384 				can_rename = false;
4385 				*pending_move = 1;
4386 			}
4387 		}
4388 
4389 		if (S_ISDIR(sctx->cur_inode_mode) && sctx->parent_root &&
4390 		    can_rename) {
4391 			ret = wait_for_parent_move(sctx, cur, is_orphan);
4392 			if (ret < 0)
4393 				goto out;
4394 			if (ret == 1) {
4395 				can_rename = false;
4396 				*pending_move = 1;
4397 			}
4398 		}
4399 
4400 		/*
4401 		 * link/move the ref to the new place. If we have an orphan
4402 		 * inode, move it and update valid_path. If not, link or move
4403 		 * it depending on the inode mode.
4404 		 */
4405 		if (is_orphan && can_rename) {
4406 			ret = rename_current_inode(sctx, valid_path, cur->full_path);
4407 			if (ret < 0)
4408 				goto out;
4409 			is_orphan = false;
4410 		} else if (can_rename) {
4411 			if (S_ISDIR(sctx->cur_inode_mode)) {
4412 				/*
4413 				 * Dirs can't be linked, so move it. For moved
4414 				 * dirs, we always have one new and one deleted
4415 				 * ref. The deleted ref is ignored later.
4416 				 */
4417 				ret = rename_current_inode(sctx, valid_path,
4418 							   cur->full_path);
4419 				if (ret < 0)
4420 					goto out;
4421 			} else {
4422 				/*
4423 				 * We might have previously orphanized an inode
4424 				 * which is an ancestor of our current inode,
4425 				 * so our reference's full path, which was
4426 				 * computed before any such orphanizations, must
4427 				 * be updated.
4428 				 */
4429 				if (orphanized_dir) {
4430 					ret = update_ref_path(sctx, cur);
4431 					if (ret < 0)
4432 						goto out;
4433 				}
4434 				ret = send_link(sctx, cur->full_path,
4435 						valid_path);
4436 				if (ret < 0)
4437 					goto out;
4438 			}
4439 		}
4440 		ret = dup_ref(cur, &check_dirs);
4441 		if (ret < 0)
4442 			goto out;
4443 	}
4444 
4445 	if (S_ISDIR(sctx->cur_inode_mode) && sctx->cur_inode_deleted) {
4446 		/*
4447 		 * Check if we can already rmdir the directory. If not,
4448 		 * orphanize it. For every dir item inside that gets deleted
4449 		 * later, we do this check again and rmdir it then if possible.
4450 		 * See the use of check_dirs for more details.
4451 		 */
4452 		ret = can_rmdir(sctx, sctx->cur_ino, sctx->cur_inode_gen);
4453 		if (ret < 0)
4454 			goto out;
4455 		if (ret) {
4456 			ret = send_rmdir(sctx, valid_path);
4457 			if (ret < 0)
4458 				goto out;
4459 		} else if (!is_orphan) {
4460 			ret = orphanize_inode(sctx, sctx->cur_ino,
4461 					sctx->cur_inode_gen, valid_path);
4462 			if (ret < 0)
4463 				goto out;
4464 			is_orphan = true;
4465 		}
4466 
4467 		list_for_each_entry(cur, &sctx->deleted_refs, list) {
4468 			ret = dup_ref(cur, &check_dirs);
4469 			if (ret < 0)
4470 				goto out;
4471 		}
4472 	} else if (S_ISDIR(sctx->cur_inode_mode) &&
4473 		   !list_empty(&sctx->deleted_refs)) {
4474 		/*
4475 		 * We have a moved dir. Add the old parent to check_dirs
4476 		 */
4477 		cur = list_first_entry(&sctx->deleted_refs, struct recorded_ref, list);
4478 		ret = dup_ref(cur, &check_dirs);
4479 		if (ret < 0)
4480 			goto out;
4481 	} else if (!S_ISDIR(sctx->cur_inode_mode)) {
4482 		/*
4483 		 * We have a non dir inode. Go through all deleted refs and
4484 		 * unlink them if they were not already overwritten by other
4485 		 * inodes.
4486 		 */
4487 		list_for_each_entry(cur, &sctx->deleted_refs, list) {
4488 			ret = did_overwrite_ref(sctx, cur->dir, cur->dir_gen,
4489 					sctx->cur_ino, sctx->cur_inode_gen,
4490 					cur->name, cur->name_len);
4491 			if (ret < 0)
4492 				goto out;
4493 			if (!ret) {
4494 				/*
4495 				 * If we orphanized any ancestor before, we need
4496 				 * to recompute the full path for deleted names,
4497 				 * since any such path was computed before we
4498 				 * processed any references and orphanized any
4499 				 * ancestor inode.
4500 				 */
4501 				if (orphanized_ancestor) {
4502 					ret = update_ref_path(sctx, cur);
4503 					if (ret < 0)
4504 						goto out;
4505 				}
4506 				ret = send_unlink(sctx, cur->full_path);
4507 				if (ret < 0)
4508 					goto out;
4509 				if (is_current_inode_path(sctx, cur->full_path))
4510 					fs_path_reset(&sctx->cur_inode_path);
4511 			}
4512 			ret = dup_ref(cur, &check_dirs);
4513 			if (ret < 0)
4514 				goto out;
4515 		}
4516 		/*
4517 		 * If the inode is still orphan, unlink the orphan. This may
4518 		 * happen when a previous inode did overwrite the first ref
4519 		 * of this inode and no new refs were added for the current
4520 		 * inode. Unlinking does not mean that the inode is deleted in
4521 		 * all cases. There may still be links to this inode in other
4522 		 * places.
4523 		 */
4524 		if (is_orphan) {
4525 			ret = send_unlink(sctx, valid_path);
4526 			if (ret < 0)
4527 				goto out;
4528 		}
4529 	}
4530 
4531 	/*
4532 	 * We did collect all parent dirs where cur_inode was once located. We
4533 	 * now go through all these dirs and check if they are pending for
4534 	 * deletion and if it's finally possible to perform the rmdir now.
4535 	 * We also update the inode stats of the parent dirs here.
4536 	 */
4537 	list_for_each_entry(cur, &check_dirs, list) {
4538 		/*
4539 		 * In case we had refs into dirs that were not processed yet,
4540 		 * we don't need to do the utime and rmdir logic for these dirs.
4541 		 * The dir will be processed later.
4542 		 */
4543 		if (cur->dir > sctx->cur_ino)
4544 			continue;
4545 
4546 		ret = get_cur_inode_state(sctx, cur->dir, cur->dir_gen, NULL, NULL);
4547 		if (ret < 0)
4548 			goto out;
4549 
4550 		if (ret == inode_state_did_create ||
4551 		    ret == inode_state_no_change) {
4552 			ret = cache_dir_utimes(sctx, cur->dir, cur->dir_gen);
4553 			if (ret < 0)
4554 				goto out;
4555 		} else if (ret == inode_state_did_delete &&
4556 			   cur->dir != last_dir_ino_rm) {
4557 			ret = can_rmdir(sctx, cur->dir, cur->dir_gen);
4558 			if (ret < 0)
4559 				goto out;
4560 			if (ret) {
4561 				ret = get_cur_path(sctx, cur->dir,
4562 						   cur->dir_gen, valid_path);
4563 				if (ret < 0)
4564 					goto out;
4565 				ret = send_rmdir(sctx, valid_path);
4566 				if (ret < 0)
4567 					goto out;
4568 				last_dir_ino_rm = cur->dir;
4569 			}
4570 		}
4571 	}
4572 
4573 	ret = 0;
4574 
4575 out:
4576 	__free_recorded_refs(&check_dirs);
4577 	free_recorded_refs(sctx);
4578 	fs_path_free(valid_path);
4579 	return ret;
4580 }
4581 
rbtree_ref_comp(const void * k,const struct rb_node * node)4582 static int rbtree_ref_comp(const void *k, const struct rb_node *node)
4583 {
4584 	const struct recorded_ref *data = k;
4585 	const struct recorded_ref *ref = rb_entry(node, struct recorded_ref, node);
4586 
4587 	if (data->dir > ref->dir)
4588 		return 1;
4589 	if (data->dir < ref->dir)
4590 		return -1;
4591 	if (data->dir_gen > ref->dir_gen)
4592 		return 1;
4593 	if (data->dir_gen < ref->dir_gen)
4594 		return -1;
4595 	if (data->name_len > ref->name_len)
4596 		return 1;
4597 	if (data->name_len < ref->name_len)
4598 		return -1;
4599 	return strcmp(data->name, ref->name);
4600 }
4601 
rbtree_ref_less(struct rb_node * node,const struct rb_node * parent)4602 static bool rbtree_ref_less(struct rb_node *node, const struct rb_node *parent)
4603 {
4604 	const struct recorded_ref *entry = rb_entry(node, struct recorded_ref, node);
4605 
4606 	return rbtree_ref_comp(entry, parent) < 0;
4607 }
4608 
record_ref_in_tree(struct rb_root * root,struct list_head * refs,struct fs_path * name,u64 dir,u64 dir_gen,struct send_ctx * sctx)4609 static int record_ref_in_tree(struct rb_root *root, struct list_head *refs,
4610 			      struct fs_path *name, u64 dir, u64 dir_gen,
4611 			      struct send_ctx *sctx)
4612 {
4613 	int ret = 0;
4614 	struct fs_path *path = NULL;
4615 	struct recorded_ref *ref = NULL;
4616 
4617 	path = fs_path_alloc();
4618 	if (!path) {
4619 		ret = -ENOMEM;
4620 		goto out;
4621 	}
4622 
4623 	ref = recorded_ref_alloc();
4624 	if (!ref) {
4625 		ret = -ENOMEM;
4626 		goto out;
4627 	}
4628 
4629 	ret = get_cur_path(sctx, dir, dir_gen, path);
4630 	if (ret < 0)
4631 		goto out;
4632 	ret = fs_path_add_path(path, name);
4633 	if (ret < 0)
4634 		goto out;
4635 
4636 	ref->dir = dir;
4637 	ref->dir_gen = dir_gen;
4638 	set_ref_path(ref, path);
4639 	list_add_tail(&ref->list, refs);
4640 	rb_add(&ref->node, root, rbtree_ref_less);
4641 	ref->root = root;
4642 out:
4643 	if (ret) {
4644 		if (path && (!ref || !ref->full_path))
4645 			fs_path_free(path);
4646 		recorded_ref_free(ref);
4647 	}
4648 	return ret;
4649 }
4650 
record_new_ref_if_needed(u64 dir,struct fs_path * name,void * ctx)4651 static int record_new_ref_if_needed(u64 dir, struct fs_path *name, void *ctx)
4652 {
4653 	int ret;
4654 	struct send_ctx *sctx = ctx;
4655 	struct rb_node *node = NULL;
4656 	struct recorded_ref data;
4657 	struct recorded_ref *ref;
4658 	u64 dir_gen;
4659 
4660 	ret = get_inode_gen(sctx->send_root, dir, &dir_gen);
4661 	if (ret < 0)
4662 		return ret;
4663 
4664 	data.dir = dir;
4665 	data.dir_gen = dir_gen;
4666 	set_ref_path(&data, name);
4667 	node = rb_find(&data, &sctx->rbtree_deleted_refs, rbtree_ref_comp);
4668 	if (node) {
4669 		ref = rb_entry(node, struct recorded_ref, node);
4670 		recorded_ref_free(ref);
4671 	} else {
4672 		ret = record_ref_in_tree(&sctx->rbtree_new_refs,
4673 					 &sctx->new_refs, name, dir, dir_gen,
4674 					 sctx);
4675 	}
4676 
4677 	return ret;
4678 }
4679 
record_deleted_ref_if_needed(u64 dir,struct fs_path * name,void * ctx)4680 static int record_deleted_ref_if_needed(u64 dir, struct fs_path *name, void *ctx)
4681 {
4682 	int ret;
4683 	struct send_ctx *sctx = ctx;
4684 	struct rb_node *node = NULL;
4685 	struct recorded_ref data;
4686 	struct recorded_ref *ref;
4687 	u64 dir_gen;
4688 
4689 	ret = get_inode_gen(sctx->parent_root, dir, &dir_gen);
4690 	if (ret < 0)
4691 		return ret;
4692 
4693 	data.dir = dir;
4694 	data.dir_gen = dir_gen;
4695 	set_ref_path(&data, name);
4696 	node = rb_find(&data, &sctx->rbtree_new_refs, rbtree_ref_comp);
4697 	if (node) {
4698 		ref = rb_entry(node, struct recorded_ref, node);
4699 		recorded_ref_free(ref);
4700 	} else {
4701 		ret = record_ref_in_tree(&sctx->rbtree_deleted_refs,
4702 					 &sctx->deleted_refs, name, dir,
4703 					 dir_gen, sctx);
4704 	}
4705 
4706 	return ret;
4707 }
4708 
record_new_ref(struct send_ctx * sctx)4709 static int record_new_ref(struct send_ctx *sctx)
4710 {
4711 	int ret;
4712 
4713 	ret = iterate_inode_ref(sctx->send_root, sctx->left_path, sctx->cmp_key,
4714 				false, record_new_ref_if_needed, sctx);
4715 	if (ret < 0)
4716 		return ret;
4717 
4718 	return 0;
4719 }
4720 
record_deleted_ref(struct send_ctx * sctx)4721 static int record_deleted_ref(struct send_ctx *sctx)
4722 {
4723 	int ret;
4724 
4725 	ret = iterate_inode_ref(sctx->parent_root, sctx->right_path, sctx->cmp_key,
4726 				false, record_deleted_ref_if_needed, sctx);
4727 	if (ret < 0)
4728 		return ret;
4729 
4730 	return 0;
4731 }
4732 
record_changed_ref(struct send_ctx * sctx)4733 static int record_changed_ref(struct send_ctx *sctx)
4734 {
4735 	int ret;
4736 
4737 	ret = iterate_inode_ref(sctx->send_root, sctx->left_path, sctx->cmp_key,
4738 				false, record_new_ref_if_needed, sctx);
4739 	if (ret < 0)
4740 		return ret;
4741 	ret = iterate_inode_ref(sctx->parent_root, sctx->right_path, sctx->cmp_key,
4742 				false, record_deleted_ref_if_needed, sctx);
4743 	if (ret < 0)
4744 		return ret;
4745 
4746 	return 0;
4747 }
4748 
4749 /*
4750  * Record and process all refs at once. Needed when an inode changes the
4751  * generation number, which means that it was deleted and recreated.
4752  */
process_all_refs(struct send_ctx * sctx,enum btrfs_compare_tree_result cmd)4753 static int process_all_refs(struct send_ctx *sctx,
4754 			    enum btrfs_compare_tree_result cmd)
4755 {
4756 	int ret = 0;
4757 	int iter_ret = 0;
4758 	struct btrfs_root *root;
4759 	BTRFS_PATH_AUTO_FREE(path);
4760 	struct btrfs_key key;
4761 	struct btrfs_key found_key;
4762 	iterate_inode_ref_t cb;
4763 	int pending_move = 0;
4764 
4765 	path = alloc_path_for_send();
4766 	if (!path)
4767 		return -ENOMEM;
4768 
4769 	if (cmd == BTRFS_COMPARE_TREE_NEW) {
4770 		root = sctx->send_root;
4771 		cb = record_new_ref_if_needed;
4772 	} else if (cmd == BTRFS_COMPARE_TREE_DELETED) {
4773 		root = sctx->parent_root;
4774 		cb = record_deleted_ref_if_needed;
4775 	} else {
4776 		btrfs_err(sctx->send_root->fs_info,
4777 				"Wrong command %d in process_all_refs", cmd);
4778 		return -EINVAL;
4779 	}
4780 
4781 	key.objectid = sctx->cmp_key->objectid;
4782 	key.type = BTRFS_INODE_REF_KEY;
4783 	key.offset = 0;
4784 	btrfs_for_each_slot(root, &key, &found_key, path, iter_ret) {
4785 		if (found_key.objectid != key.objectid ||
4786 		    (found_key.type != BTRFS_INODE_REF_KEY &&
4787 		     found_key.type != BTRFS_INODE_EXTREF_KEY))
4788 			break;
4789 
4790 		ret = iterate_inode_ref(root, path, &found_key, false, cb, sctx);
4791 		if (ret < 0)
4792 			return ret;
4793 	}
4794 	/* Catch error found during iteration */
4795 	if (iter_ret < 0)
4796 		return iter_ret;
4797 
4798 	btrfs_release_path(path);
4799 
4800 	/*
4801 	 * We don't actually care about pending_move as we are simply
4802 	 * re-creating this inode and will be rename'ing it into place once we
4803 	 * rename the parent directory.
4804 	 */
4805 	return process_recorded_refs(sctx, &pending_move);
4806 }
4807 
send_set_xattr(struct send_ctx * sctx,const char * name,int name_len,const char * data,int data_len)4808 static int send_set_xattr(struct send_ctx *sctx,
4809 			  const char *name, int name_len,
4810 			  const char *data, int data_len)
4811 {
4812 	struct fs_path *path;
4813 	int ret;
4814 
4815 	path = get_cur_inode_path(sctx);
4816 	if (IS_ERR(path))
4817 		return PTR_ERR(path);
4818 
4819 	ret = begin_cmd(sctx, BTRFS_SEND_C_SET_XATTR);
4820 	if (ret < 0)
4821 		return ret;
4822 
4823 	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path);
4824 	TLV_PUT_STRING(sctx, BTRFS_SEND_A_XATTR_NAME, name, name_len);
4825 	TLV_PUT(sctx, BTRFS_SEND_A_XATTR_DATA, data, data_len);
4826 
4827 	ret = send_cmd(sctx);
4828 
4829 tlv_put_failure:
4830 	return ret;
4831 }
4832 
send_remove_xattr(struct send_ctx * sctx,struct fs_path * path,const char * name,int name_len)4833 static int send_remove_xattr(struct send_ctx *sctx,
4834 			  struct fs_path *path,
4835 			  const char *name, int name_len)
4836 {
4837 	int ret;
4838 
4839 	ret = begin_cmd(sctx, BTRFS_SEND_C_REMOVE_XATTR);
4840 	if (ret < 0)
4841 		return ret;
4842 
4843 	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path);
4844 	TLV_PUT_STRING(sctx, BTRFS_SEND_A_XATTR_NAME, name, name_len);
4845 
4846 	ret = send_cmd(sctx);
4847 
4848 tlv_put_failure:
4849 	return ret;
4850 }
4851 
__process_new_xattr(int num,struct btrfs_key * di_key,const char * name,int name_len,const char * data,int data_len,void * ctx)4852 static int __process_new_xattr(int num, struct btrfs_key *di_key,
4853 			       const char *name, int name_len, const char *data,
4854 			       int data_len, void *ctx)
4855 {
4856 	struct send_ctx *sctx = ctx;
4857 	struct posix_acl_xattr_header dummy_acl;
4858 
4859 	/* Capabilities are emitted by finish_inode_if_needed */
4860 	if (!strncmp(name, XATTR_NAME_CAPS, name_len))
4861 		return 0;
4862 
4863 	/*
4864 	 * This hack is needed because empty acls are stored as zero byte
4865 	 * data in xattrs. Problem with that is, that receiving these zero byte
4866 	 * acls will fail later. To fix this, we send a dummy acl list that
4867 	 * only contains the version number and no entries.
4868 	 */
4869 	if (!strncmp(name, XATTR_NAME_POSIX_ACL_ACCESS, name_len) ||
4870 	    !strncmp(name, XATTR_NAME_POSIX_ACL_DEFAULT, name_len)) {
4871 		if (data_len == 0) {
4872 			dummy_acl.a_version =
4873 					cpu_to_le32(POSIX_ACL_XATTR_VERSION);
4874 			data = (char *)&dummy_acl;
4875 			data_len = sizeof(dummy_acl);
4876 		}
4877 	}
4878 
4879 	return send_set_xattr(sctx, name, name_len, data, data_len);
4880 }
4881 
__process_deleted_xattr(int num,struct btrfs_key * di_key,const char * name,int name_len,const char * data,int data_len,void * ctx)4882 static int __process_deleted_xattr(int num, struct btrfs_key *di_key,
4883 				   const char *name, int name_len,
4884 				   const char *data, int data_len, void *ctx)
4885 {
4886 	struct send_ctx *sctx = ctx;
4887 	struct fs_path *p;
4888 
4889 	p = get_cur_inode_path(sctx);
4890 	if (IS_ERR(p))
4891 		return PTR_ERR(p);
4892 
4893 	return send_remove_xattr(sctx, p, name, name_len);
4894 }
4895 
process_new_xattr(struct send_ctx * sctx)4896 static int process_new_xattr(struct send_ctx *sctx)
4897 {
4898 	return iterate_dir_item(sctx->send_root, sctx->left_path,
4899 				__process_new_xattr, sctx);
4900 }
4901 
process_deleted_xattr(struct send_ctx * sctx)4902 static int process_deleted_xattr(struct send_ctx *sctx)
4903 {
4904 	return iterate_dir_item(sctx->parent_root, sctx->right_path,
4905 				__process_deleted_xattr, sctx);
4906 }
4907 
4908 struct find_xattr_ctx {
4909 	const char *name;
4910 	int name_len;
4911 	int found_idx;
4912 	char *found_data;
4913 	int found_data_len;
4914 };
4915 
__find_xattr(int num,struct btrfs_key * di_key,const char * name,int name_len,const char * data,int data_len,void * vctx)4916 static int __find_xattr(int num, struct btrfs_key *di_key, const char *name,
4917 			int name_len, const char *data, int data_len, void *vctx)
4918 {
4919 	struct find_xattr_ctx *ctx = vctx;
4920 
4921 	if (name_len == ctx->name_len &&
4922 	    strncmp(name, ctx->name, name_len) == 0) {
4923 		ctx->found_idx = num;
4924 		ctx->found_data_len = data_len;
4925 		ctx->found_data = kmemdup(data, data_len, GFP_KERNEL);
4926 		if (!ctx->found_data)
4927 			return -ENOMEM;
4928 		return 1;
4929 	}
4930 	return 0;
4931 }
4932 
find_xattr(struct btrfs_root * root,struct btrfs_path * path,struct btrfs_key * key,const char * name,int name_len,char ** data,int * data_len)4933 static int find_xattr(struct btrfs_root *root,
4934 		      struct btrfs_path *path,
4935 		      struct btrfs_key *key,
4936 		      const char *name, int name_len,
4937 		      char **data, int *data_len)
4938 {
4939 	int ret;
4940 	struct find_xattr_ctx ctx;
4941 
4942 	ctx.name = name;
4943 	ctx.name_len = name_len;
4944 	ctx.found_idx = -1;
4945 	ctx.found_data = NULL;
4946 	ctx.found_data_len = 0;
4947 
4948 	ret = iterate_dir_item(root, path, __find_xattr, &ctx);
4949 	if (ret < 0)
4950 		return ret;
4951 
4952 	if (ctx.found_idx == -1)
4953 		return -ENOENT;
4954 	if (data) {
4955 		*data = ctx.found_data;
4956 		*data_len = ctx.found_data_len;
4957 	} else {
4958 		kfree(ctx.found_data);
4959 	}
4960 	return ctx.found_idx;
4961 }
4962 
4963 
__process_changed_new_xattr(int num,struct btrfs_key * di_key,const char * name,int name_len,const char * data,int data_len,void * ctx)4964 static int __process_changed_new_xattr(int num, struct btrfs_key *di_key,
4965 				       const char *name, int name_len,
4966 				       const char *data, int data_len,
4967 				       void *ctx)
4968 {
4969 	int ret;
4970 	struct send_ctx *sctx = ctx;
4971 	char *found_data = NULL;
4972 	int found_data_len  = 0;
4973 
4974 	ret = find_xattr(sctx->parent_root, sctx->right_path,
4975 			 sctx->cmp_key, name, name_len, &found_data,
4976 			 &found_data_len);
4977 	if (ret == -ENOENT) {
4978 		ret = __process_new_xattr(num, di_key, name, name_len, data,
4979 					  data_len, ctx);
4980 	} else if (ret >= 0) {
4981 		if (data_len != found_data_len ||
4982 		    memcmp(data, found_data, data_len)) {
4983 			ret = __process_new_xattr(num, di_key, name, name_len,
4984 						  data, data_len, ctx);
4985 		} else {
4986 			ret = 0;
4987 		}
4988 	}
4989 
4990 	kfree(found_data);
4991 	return ret;
4992 }
4993 
__process_changed_deleted_xattr(int num,struct btrfs_key * di_key,const char * name,int name_len,const char * data,int data_len,void * ctx)4994 static int __process_changed_deleted_xattr(int num, struct btrfs_key *di_key,
4995 					   const char *name, int name_len,
4996 					   const char *data, int data_len,
4997 					   void *ctx)
4998 {
4999 	int ret;
5000 	struct send_ctx *sctx = ctx;
5001 
5002 	ret = find_xattr(sctx->send_root, sctx->left_path, sctx->cmp_key,
5003 			 name, name_len, NULL, NULL);
5004 	if (ret == -ENOENT)
5005 		ret = __process_deleted_xattr(num, di_key, name, name_len, data,
5006 					      data_len, ctx);
5007 	else if (ret >= 0)
5008 		ret = 0;
5009 
5010 	return ret;
5011 }
5012 
process_changed_xattr(struct send_ctx * sctx)5013 static int process_changed_xattr(struct send_ctx *sctx)
5014 {
5015 	int ret;
5016 
5017 	ret = iterate_dir_item(sctx->send_root, sctx->left_path,
5018 			__process_changed_new_xattr, sctx);
5019 	if (ret < 0)
5020 		return ret;
5021 
5022 	return iterate_dir_item(sctx->parent_root, sctx->right_path,
5023 				__process_changed_deleted_xattr, sctx);
5024 }
5025 
process_all_new_xattrs(struct send_ctx * sctx)5026 static int process_all_new_xattrs(struct send_ctx *sctx)
5027 {
5028 	int ret = 0;
5029 	int iter_ret = 0;
5030 	struct btrfs_root *root;
5031 	BTRFS_PATH_AUTO_FREE(path);
5032 	struct btrfs_key key;
5033 	struct btrfs_key found_key;
5034 
5035 	path = alloc_path_for_send();
5036 	if (!path)
5037 		return -ENOMEM;
5038 
5039 	root = sctx->send_root;
5040 
5041 	key.objectid = sctx->cmp_key->objectid;
5042 	key.type = BTRFS_XATTR_ITEM_KEY;
5043 	key.offset = 0;
5044 	btrfs_for_each_slot(root, &key, &found_key, path, iter_ret) {
5045 		if (found_key.objectid != key.objectid ||
5046 		    found_key.type != key.type) {
5047 			ret = 0;
5048 			break;
5049 		}
5050 
5051 		ret = iterate_dir_item(root, path, __process_new_xattr, sctx);
5052 		if (ret < 0)
5053 			break;
5054 	}
5055 	/* Catch error found during iteration */
5056 	if (iter_ret < 0)
5057 		ret = iter_ret;
5058 
5059 	return ret;
5060 }
5061 
send_verity(struct send_ctx * sctx,struct fs_path * path,struct fsverity_descriptor * desc)5062 static int send_verity(struct send_ctx *sctx, struct fs_path *path,
5063 		       struct fsverity_descriptor *desc)
5064 {
5065 	int ret;
5066 
5067 	ret = begin_cmd(sctx, BTRFS_SEND_C_ENABLE_VERITY);
5068 	if (ret < 0)
5069 		return ret;
5070 
5071 	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path);
5072 	TLV_PUT_U8(sctx, BTRFS_SEND_A_VERITY_ALGORITHM,
5073 			le8_to_cpu(desc->hash_algorithm));
5074 	TLV_PUT_U32(sctx, BTRFS_SEND_A_VERITY_BLOCK_SIZE,
5075 			1U << le8_to_cpu(desc->log_blocksize));
5076 	TLV_PUT(sctx, BTRFS_SEND_A_VERITY_SALT_DATA, desc->salt,
5077 			le8_to_cpu(desc->salt_size));
5078 	TLV_PUT(sctx, BTRFS_SEND_A_VERITY_SIG_DATA, desc->signature,
5079 			le32_to_cpu(desc->sig_size));
5080 
5081 	ret = send_cmd(sctx);
5082 
5083 tlv_put_failure:
5084 	return ret;
5085 }
5086 
process_verity(struct send_ctx * sctx)5087 static int process_verity(struct send_ctx *sctx)
5088 {
5089 	int ret = 0;
5090 	struct btrfs_inode *inode;
5091 	struct fs_path *p;
5092 
5093 	inode = btrfs_iget(sctx->cur_ino, sctx->send_root);
5094 	if (IS_ERR(inode))
5095 		return PTR_ERR(inode);
5096 
5097 	ret = btrfs_get_verity_descriptor(&inode->vfs_inode, NULL, 0);
5098 	if (ret < 0)
5099 		goto iput;
5100 
5101 	if (ret > FS_VERITY_MAX_DESCRIPTOR_SIZE) {
5102 		ret = -EMSGSIZE;
5103 		goto iput;
5104 	}
5105 	if (!sctx->verity_descriptor) {
5106 		sctx->verity_descriptor = kvmalloc(FS_VERITY_MAX_DESCRIPTOR_SIZE,
5107 						   GFP_KERNEL);
5108 		if (!sctx->verity_descriptor) {
5109 			ret = -ENOMEM;
5110 			goto iput;
5111 		}
5112 	}
5113 
5114 	ret = btrfs_get_verity_descriptor(&inode->vfs_inode, sctx->verity_descriptor, ret);
5115 	if (ret < 0)
5116 		goto iput;
5117 
5118 	p = get_cur_inode_path(sctx);
5119 	if (IS_ERR(p)) {
5120 		ret = PTR_ERR(p);
5121 		goto iput;
5122 	}
5123 
5124 	ret = send_verity(sctx, p, sctx->verity_descriptor);
5125 iput:
5126 	iput(&inode->vfs_inode);
5127 	return ret;
5128 }
5129 
max_send_read_size(const struct send_ctx * sctx)5130 static inline u64 max_send_read_size(const struct send_ctx *sctx)
5131 {
5132 	return sctx->send_max_size - SZ_16K;
5133 }
5134 
put_data_header(struct send_ctx * sctx,u32 len)5135 static int put_data_header(struct send_ctx *sctx, u32 len)
5136 {
5137 	if (WARN_ON_ONCE(sctx->put_data))
5138 		return -EINVAL;
5139 	sctx->put_data = true;
5140 	if (sctx->proto >= 2) {
5141 		/*
5142 		 * Since v2, the data attribute header doesn't include a length,
5143 		 * it is implicitly to the end of the command.
5144 		 */
5145 		if (sctx->send_max_size - sctx->send_size < sizeof(__le16) + len)
5146 			return -EOVERFLOW;
5147 		put_unaligned_le16(BTRFS_SEND_A_DATA, sctx->send_buf + sctx->send_size);
5148 		sctx->send_size += sizeof(__le16);
5149 	} else {
5150 		struct btrfs_tlv_header *hdr;
5151 
5152 		if (sctx->send_max_size - sctx->send_size < sizeof(*hdr) + len)
5153 			return -EOVERFLOW;
5154 		hdr = (struct btrfs_tlv_header *)(sctx->send_buf + sctx->send_size);
5155 		put_unaligned_le16(BTRFS_SEND_A_DATA, &hdr->tlv_type);
5156 		put_unaligned_le16(len, &hdr->tlv_len);
5157 		sctx->send_size += sizeof(*hdr);
5158 	}
5159 	return 0;
5160 }
5161 
put_file_data(struct send_ctx * sctx,u64 offset,u32 len)5162 static int put_file_data(struct send_ctx *sctx, u64 offset, u32 len)
5163 {
5164 	struct btrfs_root *root = sctx->send_root;
5165 	struct btrfs_fs_info *fs_info = root->fs_info;
5166 	u64 cur = offset;
5167 	const u64 end = offset + len;
5168 	const pgoff_t last_index = ((end - 1) >> PAGE_SHIFT);
5169 	struct address_space *mapping = sctx->cur_inode->i_mapping;
5170 	int ret;
5171 
5172 	ret = put_data_header(sctx, len);
5173 	if (ret)
5174 		return ret;
5175 
5176 	while (cur < end) {
5177 		pgoff_t index = (cur >> PAGE_SHIFT);
5178 		unsigned int cur_len;
5179 		unsigned int pg_offset;
5180 		struct folio *folio;
5181 
5182 		folio = filemap_lock_folio(mapping, index);
5183 		if (IS_ERR(folio)) {
5184 			page_cache_sync_readahead(mapping,
5185 						  &sctx->ra, NULL, index,
5186 						  last_index + 1 - index);
5187 
5188 	                folio = filemap_grab_folio(mapping, index);
5189 			if (IS_ERR(folio)) {
5190 				ret = PTR_ERR(folio);
5191 				break;
5192 			}
5193 		}
5194 		pg_offset = offset_in_folio(folio, cur);
5195 		cur_len = min_t(unsigned int, end - cur, folio_size(folio) - pg_offset);
5196 
5197 		if (folio_test_readahead(folio))
5198 			page_cache_async_readahead(mapping, &sctx->ra, NULL, folio,
5199 						   last_index + 1 - index);
5200 
5201 		if (!folio_test_uptodate(folio)) {
5202 			btrfs_read_folio(NULL, folio);
5203 			folio_lock(folio);
5204 			if (unlikely(!folio_test_uptodate(folio))) {
5205 				folio_unlock(folio);
5206 				btrfs_err(fs_info,
5207 			"send: IO error at offset %llu for inode %llu root %llu",
5208 					folio_pos(folio), sctx->cur_ino,
5209 					btrfs_root_id(sctx->send_root));
5210 				folio_put(folio);
5211 				ret = -EIO;
5212 				break;
5213 			}
5214 			if (folio->mapping != mapping) {
5215 				folio_unlock(folio);
5216 				folio_put(folio);
5217 				continue;
5218 			}
5219 		}
5220 
5221 		memcpy_from_folio(sctx->send_buf + sctx->send_size, folio,
5222 				  pg_offset, cur_len);
5223 		folio_unlock(folio);
5224 		folio_put(folio);
5225 		cur += cur_len;
5226 		sctx->send_size += cur_len;
5227 	}
5228 
5229 	return ret;
5230 }
5231 
5232 /*
5233  * Read some bytes from the current inode/file and send a write command to
5234  * user space.
5235  */
send_write(struct send_ctx * sctx,u64 offset,u32 len)5236 static int send_write(struct send_ctx *sctx, u64 offset, u32 len)
5237 {
5238 	int ret = 0;
5239 	struct fs_path *p;
5240 
5241 	p = get_cur_inode_path(sctx);
5242 	if (IS_ERR(p))
5243 		return PTR_ERR(p);
5244 
5245 	ret = begin_cmd(sctx, BTRFS_SEND_C_WRITE);
5246 	if (ret < 0)
5247 		return ret;
5248 
5249 	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
5250 	TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset);
5251 	ret = put_file_data(sctx, offset, len);
5252 	if (ret < 0)
5253 		return ret;
5254 
5255 	ret = send_cmd(sctx);
5256 
5257 tlv_put_failure:
5258 	return ret;
5259 }
5260 
5261 /*
5262  * Send a clone command to user space.
5263  */
send_clone(struct send_ctx * sctx,u64 offset,u32 len,struct clone_root * clone_root)5264 static int send_clone(struct send_ctx *sctx,
5265 		      u64 offset, u32 len,
5266 		      struct clone_root *clone_root)
5267 {
5268 	int ret = 0;
5269 	struct fs_path *p;
5270 	struct fs_path *cur_inode_path;
5271 	u64 gen;
5272 
5273 	cur_inode_path = get_cur_inode_path(sctx);
5274 	if (IS_ERR(cur_inode_path))
5275 		return PTR_ERR(cur_inode_path);
5276 
5277 	p = fs_path_alloc();
5278 	if (!p)
5279 		return -ENOMEM;
5280 
5281 	ret = begin_cmd(sctx, BTRFS_SEND_C_CLONE);
5282 	if (ret < 0)
5283 		goto out;
5284 
5285 	TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset);
5286 	TLV_PUT_U64(sctx, BTRFS_SEND_A_CLONE_LEN, len);
5287 	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, cur_inode_path);
5288 
5289 	if (clone_root->root == sctx->send_root) {
5290 		ret = get_inode_gen(sctx->send_root, clone_root->ino, &gen);
5291 		if (ret < 0)
5292 			goto out;
5293 		ret = get_cur_path(sctx, clone_root->ino, gen, p);
5294 	} else {
5295 		ret = get_inode_path(clone_root->root, clone_root->ino, p);
5296 	}
5297 	if (ret < 0)
5298 		goto out;
5299 
5300 	/*
5301 	 * If the parent we're using has a received_uuid set then use that as
5302 	 * our clone source as that is what we will look for when doing a
5303 	 * receive.
5304 	 *
5305 	 * This covers the case that we create a snapshot off of a received
5306 	 * subvolume and then use that as the parent and try to receive on a
5307 	 * different host.
5308 	 */
5309 	if (!btrfs_is_empty_uuid(clone_root->root->root_item.received_uuid))
5310 		TLV_PUT_UUID(sctx, BTRFS_SEND_A_CLONE_UUID,
5311 			     clone_root->root->root_item.received_uuid);
5312 	else
5313 		TLV_PUT_UUID(sctx, BTRFS_SEND_A_CLONE_UUID,
5314 			     clone_root->root->root_item.uuid);
5315 	TLV_PUT_U64(sctx, BTRFS_SEND_A_CLONE_CTRANSID,
5316 		    btrfs_root_ctransid(&clone_root->root->root_item));
5317 	TLV_PUT_PATH(sctx, BTRFS_SEND_A_CLONE_PATH, p);
5318 	TLV_PUT_U64(sctx, BTRFS_SEND_A_CLONE_OFFSET,
5319 			clone_root->offset);
5320 
5321 	ret = send_cmd(sctx);
5322 
5323 tlv_put_failure:
5324 out:
5325 	fs_path_free(p);
5326 	return ret;
5327 }
5328 
5329 /*
5330  * Send an update extent command to user space.
5331  */
send_update_extent(struct send_ctx * sctx,u64 offset,u32 len)5332 static int send_update_extent(struct send_ctx *sctx,
5333 			      u64 offset, u32 len)
5334 {
5335 	int ret = 0;
5336 	struct fs_path *p;
5337 
5338 	p = get_cur_inode_path(sctx);
5339 	if (IS_ERR(p))
5340 		return PTR_ERR(p);
5341 
5342 	ret = begin_cmd(sctx, BTRFS_SEND_C_UPDATE_EXTENT);
5343 	if (ret < 0)
5344 		return ret;
5345 
5346 	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
5347 	TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset);
5348 	TLV_PUT_U64(sctx, BTRFS_SEND_A_SIZE, len);
5349 
5350 	ret = send_cmd(sctx);
5351 
5352 tlv_put_failure:
5353 	return ret;
5354 }
5355 
send_fallocate(struct send_ctx * sctx,u32 mode,u64 offset,u64 len)5356 static int send_fallocate(struct send_ctx *sctx, u32 mode, u64 offset, u64 len)
5357 {
5358 	struct fs_path *path;
5359 	int ret;
5360 
5361 	path = get_cur_inode_path(sctx);
5362 	if (IS_ERR(path))
5363 		return PTR_ERR(path);
5364 
5365 	ret = begin_cmd(sctx, BTRFS_SEND_C_FALLOCATE);
5366 	if (ret < 0)
5367 		return ret;
5368 
5369 	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path);
5370 	TLV_PUT_U32(sctx, BTRFS_SEND_A_FALLOCATE_MODE, mode);
5371 	TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset);
5372 	TLV_PUT_U64(sctx, BTRFS_SEND_A_SIZE, len);
5373 
5374 	ret = send_cmd(sctx);
5375 
5376 tlv_put_failure:
5377 	return ret;
5378 }
5379 
send_hole(struct send_ctx * sctx,u64 end)5380 static int send_hole(struct send_ctx *sctx, u64 end)
5381 {
5382 	struct fs_path *p = NULL;
5383 	u64 read_size = max_send_read_size(sctx);
5384 	u64 offset = sctx->cur_inode_last_extent;
5385 	int ret = 0;
5386 
5387 	/*
5388 	 * Starting with send stream v2 we have fallocate and can use it to
5389 	 * punch holes instead of sending writes full of zeroes.
5390 	 */
5391 	if (proto_cmd_ok(sctx, BTRFS_SEND_C_FALLOCATE))
5392 		return send_fallocate(sctx, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
5393 				      offset, end - offset);
5394 
5395 	/*
5396 	 * A hole that starts at EOF or beyond it. Since we do not yet support
5397 	 * fallocate (for extent preallocation and hole punching), sending a
5398 	 * write of zeroes starting at EOF or beyond would later require issuing
5399 	 * a truncate operation which would undo the write and achieve nothing.
5400 	 */
5401 	if (offset >= sctx->cur_inode_size)
5402 		return 0;
5403 
5404 	/*
5405 	 * Don't go beyond the inode's i_size due to prealloc extents that start
5406 	 * after the i_size.
5407 	 */
5408 	end = min_t(u64, end, sctx->cur_inode_size);
5409 
5410 	if (sctx->flags & BTRFS_SEND_FLAG_NO_FILE_DATA)
5411 		return send_update_extent(sctx, offset, end - offset);
5412 
5413 	p = get_cur_inode_path(sctx);
5414 	if (IS_ERR(p))
5415 		return PTR_ERR(p);
5416 
5417 	while (offset < end) {
5418 		u64 len = min(end - offset, read_size);
5419 
5420 		ret = begin_cmd(sctx, BTRFS_SEND_C_WRITE);
5421 		if (ret < 0)
5422 			break;
5423 		TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
5424 		TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset);
5425 		ret = put_data_header(sctx, len);
5426 		if (ret < 0)
5427 			break;
5428 		memset(sctx->send_buf + sctx->send_size, 0, len);
5429 		sctx->send_size += len;
5430 		ret = send_cmd(sctx);
5431 		if (ret < 0)
5432 			break;
5433 		offset += len;
5434 	}
5435 	sctx->cur_inode_next_write_offset = offset;
5436 tlv_put_failure:
5437 	return ret;
5438 }
5439 
send_encoded_inline_extent(struct send_ctx * sctx,struct btrfs_path * path,u64 offset,u64 len)5440 static int send_encoded_inline_extent(struct send_ctx *sctx,
5441 				      struct btrfs_path *path, u64 offset,
5442 				      u64 len)
5443 {
5444 	struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
5445 	struct fs_path *fspath;
5446 	struct extent_buffer *leaf = path->nodes[0];
5447 	struct btrfs_key key;
5448 	struct btrfs_file_extent_item *ei;
5449 	u64 ram_bytes;
5450 	size_t inline_size;
5451 	int ret;
5452 
5453 	fspath = get_cur_inode_path(sctx);
5454 	if (IS_ERR(fspath))
5455 		return PTR_ERR(fspath);
5456 
5457 	ret = begin_cmd(sctx, BTRFS_SEND_C_ENCODED_WRITE);
5458 	if (ret < 0)
5459 		return ret;
5460 
5461 	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
5462 	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item);
5463 	ram_bytes = btrfs_file_extent_ram_bytes(leaf, ei);
5464 	inline_size = btrfs_file_extent_inline_item_len(leaf, path->slots[0]);
5465 
5466 	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, fspath);
5467 	TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset);
5468 	TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_FILE_LEN,
5469 		    min(key.offset + ram_bytes - offset, len));
5470 	TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_LEN, ram_bytes);
5471 	TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_OFFSET, offset - key.offset);
5472 	ret = btrfs_encoded_io_compression_from_extent(fs_info,
5473 				btrfs_file_extent_compression(leaf, ei));
5474 	if (ret < 0)
5475 		return ret;
5476 	TLV_PUT_U32(sctx, BTRFS_SEND_A_COMPRESSION, ret);
5477 
5478 	ret = put_data_header(sctx, inline_size);
5479 	if (ret < 0)
5480 		return ret;
5481 	read_extent_buffer(leaf, sctx->send_buf + sctx->send_size,
5482 			   btrfs_file_extent_inline_start(ei), inline_size);
5483 	sctx->send_size += inline_size;
5484 
5485 	ret = send_cmd(sctx);
5486 
5487 tlv_put_failure:
5488 	return ret;
5489 }
5490 
send_encoded_extent(struct send_ctx * sctx,struct btrfs_path * path,u64 offset,u64 len)5491 static int send_encoded_extent(struct send_ctx *sctx, struct btrfs_path *path,
5492 			       u64 offset, u64 len)
5493 {
5494 	struct btrfs_root *root = sctx->send_root;
5495 	struct btrfs_fs_info *fs_info = root->fs_info;
5496 	struct btrfs_inode *inode;
5497 	struct fs_path *fspath;
5498 	struct extent_buffer *leaf = path->nodes[0];
5499 	struct btrfs_key key;
5500 	struct btrfs_file_extent_item *ei;
5501 	u64 disk_bytenr, disk_num_bytes;
5502 	u32 data_offset;
5503 	struct btrfs_cmd_header *hdr;
5504 	u32 crc;
5505 	int ret;
5506 
5507 	inode = btrfs_iget(sctx->cur_ino, root);
5508 	if (IS_ERR(inode))
5509 		return PTR_ERR(inode);
5510 
5511 	fspath = get_cur_inode_path(sctx);
5512 	if (IS_ERR(fspath)) {
5513 		ret = PTR_ERR(fspath);
5514 		goto out;
5515 	}
5516 
5517 	ret = begin_cmd(sctx, BTRFS_SEND_C_ENCODED_WRITE);
5518 	if (ret < 0)
5519 		goto out;
5520 
5521 	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
5522 	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item);
5523 	disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, ei);
5524 	disk_num_bytes = btrfs_file_extent_disk_num_bytes(leaf, ei);
5525 
5526 	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, fspath);
5527 	TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset);
5528 	TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_FILE_LEN,
5529 		    min(key.offset + btrfs_file_extent_num_bytes(leaf, ei) - offset,
5530 			len));
5531 	TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_LEN,
5532 		    btrfs_file_extent_ram_bytes(leaf, ei));
5533 	TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_OFFSET,
5534 		    offset - key.offset + btrfs_file_extent_offset(leaf, ei));
5535 	ret = btrfs_encoded_io_compression_from_extent(fs_info,
5536 				btrfs_file_extent_compression(leaf, ei));
5537 	if (ret < 0)
5538 		goto out;
5539 	TLV_PUT_U32(sctx, BTRFS_SEND_A_COMPRESSION, ret);
5540 	TLV_PUT_U32(sctx, BTRFS_SEND_A_ENCRYPTION, 0);
5541 
5542 	ret = put_data_header(sctx, disk_num_bytes);
5543 	if (ret < 0)
5544 		goto out;
5545 
5546 	/*
5547 	 * We want to do I/O directly into the send buffer, so get the next page
5548 	 * boundary in the send buffer. This means that there may be a gap
5549 	 * between the beginning of the command and the file data.
5550 	 */
5551 	data_offset = PAGE_ALIGN(sctx->send_size);
5552 	if (data_offset > sctx->send_max_size ||
5553 	    sctx->send_max_size - data_offset < disk_num_bytes) {
5554 		ret = -EOVERFLOW;
5555 		goto out;
5556 	}
5557 
5558 	/*
5559 	 * Note that send_buf is a mapping of send_buf_pages, so this is really
5560 	 * reading into send_buf.
5561 	 */
5562 	ret = btrfs_encoded_read_regular_fill_pages(inode,
5563 						    disk_bytenr, disk_num_bytes,
5564 						    sctx->send_buf_pages +
5565 						    (data_offset >> PAGE_SHIFT),
5566 						    NULL);
5567 	if (ret)
5568 		goto out;
5569 
5570 	hdr = (struct btrfs_cmd_header *)sctx->send_buf;
5571 	hdr->len = cpu_to_le32(sctx->send_size + disk_num_bytes - sizeof(*hdr));
5572 	hdr->crc = 0;
5573 	crc = crc32c(0, sctx->send_buf, sctx->send_size);
5574 	crc = crc32c(crc, sctx->send_buf + data_offset, disk_num_bytes);
5575 	hdr->crc = cpu_to_le32(crc);
5576 
5577 	ret = write_buf(sctx->send_filp, sctx->send_buf, sctx->send_size,
5578 			&sctx->send_off);
5579 	if (!ret) {
5580 		ret = write_buf(sctx->send_filp, sctx->send_buf + data_offset,
5581 				disk_num_bytes, &sctx->send_off);
5582 	}
5583 	sctx->send_size = 0;
5584 	sctx->put_data = false;
5585 
5586 tlv_put_failure:
5587 out:
5588 	iput(&inode->vfs_inode);
5589 	return ret;
5590 }
5591 
send_extent_data(struct send_ctx * sctx,struct btrfs_path * path,const u64 offset,const u64 len)5592 static int send_extent_data(struct send_ctx *sctx, struct btrfs_path *path,
5593 			    const u64 offset, const u64 len)
5594 {
5595 	const u64 end = offset + len;
5596 	struct extent_buffer *leaf = path->nodes[0];
5597 	struct btrfs_file_extent_item *ei;
5598 	u64 read_size = max_send_read_size(sctx);
5599 	u64 sent = 0;
5600 
5601 	if (sctx->flags & BTRFS_SEND_FLAG_NO_FILE_DATA)
5602 		return send_update_extent(sctx, offset, len);
5603 
5604 	ei = btrfs_item_ptr(leaf, path->slots[0],
5605 			    struct btrfs_file_extent_item);
5606 	/*
5607 	 * Do not go through encoded read for bs > ps cases.
5608 	 *
5609 	 * Encoded send is using vmallocated pages as buffer, which we can
5610 	 * not ensure every folio is large enough to contain a block.
5611 	 */
5612 	if (sctx->send_root->fs_info->sectorsize <= PAGE_SIZE &&
5613 	    (sctx->flags & BTRFS_SEND_FLAG_COMPRESSED) &&
5614 	    btrfs_file_extent_compression(leaf, ei) != BTRFS_COMPRESS_NONE) {
5615 		bool is_inline = (btrfs_file_extent_type(leaf, ei) ==
5616 				  BTRFS_FILE_EXTENT_INLINE);
5617 
5618 		/*
5619 		 * Send the compressed extent unless the compressed data is
5620 		 * larger than the decompressed data. This can happen if we're
5621 		 * not sending the entire extent, either because it has been
5622 		 * partially overwritten/truncated or because this is a part of
5623 		 * the extent that we couldn't clone in clone_range().
5624 		 */
5625 		if (is_inline &&
5626 		    btrfs_file_extent_inline_item_len(leaf,
5627 						      path->slots[0]) <= len) {
5628 			return send_encoded_inline_extent(sctx, path, offset,
5629 							  len);
5630 		} else if (!is_inline &&
5631 			   btrfs_file_extent_disk_num_bytes(leaf, ei) <= len) {
5632 			return send_encoded_extent(sctx, path, offset, len);
5633 		}
5634 	}
5635 
5636 	if (sctx->cur_inode == NULL) {
5637 		struct btrfs_inode *btrfs_inode;
5638 		struct btrfs_root *root = sctx->send_root;
5639 
5640 		btrfs_inode = btrfs_iget(sctx->cur_ino, root);
5641 		if (IS_ERR(btrfs_inode))
5642 			return PTR_ERR(btrfs_inode);
5643 
5644 		sctx->cur_inode = &btrfs_inode->vfs_inode;
5645 		memset(&sctx->ra, 0, sizeof(struct file_ra_state));
5646 		file_ra_state_init(&sctx->ra, sctx->cur_inode->i_mapping);
5647 
5648 		/*
5649 		 * It's very likely there are no pages from this inode in the page
5650 		 * cache, so after reading extents and sending their data, we clean
5651 		 * the page cache to avoid trashing the page cache (adding pressure
5652 		 * to the page cache and forcing eviction of other data more useful
5653 		 * for applications).
5654 		 *
5655 		 * We decide if we should clean the page cache simply by checking
5656 		 * if the inode's mapping nrpages is 0 when we first open it, and
5657 		 * not by using something like filemap_range_has_page() before
5658 		 * reading an extent because when we ask the readahead code to
5659 		 * read a given file range, it may (and almost always does) read
5660 		 * pages from beyond that range (see the documentation for
5661 		 * page_cache_sync_readahead()), so it would not be reliable,
5662 		 * because after reading the first extent future calls to
5663 		 * filemap_range_has_page() would return true because the readahead
5664 		 * on the previous extent resulted in reading pages of the current
5665 		 * extent as well.
5666 		 */
5667 		sctx->clean_page_cache = (sctx->cur_inode->i_mapping->nrpages == 0);
5668 		sctx->page_cache_clear_start = round_down(offset, PAGE_SIZE);
5669 	}
5670 
5671 	while (sent < len) {
5672 		u64 size = min(len - sent, read_size);
5673 		int ret;
5674 
5675 		ret = send_write(sctx, offset + sent, size);
5676 		if (ret < 0)
5677 			return ret;
5678 		sent += size;
5679 	}
5680 
5681 	if (sctx->clean_page_cache && PAGE_ALIGNED(end)) {
5682 		/*
5683 		 * Always operate only on ranges that are a multiple of the page
5684 		 * size. This is not only to prevent zeroing parts of a page in
5685 		 * the case of subpage sector size, but also to guarantee we evict
5686 		 * pages, as passing a range that is smaller than page size does
5687 		 * not evict the respective page (only zeroes part of its content).
5688 		 *
5689 		 * Always start from the end offset of the last range cleared.
5690 		 * This is because the readahead code may (and very often does)
5691 		 * reads pages beyond the range we request for readahead. So if
5692 		 * we have an extent layout like this:
5693 		 *
5694 		 *            [ extent A ] [ extent B ] [ extent C ]
5695 		 *
5696 		 * When we ask page_cache_sync_readahead() to read extent A, it
5697 		 * may also trigger reads for pages of extent B. If we are doing
5698 		 * an incremental send and extent B has not changed between the
5699 		 * parent and send snapshots, some or all of its pages may end
5700 		 * up being read and placed in the page cache. So when truncating
5701 		 * the page cache we always start from the end offset of the
5702 		 * previously processed extent up to the end of the current
5703 		 * extent.
5704 		 */
5705 		truncate_inode_pages_range(&sctx->cur_inode->i_data,
5706 					   sctx->page_cache_clear_start,
5707 					   end - 1);
5708 		sctx->page_cache_clear_start = end;
5709 	}
5710 
5711 	return 0;
5712 }
5713 
5714 /*
5715  * Search for a capability xattr related to sctx->cur_ino. If the capability is
5716  * found, call send_set_xattr function to emit it.
5717  *
5718  * Return 0 if there isn't a capability, or when the capability was emitted
5719  * successfully, or < 0 if an error occurred.
5720  */
send_capabilities(struct send_ctx * sctx)5721 static int send_capabilities(struct send_ctx *sctx)
5722 {
5723 	BTRFS_PATH_AUTO_FREE(path);
5724 	struct btrfs_dir_item *di;
5725 	struct extent_buffer *leaf;
5726 	unsigned long data_ptr;
5727 	char *buf = NULL;
5728 	int buf_len;
5729 	int ret = 0;
5730 
5731 	path = alloc_path_for_send();
5732 	if (!path)
5733 		return -ENOMEM;
5734 
5735 	di = btrfs_lookup_xattr(NULL, sctx->send_root, path, sctx->cur_ino,
5736 				XATTR_NAME_CAPS, strlen(XATTR_NAME_CAPS), 0);
5737 	if (!di) {
5738 		/* There is no xattr for this inode */
5739 		goto out;
5740 	} else if (IS_ERR(di)) {
5741 		ret = PTR_ERR(di);
5742 		goto out;
5743 	}
5744 
5745 	leaf = path->nodes[0];
5746 	buf_len = btrfs_dir_data_len(leaf, di);
5747 
5748 	buf = kmalloc(buf_len, GFP_KERNEL);
5749 	if (!buf) {
5750 		ret = -ENOMEM;
5751 		goto out;
5752 	}
5753 
5754 	data_ptr = (unsigned long)(di + 1) + btrfs_dir_name_len(leaf, di);
5755 	read_extent_buffer(leaf, buf, data_ptr, buf_len);
5756 
5757 	ret = send_set_xattr(sctx, XATTR_NAME_CAPS,
5758 			strlen(XATTR_NAME_CAPS), buf, buf_len);
5759 out:
5760 	kfree(buf);
5761 	return ret;
5762 }
5763 
clone_range(struct send_ctx * sctx,struct btrfs_path * dst_path,struct clone_root * clone_root,const u64 disk_byte,u64 data_offset,u64 offset,u64 len)5764 static int clone_range(struct send_ctx *sctx, struct btrfs_path *dst_path,
5765 		       struct clone_root *clone_root, const u64 disk_byte,
5766 		       u64 data_offset, u64 offset, u64 len)
5767 {
5768 	BTRFS_PATH_AUTO_FREE(path);
5769 	struct btrfs_key key;
5770 	int ret;
5771 	struct btrfs_inode_info info;
5772 	u64 clone_src_i_size = 0;
5773 
5774 	/*
5775 	 * Prevent cloning from a zero offset with a length matching the sector
5776 	 * size because in some scenarios this will make the receiver fail.
5777 	 *
5778 	 * For example, if in the source filesystem the extent at offset 0
5779 	 * has a length of sectorsize and it was written using direct IO, then
5780 	 * it can never be an inline extent (even if compression is enabled).
5781 	 * Then this extent can be cloned in the original filesystem to a non
5782 	 * zero file offset, but it may not be possible to clone in the
5783 	 * destination filesystem because it can be inlined due to compression
5784 	 * on the destination filesystem (as the receiver's write operations are
5785 	 * always done using buffered IO). The same happens when the original
5786 	 * filesystem does not have compression enabled but the destination
5787 	 * filesystem has.
5788 	 */
5789 	if (clone_root->offset == 0 &&
5790 	    len == sctx->send_root->fs_info->sectorsize)
5791 		return send_extent_data(sctx, dst_path, offset, len);
5792 
5793 	path = alloc_path_for_send();
5794 	if (!path)
5795 		return -ENOMEM;
5796 
5797 	/*
5798 	 * There are inodes that have extents that lie behind its i_size. Don't
5799 	 * accept clones from these extents.
5800 	 */
5801 	ret = get_inode_info(clone_root->root, clone_root->ino, &info);
5802 	btrfs_release_path(path);
5803 	if (ret < 0)
5804 		return ret;
5805 	clone_src_i_size = info.size;
5806 
5807 	/*
5808 	 * We can't send a clone operation for the entire range if we find
5809 	 * extent items in the respective range in the source file that
5810 	 * refer to different extents or if we find holes.
5811 	 * So check for that and do a mix of clone and regular write/copy
5812 	 * operations if needed.
5813 	 *
5814 	 * Example:
5815 	 *
5816 	 * mkfs.btrfs -f /dev/sda
5817 	 * mount /dev/sda /mnt
5818 	 * xfs_io -f -c "pwrite -S 0xaa 0K 100K" /mnt/foo
5819 	 * cp --reflink=always /mnt/foo /mnt/bar
5820 	 * xfs_io -c "pwrite -S 0xbb 50K 50K" /mnt/foo
5821 	 * btrfs subvolume snapshot -r /mnt /mnt/snap
5822 	 *
5823 	 * If when we send the snapshot and we are processing file bar (which
5824 	 * has a higher inode number than foo) we blindly send a clone operation
5825 	 * for the [0, 100K[ range from foo to bar, the receiver ends up getting
5826 	 * a file bar that matches the content of file foo - iow, doesn't match
5827 	 * the content from bar in the original filesystem.
5828 	 */
5829 	key.objectid = clone_root->ino;
5830 	key.type = BTRFS_EXTENT_DATA_KEY;
5831 	key.offset = clone_root->offset;
5832 	ret = btrfs_search_slot(NULL, clone_root->root, &key, path, 0, 0);
5833 	if (ret < 0)
5834 		return ret;
5835 	if (ret > 0 && path->slots[0] > 0) {
5836 		btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0] - 1);
5837 		if (key.objectid == clone_root->ino &&
5838 		    key.type == BTRFS_EXTENT_DATA_KEY)
5839 			path->slots[0]--;
5840 	}
5841 
5842 	while (true) {
5843 		struct extent_buffer *leaf = path->nodes[0];
5844 		int slot = path->slots[0];
5845 		struct btrfs_file_extent_item *ei;
5846 		u8 type;
5847 		u64 ext_len;
5848 		u64 clone_len;
5849 		u64 clone_data_offset;
5850 		bool crossed_src_i_size = false;
5851 
5852 		if (slot >= btrfs_header_nritems(leaf)) {
5853 			ret = btrfs_next_leaf(clone_root->root, path);
5854 			if (ret < 0)
5855 				return ret;
5856 			else if (ret > 0)
5857 				break;
5858 			continue;
5859 		}
5860 
5861 		btrfs_item_key_to_cpu(leaf, &key, slot);
5862 
5863 		/*
5864 		 * We might have an implicit trailing hole (NO_HOLES feature
5865 		 * enabled). We deal with it after leaving this loop.
5866 		 */
5867 		if (key.objectid != clone_root->ino ||
5868 		    key.type != BTRFS_EXTENT_DATA_KEY)
5869 			break;
5870 
5871 		ei = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
5872 		type = btrfs_file_extent_type(leaf, ei);
5873 		if (type == BTRFS_FILE_EXTENT_INLINE) {
5874 			ext_len = btrfs_file_extent_ram_bytes(leaf, ei);
5875 			ext_len = PAGE_ALIGN(ext_len);
5876 		} else {
5877 			ext_len = btrfs_file_extent_num_bytes(leaf, ei);
5878 		}
5879 
5880 		if (key.offset + ext_len <= clone_root->offset)
5881 			goto next;
5882 
5883 		if (key.offset > clone_root->offset) {
5884 			/* Implicit hole, NO_HOLES feature enabled. */
5885 			u64 hole_len = key.offset - clone_root->offset;
5886 
5887 			if (hole_len > len)
5888 				hole_len = len;
5889 			ret = send_extent_data(sctx, dst_path, offset,
5890 					       hole_len);
5891 			if (ret < 0)
5892 				return ret;
5893 
5894 			len -= hole_len;
5895 			if (len == 0)
5896 				break;
5897 			offset += hole_len;
5898 			clone_root->offset += hole_len;
5899 			data_offset += hole_len;
5900 		}
5901 
5902 		if (key.offset >= clone_root->offset + len)
5903 			break;
5904 
5905 		if (key.offset >= clone_src_i_size)
5906 			break;
5907 
5908 		if (key.offset + ext_len > clone_src_i_size) {
5909 			ext_len = clone_src_i_size - key.offset;
5910 			crossed_src_i_size = true;
5911 		}
5912 
5913 		clone_data_offset = btrfs_file_extent_offset(leaf, ei);
5914 		if (btrfs_file_extent_disk_bytenr(leaf, ei) == disk_byte) {
5915 			clone_root->offset = key.offset;
5916 			if (clone_data_offset < data_offset &&
5917 				clone_data_offset + ext_len > data_offset) {
5918 				u64 extent_offset;
5919 
5920 				extent_offset = data_offset - clone_data_offset;
5921 				ext_len -= extent_offset;
5922 				clone_data_offset += extent_offset;
5923 				clone_root->offset += extent_offset;
5924 			}
5925 		}
5926 
5927 		clone_len = min_t(u64, ext_len, len);
5928 
5929 		if (btrfs_file_extent_disk_bytenr(leaf, ei) == disk_byte &&
5930 		    clone_data_offset == data_offset) {
5931 			const u64 src_end = clone_root->offset + clone_len;
5932 			const u64 sectorsize = SZ_64K;
5933 
5934 			/*
5935 			 * We can't clone the last block, when its size is not
5936 			 * sector size aligned, into the middle of a file. If we
5937 			 * do so, the receiver will get a failure (-EINVAL) when
5938 			 * trying to clone or will silently corrupt the data in
5939 			 * the destination file if it's on a kernel without the
5940 			 * fix introduced by commit ac765f83f1397646
5941 			 * ("Btrfs: fix data corruption due to cloning of eof
5942 			 * block).
5943 			 *
5944 			 * So issue a clone of the aligned down range plus a
5945 			 * regular write for the eof block, if we hit that case.
5946 			 *
5947 			 * Also, we use the maximum possible sector size, 64K,
5948 			 * because we don't know what's the sector size of the
5949 			 * filesystem that receives the stream, so we have to
5950 			 * assume the largest possible sector size.
5951 			 */
5952 			if (src_end == clone_src_i_size &&
5953 			    !IS_ALIGNED(src_end, sectorsize) &&
5954 			    offset + clone_len < sctx->cur_inode_size) {
5955 				u64 slen;
5956 
5957 				slen = ALIGN_DOWN(src_end - clone_root->offset,
5958 						  sectorsize);
5959 				if (slen > 0) {
5960 					ret = send_clone(sctx, offset, slen,
5961 							 clone_root);
5962 					if (ret < 0)
5963 						return ret;
5964 				}
5965 				ret = send_extent_data(sctx, dst_path,
5966 						       offset + slen,
5967 						       clone_len - slen);
5968 			} else {
5969 				ret = send_clone(sctx, offset, clone_len,
5970 						 clone_root);
5971 			}
5972 		} else if (crossed_src_i_size && clone_len < len) {
5973 			/*
5974 			 * If we are at i_size of the clone source inode and we
5975 			 * can not clone from it, terminate the loop. This is
5976 			 * to avoid sending two write operations, one with a
5977 			 * length matching clone_len and the final one after
5978 			 * this loop with a length of len - clone_len.
5979 			 *
5980 			 * When using encoded writes (BTRFS_SEND_FLAG_COMPRESSED
5981 			 * was passed to the send ioctl), this helps avoid
5982 			 * sending an encoded write for an offset that is not
5983 			 * sector size aligned, in case the i_size of the source
5984 			 * inode is not sector size aligned. That will make the
5985 			 * receiver fallback to decompression of the data and
5986 			 * writing it using regular buffered IO, therefore while
5987 			 * not incorrect, it's not optimal due decompression and
5988 			 * possible re-compression at the receiver.
5989 			 */
5990 			break;
5991 		} else {
5992 			ret = send_extent_data(sctx, dst_path, offset,
5993 					       clone_len);
5994 		}
5995 
5996 		if (ret < 0)
5997 			return ret;
5998 
5999 		len -= clone_len;
6000 		if (len == 0)
6001 			break;
6002 		offset += clone_len;
6003 		clone_root->offset += clone_len;
6004 
6005 		/*
6006 		 * If we are cloning from the file we are currently processing,
6007 		 * and using the send root as the clone root, we must stop once
6008 		 * the current clone offset reaches the current eof of the file
6009 		 * at the receiver, otherwise we would issue an invalid clone
6010 		 * operation (source range going beyond eof) and cause the
6011 		 * receiver to fail. So if we reach the current eof, bail out
6012 		 * and fallback to a regular write.
6013 		 */
6014 		if (clone_root->root == sctx->send_root &&
6015 		    clone_root->ino == sctx->cur_ino &&
6016 		    clone_root->offset >= sctx->cur_inode_next_write_offset)
6017 			break;
6018 
6019 		data_offset += clone_len;
6020 next:
6021 		path->slots[0]++;
6022 	}
6023 
6024 	if (len > 0)
6025 		ret = send_extent_data(sctx, dst_path, offset, len);
6026 	else
6027 		ret = 0;
6028 	return ret;
6029 }
6030 
send_write_or_clone(struct send_ctx * sctx,struct btrfs_path * path,struct btrfs_key * key,struct clone_root * clone_root)6031 static int send_write_or_clone(struct send_ctx *sctx,
6032 			       struct btrfs_path *path,
6033 			       struct btrfs_key *key,
6034 			       struct clone_root *clone_root)
6035 {
6036 	int ret = 0;
6037 	u64 offset = key->offset;
6038 	u64 end;
6039 	u64 bs = sctx->send_root->fs_info->sectorsize;
6040 	struct btrfs_file_extent_item *ei;
6041 	u64 disk_byte;
6042 	u64 data_offset;
6043 	u64 num_bytes;
6044 	struct btrfs_inode_info info = { 0 };
6045 
6046 	end = min_t(u64, btrfs_file_extent_end(path), sctx->cur_inode_size);
6047 	if (offset >= end)
6048 		return 0;
6049 
6050 	num_bytes = end - offset;
6051 
6052 	if (!clone_root)
6053 		goto write_data;
6054 
6055 	if (IS_ALIGNED(end, bs))
6056 		goto clone_data;
6057 
6058 	/*
6059 	 * If the extent end is not aligned, we can clone if the extent ends at
6060 	 * the i_size of the inode and the clone range ends at the i_size of the
6061 	 * source inode, otherwise the clone operation fails with -EINVAL.
6062 	 */
6063 	if (end != sctx->cur_inode_size)
6064 		goto write_data;
6065 
6066 	ret = get_inode_info(clone_root->root, clone_root->ino, &info);
6067 	if (ret < 0)
6068 		return ret;
6069 
6070 	if (clone_root->offset + num_bytes == info.size) {
6071 		/*
6072 		 * The final size of our file matches the end offset, but it may
6073 		 * be that its current size is larger, so we have to truncate it
6074 		 * to any value between the start offset of the range and the
6075 		 * final i_size, otherwise the clone operation is invalid
6076 		 * because it's unaligned and it ends before the current EOF.
6077 		 * We do this truncate to the final i_size when we finish
6078 		 * processing the inode, but it's too late by then. And here we
6079 		 * truncate to the start offset of the range because it's always
6080 		 * sector size aligned while if it were the final i_size it
6081 		 * would result in dirtying part of a page, filling part of a
6082 		 * page with zeroes and then having the clone operation at the
6083 		 * receiver trigger IO and wait for it due to the dirty page.
6084 		 */
6085 		if (sctx->parent_root != NULL) {
6086 			ret = send_truncate(sctx, sctx->cur_ino,
6087 					    sctx->cur_inode_gen, offset);
6088 			if (ret < 0)
6089 				return ret;
6090 		}
6091 		goto clone_data;
6092 	}
6093 
6094 write_data:
6095 	ret = send_extent_data(sctx, path, offset, num_bytes);
6096 	sctx->cur_inode_next_write_offset = end;
6097 	return ret;
6098 
6099 clone_data:
6100 	ei = btrfs_item_ptr(path->nodes[0], path->slots[0],
6101 			    struct btrfs_file_extent_item);
6102 	disk_byte = btrfs_file_extent_disk_bytenr(path->nodes[0], ei);
6103 	data_offset = btrfs_file_extent_offset(path->nodes[0], ei);
6104 	ret = clone_range(sctx, path, clone_root, disk_byte, data_offset, offset,
6105 			  num_bytes);
6106 	sctx->cur_inode_next_write_offset = end;
6107 	return ret;
6108 }
6109 
is_extent_unchanged(struct send_ctx * sctx,struct btrfs_path * left_path,struct btrfs_key * ekey)6110 static int is_extent_unchanged(struct send_ctx *sctx,
6111 			       struct btrfs_path *left_path,
6112 			       struct btrfs_key *ekey)
6113 {
6114 	int ret = 0;
6115 	struct btrfs_key key;
6116 	BTRFS_PATH_AUTO_FREE(path);
6117 	struct extent_buffer *eb;
6118 	int slot;
6119 	struct btrfs_key found_key;
6120 	struct btrfs_file_extent_item *ei;
6121 	u64 left_disknr;
6122 	u64 right_disknr;
6123 	u64 left_offset;
6124 	u64 right_offset;
6125 	u64 left_offset_fixed;
6126 	u64 left_len;
6127 	u64 right_len;
6128 	u64 left_gen;
6129 	u64 right_gen;
6130 	u8 left_type;
6131 	u8 right_type;
6132 
6133 	path = alloc_path_for_send();
6134 	if (!path)
6135 		return -ENOMEM;
6136 
6137 	eb = left_path->nodes[0];
6138 	slot = left_path->slots[0];
6139 	ei = btrfs_item_ptr(eb, slot, struct btrfs_file_extent_item);
6140 	left_type = btrfs_file_extent_type(eb, ei);
6141 
6142 	if (left_type != BTRFS_FILE_EXTENT_REG)
6143 		return 0;
6144 
6145 	left_disknr = btrfs_file_extent_disk_bytenr(eb, ei);
6146 	left_len = btrfs_file_extent_num_bytes(eb, ei);
6147 	left_offset = btrfs_file_extent_offset(eb, ei);
6148 	left_gen = btrfs_file_extent_generation(eb, ei);
6149 
6150 	/*
6151 	 * Following comments will refer to these graphics. L is the left
6152 	 * extents which we are checking at the moment. 1-8 are the right
6153 	 * extents that we iterate.
6154 	 *
6155 	 *       |-----L-----|
6156 	 * |-1-|-2a-|-3-|-4-|-5-|-6-|
6157 	 *
6158 	 *       |-----L-----|
6159 	 * |--1--|-2b-|...(same as above)
6160 	 *
6161 	 * Alternative situation. Happens on files where extents got split.
6162 	 *       |-----L-----|
6163 	 * |-----------7-----------|-6-|
6164 	 *
6165 	 * Alternative situation. Happens on files which got larger.
6166 	 *       |-----L-----|
6167 	 * |-8-|
6168 	 * Nothing follows after 8.
6169 	 */
6170 
6171 	key.objectid = ekey->objectid;
6172 	key.type = BTRFS_EXTENT_DATA_KEY;
6173 	key.offset = ekey->offset;
6174 	ret = btrfs_search_slot_for_read(sctx->parent_root, &key, path, 0, 0);
6175 	if (ret < 0)
6176 		return ret;
6177 	if (ret)
6178 		return 0;
6179 
6180 	/*
6181 	 * Handle special case where the right side has no extents at all.
6182 	 */
6183 	eb = path->nodes[0];
6184 	slot = path->slots[0];
6185 	btrfs_item_key_to_cpu(eb, &found_key, slot);
6186 	if (found_key.objectid != key.objectid ||
6187 	    found_key.type != key.type)
6188 		/* If we're a hole then just pretend nothing changed */
6189 		return (left_disknr ? 0 : 1);
6190 
6191 	/*
6192 	 * We're now on 2a, 2b or 7.
6193 	 */
6194 	key = found_key;
6195 	while (key.offset < ekey->offset + left_len) {
6196 		ei = btrfs_item_ptr(eb, slot, struct btrfs_file_extent_item);
6197 		right_type = btrfs_file_extent_type(eb, ei);
6198 		if (right_type != BTRFS_FILE_EXTENT_REG &&
6199 		    right_type != BTRFS_FILE_EXTENT_INLINE)
6200 			return 0;
6201 
6202 		if (right_type == BTRFS_FILE_EXTENT_INLINE) {
6203 			right_len = btrfs_file_extent_ram_bytes(eb, ei);
6204 			right_len = PAGE_ALIGN(right_len);
6205 		} else {
6206 			right_len = btrfs_file_extent_num_bytes(eb, ei);
6207 		}
6208 
6209 		/*
6210 		 * Are we at extent 8? If yes, we know the extent is changed.
6211 		 * This may only happen on the first iteration.
6212 		 */
6213 		if (found_key.offset + right_len <= ekey->offset)
6214 			/* If we're a hole just pretend nothing changed */
6215 			return (left_disknr ? 0 : 1);
6216 
6217 		/*
6218 		 * We just wanted to see if when we have an inline extent, what
6219 		 * follows it is a regular extent (wanted to check the above
6220 		 * condition for inline extents too). This should normally not
6221 		 * happen but it's possible for example when we have an inline
6222 		 * compressed extent representing data with a size matching
6223 		 * the page size (currently the same as sector size).
6224 		 */
6225 		if (right_type == BTRFS_FILE_EXTENT_INLINE)
6226 			return 0;
6227 
6228 		right_disknr = btrfs_file_extent_disk_bytenr(eb, ei);
6229 		right_offset = btrfs_file_extent_offset(eb, ei);
6230 		right_gen = btrfs_file_extent_generation(eb, ei);
6231 
6232 		left_offset_fixed = left_offset;
6233 		if (key.offset < ekey->offset) {
6234 			/* Fix the right offset for 2a and 7. */
6235 			right_offset += ekey->offset - key.offset;
6236 		} else {
6237 			/* Fix the left offset for all behind 2a and 2b */
6238 			left_offset_fixed += key.offset - ekey->offset;
6239 		}
6240 
6241 		/*
6242 		 * Check if we have the same extent.
6243 		 */
6244 		if (left_disknr != right_disknr ||
6245 		    left_offset_fixed != right_offset ||
6246 		    left_gen != right_gen)
6247 			return 0;
6248 
6249 		/*
6250 		 * Go to the next extent.
6251 		 */
6252 		ret = btrfs_next_item(sctx->parent_root, path);
6253 		if (ret < 0)
6254 			return ret;
6255 		if (!ret) {
6256 			eb = path->nodes[0];
6257 			slot = path->slots[0];
6258 			btrfs_item_key_to_cpu(eb, &found_key, slot);
6259 		}
6260 		if (ret || found_key.objectid != key.objectid ||
6261 		    found_key.type != key.type) {
6262 			key.offset += right_len;
6263 			break;
6264 		}
6265 		if (found_key.offset != key.offset + right_len)
6266 			return 0;
6267 
6268 		key = found_key;
6269 	}
6270 
6271 	/*
6272 	 * We're now behind the left extent (treat as unchanged) or at the end
6273 	 * of the right side (treat as changed).
6274 	 */
6275 	if (key.offset >= ekey->offset + left_len)
6276 		ret = 1;
6277 	else
6278 		ret = 0;
6279 
6280 	return ret;
6281 }
6282 
get_last_extent(struct send_ctx * sctx,u64 offset)6283 static int get_last_extent(struct send_ctx *sctx, u64 offset)
6284 {
6285 	BTRFS_PATH_AUTO_FREE(path);
6286 	struct btrfs_root *root = sctx->send_root;
6287 	struct btrfs_key key;
6288 	int ret;
6289 
6290 	path = alloc_path_for_send();
6291 	if (!path)
6292 		return -ENOMEM;
6293 
6294 	sctx->cur_inode_last_extent = 0;
6295 
6296 	key.objectid = sctx->cur_ino;
6297 	key.type = BTRFS_EXTENT_DATA_KEY;
6298 	key.offset = offset;
6299 	ret = btrfs_search_slot_for_read(root, &key, path, 0, 1);
6300 	if (ret < 0)
6301 		return ret;
6302 	ret = 0;
6303 	btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
6304 	if (key.objectid != sctx->cur_ino || key.type != BTRFS_EXTENT_DATA_KEY)
6305 		return ret;
6306 
6307 	sctx->cur_inode_last_extent = btrfs_file_extent_end(path);
6308 	return ret;
6309 }
6310 
range_is_hole_in_parent(struct send_ctx * sctx,const u64 start,const u64 end)6311 static int range_is_hole_in_parent(struct send_ctx *sctx,
6312 				   const u64 start,
6313 				   const u64 end)
6314 {
6315 	BTRFS_PATH_AUTO_FREE(path);
6316 	struct btrfs_key key;
6317 	struct btrfs_root *root = sctx->parent_root;
6318 	u64 search_start = start;
6319 	int ret;
6320 
6321 	path = alloc_path_for_send();
6322 	if (!path)
6323 		return -ENOMEM;
6324 
6325 	key.objectid = sctx->cur_ino;
6326 	key.type = BTRFS_EXTENT_DATA_KEY;
6327 	key.offset = search_start;
6328 	ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
6329 	if (ret < 0)
6330 		return ret;
6331 	if (ret > 0 && path->slots[0] > 0)
6332 		path->slots[0]--;
6333 
6334 	while (search_start < end) {
6335 		struct extent_buffer *leaf = path->nodes[0];
6336 		int slot = path->slots[0];
6337 		struct btrfs_file_extent_item *fi;
6338 		u64 extent_end;
6339 
6340 		if (slot >= btrfs_header_nritems(leaf)) {
6341 			ret = btrfs_next_leaf(root, path);
6342 			if (ret < 0)
6343 				return ret;
6344 			if (ret > 0)
6345 				break;
6346 			continue;
6347 		}
6348 
6349 		btrfs_item_key_to_cpu(leaf, &key, slot);
6350 		if (key.objectid < sctx->cur_ino ||
6351 		    key.type < BTRFS_EXTENT_DATA_KEY)
6352 			goto next;
6353 		if (key.objectid > sctx->cur_ino ||
6354 		    key.type > BTRFS_EXTENT_DATA_KEY ||
6355 		    key.offset >= end)
6356 			break;
6357 
6358 		fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
6359 		extent_end = btrfs_file_extent_end(path);
6360 		if (extent_end <= start)
6361 			goto next;
6362 		if (btrfs_file_extent_disk_bytenr(leaf, fi) == 0) {
6363 			search_start = extent_end;
6364 			goto next;
6365 		}
6366 		return 0;
6367 next:
6368 		path->slots[0]++;
6369 	}
6370 	return 1;
6371 }
6372 
maybe_send_hole(struct send_ctx * sctx,struct btrfs_path * path,struct btrfs_key * key)6373 static int maybe_send_hole(struct send_ctx *sctx, struct btrfs_path *path,
6374 			   struct btrfs_key *key)
6375 {
6376 	int ret = 0;
6377 
6378 	if (sctx->cur_ino != key->objectid || !need_send_hole(sctx))
6379 		return 0;
6380 
6381 	/*
6382 	 * Get last extent's end offset (exclusive) if we haven't determined it
6383 	 * yet (we're processing the first file extent item that is new), or if
6384 	 * we're at the first slot of a leaf and the last extent's end is less
6385 	 * than the current extent's offset, because we might have skipped
6386 	 * entire leaves that contained only file extent items for our current
6387 	 * inode. These leaves have a generation number smaller (older) than the
6388 	 * one in the current leaf and the leaf our last extent came from, and
6389 	 * are located between these 2 leaves.
6390 	 */
6391 	if ((sctx->cur_inode_last_extent == (u64)-1) ||
6392 	    (path->slots[0] == 0 && sctx->cur_inode_last_extent < key->offset)) {
6393 		ret = get_last_extent(sctx, key->offset - 1);
6394 		if (ret)
6395 			return ret;
6396 	}
6397 
6398 	if (sctx->cur_inode_last_extent < key->offset) {
6399 		ret = range_is_hole_in_parent(sctx,
6400 					      sctx->cur_inode_last_extent,
6401 					      key->offset);
6402 		if (ret < 0)
6403 			return ret;
6404 		else if (ret == 0)
6405 			ret = send_hole(sctx, key->offset);
6406 		else
6407 			ret = 0;
6408 	}
6409 	sctx->cur_inode_last_extent = btrfs_file_extent_end(path);
6410 	return ret;
6411 }
6412 
process_extent(struct send_ctx * sctx,struct btrfs_path * path,struct btrfs_key * key)6413 static int process_extent(struct send_ctx *sctx,
6414 			  struct btrfs_path *path,
6415 			  struct btrfs_key *key)
6416 {
6417 	struct clone_root *found_clone = NULL;
6418 	int ret = 0;
6419 
6420 	if (S_ISLNK(sctx->cur_inode_mode))
6421 		return 0;
6422 
6423 	if (sctx->parent_root && !sctx->cur_inode_new) {
6424 		ret = is_extent_unchanged(sctx, path, key);
6425 		if (ret < 0)
6426 			goto out;
6427 		if (ret) {
6428 			ret = 0;
6429 			goto out_hole;
6430 		}
6431 	} else {
6432 		struct btrfs_file_extent_item *ei;
6433 		u8 type;
6434 
6435 		ei = btrfs_item_ptr(path->nodes[0], path->slots[0],
6436 				    struct btrfs_file_extent_item);
6437 		type = btrfs_file_extent_type(path->nodes[0], ei);
6438 		if (type == BTRFS_FILE_EXTENT_PREALLOC ||
6439 		    type == BTRFS_FILE_EXTENT_REG) {
6440 			/*
6441 			 * The send spec does not have a prealloc command yet,
6442 			 * so just leave a hole for prealloc'ed extents until
6443 			 * we have enough commands queued up to justify rev'ing
6444 			 * the send spec.
6445 			 */
6446 			if (type == BTRFS_FILE_EXTENT_PREALLOC) {
6447 				ret = 0;
6448 				goto out;
6449 			}
6450 
6451 			/* Have a hole, just skip it. */
6452 			if (btrfs_file_extent_disk_bytenr(path->nodes[0], ei) == 0) {
6453 				ret = 0;
6454 				goto out;
6455 			}
6456 		}
6457 	}
6458 
6459 	ret = find_extent_clone(sctx, path, key->objectid, key->offset,
6460 			sctx->cur_inode_size, &found_clone);
6461 	if (ret != -ENOENT && ret < 0)
6462 		goto out;
6463 
6464 	ret = send_write_or_clone(sctx, path, key, found_clone);
6465 	if (ret)
6466 		goto out;
6467 out_hole:
6468 	ret = maybe_send_hole(sctx, path, key);
6469 out:
6470 	return ret;
6471 }
6472 
process_all_extents(struct send_ctx * sctx)6473 static int process_all_extents(struct send_ctx *sctx)
6474 {
6475 	int ret = 0;
6476 	int iter_ret = 0;
6477 	struct btrfs_root *root;
6478 	BTRFS_PATH_AUTO_FREE(path);
6479 	struct btrfs_key key;
6480 	struct btrfs_key found_key;
6481 
6482 	root = sctx->send_root;
6483 	path = alloc_path_for_send();
6484 	if (!path)
6485 		return -ENOMEM;
6486 
6487 	key.objectid = sctx->cmp_key->objectid;
6488 	key.type = BTRFS_EXTENT_DATA_KEY;
6489 	key.offset = 0;
6490 	btrfs_for_each_slot(root, &key, &found_key, path, iter_ret) {
6491 		if (found_key.objectid != key.objectid ||
6492 		    found_key.type != key.type) {
6493 			ret = 0;
6494 			break;
6495 		}
6496 
6497 		ret = process_extent(sctx, path, &found_key);
6498 		if (ret < 0)
6499 			break;
6500 	}
6501 	/* Catch error found during iteration */
6502 	if (iter_ret < 0)
6503 		ret = iter_ret;
6504 
6505 	return ret;
6506 }
6507 
process_recorded_refs_if_needed(struct send_ctx * sctx,bool at_end,int * pending_move,int * refs_processed)6508 static int process_recorded_refs_if_needed(struct send_ctx *sctx, bool at_end,
6509 					   int *pending_move,
6510 					   int *refs_processed)
6511 {
6512 	int ret = 0;
6513 
6514 	if (sctx->cur_ino == 0)
6515 		goto out;
6516 	if (!at_end && sctx->cur_ino == sctx->cmp_key->objectid &&
6517 	    sctx->cmp_key->type <= BTRFS_INODE_EXTREF_KEY)
6518 		goto out;
6519 	if (list_empty(&sctx->new_refs) && list_empty(&sctx->deleted_refs))
6520 		goto out;
6521 
6522 	ret = process_recorded_refs(sctx, pending_move);
6523 	if (ret < 0)
6524 		goto out;
6525 
6526 	*refs_processed = 1;
6527 out:
6528 	return ret;
6529 }
6530 
finish_inode_if_needed(struct send_ctx * sctx,bool at_end)6531 static int finish_inode_if_needed(struct send_ctx *sctx, bool at_end)
6532 {
6533 	int ret = 0;
6534 	struct btrfs_inode_info info;
6535 	u64 left_mode;
6536 	u64 left_uid;
6537 	u64 left_gid;
6538 	u64 left_fileattr;
6539 	u64 right_mode;
6540 	u64 right_uid;
6541 	u64 right_gid;
6542 	u64 right_fileattr;
6543 	int need_chmod = 0;
6544 	int need_chown = 0;
6545 	bool need_fileattr = false;
6546 	int need_truncate = 1;
6547 	int pending_move = 0;
6548 	int refs_processed = 0;
6549 
6550 	if (sctx->ignore_cur_inode)
6551 		return 0;
6552 
6553 	ret = process_recorded_refs_if_needed(sctx, at_end, &pending_move,
6554 					      &refs_processed);
6555 	if (ret < 0)
6556 		goto out;
6557 
6558 	/*
6559 	 * We have processed the refs and thus need to advance send_progress.
6560 	 * Now, calls to get_cur_xxx will take the updated refs of the current
6561 	 * inode into account.
6562 	 *
6563 	 * On the other hand, if our current inode is a directory and couldn't
6564 	 * be moved/renamed because its parent was renamed/moved too and it has
6565 	 * a higher inode number, we can only move/rename our current inode
6566 	 * after we moved/renamed its parent. Therefore in this case operate on
6567 	 * the old path (pre move/rename) of our current inode, and the
6568 	 * move/rename will be performed later.
6569 	 */
6570 	if (refs_processed && !pending_move)
6571 		sctx->send_progress = sctx->cur_ino + 1;
6572 
6573 	if (sctx->cur_ino == 0 || sctx->cur_inode_deleted)
6574 		goto out;
6575 	if (!at_end && sctx->cmp_key->objectid == sctx->cur_ino)
6576 		goto out;
6577 	ret = get_inode_info(sctx->send_root, sctx->cur_ino, &info);
6578 	if (ret < 0)
6579 		goto out;
6580 	left_mode = info.mode;
6581 	left_uid = info.uid;
6582 	left_gid = info.gid;
6583 	left_fileattr = info.fileattr;
6584 
6585 	if (!sctx->parent_root || sctx->cur_inode_new) {
6586 		need_chown = 1;
6587 		if (!S_ISLNK(sctx->cur_inode_mode))
6588 			need_chmod = 1;
6589 		if (sctx->cur_inode_next_write_offset == sctx->cur_inode_size)
6590 			need_truncate = 0;
6591 	} else {
6592 		u64 old_size;
6593 
6594 		ret = get_inode_info(sctx->parent_root, sctx->cur_ino, &info);
6595 		if (ret < 0)
6596 			goto out;
6597 		old_size = info.size;
6598 		right_mode = info.mode;
6599 		right_uid = info.uid;
6600 		right_gid = info.gid;
6601 		right_fileattr = info.fileattr;
6602 
6603 		if (left_uid != right_uid || left_gid != right_gid)
6604 			need_chown = 1;
6605 		if (!S_ISLNK(sctx->cur_inode_mode) && left_mode != right_mode)
6606 			need_chmod = 1;
6607 		if (!S_ISLNK(sctx->cur_inode_mode) && left_fileattr != right_fileattr)
6608 			need_fileattr = true;
6609 		if ((old_size == sctx->cur_inode_size) ||
6610 		    (sctx->cur_inode_size > old_size &&
6611 		     sctx->cur_inode_next_write_offset == sctx->cur_inode_size))
6612 			need_truncate = 0;
6613 	}
6614 
6615 	if (S_ISREG(sctx->cur_inode_mode)) {
6616 		if (need_send_hole(sctx)) {
6617 			if (sctx->cur_inode_last_extent == (u64)-1 ||
6618 			    sctx->cur_inode_last_extent <
6619 			    sctx->cur_inode_size) {
6620 				ret = get_last_extent(sctx, (u64)-1);
6621 				if (ret)
6622 					goto out;
6623 			}
6624 			if (sctx->cur_inode_last_extent < sctx->cur_inode_size) {
6625 				ret = range_is_hole_in_parent(sctx,
6626 						      sctx->cur_inode_last_extent,
6627 						      sctx->cur_inode_size);
6628 				if (ret < 0) {
6629 					goto out;
6630 				} else if (ret == 0) {
6631 					ret = send_hole(sctx, sctx->cur_inode_size);
6632 					if (ret < 0)
6633 						goto out;
6634 				} else {
6635 					/* Range is already a hole, skip. */
6636 					ret = 0;
6637 				}
6638 			}
6639 		}
6640 		if (need_truncate) {
6641 			ret = send_truncate(sctx, sctx->cur_ino,
6642 					    sctx->cur_inode_gen,
6643 					    sctx->cur_inode_size);
6644 			if (ret < 0)
6645 				goto out;
6646 		}
6647 	}
6648 
6649 	if (need_chown) {
6650 		ret = send_chown(sctx, sctx->cur_ino, sctx->cur_inode_gen,
6651 				left_uid, left_gid);
6652 		if (ret < 0)
6653 			goto out;
6654 	}
6655 	if (need_chmod) {
6656 		ret = send_chmod(sctx, sctx->cur_ino, sctx->cur_inode_gen,
6657 				left_mode);
6658 		if (ret < 0)
6659 			goto out;
6660 	}
6661 	if (need_fileattr) {
6662 		ret = send_fileattr(sctx, sctx->cur_ino, sctx->cur_inode_gen,
6663 				    left_fileattr);
6664 		if (ret < 0)
6665 			goto out;
6666 	}
6667 
6668 	if (proto_cmd_ok(sctx, BTRFS_SEND_C_ENABLE_VERITY)
6669 	    && sctx->cur_inode_needs_verity) {
6670 		ret = process_verity(sctx);
6671 		if (ret < 0)
6672 			goto out;
6673 	}
6674 
6675 	ret = send_capabilities(sctx);
6676 	if (ret < 0)
6677 		goto out;
6678 
6679 	/*
6680 	 * If other directory inodes depended on our current directory
6681 	 * inode's move/rename, now do their move/rename operations.
6682 	 */
6683 	if (!is_waiting_for_move(sctx, sctx->cur_ino)) {
6684 		ret = apply_children_dir_moves(sctx);
6685 		if (ret)
6686 			goto out;
6687 		/*
6688 		 * Need to send that every time, no matter if it actually
6689 		 * changed between the two trees as we have done changes to
6690 		 * the inode before. If our inode is a directory and it's
6691 		 * waiting to be moved/renamed, we will send its utimes when
6692 		 * it's moved/renamed, therefore we don't need to do it here.
6693 		 */
6694 		sctx->send_progress = sctx->cur_ino + 1;
6695 
6696 		/*
6697 		 * If the current inode is a non-empty directory, delay issuing
6698 		 * the utimes command for it, as it's very likely we have inodes
6699 		 * with an higher number inside it. We want to issue the utimes
6700 		 * command only after adding all dentries to it.
6701 		 */
6702 		if (S_ISDIR(sctx->cur_inode_mode) && sctx->cur_inode_size > 0)
6703 			ret = cache_dir_utimes(sctx, sctx->cur_ino, sctx->cur_inode_gen);
6704 		else
6705 			ret = send_utimes(sctx, sctx->cur_ino, sctx->cur_inode_gen);
6706 
6707 		if (ret < 0)
6708 			goto out;
6709 	}
6710 
6711 out:
6712 	if (!ret)
6713 		ret = trim_dir_utimes_cache(sctx);
6714 
6715 	return ret;
6716 }
6717 
close_current_inode(struct send_ctx * sctx)6718 static void close_current_inode(struct send_ctx *sctx)
6719 {
6720 	u64 i_size;
6721 
6722 	if (sctx->cur_inode == NULL)
6723 		return;
6724 
6725 	i_size = i_size_read(sctx->cur_inode);
6726 
6727 	/*
6728 	 * If we are doing an incremental send, we may have extents between the
6729 	 * last processed extent and the i_size that have not been processed
6730 	 * because they haven't changed but we may have read some of their pages
6731 	 * through readahead, see the comments at send_extent_data().
6732 	 */
6733 	if (sctx->clean_page_cache && sctx->page_cache_clear_start < i_size)
6734 		truncate_inode_pages_range(&sctx->cur_inode->i_data,
6735 					   sctx->page_cache_clear_start,
6736 					   round_up(i_size, PAGE_SIZE) - 1);
6737 
6738 	iput(sctx->cur_inode);
6739 	sctx->cur_inode = NULL;
6740 }
6741 
changed_inode(struct send_ctx * sctx,enum btrfs_compare_tree_result result)6742 static int changed_inode(struct send_ctx *sctx,
6743 			 enum btrfs_compare_tree_result result)
6744 {
6745 	int ret = 0;
6746 	struct btrfs_key *key = sctx->cmp_key;
6747 	struct btrfs_inode_item *left_ii = NULL;
6748 	struct btrfs_inode_item *right_ii = NULL;
6749 	u64 left_gen = 0;
6750 	u64 right_gen = 0;
6751 
6752 	close_current_inode(sctx);
6753 
6754 	sctx->cur_ino = key->objectid;
6755 	sctx->cur_inode_new_gen = false;
6756 	sctx->cur_inode_last_extent = (u64)-1;
6757 	sctx->cur_inode_next_write_offset = 0;
6758 	sctx->ignore_cur_inode = false;
6759 	fs_path_reset(&sctx->cur_inode_path);
6760 
6761 	/*
6762 	 * Set send_progress to current inode. This will tell all get_cur_xxx
6763 	 * functions that the current inode's refs are not updated yet. Later,
6764 	 * when process_recorded_refs is finished, it is set to cur_ino + 1.
6765 	 */
6766 	sctx->send_progress = sctx->cur_ino;
6767 
6768 	if (result == BTRFS_COMPARE_TREE_NEW ||
6769 	    result == BTRFS_COMPARE_TREE_CHANGED) {
6770 		left_ii = btrfs_item_ptr(sctx->left_path->nodes[0],
6771 				sctx->left_path->slots[0],
6772 				struct btrfs_inode_item);
6773 		left_gen = btrfs_inode_generation(sctx->left_path->nodes[0],
6774 				left_ii);
6775 	} else {
6776 		right_ii = btrfs_item_ptr(sctx->right_path->nodes[0],
6777 				sctx->right_path->slots[0],
6778 				struct btrfs_inode_item);
6779 		right_gen = btrfs_inode_generation(sctx->right_path->nodes[0],
6780 				right_ii);
6781 	}
6782 	if (result == BTRFS_COMPARE_TREE_CHANGED) {
6783 		right_ii = btrfs_item_ptr(sctx->right_path->nodes[0],
6784 				sctx->right_path->slots[0],
6785 				struct btrfs_inode_item);
6786 
6787 		right_gen = btrfs_inode_generation(sctx->right_path->nodes[0],
6788 				right_ii);
6789 
6790 		/*
6791 		 * The cur_ino = root dir case is special here. We can't treat
6792 		 * the inode as deleted+reused because it would generate a
6793 		 * stream that tries to delete/mkdir the root dir.
6794 		 */
6795 		if (left_gen != right_gen &&
6796 		    sctx->cur_ino != BTRFS_FIRST_FREE_OBJECTID)
6797 			sctx->cur_inode_new_gen = true;
6798 	}
6799 
6800 	/*
6801 	 * Normally we do not find inodes with a link count of zero (orphans)
6802 	 * because the most common case is to create a snapshot and use it
6803 	 * for a send operation. However other less common use cases involve
6804 	 * using a subvolume and send it after turning it to RO mode just
6805 	 * after deleting all hard links of a file while holding an open
6806 	 * file descriptor against it or turning a RO snapshot into RW mode,
6807 	 * keep an open file descriptor against a file, delete it and then
6808 	 * turn the snapshot back to RO mode before using it for a send
6809 	 * operation. The former is what the receiver operation does.
6810 	 * Therefore, if we want to send these snapshots soon after they're
6811 	 * received, we need to handle orphan inodes as well. Moreover, orphans
6812 	 * can appear not only in the send snapshot but also in the parent
6813 	 * snapshot. Here are several cases:
6814 	 *
6815 	 * Case 1: BTRFS_COMPARE_TREE_NEW
6816 	 *       |  send snapshot  | action
6817 	 * --------------------------------
6818 	 * nlink |        0        | ignore
6819 	 *
6820 	 * Case 2: BTRFS_COMPARE_TREE_DELETED
6821 	 *       | parent snapshot | action
6822 	 * ----------------------------------
6823 	 * nlink |        0        | as usual
6824 	 * Note: No unlinks will be sent because there're no paths for it.
6825 	 *
6826 	 * Case 3: BTRFS_COMPARE_TREE_CHANGED
6827 	 *           |       | parent snapshot | send snapshot | action
6828 	 * -----------------------------------------------------------------------
6829 	 * subcase 1 | nlink |        0        |       0       | ignore
6830 	 * subcase 2 | nlink |       >0        |       0       | new_gen(deletion)
6831 	 * subcase 3 | nlink |        0        |      >0       | new_gen(creation)
6832 	 *
6833 	 */
6834 	if (result == BTRFS_COMPARE_TREE_NEW) {
6835 		if (btrfs_inode_nlink(sctx->left_path->nodes[0], left_ii) == 0) {
6836 			sctx->ignore_cur_inode = true;
6837 			goto out;
6838 		}
6839 		sctx->cur_inode_gen = left_gen;
6840 		sctx->cur_inode_new = true;
6841 		sctx->cur_inode_deleted = false;
6842 		sctx->cur_inode_size = btrfs_inode_size(
6843 				sctx->left_path->nodes[0], left_ii);
6844 		sctx->cur_inode_mode = btrfs_inode_mode(
6845 				sctx->left_path->nodes[0], left_ii);
6846 		sctx->cur_inode_rdev = btrfs_inode_rdev(
6847 				sctx->left_path->nodes[0], left_ii);
6848 		if (sctx->cur_ino != BTRFS_FIRST_FREE_OBJECTID)
6849 			ret = send_create_inode_if_needed(sctx);
6850 	} else if (result == BTRFS_COMPARE_TREE_DELETED) {
6851 		sctx->cur_inode_gen = right_gen;
6852 		sctx->cur_inode_new = false;
6853 		sctx->cur_inode_deleted = true;
6854 		sctx->cur_inode_size = btrfs_inode_size(
6855 				sctx->right_path->nodes[0], right_ii);
6856 		sctx->cur_inode_mode = btrfs_inode_mode(
6857 				sctx->right_path->nodes[0], right_ii);
6858 	} else if (result == BTRFS_COMPARE_TREE_CHANGED) {
6859 		u32 new_nlinks, old_nlinks;
6860 
6861 		new_nlinks = btrfs_inode_nlink(sctx->left_path->nodes[0], left_ii);
6862 		old_nlinks = btrfs_inode_nlink(sctx->right_path->nodes[0], right_ii);
6863 		if (new_nlinks == 0 && old_nlinks == 0) {
6864 			sctx->ignore_cur_inode = true;
6865 			goto out;
6866 		} else if (new_nlinks == 0 || old_nlinks == 0) {
6867 			sctx->cur_inode_new_gen = 1;
6868 		}
6869 		/*
6870 		 * We need to do some special handling in case the inode was
6871 		 * reported as changed with a changed generation number. This
6872 		 * means that the original inode was deleted and new inode
6873 		 * reused the same inum. So we have to treat the old inode as
6874 		 * deleted and the new one as new.
6875 		 */
6876 		if (sctx->cur_inode_new_gen) {
6877 			/*
6878 			 * First, process the inode as if it was deleted.
6879 			 */
6880 			if (old_nlinks > 0) {
6881 				sctx->cur_inode_gen = right_gen;
6882 				sctx->cur_inode_new = false;
6883 				sctx->cur_inode_deleted = true;
6884 				sctx->cur_inode_size = btrfs_inode_size(
6885 						sctx->right_path->nodes[0], right_ii);
6886 				sctx->cur_inode_mode = btrfs_inode_mode(
6887 						sctx->right_path->nodes[0], right_ii);
6888 				ret = process_all_refs(sctx,
6889 						BTRFS_COMPARE_TREE_DELETED);
6890 				if (ret < 0)
6891 					goto out;
6892 			}
6893 
6894 			/*
6895 			 * Now process the inode as if it was new.
6896 			 */
6897 			if (new_nlinks > 0) {
6898 				sctx->cur_inode_gen = left_gen;
6899 				sctx->cur_inode_new = true;
6900 				sctx->cur_inode_deleted = false;
6901 				sctx->cur_inode_size = btrfs_inode_size(
6902 						sctx->left_path->nodes[0],
6903 						left_ii);
6904 				sctx->cur_inode_mode = btrfs_inode_mode(
6905 						sctx->left_path->nodes[0],
6906 						left_ii);
6907 				sctx->cur_inode_rdev = btrfs_inode_rdev(
6908 						sctx->left_path->nodes[0],
6909 						left_ii);
6910 				ret = send_create_inode_if_needed(sctx);
6911 				if (ret < 0)
6912 					goto out;
6913 
6914 				ret = process_all_refs(sctx, BTRFS_COMPARE_TREE_NEW);
6915 				if (ret < 0)
6916 					goto out;
6917 				/*
6918 				 * Advance send_progress now as we did not get
6919 				 * into process_recorded_refs_if_needed in the
6920 				 * new_gen case.
6921 				 */
6922 				sctx->send_progress = sctx->cur_ino + 1;
6923 
6924 				/*
6925 				 * Now process all extents and xattrs of the
6926 				 * inode as if they were all new.
6927 				 */
6928 				ret = process_all_extents(sctx);
6929 				if (ret < 0)
6930 					goto out;
6931 				ret = process_all_new_xattrs(sctx);
6932 				if (ret < 0)
6933 					goto out;
6934 			}
6935 		} else {
6936 			sctx->cur_inode_gen = left_gen;
6937 			sctx->cur_inode_new = false;
6938 			sctx->cur_inode_new_gen = false;
6939 			sctx->cur_inode_deleted = false;
6940 			sctx->cur_inode_size = btrfs_inode_size(
6941 					sctx->left_path->nodes[0], left_ii);
6942 			sctx->cur_inode_mode = btrfs_inode_mode(
6943 					sctx->left_path->nodes[0], left_ii);
6944 		}
6945 	}
6946 
6947 out:
6948 	return ret;
6949 }
6950 
6951 /*
6952  * We have to process new refs before deleted refs, but compare_trees gives us
6953  * the new and deleted refs mixed. To fix this, we record the new/deleted refs
6954  * first and later process them in process_recorded_refs.
6955  * For the cur_inode_new_gen case, we skip recording completely because
6956  * changed_inode did already initiate processing of refs. The reason for this is
6957  * that in this case, compare_tree actually compares the refs of 2 different
6958  * inodes. To fix this, process_all_refs is used in changed_inode to handle all
6959  * refs of the right tree as deleted and all refs of the left tree as new.
6960  */
changed_ref(struct send_ctx * sctx,enum btrfs_compare_tree_result result)6961 static int changed_ref(struct send_ctx *sctx,
6962 		       enum btrfs_compare_tree_result result)
6963 {
6964 	int ret = 0;
6965 
6966 	if (unlikely(sctx->cur_ino != sctx->cmp_key->objectid)) {
6967 		inconsistent_snapshot_error(sctx, result, "reference");
6968 		return -EIO;
6969 	}
6970 
6971 	if (!sctx->cur_inode_new_gen &&
6972 	    sctx->cur_ino != BTRFS_FIRST_FREE_OBJECTID) {
6973 		if (result == BTRFS_COMPARE_TREE_NEW)
6974 			ret = record_new_ref(sctx);
6975 		else if (result == BTRFS_COMPARE_TREE_DELETED)
6976 			ret = record_deleted_ref(sctx);
6977 		else if (result == BTRFS_COMPARE_TREE_CHANGED)
6978 			ret = record_changed_ref(sctx);
6979 	}
6980 
6981 	return ret;
6982 }
6983 
6984 /*
6985  * Process new/deleted/changed xattrs. We skip processing in the
6986  * cur_inode_new_gen case because changed_inode did already initiate processing
6987  * of xattrs. The reason is the same as in changed_ref
6988  */
changed_xattr(struct send_ctx * sctx,enum btrfs_compare_tree_result result)6989 static int changed_xattr(struct send_ctx *sctx,
6990 			 enum btrfs_compare_tree_result result)
6991 {
6992 	int ret = 0;
6993 
6994 	if (unlikely(sctx->cur_ino != sctx->cmp_key->objectid)) {
6995 		inconsistent_snapshot_error(sctx, result, "xattr");
6996 		return -EIO;
6997 	}
6998 
6999 	if (!sctx->cur_inode_new_gen && !sctx->cur_inode_deleted) {
7000 		if (result == BTRFS_COMPARE_TREE_NEW)
7001 			ret = process_new_xattr(sctx);
7002 		else if (result == BTRFS_COMPARE_TREE_DELETED)
7003 			ret = process_deleted_xattr(sctx);
7004 		else if (result == BTRFS_COMPARE_TREE_CHANGED)
7005 			ret = process_changed_xattr(sctx);
7006 	}
7007 
7008 	return ret;
7009 }
7010 
7011 /*
7012  * Process new/deleted/changed extents. We skip processing in the
7013  * cur_inode_new_gen case because changed_inode did already initiate processing
7014  * of extents. The reason is the same as in changed_ref
7015  */
changed_extent(struct send_ctx * sctx,enum btrfs_compare_tree_result result)7016 static int changed_extent(struct send_ctx *sctx,
7017 			  enum btrfs_compare_tree_result result)
7018 {
7019 	int ret = 0;
7020 
7021 	/*
7022 	 * We have found an extent item that changed without the inode item
7023 	 * having changed. This can happen either after relocation (where the
7024 	 * disk_bytenr of an extent item is replaced at
7025 	 * relocation.c:replace_file_extents()) or after deduplication into a
7026 	 * file in both the parent and send snapshots (where an extent item can
7027 	 * get modified or replaced with a new one). Note that deduplication
7028 	 * updates the inode item, but it only changes the iversion (sequence
7029 	 * field in the inode item) of the inode, so if a file is deduplicated
7030 	 * the same amount of times in both the parent and send snapshots, its
7031 	 * iversion becomes the same in both snapshots, whence the inode item is
7032 	 * the same on both snapshots.
7033 	 */
7034 	if (sctx->cur_ino != sctx->cmp_key->objectid)
7035 		return 0;
7036 
7037 	if (!sctx->cur_inode_new_gen && !sctx->cur_inode_deleted) {
7038 		if (result != BTRFS_COMPARE_TREE_DELETED)
7039 			ret = process_extent(sctx, sctx->left_path,
7040 					sctx->cmp_key);
7041 	}
7042 
7043 	return ret;
7044 }
7045 
changed_verity(struct send_ctx * sctx,enum btrfs_compare_tree_result result)7046 static int changed_verity(struct send_ctx *sctx, enum btrfs_compare_tree_result result)
7047 {
7048 	if (!sctx->cur_inode_new_gen && !sctx->cur_inode_deleted) {
7049 		if (result == BTRFS_COMPARE_TREE_NEW)
7050 			sctx->cur_inode_needs_verity = true;
7051 	}
7052 	return 0;
7053 }
7054 
dir_changed(struct send_ctx * sctx,u64 dir)7055 static int dir_changed(struct send_ctx *sctx, u64 dir)
7056 {
7057 	u64 orig_gen, new_gen;
7058 	int ret;
7059 
7060 	ret = get_inode_gen(sctx->send_root, dir, &new_gen);
7061 	if (ret)
7062 		return ret;
7063 
7064 	ret = get_inode_gen(sctx->parent_root, dir, &orig_gen);
7065 	if (ret)
7066 		return ret;
7067 
7068 	return (orig_gen != new_gen) ? 1 : 0;
7069 }
7070 
compare_refs(struct send_ctx * sctx,struct btrfs_path * path,struct btrfs_key * key)7071 static int compare_refs(struct send_ctx *sctx, struct btrfs_path *path,
7072 			struct btrfs_key *key)
7073 {
7074 	struct btrfs_inode_extref *extref;
7075 	struct extent_buffer *leaf;
7076 	u64 dirid = 0, last_dirid = 0;
7077 	unsigned long ptr;
7078 	u32 item_size;
7079 	u32 cur_offset = 0;
7080 	int ref_name_len;
7081 	int ret = 0;
7082 
7083 	/* Easy case, just check this one dirid */
7084 	if (key->type == BTRFS_INODE_REF_KEY) {
7085 		dirid = key->offset;
7086 
7087 		ret = dir_changed(sctx, dirid);
7088 		goto out;
7089 	}
7090 
7091 	leaf = path->nodes[0];
7092 	item_size = btrfs_item_size(leaf, path->slots[0]);
7093 	ptr = btrfs_item_ptr_offset(leaf, path->slots[0]);
7094 	while (cur_offset < item_size) {
7095 		extref = (struct btrfs_inode_extref *)(ptr +
7096 						       cur_offset);
7097 		dirid = btrfs_inode_extref_parent(leaf, extref);
7098 		ref_name_len = btrfs_inode_extref_name_len(leaf, extref);
7099 		cur_offset += ref_name_len + sizeof(*extref);
7100 		if (dirid == last_dirid)
7101 			continue;
7102 		ret = dir_changed(sctx, dirid);
7103 		if (ret)
7104 			break;
7105 		last_dirid = dirid;
7106 	}
7107 out:
7108 	return ret;
7109 }
7110 
7111 /*
7112  * Updates compare related fields in sctx and simply forwards to the actual
7113  * changed_xxx functions.
7114  */
changed_cb(struct btrfs_path * left_path,struct btrfs_path * right_path,struct btrfs_key * key,enum btrfs_compare_tree_result result,struct send_ctx * sctx)7115 static int changed_cb(struct btrfs_path *left_path,
7116 		      struct btrfs_path *right_path,
7117 		      struct btrfs_key *key,
7118 		      enum btrfs_compare_tree_result result,
7119 		      struct send_ctx *sctx)
7120 {
7121 	int ret;
7122 
7123 	/*
7124 	 * We can not hold the commit root semaphore here. This is because in
7125 	 * the case of sending and receiving to the same filesystem, using a
7126 	 * pipe, could result in a deadlock:
7127 	 *
7128 	 * 1) The task running send blocks on the pipe because it's full;
7129 	 *
7130 	 * 2) The task running receive, which is the only consumer of the pipe,
7131 	 *    is waiting for a transaction commit (for example due to a space
7132 	 *    reservation when doing a write or triggering a transaction commit
7133 	 *    when creating a subvolume);
7134 	 *
7135 	 * 3) The transaction is waiting to write lock the commit root semaphore,
7136 	 *    but can not acquire it since it's being held at 1).
7137 	 *
7138 	 * Down this call chain we write to the pipe through kernel_write().
7139 	 * The same type of problem can also happen when sending to a file that
7140 	 * is stored in the same filesystem - when reserving space for a write
7141 	 * into the file, we can trigger a transaction commit.
7142 	 *
7143 	 * Our caller has supplied us with clones of leaves from the send and
7144 	 * parent roots, so we're safe here from a concurrent relocation and
7145 	 * further reallocation of metadata extents while we are here. Below we
7146 	 * also assert that the leaves are clones.
7147 	 */
7148 	lockdep_assert_not_held(&sctx->send_root->fs_info->commit_root_sem);
7149 
7150 	/*
7151 	 * We always have a send root, so left_path is never NULL. We will not
7152 	 * have a leaf when we have reached the end of the send root but have
7153 	 * not yet reached the end of the parent root.
7154 	 */
7155 	if (left_path->nodes[0])
7156 		ASSERT(test_bit(EXTENT_BUFFER_UNMAPPED,
7157 				&left_path->nodes[0]->bflags));
7158 	/*
7159 	 * When doing a full send we don't have a parent root, so right_path is
7160 	 * NULL. When doing an incremental send, we may have reached the end of
7161 	 * the parent root already, so we don't have a leaf at right_path.
7162 	 */
7163 	if (right_path && right_path->nodes[0])
7164 		ASSERT(test_bit(EXTENT_BUFFER_UNMAPPED,
7165 				&right_path->nodes[0]->bflags));
7166 
7167 	if (result == BTRFS_COMPARE_TREE_SAME) {
7168 		if (key->type == BTRFS_INODE_REF_KEY ||
7169 		    key->type == BTRFS_INODE_EXTREF_KEY) {
7170 			ret = compare_refs(sctx, left_path, key);
7171 			if (!ret)
7172 				return 0;
7173 			if (ret < 0)
7174 				return ret;
7175 		} else if (key->type == BTRFS_EXTENT_DATA_KEY) {
7176 			return maybe_send_hole(sctx, left_path, key);
7177 		} else {
7178 			return 0;
7179 		}
7180 		result = BTRFS_COMPARE_TREE_CHANGED;
7181 	}
7182 
7183 	sctx->left_path = left_path;
7184 	sctx->right_path = right_path;
7185 	sctx->cmp_key = key;
7186 
7187 	ret = finish_inode_if_needed(sctx, 0);
7188 	if (ret < 0)
7189 		goto out;
7190 
7191 	/* Ignore non-FS objects */
7192 	if (key->objectid == BTRFS_FREE_INO_OBJECTID ||
7193 	    key->objectid == BTRFS_FREE_SPACE_OBJECTID)
7194 		goto out;
7195 
7196 	if (key->type == BTRFS_INODE_ITEM_KEY) {
7197 		ret = changed_inode(sctx, result);
7198 	} else if (!sctx->ignore_cur_inode) {
7199 		if (key->type == BTRFS_INODE_REF_KEY ||
7200 		    key->type == BTRFS_INODE_EXTREF_KEY)
7201 			ret = changed_ref(sctx, result);
7202 		else if (key->type == BTRFS_XATTR_ITEM_KEY)
7203 			ret = changed_xattr(sctx, result);
7204 		else if (key->type == BTRFS_EXTENT_DATA_KEY)
7205 			ret = changed_extent(sctx, result);
7206 		else if (key->type == BTRFS_VERITY_DESC_ITEM_KEY &&
7207 			 key->offset == 0)
7208 			ret = changed_verity(sctx, result);
7209 	}
7210 
7211 out:
7212 	return ret;
7213 }
7214 
search_key_again(const struct send_ctx * sctx,struct btrfs_root * root,struct btrfs_path * path,const struct btrfs_key * key)7215 static int search_key_again(const struct send_ctx *sctx,
7216 			    struct btrfs_root *root,
7217 			    struct btrfs_path *path,
7218 			    const struct btrfs_key *key)
7219 {
7220 	int ret;
7221 
7222 	if (!path->need_commit_sem)
7223 		lockdep_assert_held_read(&root->fs_info->commit_root_sem);
7224 
7225 	/*
7226 	 * Roots used for send operations are readonly and no one can add,
7227 	 * update or remove keys from them, so we should be able to find our
7228 	 * key again. The only exception is deduplication, which can operate on
7229 	 * readonly roots and add, update or remove keys to/from them - but at
7230 	 * the moment we don't allow it to run in parallel with send.
7231 	 */
7232 	ret = btrfs_search_slot(NULL, root, key, path, 0, 0);
7233 	ASSERT(ret <= 0);
7234 	if (unlikely(ret > 0)) {
7235 		btrfs_print_tree(path->nodes[path->lowest_level], false);
7236 		btrfs_err(root->fs_info,
7237 "send: key (%llu %u %llu) not found in %s root %llu, lowest_level %d, slot %d",
7238 			  key->objectid, key->type, key->offset,
7239 			  (root == sctx->parent_root ? "parent" : "send"),
7240 			  btrfs_root_id(root), path->lowest_level,
7241 			  path->slots[path->lowest_level]);
7242 		return -EUCLEAN;
7243 	}
7244 
7245 	return ret;
7246 }
7247 
full_send_tree(struct send_ctx * sctx)7248 static int full_send_tree(struct send_ctx *sctx)
7249 {
7250 	int ret;
7251 	struct btrfs_root *send_root = sctx->send_root;
7252 	struct btrfs_key key;
7253 	struct btrfs_fs_info *fs_info = send_root->fs_info;
7254 	BTRFS_PATH_AUTO_FREE(path);
7255 
7256 	path = alloc_path_for_send();
7257 	if (!path)
7258 		return -ENOMEM;
7259 	path->reada = READA_FORWARD_ALWAYS;
7260 
7261 	key.objectid = BTRFS_FIRST_FREE_OBJECTID;
7262 	key.type = BTRFS_INODE_ITEM_KEY;
7263 	key.offset = 0;
7264 
7265 	down_read(&fs_info->commit_root_sem);
7266 	sctx->last_reloc_trans = fs_info->last_reloc_trans;
7267 	up_read(&fs_info->commit_root_sem);
7268 
7269 	ret = btrfs_search_slot_for_read(send_root, &key, path, 1, 0);
7270 	if (ret < 0)
7271 		return ret;
7272 	if (ret)
7273 		goto out_finish;
7274 
7275 	while (1) {
7276 		btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
7277 
7278 		ret = changed_cb(path, NULL, &key,
7279 				 BTRFS_COMPARE_TREE_NEW, sctx);
7280 		if (ret < 0)
7281 			return ret;
7282 
7283 		down_read(&fs_info->commit_root_sem);
7284 		if (fs_info->last_reloc_trans > sctx->last_reloc_trans) {
7285 			sctx->last_reloc_trans = fs_info->last_reloc_trans;
7286 			up_read(&fs_info->commit_root_sem);
7287 			/*
7288 			 * A transaction used for relocating a block group was
7289 			 * committed or is about to finish its commit. Release
7290 			 * our path (leaf) and restart the search, so that we
7291 			 * avoid operating on any file extent items that are
7292 			 * stale, with a disk_bytenr that reflects a pre
7293 			 * relocation value. This way we avoid as much as
7294 			 * possible to fallback to regular writes when checking
7295 			 * if we can clone file ranges.
7296 			 */
7297 			btrfs_release_path(path);
7298 			ret = search_key_again(sctx, send_root, path, &key);
7299 			if (ret < 0)
7300 				return ret;
7301 		} else {
7302 			up_read(&fs_info->commit_root_sem);
7303 		}
7304 
7305 		ret = btrfs_next_item(send_root, path);
7306 		if (ret < 0)
7307 			return ret;
7308 		if (ret) {
7309 			ret  = 0;
7310 			break;
7311 		}
7312 	}
7313 
7314 out_finish:
7315 	return finish_inode_if_needed(sctx, 1);
7316 }
7317 
replace_node_with_clone(struct btrfs_path * path,int level)7318 static int replace_node_with_clone(struct btrfs_path *path, int level)
7319 {
7320 	struct extent_buffer *clone;
7321 
7322 	clone = btrfs_clone_extent_buffer(path->nodes[level]);
7323 	if (!clone)
7324 		return -ENOMEM;
7325 
7326 	free_extent_buffer(path->nodes[level]);
7327 	path->nodes[level] = clone;
7328 
7329 	return 0;
7330 }
7331 
tree_move_down(struct btrfs_path * path,int * level,u64 reada_min_gen)7332 static int tree_move_down(struct btrfs_path *path, int *level, u64 reada_min_gen)
7333 {
7334 	struct extent_buffer *eb;
7335 	struct extent_buffer *parent = path->nodes[*level];
7336 	int slot = path->slots[*level];
7337 	const int nritems = btrfs_header_nritems(parent);
7338 	u64 reada_max;
7339 	u64 reada_done = 0;
7340 
7341 	lockdep_assert_held_read(&parent->fs_info->commit_root_sem);
7342 	ASSERT(*level != 0);
7343 
7344 	eb = btrfs_read_node_slot(parent, slot);
7345 	if (IS_ERR(eb))
7346 		return PTR_ERR(eb);
7347 
7348 	/*
7349 	 * Trigger readahead for the next leaves we will process, so that it is
7350 	 * very likely that when we need them they are already in memory and we
7351 	 * will not block on disk IO. For nodes we only do readahead for one,
7352 	 * since the time window between processing nodes is typically larger.
7353 	 */
7354 	reada_max = (*level == 1 ? SZ_128K : eb->fs_info->nodesize);
7355 
7356 	for (slot++; slot < nritems && reada_done < reada_max; slot++) {
7357 		if (btrfs_node_ptr_generation(parent, slot) > reada_min_gen) {
7358 			btrfs_readahead_node_child(parent, slot);
7359 			reada_done += eb->fs_info->nodesize;
7360 		}
7361 	}
7362 
7363 	path->nodes[*level - 1] = eb;
7364 	path->slots[*level - 1] = 0;
7365 	(*level)--;
7366 
7367 	if (*level == 0)
7368 		return replace_node_with_clone(path, 0);
7369 
7370 	return 0;
7371 }
7372 
tree_move_next_or_upnext(struct btrfs_path * path,int * level,int root_level)7373 static int tree_move_next_or_upnext(struct btrfs_path *path,
7374 				    int *level, int root_level)
7375 {
7376 	int ret = 0;
7377 	int nritems;
7378 	nritems = btrfs_header_nritems(path->nodes[*level]);
7379 
7380 	path->slots[*level]++;
7381 
7382 	while (path->slots[*level] >= nritems) {
7383 		if (*level == root_level) {
7384 			path->slots[*level] = nritems - 1;
7385 			return -1;
7386 		}
7387 
7388 		/* move upnext */
7389 		path->slots[*level] = 0;
7390 		free_extent_buffer(path->nodes[*level]);
7391 		path->nodes[*level] = NULL;
7392 		(*level)++;
7393 		path->slots[*level]++;
7394 
7395 		nritems = btrfs_header_nritems(path->nodes[*level]);
7396 		ret = 1;
7397 	}
7398 	return ret;
7399 }
7400 
7401 /*
7402  * Returns 1 if it had to move up and next. 0 is returned if it moved only next
7403  * or down.
7404  */
tree_advance(struct btrfs_path * path,int * level,int root_level,int allow_down,struct btrfs_key * key,u64 reada_min_gen)7405 static int tree_advance(struct btrfs_path *path,
7406 			int *level, int root_level,
7407 			int allow_down,
7408 			struct btrfs_key *key,
7409 			u64 reada_min_gen)
7410 {
7411 	int ret;
7412 
7413 	if (*level == 0 || !allow_down) {
7414 		ret = tree_move_next_or_upnext(path, level, root_level);
7415 	} else {
7416 		ret = tree_move_down(path, level, reada_min_gen);
7417 	}
7418 
7419 	/*
7420 	 * Even if we have reached the end of a tree, ret is -1, update the key
7421 	 * anyway, so that in case we need to restart due to a block group
7422 	 * relocation, we can assert that the last key of the root node still
7423 	 * exists in the tree.
7424 	 */
7425 	if (*level == 0)
7426 		btrfs_item_key_to_cpu(path->nodes[*level], key,
7427 				      path->slots[*level]);
7428 	else
7429 		btrfs_node_key_to_cpu(path->nodes[*level], key,
7430 				      path->slots[*level]);
7431 
7432 	return ret;
7433 }
7434 
tree_compare_item(struct btrfs_path * left_path,struct btrfs_path * right_path,char * tmp_buf)7435 static int tree_compare_item(struct btrfs_path *left_path,
7436 			     struct btrfs_path *right_path,
7437 			     char *tmp_buf)
7438 {
7439 	int cmp;
7440 	int len1, len2;
7441 	unsigned long off1, off2;
7442 
7443 	len1 = btrfs_item_size(left_path->nodes[0], left_path->slots[0]);
7444 	len2 = btrfs_item_size(right_path->nodes[0], right_path->slots[0]);
7445 	if (len1 != len2)
7446 		return 1;
7447 
7448 	off1 = btrfs_item_ptr_offset(left_path->nodes[0], left_path->slots[0]);
7449 	off2 = btrfs_item_ptr_offset(right_path->nodes[0],
7450 				right_path->slots[0]);
7451 
7452 	read_extent_buffer(left_path->nodes[0], tmp_buf, off1, len1);
7453 
7454 	cmp = memcmp_extent_buffer(right_path->nodes[0], tmp_buf, off2, len1);
7455 	if (cmp)
7456 		return 1;
7457 	return 0;
7458 }
7459 
7460 /*
7461  * A transaction used for relocating a block group was committed or is about to
7462  * finish its commit. Release our paths and restart the search, so that we are
7463  * not using stale extent buffers:
7464  *
7465  * 1) For levels > 0, we are only holding references of extent buffers, without
7466  *    any locks on them, which does not prevent them from having been relocated
7467  *    and reallocated after the last time we released the commit root semaphore.
7468  *    The exception are the root nodes, for which we always have a clone, see
7469  *    the comment at btrfs_compare_trees();
7470  *
7471  * 2) For leaves, level 0, we are holding copies (clones) of extent buffers, so
7472  *    we are safe from the concurrent relocation and reallocation. However they
7473  *    can have file extent items with a pre relocation disk_bytenr value, so we
7474  *    restart the start from the current commit roots and clone the new leaves so
7475  *    that we get the post relocation disk_bytenr values. Not doing so, could
7476  *    make us clone the wrong data in case there are new extents using the old
7477  *    disk_bytenr that happen to be shared.
7478  */
restart_after_relocation(struct btrfs_path * left_path,struct btrfs_path * right_path,const struct btrfs_key * left_key,const struct btrfs_key * right_key,int left_level,int right_level,const struct send_ctx * sctx)7479 static int restart_after_relocation(struct btrfs_path *left_path,
7480 				    struct btrfs_path *right_path,
7481 				    const struct btrfs_key *left_key,
7482 				    const struct btrfs_key *right_key,
7483 				    int left_level,
7484 				    int right_level,
7485 				    const struct send_ctx *sctx)
7486 {
7487 	int root_level;
7488 	int ret;
7489 
7490 	lockdep_assert_held_read(&sctx->send_root->fs_info->commit_root_sem);
7491 
7492 	btrfs_release_path(left_path);
7493 	btrfs_release_path(right_path);
7494 
7495 	/*
7496 	 * Since keys can not be added or removed to/from our roots because they
7497 	 * are readonly and we do not allow deduplication to run in parallel
7498 	 * (which can add, remove or change keys), the layout of the trees should
7499 	 * not change.
7500 	 */
7501 	left_path->lowest_level = left_level;
7502 	ret = search_key_again(sctx, sctx->send_root, left_path, left_key);
7503 	if (ret < 0)
7504 		return ret;
7505 
7506 	right_path->lowest_level = right_level;
7507 	ret = search_key_again(sctx, sctx->parent_root, right_path, right_key);
7508 	if (ret < 0)
7509 		return ret;
7510 
7511 	/*
7512 	 * If the lowest level nodes are leaves, clone them so that they can be
7513 	 * safely used by changed_cb() while not under the protection of the
7514 	 * commit root semaphore, even if relocation and reallocation happens in
7515 	 * parallel.
7516 	 */
7517 	if (left_level == 0) {
7518 		ret = replace_node_with_clone(left_path, 0);
7519 		if (ret < 0)
7520 			return ret;
7521 	}
7522 
7523 	if (right_level == 0) {
7524 		ret = replace_node_with_clone(right_path, 0);
7525 		if (ret < 0)
7526 			return ret;
7527 	}
7528 
7529 	/*
7530 	 * Now clone the root nodes (unless they happen to be the leaves we have
7531 	 * already cloned). This is to protect against concurrent snapshotting of
7532 	 * the send and parent roots (see the comment at btrfs_compare_trees()).
7533 	 */
7534 	root_level = btrfs_header_level(sctx->send_root->commit_root);
7535 	if (root_level > 0) {
7536 		ret = replace_node_with_clone(left_path, root_level);
7537 		if (ret < 0)
7538 			return ret;
7539 	}
7540 
7541 	root_level = btrfs_header_level(sctx->parent_root->commit_root);
7542 	if (root_level > 0) {
7543 		ret = replace_node_with_clone(right_path, root_level);
7544 		if (ret < 0)
7545 			return ret;
7546 	}
7547 
7548 	return 0;
7549 }
7550 
7551 /*
7552  * This function compares two trees and calls the provided callback for
7553  * every changed/new/deleted item it finds.
7554  * If shared tree blocks are encountered, whole subtrees are skipped, making
7555  * the compare pretty fast on snapshotted subvolumes.
7556  *
7557  * This currently works on commit roots only. As commit roots are read only,
7558  * we don't do any locking. The commit roots are protected with transactions.
7559  * Transactions are ended and rejoined when a commit is tried in between.
7560  *
7561  * This function checks for modifications done to the trees while comparing.
7562  * If it detects a change, it aborts immediately.
7563  */
btrfs_compare_trees(struct btrfs_root * left_root,struct btrfs_root * right_root,struct send_ctx * sctx)7564 static int btrfs_compare_trees(struct btrfs_root *left_root,
7565 			struct btrfs_root *right_root, struct send_ctx *sctx)
7566 {
7567 	struct btrfs_fs_info *fs_info = left_root->fs_info;
7568 	int ret;
7569 	int cmp;
7570 	BTRFS_PATH_AUTO_FREE(left_path);
7571 	BTRFS_PATH_AUTO_FREE(right_path);
7572 	struct btrfs_key left_key;
7573 	struct btrfs_key right_key;
7574 	char *tmp_buf = NULL;
7575 	int left_root_level;
7576 	int right_root_level;
7577 	int left_level;
7578 	int right_level;
7579 	int left_end_reached = 0;
7580 	int right_end_reached = 0;
7581 	int advance_left = 0;
7582 	int advance_right = 0;
7583 	u64 left_blockptr;
7584 	u64 right_blockptr;
7585 	u64 left_gen;
7586 	u64 right_gen;
7587 	u64 reada_min_gen;
7588 
7589 	left_path = btrfs_alloc_path();
7590 	if (!left_path) {
7591 		ret = -ENOMEM;
7592 		goto out;
7593 	}
7594 	right_path = btrfs_alloc_path();
7595 	if (!right_path) {
7596 		ret = -ENOMEM;
7597 		goto out;
7598 	}
7599 
7600 	tmp_buf = kvmalloc(fs_info->nodesize, GFP_KERNEL);
7601 	if (!tmp_buf) {
7602 		ret = -ENOMEM;
7603 		goto out;
7604 	}
7605 
7606 	left_path->search_commit_root = 1;
7607 	left_path->skip_locking = 1;
7608 	right_path->search_commit_root = 1;
7609 	right_path->skip_locking = 1;
7610 
7611 	/*
7612 	 * Strategy: Go to the first items of both trees. Then do
7613 	 *
7614 	 * If both trees are at level 0
7615 	 *   Compare keys of current items
7616 	 *     If left < right treat left item as new, advance left tree
7617 	 *       and repeat
7618 	 *     If left > right treat right item as deleted, advance right tree
7619 	 *       and repeat
7620 	 *     If left == right do deep compare of items, treat as changed if
7621 	 *       needed, advance both trees and repeat
7622 	 * If both trees are at the same level but not at level 0
7623 	 *   Compare keys of current nodes/leafs
7624 	 *     If left < right advance left tree and repeat
7625 	 *     If left > right advance right tree and repeat
7626 	 *     If left == right compare blockptrs of the next nodes/leafs
7627 	 *       If they match advance both trees but stay at the same level
7628 	 *         and repeat
7629 	 *       If they don't match advance both trees while allowing to go
7630 	 *         deeper and repeat
7631 	 * If tree levels are different
7632 	 *   Advance the tree that needs it and repeat
7633 	 *
7634 	 * Advancing a tree means:
7635 	 *   If we are at level 0, try to go to the next slot. If that's not
7636 	 *   possible, go one level up and repeat. Stop when we found a level
7637 	 *   where we could go to the next slot. We may at this point be on a
7638 	 *   node or a leaf.
7639 	 *
7640 	 *   If we are not at level 0 and not on shared tree blocks, go one
7641 	 *   level deeper.
7642 	 *
7643 	 *   If we are not at level 0 and on shared tree blocks, go one slot to
7644 	 *   the right if possible or go up and right.
7645 	 */
7646 
7647 	down_read(&fs_info->commit_root_sem);
7648 	left_level = btrfs_header_level(left_root->commit_root);
7649 	left_root_level = left_level;
7650 	/*
7651 	 * We clone the root node of the send and parent roots to prevent races
7652 	 * with snapshot creation of these roots. Snapshot creation COWs the
7653 	 * root node of a tree, so after the transaction is committed the old
7654 	 * extent can be reallocated while this send operation is still ongoing.
7655 	 * So we clone them, under the commit root semaphore, to be race free.
7656 	 */
7657 	left_path->nodes[left_level] =
7658 			btrfs_clone_extent_buffer(left_root->commit_root);
7659 	if (!left_path->nodes[left_level]) {
7660 		ret = -ENOMEM;
7661 		goto out_unlock;
7662 	}
7663 
7664 	right_level = btrfs_header_level(right_root->commit_root);
7665 	right_root_level = right_level;
7666 	right_path->nodes[right_level] =
7667 			btrfs_clone_extent_buffer(right_root->commit_root);
7668 	if (!right_path->nodes[right_level]) {
7669 		ret = -ENOMEM;
7670 		goto out_unlock;
7671 	}
7672 	/*
7673 	 * Our right root is the parent root, while the left root is the "send"
7674 	 * root. We know that all new nodes/leaves in the left root must have
7675 	 * a generation greater than the right root's generation, so we trigger
7676 	 * readahead for those nodes and leaves of the left root, as we know we
7677 	 * will need to read them at some point.
7678 	 */
7679 	reada_min_gen = btrfs_header_generation(right_root->commit_root);
7680 
7681 	if (left_level == 0)
7682 		btrfs_item_key_to_cpu(left_path->nodes[left_level],
7683 				&left_key, left_path->slots[left_level]);
7684 	else
7685 		btrfs_node_key_to_cpu(left_path->nodes[left_level],
7686 				&left_key, left_path->slots[left_level]);
7687 	if (right_level == 0)
7688 		btrfs_item_key_to_cpu(right_path->nodes[right_level],
7689 				&right_key, right_path->slots[right_level]);
7690 	else
7691 		btrfs_node_key_to_cpu(right_path->nodes[right_level],
7692 				&right_key, right_path->slots[right_level]);
7693 
7694 	sctx->last_reloc_trans = fs_info->last_reloc_trans;
7695 
7696 	while (1) {
7697 		if (need_resched() ||
7698 		    rwsem_is_contended(&fs_info->commit_root_sem)) {
7699 			up_read(&fs_info->commit_root_sem);
7700 			cond_resched();
7701 			down_read(&fs_info->commit_root_sem);
7702 		}
7703 
7704 		if (fs_info->last_reloc_trans > sctx->last_reloc_trans) {
7705 			ret = restart_after_relocation(left_path, right_path,
7706 						       &left_key, &right_key,
7707 						       left_level, right_level,
7708 						       sctx);
7709 			if (ret < 0)
7710 				goto out_unlock;
7711 			sctx->last_reloc_trans = fs_info->last_reloc_trans;
7712 		}
7713 
7714 		if (advance_left && !left_end_reached) {
7715 			ret = tree_advance(left_path, &left_level,
7716 					left_root_level,
7717 					advance_left != ADVANCE_ONLY_NEXT,
7718 					&left_key, reada_min_gen);
7719 			if (ret == -1)
7720 				left_end_reached = ADVANCE;
7721 			else if (ret < 0)
7722 				goto out_unlock;
7723 			advance_left = 0;
7724 		}
7725 		if (advance_right && !right_end_reached) {
7726 			ret = tree_advance(right_path, &right_level,
7727 					right_root_level,
7728 					advance_right != ADVANCE_ONLY_NEXT,
7729 					&right_key, reada_min_gen);
7730 			if (ret == -1)
7731 				right_end_reached = ADVANCE;
7732 			else if (ret < 0)
7733 				goto out_unlock;
7734 			advance_right = 0;
7735 		}
7736 
7737 		if (left_end_reached && right_end_reached) {
7738 			ret = 0;
7739 			goto out_unlock;
7740 		} else if (left_end_reached) {
7741 			if (right_level == 0) {
7742 				up_read(&fs_info->commit_root_sem);
7743 				ret = changed_cb(left_path, right_path,
7744 						&right_key,
7745 						BTRFS_COMPARE_TREE_DELETED,
7746 						sctx);
7747 				if (ret < 0)
7748 					goto out;
7749 				down_read(&fs_info->commit_root_sem);
7750 			}
7751 			advance_right = ADVANCE;
7752 			continue;
7753 		} else if (right_end_reached) {
7754 			if (left_level == 0) {
7755 				up_read(&fs_info->commit_root_sem);
7756 				ret = changed_cb(left_path, right_path,
7757 						&left_key,
7758 						BTRFS_COMPARE_TREE_NEW,
7759 						sctx);
7760 				if (ret < 0)
7761 					goto out;
7762 				down_read(&fs_info->commit_root_sem);
7763 			}
7764 			advance_left = ADVANCE;
7765 			continue;
7766 		}
7767 
7768 		if (left_level == 0 && right_level == 0) {
7769 			up_read(&fs_info->commit_root_sem);
7770 			cmp = btrfs_comp_cpu_keys(&left_key, &right_key);
7771 			if (cmp < 0) {
7772 				ret = changed_cb(left_path, right_path,
7773 						&left_key,
7774 						BTRFS_COMPARE_TREE_NEW,
7775 						sctx);
7776 				advance_left = ADVANCE;
7777 			} else if (cmp > 0) {
7778 				ret = changed_cb(left_path, right_path,
7779 						&right_key,
7780 						BTRFS_COMPARE_TREE_DELETED,
7781 						sctx);
7782 				advance_right = ADVANCE;
7783 			} else {
7784 				enum btrfs_compare_tree_result result;
7785 
7786 				WARN_ON(!extent_buffer_uptodate(left_path->nodes[0]));
7787 				ret = tree_compare_item(left_path, right_path,
7788 							tmp_buf);
7789 				if (ret)
7790 					result = BTRFS_COMPARE_TREE_CHANGED;
7791 				else
7792 					result = BTRFS_COMPARE_TREE_SAME;
7793 				ret = changed_cb(left_path, right_path,
7794 						 &left_key, result, sctx);
7795 				advance_left = ADVANCE;
7796 				advance_right = ADVANCE;
7797 			}
7798 
7799 			if (ret < 0)
7800 				goto out;
7801 			down_read(&fs_info->commit_root_sem);
7802 		} else if (left_level == right_level) {
7803 			cmp = btrfs_comp_cpu_keys(&left_key, &right_key);
7804 			if (cmp < 0) {
7805 				advance_left = ADVANCE;
7806 			} else if (cmp > 0) {
7807 				advance_right = ADVANCE;
7808 			} else {
7809 				left_blockptr = btrfs_node_blockptr(
7810 						left_path->nodes[left_level],
7811 						left_path->slots[left_level]);
7812 				right_blockptr = btrfs_node_blockptr(
7813 						right_path->nodes[right_level],
7814 						right_path->slots[right_level]);
7815 				left_gen = btrfs_node_ptr_generation(
7816 						left_path->nodes[left_level],
7817 						left_path->slots[left_level]);
7818 				right_gen = btrfs_node_ptr_generation(
7819 						right_path->nodes[right_level],
7820 						right_path->slots[right_level]);
7821 				if (left_blockptr == right_blockptr &&
7822 				    left_gen == right_gen) {
7823 					/*
7824 					 * As we're on a shared block, don't
7825 					 * allow to go deeper.
7826 					 */
7827 					advance_left = ADVANCE_ONLY_NEXT;
7828 					advance_right = ADVANCE_ONLY_NEXT;
7829 				} else {
7830 					advance_left = ADVANCE;
7831 					advance_right = ADVANCE;
7832 				}
7833 			}
7834 		} else if (left_level < right_level) {
7835 			advance_right = ADVANCE;
7836 		} else {
7837 			advance_left = ADVANCE;
7838 		}
7839 	}
7840 
7841 out_unlock:
7842 	up_read(&fs_info->commit_root_sem);
7843 out:
7844 	kvfree(tmp_buf);
7845 	return ret;
7846 }
7847 
send_subvol(struct send_ctx * sctx)7848 static int send_subvol(struct send_ctx *sctx)
7849 {
7850 	int ret;
7851 
7852 	if (!(sctx->flags & BTRFS_SEND_FLAG_OMIT_STREAM_HEADER)) {
7853 		ret = send_header(sctx);
7854 		if (ret < 0)
7855 			goto out;
7856 	}
7857 
7858 	ret = send_subvol_begin(sctx);
7859 	if (ret < 0)
7860 		goto out;
7861 
7862 	if (sctx->parent_root) {
7863 		ret = btrfs_compare_trees(sctx->send_root, sctx->parent_root, sctx);
7864 		if (ret < 0)
7865 			goto out;
7866 		ret = finish_inode_if_needed(sctx, 1);
7867 		if (ret < 0)
7868 			goto out;
7869 	} else {
7870 		ret = full_send_tree(sctx);
7871 		if (ret < 0)
7872 			goto out;
7873 	}
7874 
7875 out:
7876 	free_recorded_refs(sctx);
7877 	return ret;
7878 }
7879 
7880 /*
7881  * If orphan cleanup did remove any orphans from a root, it means the tree
7882  * was modified and therefore the commit root is not the same as the current
7883  * root anymore. This is a problem, because send uses the commit root and
7884  * therefore can see inode items that don't exist in the current root anymore,
7885  * and for example make calls to btrfs_iget, which will do tree lookups based
7886  * on the current root and not on the commit root. Those lookups will fail,
7887  * returning a -ESTALE error, and making send fail with that error. So make
7888  * sure a send does not see any orphans we have just removed, and that it will
7889  * see the same inodes regardless of whether a transaction commit happened
7890  * before it started (meaning that the commit root will be the same as the
7891  * current root) or not.
7892  */
ensure_commit_roots_uptodate(struct send_ctx * sctx)7893 static int ensure_commit_roots_uptodate(struct send_ctx *sctx)
7894 {
7895 	struct btrfs_root *root = sctx->parent_root;
7896 
7897 	if (root && root->node != root->commit_root)
7898 		return btrfs_commit_current_transaction(root);
7899 
7900 	for (int i = 0; i < sctx->clone_roots_cnt; i++) {
7901 		root = sctx->clone_roots[i].root;
7902 		if (root->node != root->commit_root)
7903 			return btrfs_commit_current_transaction(root);
7904 	}
7905 
7906 	return 0;
7907 }
7908 
7909 /*
7910  * Make sure any existing delalloc is flushed for any root used by a send
7911  * operation so that we do not miss any data and we do not race with writeback
7912  * finishing and changing a tree while send is using the tree. This could
7913  * happen if a subvolume is in RW mode, has delalloc, is turned to RO mode and
7914  * a send operation then uses the subvolume.
7915  * After flushing delalloc ensure_commit_roots_uptodate() must be called.
7916  */
flush_delalloc_roots(struct send_ctx * sctx)7917 static int flush_delalloc_roots(struct send_ctx *sctx)
7918 {
7919 	struct btrfs_root *root = sctx->parent_root;
7920 	int ret;
7921 	int i;
7922 
7923 	if (root) {
7924 		ret = btrfs_start_delalloc_snapshot(root, false);
7925 		if (ret)
7926 			return ret;
7927 		btrfs_wait_ordered_extents(root, U64_MAX, NULL);
7928 	}
7929 
7930 	for (i = 0; i < sctx->clone_roots_cnt; i++) {
7931 		root = sctx->clone_roots[i].root;
7932 		ret = btrfs_start_delalloc_snapshot(root, false);
7933 		if (ret)
7934 			return ret;
7935 		btrfs_wait_ordered_extents(root, U64_MAX, NULL);
7936 	}
7937 
7938 	return 0;
7939 }
7940 
btrfs_root_dec_send_in_progress(struct btrfs_root * root)7941 static void btrfs_root_dec_send_in_progress(struct btrfs_root* root)
7942 {
7943 	spin_lock(&root->root_item_lock);
7944 	root->send_in_progress--;
7945 	/*
7946 	 * Not much left to do, we don't know why it's unbalanced and
7947 	 * can't blindly reset it to 0.
7948 	 */
7949 	if (root->send_in_progress < 0)
7950 		btrfs_err(root->fs_info,
7951 			  "send_in_progress unbalanced %d root %llu",
7952 			  root->send_in_progress, btrfs_root_id(root));
7953 	spin_unlock(&root->root_item_lock);
7954 }
7955 
dedupe_in_progress_warn(const struct btrfs_root * root)7956 static void dedupe_in_progress_warn(const struct btrfs_root *root)
7957 {
7958 	btrfs_warn_rl(root->fs_info,
7959 "cannot use root %llu for send while deduplications on it are in progress (%d in progress)",
7960 		      btrfs_root_id(root), root->dedupe_in_progress);
7961 }
7962 
btrfs_ioctl_send(struct btrfs_root * send_root,const struct btrfs_ioctl_send_args * arg)7963 long btrfs_ioctl_send(struct btrfs_root *send_root, const struct btrfs_ioctl_send_args *arg)
7964 {
7965 	int ret = 0;
7966 	struct btrfs_fs_info *fs_info = send_root->fs_info;
7967 	struct btrfs_root *clone_root;
7968 	struct send_ctx *sctx = NULL;
7969 	u32 i;
7970 	u64 *clone_sources_tmp = NULL;
7971 	int clone_sources_to_rollback = 0;
7972 	size_t alloc_size;
7973 	int sort_clone_roots = 0;
7974 	struct btrfs_lru_cache_entry *entry;
7975 	struct btrfs_lru_cache_entry *tmp;
7976 
7977 	if (!capable(CAP_SYS_ADMIN))
7978 		return -EPERM;
7979 
7980 	/*
7981 	 * The subvolume must remain read-only during send, protect against
7982 	 * making it RW. This also protects against deletion.
7983 	 */
7984 	spin_lock(&send_root->root_item_lock);
7985 	/*
7986 	 * Unlikely but possible, if the subvolume is marked for deletion but
7987 	 * is slow to remove the directory entry, send can still be started.
7988 	 */
7989 	if (btrfs_root_dead(send_root)) {
7990 		spin_unlock(&send_root->root_item_lock);
7991 		return -EPERM;
7992 	}
7993 	/* Userspace tools do the checks and warn the user if it's not RO. */
7994 	if (!btrfs_root_readonly(send_root)) {
7995 		spin_unlock(&send_root->root_item_lock);
7996 		return -EPERM;
7997 	}
7998 	if (send_root->dedupe_in_progress) {
7999 		dedupe_in_progress_warn(send_root);
8000 		spin_unlock(&send_root->root_item_lock);
8001 		return -EAGAIN;
8002 	}
8003 	send_root->send_in_progress++;
8004 	spin_unlock(&send_root->root_item_lock);
8005 
8006 	/*
8007 	 * Check that we don't overflow at later allocations, we request
8008 	 * clone_sources_count + 1 items, and compare to unsigned long inside
8009 	 * access_ok. Also set an upper limit for allocation size so this can't
8010 	 * easily exhaust memory. Max number of clone sources is about 200K.
8011 	 */
8012 	if (arg->clone_sources_count > SZ_8M / sizeof(struct clone_root)) {
8013 		ret = -EINVAL;
8014 		goto out;
8015 	}
8016 
8017 	if (arg->flags & ~BTRFS_SEND_FLAG_MASK) {
8018 		ret = -EOPNOTSUPP;
8019 		goto out;
8020 	}
8021 
8022 	sctx = kzalloc(sizeof(struct send_ctx), GFP_KERNEL);
8023 	if (!sctx) {
8024 		ret = -ENOMEM;
8025 		goto out;
8026 	}
8027 
8028 	init_path(&sctx->cur_inode_path);
8029 	INIT_LIST_HEAD(&sctx->new_refs);
8030 	INIT_LIST_HEAD(&sctx->deleted_refs);
8031 
8032 	btrfs_lru_cache_init(&sctx->name_cache, SEND_MAX_NAME_CACHE_SIZE);
8033 	btrfs_lru_cache_init(&sctx->backref_cache, SEND_MAX_BACKREF_CACHE_SIZE);
8034 	btrfs_lru_cache_init(&sctx->dir_created_cache,
8035 			     SEND_MAX_DIR_CREATED_CACHE_SIZE);
8036 	/*
8037 	 * This cache is periodically trimmed to a fixed size elsewhere, see
8038 	 * cache_dir_utimes() and trim_dir_utimes_cache().
8039 	 */
8040 	btrfs_lru_cache_init(&sctx->dir_utimes_cache, 0);
8041 
8042 	sctx->pending_dir_moves = RB_ROOT;
8043 	sctx->waiting_dir_moves = RB_ROOT;
8044 	sctx->orphan_dirs = RB_ROOT;
8045 	sctx->rbtree_new_refs = RB_ROOT;
8046 	sctx->rbtree_deleted_refs = RB_ROOT;
8047 
8048 	sctx->flags = arg->flags;
8049 
8050 	if (arg->flags & BTRFS_SEND_FLAG_VERSION) {
8051 		if (arg->version > BTRFS_SEND_STREAM_VERSION) {
8052 			ret = -EPROTO;
8053 			goto out;
8054 		}
8055 		/* Zero means "use the highest version" */
8056 		sctx->proto = arg->version ?: BTRFS_SEND_STREAM_VERSION;
8057 	} else {
8058 		sctx->proto = 1;
8059 	}
8060 	if ((arg->flags & BTRFS_SEND_FLAG_COMPRESSED) && sctx->proto < 2) {
8061 		ret = -EINVAL;
8062 		goto out;
8063 	}
8064 
8065 	sctx->send_filp = fget(arg->send_fd);
8066 	if (!sctx->send_filp || !(sctx->send_filp->f_mode & FMODE_WRITE)) {
8067 		ret = -EBADF;
8068 		goto out;
8069 	}
8070 
8071 	sctx->send_root = send_root;
8072 	sctx->clone_roots_cnt = arg->clone_sources_count;
8073 
8074 	if (sctx->proto >= 2) {
8075 		u32 send_buf_num_pages;
8076 
8077 		sctx->send_max_size = BTRFS_SEND_BUF_SIZE_V2;
8078 		sctx->send_buf = vmalloc(sctx->send_max_size);
8079 		if (!sctx->send_buf) {
8080 			ret = -ENOMEM;
8081 			goto out;
8082 		}
8083 		send_buf_num_pages = sctx->send_max_size >> PAGE_SHIFT;
8084 		sctx->send_buf_pages = kcalloc(send_buf_num_pages,
8085 					       sizeof(*sctx->send_buf_pages),
8086 					       GFP_KERNEL);
8087 		if (!sctx->send_buf_pages) {
8088 			ret = -ENOMEM;
8089 			goto out;
8090 		}
8091 		for (i = 0; i < send_buf_num_pages; i++) {
8092 			sctx->send_buf_pages[i] =
8093 				vmalloc_to_page(sctx->send_buf + (i << PAGE_SHIFT));
8094 		}
8095 	} else {
8096 		sctx->send_max_size = BTRFS_SEND_BUF_SIZE_V1;
8097 		sctx->send_buf = kvmalloc(sctx->send_max_size, GFP_KERNEL);
8098 	}
8099 	if (!sctx->send_buf) {
8100 		ret = -ENOMEM;
8101 		goto out;
8102 	}
8103 
8104 	sctx->clone_roots = kvcalloc(arg->clone_sources_count + 1,
8105 				     sizeof(*sctx->clone_roots),
8106 				     GFP_KERNEL);
8107 	if (!sctx->clone_roots) {
8108 		ret = -ENOMEM;
8109 		goto out;
8110 	}
8111 
8112 	alloc_size = array_size(sizeof(*arg->clone_sources),
8113 				arg->clone_sources_count);
8114 
8115 	if (arg->clone_sources_count) {
8116 		clone_sources_tmp = kvmalloc(alloc_size, GFP_KERNEL);
8117 		if (!clone_sources_tmp) {
8118 			ret = -ENOMEM;
8119 			goto out;
8120 		}
8121 
8122 		ret = copy_from_user(clone_sources_tmp, arg->clone_sources,
8123 				alloc_size);
8124 		if (ret) {
8125 			ret = -EFAULT;
8126 			goto out;
8127 		}
8128 
8129 		for (i = 0; i < arg->clone_sources_count; i++) {
8130 			clone_root = btrfs_get_fs_root(fs_info,
8131 						clone_sources_tmp[i], true);
8132 			if (IS_ERR(clone_root)) {
8133 				ret = PTR_ERR(clone_root);
8134 				goto out;
8135 			}
8136 			spin_lock(&clone_root->root_item_lock);
8137 			if (!btrfs_root_readonly(clone_root) ||
8138 			    btrfs_root_dead(clone_root)) {
8139 				spin_unlock(&clone_root->root_item_lock);
8140 				btrfs_put_root(clone_root);
8141 				ret = -EPERM;
8142 				goto out;
8143 			}
8144 			if (clone_root->dedupe_in_progress) {
8145 				dedupe_in_progress_warn(clone_root);
8146 				spin_unlock(&clone_root->root_item_lock);
8147 				btrfs_put_root(clone_root);
8148 				ret = -EAGAIN;
8149 				goto out;
8150 			}
8151 			clone_root->send_in_progress++;
8152 			spin_unlock(&clone_root->root_item_lock);
8153 
8154 			sctx->clone_roots[i].root = clone_root;
8155 			clone_sources_to_rollback = i + 1;
8156 		}
8157 		kvfree(clone_sources_tmp);
8158 		clone_sources_tmp = NULL;
8159 	}
8160 
8161 	if (arg->parent_root) {
8162 		sctx->parent_root = btrfs_get_fs_root(fs_info, arg->parent_root,
8163 						      true);
8164 		if (IS_ERR(sctx->parent_root)) {
8165 			ret = PTR_ERR(sctx->parent_root);
8166 			goto out;
8167 		}
8168 
8169 		spin_lock(&sctx->parent_root->root_item_lock);
8170 		sctx->parent_root->send_in_progress++;
8171 		if (!btrfs_root_readonly(sctx->parent_root) ||
8172 				btrfs_root_dead(sctx->parent_root)) {
8173 			spin_unlock(&sctx->parent_root->root_item_lock);
8174 			ret = -EPERM;
8175 			goto out;
8176 		}
8177 		if (sctx->parent_root->dedupe_in_progress) {
8178 			dedupe_in_progress_warn(sctx->parent_root);
8179 			spin_unlock(&sctx->parent_root->root_item_lock);
8180 			ret = -EAGAIN;
8181 			goto out;
8182 		}
8183 		spin_unlock(&sctx->parent_root->root_item_lock);
8184 	}
8185 
8186 	/*
8187 	 * Clones from send_root are allowed, but only if the clone source
8188 	 * is behind the current send position. This is checked while searching
8189 	 * for possible clone sources.
8190 	 */
8191 	sctx->clone_roots[sctx->clone_roots_cnt++].root =
8192 		btrfs_grab_root(sctx->send_root);
8193 
8194 	/* We do a bsearch later */
8195 	sort(sctx->clone_roots, sctx->clone_roots_cnt,
8196 			sizeof(*sctx->clone_roots), __clone_root_cmp_sort,
8197 			NULL);
8198 	sort_clone_roots = 1;
8199 
8200 	ret = flush_delalloc_roots(sctx);
8201 	if (ret)
8202 		goto out;
8203 
8204 	ret = ensure_commit_roots_uptodate(sctx);
8205 	if (ret)
8206 		goto out;
8207 
8208 	ret = send_subvol(sctx);
8209 	if (ret < 0)
8210 		goto out;
8211 
8212 	btrfs_lru_cache_for_each_entry_safe(&sctx->dir_utimes_cache, entry, tmp) {
8213 		ret = send_utimes(sctx, entry->key, entry->gen);
8214 		if (ret < 0)
8215 			goto out;
8216 		btrfs_lru_cache_remove(&sctx->dir_utimes_cache, entry);
8217 	}
8218 
8219 	if (!(sctx->flags & BTRFS_SEND_FLAG_OMIT_END_CMD)) {
8220 		ret = begin_cmd(sctx, BTRFS_SEND_C_END);
8221 		if (ret < 0)
8222 			goto out;
8223 		ret = send_cmd(sctx);
8224 		if (ret < 0)
8225 			goto out;
8226 	}
8227 
8228 out:
8229 	WARN_ON(sctx && !ret && !RB_EMPTY_ROOT(&sctx->pending_dir_moves));
8230 	while (sctx && !RB_EMPTY_ROOT(&sctx->pending_dir_moves)) {
8231 		struct rb_node *n;
8232 		struct pending_dir_move *pm;
8233 
8234 		n = rb_first(&sctx->pending_dir_moves);
8235 		pm = rb_entry(n, struct pending_dir_move, node);
8236 		while (!list_empty(&pm->list)) {
8237 			struct pending_dir_move *pm2;
8238 
8239 			pm2 = list_first_entry(&pm->list,
8240 					       struct pending_dir_move, list);
8241 			free_pending_move(sctx, pm2);
8242 		}
8243 		free_pending_move(sctx, pm);
8244 	}
8245 
8246 	WARN_ON(sctx && !ret && !RB_EMPTY_ROOT(&sctx->waiting_dir_moves));
8247 	while (sctx && !RB_EMPTY_ROOT(&sctx->waiting_dir_moves)) {
8248 		struct rb_node *n;
8249 		struct waiting_dir_move *dm;
8250 
8251 		n = rb_first(&sctx->waiting_dir_moves);
8252 		dm = rb_entry(n, struct waiting_dir_move, node);
8253 		rb_erase(&dm->node, &sctx->waiting_dir_moves);
8254 		kfree(dm);
8255 	}
8256 
8257 	WARN_ON(sctx && !ret && !RB_EMPTY_ROOT(&sctx->orphan_dirs));
8258 	while (sctx && !RB_EMPTY_ROOT(&sctx->orphan_dirs)) {
8259 		struct rb_node *n;
8260 		struct orphan_dir_info *odi;
8261 
8262 		n = rb_first(&sctx->orphan_dirs);
8263 		odi = rb_entry(n, struct orphan_dir_info, node);
8264 		free_orphan_dir_info(sctx, odi);
8265 	}
8266 
8267 	if (sort_clone_roots) {
8268 		for (i = 0; i < sctx->clone_roots_cnt; i++) {
8269 			btrfs_root_dec_send_in_progress(
8270 					sctx->clone_roots[i].root);
8271 			btrfs_put_root(sctx->clone_roots[i].root);
8272 		}
8273 	} else {
8274 		for (i = 0; sctx && i < clone_sources_to_rollback; i++) {
8275 			btrfs_root_dec_send_in_progress(
8276 					sctx->clone_roots[i].root);
8277 			btrfs_put_root(sctx->clone_roots[i].root);
8278 		}
8279 
8280 		btrfs_root_dec_send_in_progress(send_root);
8281 	}
8282 	if (sctx && !IS_ERR_OR_NULL(sctx->parent_root)) {
8283 		btrfs_root_dec_send_in_progress(sctx->parent_root);
8284 		btrfs_put_root(sctx->parent_root);
8285 	}
8286 
8287 	kvfree(clone_sources_tmp);
8288 
8289 	if (sctx) {
8290 		if (sctx->send_filp)
8291 			fput(sctx->send_filp);
8292 
8293 		kvfree(sctx->clone_roots);
8294 		kfree(sctx->send_buf_pages);
8295 		kvfree(sctx->send_buf);
8296 		kvfree(sctx->verity_descriptor);
8297 
8298 		close_current_inode(sctx);
8299 
8300 		btrfs_lru_cache_clear(&sctx->name_cache);
8301 		btrfs_lru_cache_clear(&sctx->backref_cache);
8302 		btrfs_lru_cache_clear(&sctx->dir_created_cache);
8303 		btrfs_lru_cache_clear(&sctx->dir_utimes_cache);
8304 
8305 		if (sctx->cur_inode_path.buf != sctx->cur_inode_path.inline_buf)
8306 			kfree(sctx->cur_inode_path.buf);
8307 
8308 		kfree(sctx);
8309 	}
8310 
8311 	return ret;
8312 }
8313