1 // SPDX-License-Identifier: GPL-2.0
12 #include <linux/error-injection.h>
14 #include "disk-io.h"
19 #include "async-thread.h"
20 #include "free-space-cache.h"
22 #include "print-tree.h"
23 #include "delalloc-space.h"
24 #include "block-group.h"
29 #include "inode-item.h"
30 #include "space-info.h"
33 #include "extent-tree.h"
34 #include "root-tree.h"
35 #include "file-item.h"
38 #include "tree-checker.h"
39 #include "raid-stripe-tree.h"
52  * ------------------------------------------------------------------
59  * 1.   Mark the target block group read-only
90  * map address of tree root to tree
150 	/* map start of tree root to corresponding reloc tree */
173 				 struct btrfs_backref_node *node)  in mark_block_processed()  argument
177 	if (node->level == 0 ||  in mark_block_processed()
178 	    in_range(node->bytenr, rc->block_group->start,  in mark_block_processed()
179 		     rc->block_group->length)) {  in mark_block_processed()
180 		blocksize = rc->extent_root->fs_info->nodesize;  in mark_block_processed()
181 		set_extent_bit(&rc->processed_blocks, node->bytenr,  in mark_block_processed()
182 			       node->bytenr + blocksize - 1, EXTENT_DIRTY, NULL);  in mark_block_processed()
184 	node->processed = 1;  in mark_block_processed()
188  * walk up backref nodes until reach node presents tree root
191 		struct btrfs_backref_node *node,  in walk_up_backref()  argument
197 	while (!list_empty(&node->upper)) {  in walk_up_backref()
198 		edge = list_entry(node->upper.next,  in walk_up_backref()
201 		node = edge->node[UPPER];  in walk_up_backref()
203 	BUG_ON(node->detached);  in walk_up_backref()
205 	return node;  in walk_up_backref()
219 		edge = edges[idx - 1];  in walk_down_backref()
220 		lower = edge->node[LOWER];  in walk_down_backref()
221 		if (list_is_last(&edge->list[LOWER], &lower->upper)) {  in walk_down_backref()
222 			idx--;  in walk_down_backref()
225 		edge = list_entry(edge->list[LOWER].next,  in walk_down_backref()
227 		edges[idx - 1] = edge;  in walk_down_backref()
229 		return edge->node[UPPER];  in walk_down_backref()
235 static bool reloc_root_is_dead(const struct btrfs_root *root)  in reloc_root_is_dead()  argument
243 	if (test_bit(BTRFS_ROOT_DEAD_RELOC_TREE, &root->state))  in reloc_root_is_dead()
252  * This is enough for most callers, as they don't distinguish dead reloc root
253  * from no reloc root.  But btrfs_should_ignore_reloc_root() below is a
256 static bool have_reloc_root(const struct btrfs_root *root)  in have_reloc_root()  argument
258 	if (reloc_root_is_dead(root))  in have_reloc_root()
260 	if (!root->reloc_root)  in have_reloc_root()
265 bool btrfs_should_ignore_reloc_root(const struct btrfs_root *root)  in btrfs_should_ignore_reloc_root()  argument
269 	if (!test_bit(BTRFS_ROOT_SHAREABLE, &root->state))  in btrfs_should_ignore_reloc_root()
272 	/* This root has been merged with its reloc tree, we can ignore it */  in btrfs_should_ignore_reloc_root()
273 	if (reloc_root_is_dead(root))  in btrfs_should_ignore_reloc_root()
276 	reloc_root = root->reloc_root;  in btrfs_should_ignore_reloc_root()
280 	if (btrfs_header_generation(reloc_root->commit_root) ==  in btrfs_should_ignore_reloc_root()
281 	    root->fs_info->running_transaction->transid)  in btrfs_should_ignore_reloc_root()
285 	 * backref lookup can find the reloc tree, so backref node for the fs  in btrfs_should_ignore_reloc_root()
286 	 * tree root is useless for relocation.  in btrfs_should_ignore_reloc_root()
292  * find reloc tree by address of tree root
296 	struct reloc_control *rc = fs_info->reloc_ctl;  in find_reloc_root()
298 	struct mapping_node *node;  in find_reloc_root()  local
299 	struct btrfs_root *root = NULL;  in find_reloc_root()  local
302 	spin_lock(&rc->reloc_root_tree.lock);  in find_reloc_root()
303 	rb_node = rb_simple_search(&rc->reloc_root_tree.rb_root, bytenr);  in find_reloc_root()
305 		node = rb_entry(rb_node, struct mapping_node, rb_node);  in find_reloc_root()
306 		root = node->data;  in find_reloc_root()
308 	spin_unlock(&rc->reloc_root_tree.lock);  in find_reloc_root()
309 	return btrfs_grab_root(root);  in find_reloc_root()
315  * - Cleanup the children edges and nodes
316  *   If child node is also orphan (no parent) during cleanup, then the child
317  *   node will also be cleaned up.
319  * - Freeing up leaves (level 0), keeps nodes detached
320  *   For nodes, the node is still cached as "detached"
322  * Return false if @node is not in the @useless_nodes list.
323  * Return true if @node is in the @useless_nodes list.
326 				 struct btrfs_backref_node *node)  in handle_useless_nodes()  argument
328 	struct btrfs_backref_cache *cache = &rc->backref_cache;  in handle_useless_nodes()
329 	struct list_head *useless_node = &cache->useless_node;  in handle_useless_nodes()
337 		list_del_init(&cur->list);  in handle_useless_nodes()
339 		/* Only tree root nodes can be added to @useless_nodes */  in handle_useless_nodes()
340 		ASSERT(list_empty(&cur->upper));  in handle_useless_nodes()
342 		if (cur == node)  in handle_useless_nodes()
346 		while (!list_empty(&cur->lower)) {  in handle_useless_nodes()
350 			edge = list_entry(cur->lower.next,  in handle_useless_nodes()
352 			list_del(&edge->list[UPPER]);  in handle_useless_nodes()
353 			list_del(&edge->list[LOWER]);  in handle_useless_nodes()
354 			lower = edge->node[LOWER];  in handle_useless_nodes()
357 			/* Child node is also orphan, queue for cleanup */  in handle_useless_nodes()
358 			if (list_empty(&lower->upper))  in handle_useless_nodes()
359 				list_add(&lower->list, useless_node);  in handle_useless_nodes()
369 		if (cur->level > 0) {  in handle_useless_nodes()
370 			cur->detached = 1;  in handle_useless_nodes()
372 			rb_erase(&cur->rb_node, &cache->rb_root);  in handle_useless_nodes()
380  * Build backref tree for a given tree block. Root of the backref tree
382  * b-trees that reference the tree block.
386  * these upper level blocks recursively. The recursion stops when tree root is
399 	struct btrfs_backref_cache *cache = &rc->backref_cache;  in build_backref_tree()
403 	struct btrfs_backref_node *node = NULL;  in build_backref_tree()  local
407 	iter = btrfs_backref_iter_alloc(rc->extent_root->fs_info);  in build_backref_tree()
409 		return ERR_PTR(-ENOMEM);  in build_backref_tree()
412 		ret = -ENOMEM;  in build_backref_tree()
416 	node = btrfs_backref_alloc_node(cache, bytenr, level);  in build_backref_tree()
417 	if (!node) {  in build_backref_tree()
418 		ret = -ENOMEM;  in build_backref_tree()
422 	cur = node;  in build_backref_tree()
424 	/* Breadth-first search to build backref cache */  in build_backref_tree()
431 		edge = list_first_entry_or_null(&cache->pending_edge,  in build_backref_tree()
438 			list_del_init(&edge->list[UPPER]);  in build_backref_tree()
439 			cur = edge->node[UPPER];  in build_backref_tree()
444 	ret = btrfs_backref_finish_upper_links(cache, node);  in build_backref_tree()
448 	if (handle_useless_nodes(rc, node))  in build_backref_tree()
449 		node = NULL;  in build_backref_tree()
451 	btrfs_free_path(iter->path);  in build_backref_tree()
455 		btrfs_backref_error_cleanup(cache, node);  in build_backref_tree()
458 	ASSERT(!node || !node->detached);  in build_backref_tree()
459 	ASSERT(list_empty(&cache->useless_node) &&  in build_backref_tree()
460 	       list_empty(&cache->pending_edge));  in build_backref_tree()
461 	return node;  in build_backref_tree()
465  * helper to add 'address of tree root -> reloc tree' mapping
467 static int __add_reloc_root(struct btrfs_root *root)  in __add_reloc_root()  argument
469 	struct btrfs_fs_info *fs_info = root->fs_info;  in __add_reloc_root()
471 	struct mapping_node *node;  in __add_reloc_root()  local
472 	struct reloc_control *rc = fs_info->reloc_ctl;  in __add_reloc_root()
474 	node = kmalloc(sizeof(*node), GFP_NOFS);  in __add_reloc_root()
475 	if (!node)  in __add_reloc_root()
476 		return -ENOMEM;  in __add_reloc_root()
478 	node->bytenr = root->commit_root->start;  in __add_reloc_root()
479 	node->data = root;  in __add_reloc_root()
481 	spin_lock(&rc->reloc_root_tree.lock);  in __add_reloc_root()
482 	rb_node = rb_simple_insert(&rc->reloc_root_tree.rb_root,  in __add_reloc_root()
483 				   node->bytenr, &node->rb_node);  in __add_reloc_root()
484 	spin_unlock(&rc->reloc_root_tree.lock);  in __add_reloc_root()
487 			    "Duplicate root found for start=%llu while inserting into relocation tree",  in __add_reloc_root()
488 			    node->bytenr);  in __add_reloc_root()
489 		return -EEXIST;  in __add_reloc_root()
492 	list_add_tail(&root->root_list, &rc->reloc_roots);  in __add_reloc_root()
497  * helper to delete the 'address of tree root -> reloc tree'
500 static void __del_reloc_root(struct btrfs_root *root)  in __del_reloc_root()  argument
502 	struct btrfs_fs_info *fs_info = root->fs_info;  in __del_reloc_root()
504 	struct mapping_node *node = NULL;  in __del_reloc_root()  local
505 	struct reloc_control *rc = fs_info->reloc_ctl;  in __del_reloc_root()
508 	if (rc && root->node) {  in __del_reloc_root()
509 		spin_lock(&rc->reloc_root_tree.lock);  in __del_reloc_root()
510 		rb_node = rb_simple_search(&rc->reloc_root_tree.rb_root,  in __del_reloc_root()
511 					   root->commit_root->start);  in __del_reloc_root()
513 			node = rb_entry(rb_node, struct mapping_node, rb_node);  in __del_reloc_root()
514 			rb_erase(&node->rb_node, &rc->reloc_root_tree.rb_root);  in __del_reloc_root()
515 			RB_CLEAR_NODE(&node->rb_node);  in __del_reloc_root()
517 		spin_unlock(&rc->reloc_root_tree.lock);  in __del_reloc_root()
518 		ASSERT(!node || (struct btrfs_root *)node->data == root);  in __del_reloc_root()
522 	 * We only put the reloc root here if it's on the list.  There's a lot  in __del_reloc_root()
523 	 * of places where the pattern is to splice the rc->reloc_roots, process  in __del_reloc_root()
524 	 * the reloc roots, and then add the reloc root back onto  in __del_reloc_root()
525 	 * rc->reloc_roots.  If we call __del_reloc_root while it's off of the  in __del_reloc_root()
529 	spin_lock(&fs_info->trans_lock);  in __del_reloc_root()
530 	if (!list_empty(&root->root_list)) {  in __del_reloc_root()
532 		list_del_init(&root->root_list);  in __del_reloc_root()
534 	spin_unlock(&fs_info->trans_lock);  in __del_reloc_root()
536 		btrfs_put_root(root);  in __del_reloc_root()
537 	kfree(node);  in __del_reloc_root()
541  * helper to update the 'address of tree root -> reloc tree'
544 static int __update_reloc_root(struct btrfs_root *root)  in __update_reloc_root()  argument
546 	struct btrfs_fs_info *fs_info = root->fs_info;  in __update_reloc_root()
548 	struct mapping_node *node = NULL;  in __update_reloc_root()  local
549 	struct reloc_control *rc = fs_info->reloc_ctl;  in __update_reloc_root()
551 	spin_lock(&rc->reloc_root_tree.lock);  in __update_reloc_root()
552 	rb_node = rb_simple_search(&rc->reloc_root_tree.rb_root,  in __update_reloc_root()
553 				   root->commit_root->start);  in __update_reloc_root()
555 		node = rb_entry(rb_node, struct mapping_node, rb_node);  in __update_reloc_root()
556 		rb_erase(&node->rb_node, &rc->reloc_root_tree.rb_root);  in __update_reloc_root()
558 	spin_unlock(&rc->reloc_root_tree.lock);  in __update_reloc_root()
560 	if (!node)  in __update_reloc_root()
562 	BUG_ON((struct btrfs_root *)node->data != root);  in __update_reloc_root()
564 	spin_lock(&rc->reloc_root_tree.lock);  in __update_reloc_root()
565 	node->bytenr = root->node->start;  in __update_reloc_root()
566 	rb_node = rb_simple_insert(&rc->reloc_root_tree.rb_root,  in __update_reloc_root()
567 				   node->bytenr, &node->rb_node);  in __update_reloc_root()
568 	spin_unlock(&rc->reloc_root_tree.lock);  in __update_reloc_root()
570 		btrfs_backref_panic(fs_info, node->bytenr, -EEXIST);  in __update_reloc_root()
575 					struct btrfs_root *root, u64 objectid)  in create_reloc_root()  argument
577 	struct btrfs_fs_info *fs_info = root->fs_info;  in create_reloc_root()
587 		return ERR_PTR(-ENOMEM);  in create_reloc_root()
593 	if (btrfs_root_id(root) == objectid) {  in create_reloc_root()
597 		ret = btrfs_copy_root(trans, root, root->commit_root, &eb,  in create_reloc_root()
604 		 * root - like this ctree.c:btrfs_block_can_be_shared() behaves  in create_reloc_root()
605 		 * correctly (returns true) when the relocation root is created  in create_reloc_root()
610 		commit_root_gen = btrfs_header_generation(root->commit_root);  in create_reloc_root()
611 		btrfs_set_root_last_snapshot(&root->root_item, commit_root_gen);  in create_reloc_root()
620 		ret = btrfs_copy_root(trans, root, root->node, &eb,  in create_reloc_root()
632 	memcpy(root_item, &root->root_item, sizeof(*root_item));  in create_reloc_root()
633 	btrfs_set_root_bytenr(root_item, eb->start);  in create_reloc_root()
635 	btrfs_set_root_generation(root_item, trans->transid);  in create_reloc_root()
637 	if (btrfs_root_id(root) == objectid) {  in create_reloc_root()
639 		memset(&root_item->drop_progress, 0,  in create_reloc_root()
647 	ret = btrfs_insert_root(trans, fs_info->tree_root,  in create_reloc_root()
654 	reloc_root = btrfs_read_tree_root(fs_info->tree_root, &root_key);  in create_reloc_root()
659 	set_bit(BTRFS_ROOT_SHAREABLE, &reloc_root->state);  in create_reloc_root()
660 	btrfs_set_root_last_trans(reloc_root, trans->transid);  in create_reloc_root()
672  * snapshot of the fs tree with special root objectid.
675  * root->reloc_root, and another for being on the rc->reloc_roots list.
678 			  struct btrfs_root *root)  in btrfs_init_reloc_root()  argument
680 	struct btrfs_fs_info *fs_info = root->fs_info;  in btrfs_init_reloc_root()
682 	struct reloc_control *rc = fs_info->reloc_ctl;  in btrfs_init_reloc_root()
694 	if (reloc_root_is_dead(root))  in btrfs_init_reloc_root()
700 	 * corresponding fs root, and then here we update the last trans for the  in btrfs_init_reloc_root()
701 	 * reloc root.  This means that we have to do this for the entire life  in btrfs_init_reloc_root()
702 	 * of the reloc root, regardless of which stage of the relocation we are  in btrfs_init_reloc_root()
705 	if (root->reloc_root) {  in btrfs_init_reloc_root()
706 		reloc_root = root->reloc_root;  in btrfs_init_reloc_root()
707 		btrfs_set_root_last_trans(reloc_root, trans->transid);  in btrfs_init_reloc_root()
715 	if (!rc->create_reloc_tree || btrfs_root_id(root) == BTRFS_TREE_RELOC_OBJECTID)  in btrfs_init_reloc_root()
718 	if (!trans->reloc_reserved) {  in btrfs_init_reloc_root()
719 		rsv = trans->block_rsv;  in btrfs_init_reloc_root()
720 		trans->block_rsv = rc->block_rsv;  in btrfs_init_reloc_root()
723 	reloc_root = create_reloc_root(trans, root, btrfs_root_id(root));  in btrfs_init_reloc_root()
725 		trans->block_rsv = rsv;  in btrfs_init_reloc_root()
730 	ASSERT(ret != -EEXIST);  in btrfs_init_reloc_root()
736 	root->reloc_root = btrfs_grab_root(reloc_root);  in btrfs_init_reloc_root()
741  * update root item of reloc tree
744 			    struct btrfs_root *root)  in btrfs_update_reloc_root()  argument
746 	struct btrfs_fs_info *fs_info = root->fs_info;  in btrfs_update_reloc_root()
751 	if (!have_reloc_root(root))  in btrfs_update_reloc_root()
754 	reloc_root = root->reloc_root;  in btrfs_update_reloc_root()
755 	root_item = &reloc_root->root_item;  in btrfs_update_reloc_root()
759 	 * the root.  We have the ref for root->reloc_root, but just in case  in btrfs_update_reloc_root()
760 	 * hold it while we update the reloc root.  in btrfs_update_reloc_root()
764 	/* root->reloc_root will stay until current relocation finished */  in btrfs_update_reloc_root()
765 	if (fs_info->reloc_ctl && fs_info->reloc_ctl->merge_reloc_tree &&  in btrfs_update_reloc_root()
767 		set_bit(BTRFS_ROOT_DEAD_RELOC_TREE, &root->state);  in btrfs_update_reloc_root()
776 	if (reloc_root->commit_root != reloc_root->node) {  in btrfs_update_reloc_root()
778 		btrfs_set_root_node(root_item, reloc_root->node);  in btrfs_update_reloc_root()
779 		free_extent_buffer(reloc_root->commit_root);  in btrfs_update_reloc_root()
780 		reloc_root->commit_root = btrfs_root_node(reloc_root);  in btrfs_update_reloc_root()
783 	ret = btrfs_update_root(trans, fs_info->tree_root,  in btrfs_update_reloc_root()
784 				&reloc_root->root_key, root_item);  in btrfs_update_reloc_root()
795 	struct btrfs_root *root = BTRFS_I(reloc_inode)->root;  in get_new_location()  local
803 		return -ENOMEM;  in get_new_location()
805 	bytenr -= BTRFS_I(reloc_inode)->reloc_block_group_start;  in get_new_location()
806 	ret = btrfs_lookup_file_extent(NULL, root, path,  in get_new_location()
811 		ret = -ENOENT;  in get_new_location()
815 	leaf = path->nodes[0];  in get_new_location()
816 	fi = btrfs_item_ptr(leaf, path->slots[0],  in get_new_location()
825 		ret = -EINVAL;  in get_new_location()
843 			 struct btrfs_root *root,  in replace_file_extents()  argument
846 	struct btrfs_fs_info *fs_info = root->fs_info;  in replace_file_extents()
860 	if (rc->stage != UPDATE_DATA_PTRS)  in replace_file_extents()
864 	if (btrfs_root_id(root) == BTRFS_TREE_RELOC_OBJECTID)  in replace_file_extents()
865 		parent = leaf->start;  in replace_file_extents()
885 		if (!in_range(bytenr, rc->block_group->start,  in replace_file_extents()
886 			      rc->block_group->length))  in replace_file_extents()
893 		if (btrfs_root_id(root) != BTRFS_TREE_RELOC_OBJECTID) {  in replace_file_extents()
895 				inode = btrfs_find_first_inode(root, key.objectid);  in replace_file_extents()
899 				inode = btrfs_find_first_inode(root, key.objectid);  in replace_file_extents()
907 						    fs_info->sectorsize));  in replace_file_extents()
908 				WARN_ON(!IS_ALIGNED(end, fs_info->sectorsize));  in replace_file_extents()
909 				end--;  in replace_file_extents()
911 				if (!down_read_trylock(&inode->i_mmap_lock))  in replace_file_extents()
913 				ret = try_lock_extent(&inode->io_tree, key.offset,  in replace_file_extents()
916 					up_read(&inode->i_mmap_lock);  in replace_file_extents()
921 				unlock_extent(&inode->io_tree, key.offset, end,  in replace_file_extents()
923 				up_read(&inode->i_mmap_lock);  in replace_file_extents()
927 		ret = get_new_location(rc->data_inode, &new_bytenr,  in replace_file_extents()
939 		key.offset -= btrfs_file_extent_offset(leaf, fi);  in replace_file_extents()
944 		ref.owning_root = btrfs_root_id(root);  in replace_file_extents()
947 				    btrfs_root_id(root), false);  in replace_file_extents()
958 		ref.owning_root = btrfs_root_id(root);  in replace_file_extents()
961 				    btrfs_root_id(root), false);  in replace_file_extents()
980 	btrfs_node_key(path->nodes[level], &key2, path->slots[level]);  in memcmp_node_keys()
999 	struct btrfs_fs_info *fs_info = dest->fs_info;  in replace_path()
1018 	last_snapshot = btrfs_root_last_snapshot(&src->root_item);  in replace_path()
1020 	slot = path->slots[lowest_level];  in replace_path()
1021 	btrfs_node_key_to_cpu(path->nodes[lowest_level], &key, slot);  in replace_path()
1043 		next_key->objectid = (u64)-1;  in replace_path()
1044 		next_key->type = (u8)-1;  in replace_path()
1045 		next_key->offset = (u64)-1;  in replace_path()
1057 			slot--;  in replace_path()
1063 		blocksize = fs_info->nodesize;  in replace_path()
1067 			eb = path->nodes[level];  in replace_path()
1069 							path->slots[level]);  in replace_path()
1071 							path->slots[level]);  in replace_path()
1120 		btrfs_node_key_to_cpu(path->nodes[level], &key,  in replace_path()
1121 				      path->slots[level]);  in replace_path()
1124 		path->lowest_level = level;  in replace_path()
1125 		set_bit(BTRFS_ROOT_RESET_LOCKDEP_CLASS, &src->state);  in replace_path()
1127 		clear_bit(BTRFS_ROOT_RESET_LOCKDEP_CLASS, &src->state);  in replace_path()
1128 		path->lowest_level = 0;  in replace_path()
1131 				ret = -ENOENT;  in replace_path()
1147 		 * CoW on the subtree root node before transaction commit.  in replace_path()
1150 				rc->block_group, parent, slot,  in replace_path()
1151 				path->nodes[level], path->slots[level],  in replace_path()
1161 		btrfs_set_node_blockptr(path->nodes[level],  in replace_path()
1162 					path->slots[level], old_bytenr);  in replace_path()
1163 		btrfs_set_node_ptr_generation(path->nodes[level],  in replace_path()
1164 					      path->slots[level], old_ptr_gen);  in replace_path()
1169 		ref.parent = path->nodes[level]->start;  in replace_path()
1172 		btrfs_init_tree_ref(&ref, level - 1, 0, true);  in replace_path()
1185 		btrfs_init_tree_ref(&ref, level - 1, 0, true);  in replace_path()
1196 		ref.parent = path->nodes[level]->start;  in replace_path()
1199 		btrfs_init_tree_ref(&ref, level - 1, 0, true);  in replace_path()
1213 		btrfs_init_tree_ref(&ref, level - 1, 0, true);  in replace_path()
1234 int walk_up_reloc_tree(struct btrfs_root *root, struct btrfs_path *path,  in walk_up_reloc_tree()  argument
1242 	last_snapshot = btrfs_root_last_snapshot(&root->root_item);  in walk_up_reloc_tree()
1245 		free_extent_buffer(path->nodes[i]);  in walk_up_reloc_tree()
1246 		path->nodes[i] = NULL;  in walk_up_reloc_tree()
1249 	for (i = *level; i < BTRFS_MAX_LEVEL && path->nodes[i]; i++) {  in walk_up_reloc_tree()
1250 		eb = path->nodes[i];  in walk_up_reloc_tree()
1252 		while (path->slots[i] + 1 < nritems) {  in walk_up_reloc_tree()
1253 			path->slots[i]++;  in walk_up_reloc_tree()
1254 			if (btrfs_node_ptr_generation(eb, path->slots[i]) <=  in walk_up_reloc_tree()
1261 		free_extent_buffer(path->nodes[i]);  in walk_up_reloc_tree()
1262 		path->nodes[i] = NULL;  in walk_up_reloc_tree()
1271 int walk_down_reloc_tree(struct btrfs_root *root, struct btrfs_path *path,  in walk_down_reloc_tree()  argument
1280 	last_snapshot = btrfs_root_last_snapshot(&root->root_item);  in walk_down_reloc_tree()
1282 	for (i = *level; i > 0; i--) {  in walk_down_reloc_tree()
1283 		eb = path->nodes[i];  in walk_down_reloc_tree()
1285 		while (path->slots[i] < nritems) {  in walk_down_reloc_tree()
1286 			ptr_gen = btrfs_node_ptr_generation(eb, path->slots[i]);  in walk_down_reloc_tree()
1289 			path->slots[i]++;  in walk_down_reloc_tree()
1291 		if (path->slots[i] >= nritems) {  in walk_down_reloc_tree()
1302 		eb = btrfs_read_node_slot(eb, path->slots[i]);  in walk_down_reloc_tree()
1305 		BUG_ON(btrfs_header_level(eb) != i - 1);  in walk_down_reloc_tree()
1306 		path->nodes[i - 1] = eb;  in walk_down_reloc_tree()
1307 		path->slots[i - 1] = 0;  in walk_down_reloc_tree()
1316 static int invalidate_extent_cache(struct btrfs_root *root,  in invalidate_extent_cache()  argument
1320 	struct btrfs_fs_info *fs_info = root->fs_info;  in invalidate_extent_cache()
1326 	objectid = min_key->objectid;  in invalidate_extent_cache()
1332 			iput(&inode->vfs_inode);  in invalidate_extent_cache()
1334 		if (objectid > max_key->objectid)  in invalidate_extent_cache()
1337 		inode = btrfs_find_first_inode(root, objectid);  in invalidate_extent_cache()
1342 		if (ino > max_key->objectid) {  in invalidate_extent_cache()
1343 			iput(&inode->vfs_inode);  in invalidate_extent_cache()
1348 		if (!S_ISREG(inode->vfs_inode.i_mode))  in invalidate_extent_cache()
1351 		if (unlikely(min_key->objectid == ino)) {  in invalidate_extent_cache()
1352 			if (min_key->type > BTRFS_EXTENT_DATA_KEY)  in invalidate_extent_cache()
1354 			if (min_key->type < BTRFS_EXTENT_DATA_KEY)  in invalidate_extent_cache()
1357 				start = min_key->offset;  in invalidate_extent_cache()
1358 				WARN_ON(!IS_ALIGNED(start, fs_info->sectorsize));  in invalidate_extent_cache()
1364 		if (unlikely(max_key->objectid == ino)) {  in invalidate_extent_cache()
1365 			if (max_key->type < BTRFS_EXTENT_DATA_KEY)  in invalidate_extent_cache()
1367 			if (max_key->type > BTRFS_EXTENT_DATA_KEY) {  in invalidate_extent_cache()
1368 				end = (u64)-1;  in invalidate_extent_cache()
1370 				if (max_key->offset == 0)  in invalidate_extent_cache()
1372 				end = max_key->offset;  in invalidate_extent_cache()
1373 				WARN_ON(!IS_ALIGNED(end, fs_info->sectorsize));  in invalidate_extent_cache()
1374 				end--;  in invalidate_extent_cache()
1377 			end = (u64)-1;  in invalidate_extent_cache()
1381 		lock_extent(&inode->io_tree, start, end, &cached_state);  in invalidate_extent_cache()
1383 		unlock_extent(&inode->io_tree, start, end, &cached_state);  in invalidate_extent_cache()
1393 		if (!path->nodes[level])  in find_next_key()
1395 		if (path->slots[level] + 1 <  in find_next_key()
1396 		    btrfs_header_nritems(path->nodes[level])) {  in find_next_key()
1397 			btrfs_node_key_to_cpu(path->nodes[level], key,  in find_next_key()
1398 					      path->slots[level] + 1);  in find_next_key()
1411 			       struct btrfs_root *root)  in insert_dirty_subvol()  argument
1413 	struct btrfs_root *reloc_root = root->reloc_root;  in insert_dirty_subvol()
1417 	/* @root must be a subvolume tree root with a valid reloc tree */  in insert_dirty_subvol()
1418 	ASSERT(btrfs_root_id(root) != BTRFS_TREE_RELOC_OBJECTID);  in insert_dirty_subvol()
1421 	reloc_root_item = &reloc_root->root_item;  in insert_dirty_subvol()
1422 	memset(&reloc_root_item->drop_progress, 0,  in insert_dirty_subvol()
1423 		sizeof(reloc_root_item->drop_progress));  in insert_dirty_subvol()
1426 	ret = btrfs_update_reloc_root(trans, root);  in insert_dirty_subvol()
1430 	if (list_empty(&root->reloc_dirty_list)) {  in insert_dirty_subvol()
1431 		btrfs_grab_root(root);  in insert_dirty_subvol()
1432 		list_add_tail(&root->reloc_dirty_list, &rc->dirty_subvol_roots);  in insert_dirty_subvol()
1440 	struct btrfs_root *root;  in clean_dirty_subvols()  local
1445 	list_for_each_entry_safe(root, next, &rc->dirty_subvol_roots,  in clean_dirty_subvols()
1447 		if (btrfs_root_id(root) != BTRFS_TREE_RELOC_OBJECTID) {  in clean_dirty_subvols()
1448 			/* Merged subvolume, cleanup its reloc root */  in clean_dirty_subvols()
1449 			struct btrfs_root *reloc_root = root->reloc_root;  in clean_dirty_subvols()
1451 			list_del_init(&root->reloc_dirty_list);  in clean_dirty_subvols()
1452 			root->reloc_root = NULL;  in clean_dirty_subvols()
1455 			 * root->reloc_root = NULL. Pairs with have_reloc_root.  in clean_dirty_subvols()
1458 			clear_bit(BTRFS_ROOT_DEAD_RELOC_TREE, &root->state);  in clean_dirty_subvols()
1462 				 * ->reloc_root.  If it fails however we must  in clean_dirty_subvols()
1472 			btrfs_put_root(root);  in clean_dirty_subvols()
1475 			ret2 = btrfs_drop_snapshot(root, 0, 1);  in clean_dirty_subvols()
1477 				btrfs_put_root(root);  in clean_dirty_subvols()
1491 					       struct btrfs_root *root)  in merge_reloc_root()  argument
1493 	struct btrfs_fs_info *fs_info = rc->extent_root->fs_info;  in merge_reloc_root()
1510 		return -ENOMEM;  in merge_reloc_root()
1511 	path->reada = READA_FORWARD;  in merge_reloc_root()
1513 	reloc_root = root->reloc_root;  in merge_reloc_root()
1514 	root_item = &reloc_root->root_item;  in merge_reloc_root()
1516 	if (btrfs_disk_key_objectid(&root_item->drop_progress) == 0) {  in merge_reloc_root()
1518 		atomic_inc(&reloc_root->node->refs);  in merge_reloc_root()
1519 		path->nodes[level] = reloc_root->node;  in merge_reloc_root()
1520 		path->slots[level] = 0;  in merge_reloc_root()
1522 		btrfs_disk_key_to_cpu(&key, &root_item->drop_progress);  in merge_reloc_root()
1526 		path->lowest_level = level;  in merge_reloc_root()
1528 		path->lowest_level = 0;  in merge_reloc_root()
1534 		btrfs_node_key_to_cpu(path->nodes[level], &next_key,  in merge_reloc_root()
1535 				      path->slots[level]);  in merge_reloc_root()
1544 	 * block COW, we COW at most from level 1 to root level for each tree.  in merge_reloc_root()
1550 	min_reserved = fs_info->nodesize * reserve_level * 2;  in merge_reloc_root()
1554 		ret = btrfs_block_rsv_refill(fs_info, rc->block_rsv,  in merge_reloc_root()
1559 		trans = btrfs_start_transaction(root, 0);  in merge_reloc_root()
1573 		 * btrfs_update_reloc_root() and update our root item  in merge_reloc_root()
1576 		btrfs_set_root_last_trans(reloc_root, trans->transid);  in merge_reloc_root()
1577 		trans->block_rsv = rc->block_rsv;  in merge_reloc_root()
1592 			ret = replace_path(trans, rc, root, reloc_root, path,  in merge_reloc_root()
1599 			btrfs_node_key_to_cpu(path->nodes[level], &key,  in merge_reloc_root()
1600 					      path->slots[level]);  in merge_reloc_root()
1611 		 * this is OK since root refs == 1 in this case.  in merge_reloc_root()
1613 		btrfs_node_key(path->nodes[level], &root_item->drop_progress,  in merge_reloc_root()
1614 			       path->slots[level]);  in merge_reloc_root()
1622 		if (replaced && rc->stage == UPDATE_DATA_PTRS)  in merge_reloc_root()
1623 			invalidate_extent_cache(root, &key, &next_key);  in merge_reloc_root()
1628 	 * relocated and the block is tree root.  in merge_reloc_root()
1630 	leaf = btrfs_lock_root_node(root);  in merge_reloc_root()
1631 	ret = btrfs_cow_block(trans, root, leaf, NULL, 0, &leaf,  in merge_reloc_root()
1639 		ret = insert_dirty_subvol(trans, rc, root);  in merge_reloc_root()
1649 	if (replaced && rc->stage == UPDATE_DATA_PTRS)  in merge_reloc_root()
1650 		invalidate_extent_cache(root, &key, &next_key);  in merge_reloc_root()
1658 	struct btrfs_root *root = rc->extent_root;  in prepare_to_merge()  local
1659 	struct btrfs_fs_info *fs_info = root->fs_info;  in prepare_to_merge()
1666 	mutex_lock(&fs_info->reloc_mutex);  in prepare_to_merge()
1667 	rc->merging_rsv_size += fs_info->nodesize * (BTRFS_MAX_LEVEL - 1) * 2;  in prepare_to_merge()
1668 	rc->merging_rsv_size += rc->nodes_relocated * 2;  in prepare_to_merge()
1669 	mutex_unlock(&fs_info->reloc_mutex);  in prepare_to_merge()
1673 		num_bytes = rc->merging_rsv_size;  in prepare_to_merge()
1674 		ret = btrfs_block_rsv_add(fs_info, rc->block_rsv, num_bytes,  in prepare_to_merge()
1680 	trans = btrfs_join_transaction(rc->extent_root);  in prepare_to_merge()
1683 			btrfs_block_rsv_release(fs_info, rc->block_rsv,  in prepare_to_merge()
1689 		if (num_bytes != rc->merging_rsv_size) {  in prepare_to_merge()
1691 			btrfs_block_rsv_release(fs_info, rc->block_rsv,  in prepare_to_merge()
1697 	rc->merge_reloc_tree = true;  in prepare_to_merge()
1699 	while (!list_empty(&rc->reloc_roots)) {  in prepare_to_merge()
1700 		reloc_root = list_entry(rc->reloc_roots.next,  in prepare_to_merge()
1702 		list_del_init(&reloc_root->root_list);  in prepare_to_merge()
1704 		root = btrfs_get_fs_root(fs_info, reloc_root->root_key.offset,  in prepare_to_merge()
1706 		if (IS_ERR(root)) {  in prepare_to_merge()
1708 			 * Even if we have an error we need this reloc root  in prepare_to_merge()
1711 			list_add(&reloc_root->root_list, &reloc_roots);  in prepare_to_merge()
1712 			btrfs_abort_transaction(trans, (int)PTR_ERR(root));  in prepare_to_merge()
1714 				err = PTR_ERR(root);  in prepare_to_merge()
1718 		if (unlikely(root->reloc_root != reloc_root)) {  in prepare_to_merge()
1719 			if (root->reloc_root) {  in prepare_to_merge()
1721 "reloc tree mismatch, root %lld has reloc root key (%lld %u %llu) gen %llu, expect reloc root key (…  in prepare_to_merge()
1722 					  btrfs_root_id(root),  in prepare_to_merge()
1723 					  btrfs_root_id(root->reloc_root),  in prepare_to_merge()
1724 					  root->reloc_root->root_key.type,  in prepare_to_merge()
1725 					  root->reloc_root->root_key.offset,  in prepare_to_merge()
1727 						  &root->reloc_root->root_item),  in prepare_to_merge()
1729 					  reloc_root->root_key.type,  in prepare_to_merge()
1730 					  reloc_root->root_key.offset,  in prepare_to_merge()
1732 						  &reloc_root->root_item));  in prepare_to_merge()
1735 "reloc tree mismatch, root %lld has no reloc root, expect reloc root key (%lld %u %llu) gen %llu",  in prepare_to_merge()
1736 					  btrfs_root_id(root),  in prepare_to_merge()
1738 					  reloc_root->root_key.type,  in prepare_to_merge()
1739 					  reloc_root->root_key.offset,  in prepare_to_merge()
1741 						  &reloc_root->root_item));  in prepare_to_merge()
1743 			list_add(&reloc_root->root_list, &reloc_roots);  in prepare_to_merge()
1744 			btrfs_put_root(root);  in prepare_to_merge()
1745 			btrfs_abort_transaction(trans, -EUCLEAN);  in prepare_to_merge()
1747 				err = -EUCLEAN;  in prepare_to_merge()
1756 			btrfs_set_root_refs(&reloc_root->root_item, 1);  in prepare_to_merge()
1757 		ret = btrfs_update_reloc_root(trans, root);  in prepare_to_merge()
1760 		 * Even if we have an error we need this reloc root back on our  in prepare_to_merge()
1763 		list_add(&reloc_root->root_list, &reloc_roots);  in prepare_to_merge()
1764 		btrfs_put_root(root);  in prepare_to_merge()
1774 	list_splice(&reloc_roots, &rc->reloc_roots);  in prepare_to_merge()
1795 	struct btrfs_fs_info *fs_info = rc->extent_root->fs_info;  in merge_reloc_roots()
1796 	struct btrfs_root *root;  in merge_reloc_roots()  local
1802 	root = rc->extent_root;  in merge_reloc_roots()
1810 	mutex_lock(&fs_info->reloc_mutex);  in merge_reloc_roots()
1811 	list_splice_init(&rc->reloc_roots, &reloc_roots);  in merge_reloc_roots()
1812 	mutex_unlock(&fs_info->reloc_mutex);  in merge_reloc_roots()
1819 		root = btrfs_get_fs_root(fs_info, reloc_root->root_key.offset,  in merge_reloc_roots()
1821 		if (btrfs_root_refs(&reloc_root->root_item) > 0) {  in merge_reloc_roots()
1822 			if (WARN_ON(IS_ERR(root))) {  in merge_reloc_roots()
1825 				 * and if we didn't find the root then we marked  in merge_reloc_roots()
1826 				 * the reloc root as a garbage root.  For normal  in merge_reloc_roots()
1827 				 * relocation obviously the root should exist in  in merge_reloc_roots()
1831 				ret = PTR_ERR(root);  in merge_reloc_roots()
1834 			if (WARN_ON(root->reloc_root != reloc_root)) {  in merge_reloc_roots()
1836 				 * This can happen if on-disk metadata has some  in merge_reloc_roots()
1839 				ret = -EINVAL;  in merge_reloc_roots()
1842 			ret = merge_reloc_root(rc, root);  in merge_reloc_roots()
1843 			btrfs_put_root(root);  in merge_reloc_roots()
1845 				if (list_empty(&reloc_root->root_list))  in merge_reloc_roots()
1846 					list_add_tail(&reloc_root->root_list,  in merge_reloc_roots()
1851 			if (!IS_ERR(root)) {  in merge_reloc_roots()
1852 				if (root->reloc_root == reloc_root) {  in merge_reloc_roots()
1853 					root->reloc_root = NULL;  in merge_reloc_roots()
1857 					  &root->state);  in merge_reloc_roots()
1858 				btrfs_put_root(root);  in merge_reloc_roots()
1861 			list_del_init(&reloc_root->root_list);  in merge_reloc_roots()
1862 			/* Don't forget to queue this reloc root for cleanup */  in merge_reloc_roots()
1863 			list_add_tail(&reloc_root->reloc_dirty_list,  in merge_reloc_roots()
1864 				      &rc->dirty_subvol_roots);  in merge_reloc_roots()
1877 		/* new reloc root may be added */  in merge_reloc_roots()
1878 		mutex_lock(&fs_info->reloc_mutex);  in merge_reloc_roots()
1879 		list_splice_init(&rc->reloc_roots, &reloc_roots);  in merge_reloc_roots()
1880 		mutex_unlock(&fs_info->reloc_mutex);  in merge_reloc_roots()
1887 	 * BUG_ON(!RB_EMPTY_ROOT(&rc->reloc_root_tree.rb_root));  in merge_reloc_roots()
1892 	 * fine because we're bailing here, and we hold a reference on the root  in merge_reloc_roots()
1894 	 * do the reloc_dirty_list afterwards.  Meanwhile the root->reloc_root  in merge_reloc_roots()
1915 	struct btrfs_fs_info *fs_info = reloc_root->fs_info;  in record_reloc_root_in_trans()
1916 	struct btrfs_root *root;  in record_reloc_root_in_trans()  local
1919 	if (btrfs_get_root_last_trans(reloc_root) == trans->transid)  in record_reloc_root_in_trans()
1922 	root = btrfs_get_fs_root(fs_info, reloc_root->root_key.offset, false);  in record_reloc_root_in_trans()
1925 	 * This should succeed, since we can't have a reloc root without having  in record_reloc_root_in_trans()
1926 	 * already looked up the actual root and created the reloc root for this  in record_reloc_root_in_trans()
1927 	 * root.  in record_reloc_root_in_trans()
1930 	 * reloc root without a corresponding root this could return ENOENT.  in record_reloc_root_in_trans()
1932 	if (IS_ERR(root)) {  in record_reloc_root_in_trans()
1934 		return PTR_ERR(root);  in record_reloc_root_in_trans()
1936 	if (root->reloc_root != reloc_root) {  in record_reloc_root_in_trans()
1939 			  "root %llu has two reloc roots associated with it",  in record_reloc_root_in_trans()
1940 			  reloc_root->root_key.offset);  in record_reloc_root_in_trans()
1941 		btrfs_put_root(root);  in record_reloc_root_in_trans()
1942 		return -EUCLEAN;  in record_reloc_root_in_trans()
1944 	ret = btrfs_record_root_in_trans(trans, root);  in record_reloc_root_in_trans()
1945 	btrfs_put_root(root);  in record_reloc_root_in_trans()
1953 				     struct btrfs_backref_node *node,  in select_reloc_root()  argument
1957 	struct btrfs_root *root;  in select_reloc_root()  local
1961 	next = walk_up_backref(node, edges, &index);  in select_reloc_root()
1962 	root = next->root;  in select_reloc_root()
1965 	 * If there is no root, then our references for this block are  in select_reloc_root()
1967 	 * that is owned by a root.  in select_reloc_root()
1970 	 * non-SHAREABLE root then we have backrefs that resolve improperly.  in select_reloc_root()
1975 	if (unlikely(!root)) {  in select_reloc_root()
1976 		btrfs_err(trans->fs_info,  in select_reloc_root()
1977 			  "bytenr %llu doesn't have a backref path ending in a root",  in select_reloc_root()
1978 			  node->bytenr);  in select_reloc_root()
1979 		return ERR_PTR(-EUCLEAN);  in select_reloc_root()
1981 	if (unlikely(!test_bit(BTRFS_ROOT_SHAREABLE, &root->state))) {  in select_reloc_root()
1982 		btrfs_err(trans->fs_info,  in select_reloc_root()
1983 			  "bytenr %llu has multiple refs with one ending in a non-shareable root",  in select_reloc_root()
1984 			  node->bytenr);  in select_reloc_root()
1985 		return ERR_PTR(-EUCLEAN);  in select_reloc_root()
1988 	if (btrfs_root_id(root) == BTRFS_TREE_RELOC_OBJECTID) {  in select_reloc_root()
1989 		ret = record_reloc_root_in_trans(trans, root);  in select_reloc_root()
1995 	ret = btrfs_record_root_in_trans(trans, root);  in select_reloc_root()
1998 	root = root->reloc_root;  in select_reloc_root()
2002 	 * root->reloc_root may not be set, return ENOENT in this case.  in select_reloc_root()
2004 	if (!root)  in select_reloc_root()
2005 		return ERR_PTR(-ENOENT);  in select_reloc_root()
2007 	if (next->new_bytenr) {  in select_reloc_root()
2009 		 * We just created the reloc root, so we shouldn't have  in select_reloc_root()
2010 		 * ->new_bytenr set yet. If it is then we have multiple roots  in select_reloc_root()
2014 		ASSERT(next->new_bytenr == 0);  in select_reloc_root()
2015 		btrfs_err(trans->fs_info,  in select_reloc_root()
2017 			  node->bytenr, next->bytenr);  in select_reloc_root()
2018 		return ERR_PTR(-EUCLEAN);  in select_reloc_root()
2021 	next->new_bytenr = root->node->start;  in select_reloc_root()
2022 	btrfs_put_root(next->root);  in select_reloc_root()
2023 	next->root = btrfs_grab_root(root);  in select_reloc_root()
2024 	ASSERT(next->root);  in select_reloc_root()
2027 	next = node;  in select_reloc_root()
2028 	/* setup backref node path for btrfs_reloc_cow_block */  in select_reloc_root()
2030 		rc->backref_cache.path[next->level] = next;  in select_reloc_root()
2031 		if (--index < 0)  in select_reloc_root()
2033 		next = edges[index]->node[UPPER];  in select_reloc_root()
2035 	return root;  in select_reloc_root()
2039  * Select a tree root for relocation.
2044  * Return a tree root pointer if the block is shareable.
2045  * Return -ENOENT if the block is root of reloc tree.
2048 struct btrfs_root *select_one_root(struct btrfs_backref_node *node)  in select_one_root()  argument
2051 	struct btrfs_root *root;  in select_one_root()  local
2053 	struct btrfs_backref_edge *edges[BTRFS_MAX_LEVEL - 1];  in select_one_root()
2056 	next = node;  in select_one_root()
2060 		root = next->root;  in select_one_root()
2064 		 * the way up a particular path, in this case return -EUCLEAN.  in select_one_root()
2066 		if (!root)  in select_one_root()
2067 			return ERR_PTR(-EUCLEAN);  in select_one_root()
2069 		/* No other choice for non-shareable tree */  in select_one_root()
2070 		if (!test_bit(BTRFS_ROOT_SHAREABLE, &root->state))  in select_one_root()
2071 			return root;  in select_one_root()
2073 		if (btrfs_root_id(root) != BTRFS_TREE_RELOC_OBJECTID)  in select_one_root()
2074 			fs_root = root;  in select_one_root()
2076 		if (next != node)  in select_one_root()
2080 		if (!next || next->level <= node->level)  in select_one_root()
2085 		return ERR_PTR(-ENOENT);  in select_one_root()
2090 						  struct btrfs_backref_node *node)  in calcu_metadata_size()  argument
2092 	struct btrfs_fs_info *fs_info = rc->extent_root->fs_info;  in calcu_metadata_size()
2093 	struct btrfs_backref_node *next = node;  in calcu_metadata_size()
2095 	struct btrfs_backref_edge *edges[BTRFS_MAX_LEVEL - 1];  in calcu_metadata_size()
2099 	BUG_ON(node->processed);  in calcu_metadata_size()
2104 			if (next->processed)  in calcu_metadata_size()
2107 			num_bytes += fs_info->nodesize;  in calcu_metadata_size()
2109 			if (list_empty(&next->upper))  in calcu_metadata_size()
2112 			edge = list_entry(next->upper.next,  in calcu_metadata_size()
2115 			next = edge->node[UPPER];  in calcu_metadata_size()
2125 	struct btrfs_fs_info *fs_info = trans->fs_info;  in refill_metadata_space()
2128 	trans->block_rsv = rc->block_rsv;  in refill_metadata_space()
2129 	rc->reserved_bytes += num_bytes;  in refill_metadata_space()
2133 	 * If we get an enospc just kick back -EAGAIN so we know to drop the  in refill_metadata_space()
2136 	ret = btrfs_block_rsv_refill(fs_info, rc->block_rsv, num_bytes,  in refill_metadata_space()
2139 		u64 tmp = fs_info->nodesize * RELOCATION_RESERVED_NODES;  in refill_metadata_space()
2141 		while (tmp <= rc->reserved_bytes)  in refill_metadata_space()
2150 		rc->block_rsv->size = tmp + fs_info->nodesize *  in refill_metadata_space()
2152 		return -EAGAIN;  in refill_metadata_space()
2160 				  struct btrfs_backref_node *node)  in reserve_metadata_space()  argument
2164 	num_bytes = calcu_metadata_size(rc, node) * 2;  in reserve_metadata_space()
2177 			 struct btrfs_backref_node *node,  in do_relocation()  argument
2183 	struct btrfs_backref_edge *edges[BTRFS_MAX_LEVEL - 1];  in do_relocation()
2184 	struct btrfs_root *root;  in do_relocation()  local
2195 	ASSERT(!lowest || !node->eb);  in do_relocation()
2197 	path->lowest_level = node->level + 1;  in do_relocation()
2198 	rc->backref_cache.path[node->level] = node;  in do_relocation()
2199 	list_for_each_entry(edge, &node->upper, list[LOWER]) {  in do_relocation()
2202 		upper = edge->node[UPPER];  in do_relocation()
2203 		root = select_reloc_root(trans, rc, upper, edges);  in do_relocation()
2204 		if (IS_ERR(root)) {  in do_relocation()
2205 			ret = PTR_ERR(root);  in do_relocation()
2209 		if (upper->eb && !upper->locked) {  in do_relocation()
2211 				ret = btrfs_bin_search(upper->eb, 0, key, &slot);  in do_relocation()
2215 				bytenr = btrfs_node_blockptr(upper->eb, slot);  in do_relocation()
2216 				if (node->eb->start == bytenr)  in do_relocation()
2222 		if (!upper->eb) {  in do_relocation()
2223 			ret = btrfs_search_slot(trans, root, key, path, 0, 1);  in do_relocation()
2226 					ret = -ENOENT;  in do_relocation()
2232 			if (!upper->eb) {  in do_relocation()
2233 				upper->eb = path->nodes[upper->level];  in do_relocation()
2234 				path->nodes[upper->level] = NULL;  in do_relocation()
2236 				BUG_ON(upper->eb != path->nodes[upper->level]);  in do_relocation()
2239 			upper->locked = 1;  in do_relocation()
2240 			path->locks[upper->level] = 0;  in do_relocation()
2242 			slot = path->slots[upper->level];  in do_relocation()
2245 			ret = btrfs_bin_search(upper->eb, 0, key, &slot);  in do_relocation()
2251 		bytenr = btrfs_node_blockptr(upper->eb, slot);  in do_relocation()
2253 			if (bytenr != node->bytenr) {  in do_relocation()
2254 				btrfs_err(root->fs_info,  in do_relocation()
2255 		"lowest leaf/node mismatch: bytenr %llu node->bytenr %llu slot %d upper %llu",  in do_relocation()
2256 					  bytenr, node->bytenr, slot,  in do_relocation()
2257 					  upper->eb->start);  in do_relocation()
2258 				ret = -EIO;  in do_relocation()
2262 			if (node->eb->start == bytenr)  in do_relocation()
2266 		blocksize = root->fs_info->nodesize;  in do_relocation()
2267 		eb = btrfs_read_node_slot(upper->eb, slot);  in do_relocation()
2274 		if (!node->eb) {  in do_relocation()
2275 			ret = btrfs_cow_block(trans, root, eb, upper->eb,  in do_relocation()
2283 			 * the correct backref node entry.  in do_relocation()
2285 			ASSERT(node->eb == eb);  in do_relocation()
2289 				.bytenr = node->eb->start,  in do_relocation()
2291 				.parent = upper->eb->start,  in do_relocation()
2292 				.owning_root = btrfs_header_owner(upper->eb),  in do_relocation()
2293 				.ref_root = btrfs_header_owner(upper->eb),  in do_relocation()
2296 			btrfs_set_node_blockptr(upper->eb, slot,  in do_relocation()
2297 						node->eb->start);  in do_relocation()
2298 			btrfs_set_node_ptr_generation(upper->eb, slot,  in do_relocation()
2299 						      trans->transid);  in do_relocation()
2300 			btrfs_mark_buffer_dirty(trans, upper->eb);  in do_relocation()
2302 			btrfs_init_tree_ref(&ref, node->level,  in do_relocation()
2303 					    btrfs_root_id(root), false);  in do_relocation()
2306 				ret = btrfs_drop_subtree(trans, root, eb,  in do_relocation()
2307 							 upper->eb);  in do_relocation()
2312 		if (!upper->pending)  in do_relocation()
2320 	if (!ret && node->pending) {  in do_relocation()
2321 		btrfs_backref_drop_node_buffer(node);  in do_relocation()
2322 		list_del_init(&node->list);  in do_relocation()
2323 		node->pending = 0;  in do_relocation()
2326 	path->lowest_level = 0;  in do_relocation()
2332 	ASSERT(ret != -ENOSPC);  in do_relocation()
2338 			 struct btrfs_backref_node *node,  in link_to_upper()  argument
2343 	btrfs_node_key_to_cpu(node->eb, &key, 0);  in link_to_upper()
2344 	return do_relocation(trans, rc, node, &key, path, 0);  in link_to_upper()
2352 	struct btrfs_backref_cache *cache = &rc->backref_cache;  in finish_pending_nodes()
2353 	struct btrfs_backref_node *node;  in finish_pending_nodes()  local
2358 		while (!list_empty(&cache->pending[level])) {  in finish_pending_nodes()
2359 			node = list_entry(cache->pending[level].next,  in finish_pending_nodes()
2361 			list_move_tail(&node->list, &list);  in finish_pending_nodes()
2362 			BUG_ON(!node->pending);  in finish_pending_nodes()
2365 				ret = link_to_upper(trans, rc, node, path);  in finish_pending_nodes()
2370 		list_splice_init(&list, &cache->pending[level]);  in finish_pending_nodes()
2380 				    struct btrfs_backref_node *node)  in update_processed_blocks()  argument
2382 	struct btrfs_backref_node *next = node;  in update_processed_blocks()
2384 	struct btrfs_backref_edge *edges[BTRFS_MAX_LEVEL - 1];  in update_processed_blocks()
2390 			if (next->processed)  in update_processed_blocks()
2395 			if (list_empty(&next->upper))  in update_processed_blocks()
2398 			edge = list_entry(next->upper.next,  in update_processed_blocks()
2401 			next = edge->node[UPPER];  in update_processed_blocks()
2409 	u32 blocksize = rc->extent_root->fs_info->nodesize;  in tree_block_processed()
2411 	if (test_range_bit(&rc->processed_blocks, bytenr,  in tree_block_processed()
2412 			   bytenr + blocksize - 1, EXTENT_DIRTY, NULL))  in tree_block_processed()
2421 		.level = block->level,  in get_tree_block_key()
2422 		.owner_root = block->owner,  in get_tree_block_key()
2423 		.transid = block->key.offset  in get_tree_block_key()
2427 	eb = read_tree_block(fs_info, block->bytenr, &check);  in get_tree_block_key()
2432 		return -EIO;  in get_tree_block_key()
2434 	if (block->level == 0)  in get_tree_block_key()
2435 		btrfs_item_key_to_cpu(eb, &block->key, 0);  in get_tree_block_key()
2437 		btrfs_node_key_to_cpu(eb, &block->key, 0);  in get_tree_block_key()
2439 	block->key_ready = true;  in get_tree_block_key()
2448 				struct btrfs_backref_node *node,  in relocate_tree_block()  argument
2452 	struct btrfs_root *root;  in relocate_tree_block()  local
2455 	if (!node)  in relocate_tree_block()
2462 	ret = reserve_metadata_space(trans, rc, node);  in relocate_tree_block()
2466 	BUG_ON(node->processed);  in relocate_tree_block()
2467 	root = select_one_root(node);  in relocate_tree_block()
2468 	if (IS_ERR(root)) {  in relocate_tree_block()
2469 		ret = PTR_ERR(root);  in relocate_tree_block()
2471 		/* See explanation in select_one_root for the -EUCLEAN case. */  in relocate_tree_block()
2472 		ASSERT(ret == -ENOENT);  in relocate_tree_block()
2473 		if (ret == -ENOENT) {  in relocate_tree_block()
2475 			update_processed_blocks(rc, node);  in relocate_tree_block()
2480 	if (root) {  in relocate_tree_block()
2481 		if (test_bit(BTRFS_ROOT_SHAREABLE, &root->state)) {  in relocate_tree_block()
2483 			 * This block was the root block of a root, and this is  in relocate_tree_block()
2485 			 * should not have had the ->new_bytenr modified.  in relocate_tree_block()
2494 			ASSERT(node->new_bytenr == 0);  in relocate_tree_block()
2495 			if (node->new_bytenr) {  in relocate_tree_block()
2496 				btrfs_err(root->fs_info,  in relocate_tree_block()
2498 					  node->bytenr);  in relocate_tree_block()
2499 				ret = -EUCLEAN;  in relocate_tree_block()
2502 			ret = btrfs_record_root_in_trans(trans, root);  in relocate_tree_block()
2509 			if (!root->reloc_root) {  in relocate_tree_block()
2510 				ret = -ENOENT;  in relocate_tree_block()
2513 			root = root->reloc_root;  in relocate_tree_block()
2514 			node->new_bytenr = root->node->start;  in relocate_tree_block()
2515 			btrfs_put_root(node->root);  in relocate_tree_block()
2516 			node->root = btrfs_grab_root(root);  in relocate_tree_block()
2517 			ASSERT(node->root);  in relocate_tree_block()
2519 			btrfs_err(root->fs_info,  in relocate_tree_block()
2520 				  "bytenr %llu resolved to a non-shareable root",  in relocate_tree_block()
2521 				  node->bytenr);  in relocate_tree_block()
2522 			ret = -EUCLEAN;  in relocate_tree_block()
2526 			update_processed_blocks(rc, node);  in relocate_tree_block()
2528 		ret = do_relocation(trans, rc, node, key, path, 1);  in relocate_tree_block()
2531 	if (ret || node->level == 0)  in relocate_tree_block()
2532 		btrfs_backref_cleanup_node(&rc->backref_cache, node);  in relocate_tree_block()
2540 	struct btrfs_fs_info *fs_info = trans->fs_info;  in relocate_cowonly_block()
2541 	struct btrfs_root *root;  in relocate_cowonly_block()  local
2546 	root = btrfs_get_fs_root(fs_info, block->owner, true);  in relocate_cowonly_block()
2547 	if (IS_ERR(root))  in relocate_cowonly_block()
2548 		return PTR_ERR(root);  in relocate_cowonly_block()
2550 	nr_levels = max(btrfs_header_level(root->node) - block->level, 0) + 1;  in relocate_cowonly_block()
2552 	num_bytes = fs_info->nodesize * nr_levels;  in relocate_cowonly_block()
2555 		btrfs_put_root(root);  in relocate_cowonly_block()
2558 	path->lowest_level = block->level;  in relocate_cowonly_block()
2559 	if (root == root->fs_info->chunk_root)  in relocate_cowonly_block()
2562 	ret = btrfs_search_slot(trans, root, &block->key, path, 0, 1);  in relocate_cowonly_block()
2563 	path->lowest_level = 0;  in relocate_cowonly_block()
2566 	if (root == root->fs_info->chunk_root)  in relocate_cowonly_block()
2570 	btrfs_put_root(root);  in relocate_cowonly_block()
2582 	struct btrfs_fs_info *fs_info = rc->extent_root->fs_info;  in relocate_tree_blocks()
2583 	struct btrfs_backref_node *node;  in relocate_tree_blocks()  local
2591 		ret = -ENOMEM;  in relocate_tree_blocks()
2597 		if (!block->key_ready)  in relocate_tree_blocks()
2598 			btrfs_readahead_tree_block(fs_info, block->bytenr,  in relocate_tree_blocks()
2599 						   block->owner, 0,  in relocate_tree_blocks()
2600 						   block->level);  in relocate_tree_blocks()
2605 		if (!block->key_ready) {  in relocate_tree_blocks()
2619 		if (block->owner &&  in relocate_tree_blocks()
2620 		    (!is_fstree(block->owner) ||  in relocate_tree_blocks()
2621 		     block->owner == BTRFS_DATA_RELOC_TREE_OBJECTID)) {  in relocate_tree_blocks()
2628 		node = build_backref_tree(trans, rc, &block->key,  in relocate_tree_blocks()
2629 					  block->level, block->bytenr);  in relocate_tree_blocks()
2630 		if (IS_ERR(node)) {  in relocate_tree_blocks()
2631 			ret = PTR_ERR(node);  in relocate_tree_blocks()
2635 		ret = relocate_tree_block(trans, rc, node, &block->key,  in relocate_tree_blocks()
2652 	const struct file_extent_cluster *cluster = &rc->cluster;  in prealloc_file_extent_cluster()
2653 	struct btrfs_inode *inode = BTRFS_I(rc->data_inode);  in prealloc_file_extent_cluster()
2657 	u64 offset = inode->reloc_block_group_start;  in prealloc_file_extent_cluster()
2661 	u64 i_size = i_size_read(&inode->vfs_inode);  in prealloc_file_extent_cluster()
2662 	u64 prealloc_start = cluster->start - offset;  in prealloc_file_extent_cluster()
2663 	u64 prealloc_end = cluster->end - offset;  in prealloc_file_extent_cluster()
2678 		struct address_space *mapping = inode->vfs_inode.i_mapping;  in prealloc_file_extent_cluster()
2679 		struct btrfs_fs_info *fs_info = inode->root->fs_info;  in prealloc_file_extent_cluster()
2680 		const u32 sectorsize = fs_info->sectorsize;  in prealloc_file_extent_cluster()
2692 		 * |- btrfs_lock_and_flush_ordered_range()  in prealloc_file_extent_cluster()
2693 		 *    |- btrfs_start_ordered_extent()  in prealloc_file_extent_cluster()
2694 		 *       |- extent_write_cache_pages()  in prealloc_file_extent_cluster()
2695 		 *          |- lock_page()  in prealloc_file_extent_cluster()
2709 		clear_extent_bits(&inode->io_tree, i_size,  in prealloc_file_extent_cluster()
2710 				  round_up(i_size, PAGE_SIZE) - 1,  in prealloc_file_extent_cluster()
2715 		 * will re-read the whole page anyway.  in prealloc_file_extent_cluster()
2719 					round_up(i_size, PAGE_SIZE) - i_size);  in prealloc_file_extent_cluster()
2725 	BUG_ON(cluster->start != cluster->boundary[0]);  in prealloc_file_extent_cluster()
2727 					      prealloc_end + 1 - prealloc_start);  in prealloc_file_extent_cluster()
2732 	for (nr = 0; nr < cluster->nr; nr++) {  in prealloc_file_extent_cluster()
2735 		start = cluster->boundary[nr] - offset;  in prealloc_file_extent_cluster()
2736 		if (nr + 1 < cluster->nr)  in prealloc_file_extent_cluster()
2737 			end = cluster->boundary[nr + 1] - 1 - offset;  in prealloc_file_extent_cluster()
2739 			end = cluster->end - offset;  in prealloc_file_extent_cluster()
2741 		lock_extent(&inode->io_tree, start, end, &cached_state);  in prealloc_file_extent_cluster()
2742 		num_bytes = end + 1 - start;  in prealloc_file_extent_cluster()
2743 		ret = btrfs_prealloc_file_range(&inode->vfs_inode, 0, start,  in prealloc_file_extent_cluster()
2747 		unlock_extent(&inode->io_tree, start, end, &cached_state);  in prealloc_file_extent_cluster()
2754 		btrfs_free_reserved_data_space_noquota(inode->root->fs_info,  in prealloc_file_extent_cluster()
2755 					       prealloc_end + 1 - cur_offset);  in prealloc_file_extent_cluster()
2761 	struct btrfs_inode *inode = BTRFS_I(rc->data_inode);  in setup_relocation_extent_mapping()
2764 	u64 offset = inode->reloc_block_group_start;  in setup_relocation_extent_mapping()
2765 	u64 start = rc->cluster.start - offset;  in setup_relocation_extent_mapping()
2766 	u64 end = rc->cluster.end - offset;  in setup_relocation_extent_mapping()
2771 		return -ENOMEM;  in setup_relocation_extent_mapping()
2773 	em->start = start;  in setup_relocation_extent_mapping()
2774 	em->len = end + 1 - start;  in setup_relocation_extent_mapping()
2775 	em->disk_bytenr = rc->cluster.start;  in setup_relocation_extent_mapping()
2776 	em->disk_num_bytes = em->len;  in setup_relocation_extent_mapping()
2777 	em->ram_bytes = em->len;  in setup_relocation_extent_mapping()
2778 	em->flags |= EXTENT_FLAG_PINNED;  in setup_relocation_extent_mapping()
2780 	lock_extent(&inode->io_tree, start, end, &cached_state);  in setup_relocation_extent_mapping()
2782 	unlock_extent(&inode->io_tree, start, end, &cached_state);  in setup_relocation_extent_mapping()
2793 	return atomic_read(&fs_info->balance_cancel_req) ||  in btrfs_should_cancel_balance()
2794 		atomic_read(&fs_info->reloc_cancel_req) ||  in btrfs_should_cancel_balance()
2803 	if (cluster_nr >= cluster->nr - 1)  in get_cluster_boundary_end()
2804 		return cluster->end;  in get_cluster_boundary_end()
2807 	return cluster->boundary[cluster_nr + 1] - 1;  in get_cluster_boundary_end()
2814 	const struct file_extent_cluster *cluster = &rc->cluster;  in relocate_one_folio()
2815 	struct inode *inode = rc->data_inode;  in relocate_one_folio()
2817 	u64 offset = BTRFS_I(inode)->reloc_block_group_start;  in relocate_one_folio()
2818 	const unsigned long last_index = (cluster->end - offset) >> PAGE_SHIFT;  in relocate_one_folio()
2819 	gfp_t mask = btrfs_alloc_write_mask(inode->i_mapping);  in relocate_one_folio()
2825 	const bool use_rst = btrfs_need_stripe_tree_update(fs_info, rc->block_group->flags);  in relocate_one_folio()
2829 	folio = filemap_lock_folio(inode->i_mapping, index);  in relocate_one_folio()
2842 			page_cache_sync_readahead(inode->i_mapping, ra, NULL,  in relocate_one_folio()
2843 						  index, last_index + 1 - index);  in relocate_one_folio()
2844 		folio = __filemap_get_folio(inode->i_mapping, index,  in relocate_one_folio()
2854 		page_cache_async_readahead(inode->i_mapping, ra, NULL,  in relocate_one_folio()
2855 					   folio, last_index + 1 - index);  in relocate_one_folio()
2861 			ret = -EIO;  in relocate_one_folio()
2864 		if (folio->mapping != inode->i_mapping) {  in relocate_one_folio()
2881 	folio_end = folio_start + PAGE_SIZE - 1;  in relocate_one_folio()
2887 	cur = max(folio_start, cluster->boundary[*cluster_nr] - offset);  in relocate_one_folio()
2890 		u64 extent_start = cluster->boundary[*cluster_nr] - offset;  in relocate_one_folio()
2892 						*cluster_nr) - offset;  in relocate_one_folio()
2895 		u32 clamped_len = clamped_end + 1 - clamped_start;  in relocate_one_folio()
2905 		lock_extent(&BTRFS_I(inode)->io_tree, clamped_start, clamped_end,  in relocate_one_folio()
2910 			clear_extent_bit(&BTRFS_I(inode)->io_tree,  in relocate_one_folio()
2929 		if (in_range(cluster->boundary[*cluster_nr] - offset, folio_start, PAGE_SIZE)) {  in relocate_one_folio()
2930 			u64 boundary_start = cluster->boundary[*cluster_nr] -  in relocate_one_folio()
2933 					   fs_info->sectorsize - 1;  in relocate_one_folio()
2935 			set_extent_bit(&BTRFS_I(inode)->io_tree,  in relocate_one_folio()
2939 		unlock_extent(&BTRFS_I(inode)->io_tree, clamped_start, clamped_end,  in relocate_one_folio()
2948 			if (*cluster_nr >= cluster->nr)  in relocate_one_folio()
2955 	balance_dirty_pages_ratelimited(inode->i_mapping);  in relocate_one_folio()
2958 		ret = -ECANCELED;  in relocate_one_folio()
2969 	struct inode *inode = rc->data_inode;  in relocate_file_extent_cluster()
2970 	const struct file_extent_cluster *cluster = &rc->cluster;  in relocate_file_extent_cluster()
2971 	u64 offset = BTRFS_I(inode)->reloc_block_group_start;  in relocate_file_extent_cluster()
2978 	if (!cluster->nr)  in relocate_file_extent_cluster()
2983 		return -ENOMEM;  in relocate_file_extent_cluster()
2989 	file_ra_state_init(ra, inode->i_mapping);  in relocate_file_extent_cluster()
2995 	last_index = (cluster->end - offset) >> PAGE_SHIFT;  in relocate_file_extent_cluster()
2996 	for (index = (cluster->start - offset) >> PAGE_SHIFT;  in relocate_file_extent_cluster()
3000 		WARN_ON(cluster_nr != cluster->nr);  in relocate_file_extent_cluster()
3009 	struct inode *inode = rc->data_inode;  in relocate_data_extent()
3010 	struct file_extent_cluster *cluster = &rc->cluster;  in relocate_data_extent()
3012 	struct btrfs_root *root = BTRFS_I(inode)->root;  in relocate_data_extent()  local
3014 	if (cluster->nr > 0 && extent_key->objectid != cluster->end + 1) {  in relocate_data_extent()
3018 		cluster->nr = 0;  in relocate_data_extent()
3022 	 * Under simple quotas, we set root->relocation_src_root when we find  in relocate_data_extent()
3024 	 * them while relocating. Handle this by storing the owning root that  in relocate_data_extent()
3025 	 * started a cluster and if we see an extent from a different root break  in relocate_data_extent()
3026 	 * cluster formation (just like the above case of non-adjacent extents).  in relocate_data_extent()
3032 	if (cluster->nr > 0 && cluster->owning_root != root->relocation_src_root) {  in relocate_data_extent()
3033 		u64 tmp = root->relocation_src_root;  in relocate_data_extent()
3036 		 * root->relocation_src_root is the state that actually affects  in relocate_data_extent()
3037 		 * the preallocation we do here, so set it to the root owning  in relocate_data_extent()
3040 		root->relocation_src_root = cluster->owning_root;  in relocate_data_extent()
3044 		cluster->nr = 0;  in relocate_data_extent()
3045 		/* And reset it back for the current extent's owning root. */  in relocate_data_extent()
3046 		root->relocation_src_root = tmp;  in relocate_data_extent()
3049 	if (!cluster->nr) {  in relocate_data_extent()
3050 		cluster->start = extent_key->objectid;  in relocate_data_extent()
3051 		cluster->owning_root = root->relocation_src_root;  in relocate_data_extent()
3054 		BUG_ON(cluster->nr >= MAX_EXTENTS);  in relocate_data_extent()
3055 	cluster->end = extent_key->objectid + extent_key->offset - 1;  in relocate_data_extent()
3056 	cluster->boundary[cluster->nr] = extent_key->objectid;  in relocate_data_extent()
3057 	cluster->nr++;  in relocate_data_extent()
3059 	if (cluster->nr >= MAX_EXTENTS) {  in relocate_data_extent()
3063 		cluster->nr = 0;  in relocate_data_extent()
3083 	int level = -1;  in add_tree_block()
3087 	eb =  path->nodes[0];  in add_tree_block()
3088 	item_size = btrfs_item_size(eb, path->slots[0]);  in add_tree_block()
3090 	if (extent_key->type == BTRFS_METADATA_ITEM_KEY ||  in add_tree_block()
3094 		ei = btrfs_item_ptr(eb, path->slots[0],  in add_tree_block()
3097 		if (extent_key->type == BTRFS_EXTENT_ITEM_KEY) {  in add_tree_block()
3102 			level = (int)extent_key->offset;  in add_tree_block()
3112 		 * inline ref offset.  We know it's an fs root if  in add_tree_block()
3133 				return -EINVAL;  in add_tree_block()
3139 		btrfs_err(rc->block_group->fs_info,  in add_tree_block()
3141 			  eb->start, path->slots[0]);  in add_tree_block()
3143 		return -EUCLEAN;  in add_tree_block()
3148 	BUG_ON(level == -1);  in add_tree_block()
3152 		return -ENOMEM;  in add_tree_block()
3154 	block->bytenr = extent_key->objectid;  in add_tree_block()
3155 	block->key.objectid = rc->extent_root->fs_info->nodesize;  in add_tree_block()
3156 	block->key.offset = generation;  in add_tree_block()
3157 	block->level = level;  in add_tree_block()
3158 	block->key_ready = false;  in add_tree_block()
3159 	block->owner = owner;  in add_tree_block()
3161 	rb_node = rb_simple_insert(blocks, block->bytenr, &block->rb_node);  in add_tree_block()
3163 		btrfs_backref_panic(rc->extent_root->fs_info, block->bytenr,  in add_tree_block()
3164 				    -EEXIST);  in add_tree_block()
3176 	struct btrfs_fs_info *fs_info = rc->extent_root->fs_info;  in __add_tree_block()
3190 		return -ENOMEM;  in __add_tree_block()
3195 		key.offset = (u64)-1;  in __add_tree_block()
3201 	path->search_commit_root = 1;  in __add_tree_block()
3202 	path->skip_locking = 1;  in __add_tree_block()
3203 	ret = btrfs_search_slot(NULL, rc->extent_root, &key, path, 0, 0);  in __add_tree_block()
3208 		if (path->slots[0]) {  in __add_tree_block()
3209 			path->slots[0]--;  in __add_tree_block()
3210 			btrfs_item_key_to_cpu(path->nodes[0], &key,  in __add_tree_block()
3211 					      path->slots[0]);  in __add_tree_block()
3227 		btrfs_print_leaf(path->nodes[0]);  in __add_tree_block()
3232 		ret = -EINVAL;  in __add_tree_block()
3246 	struct btrfs_fs_info *fs_info = block_group->fs_info;  in delete_block_group_cache()
3247 	struct btrfs_root *root = fs_info->tree_root;  in delete_block_group_cache()  local
3255 	btrfs_inode = btrfs_iget(ino, root);  in delete_block_group_cache()
3257 		return -ENOENT;  in delete_block_group_cache()
3258 	inode = &btrfs_inode->vfs_inode;  in delete_block_group_cache()
3262 						 &fs_info->global_block_rsv);  in delete_block_group_cache()
3266 	trans = btrfs_join_transaction(root);  in delete_block_group_cache()
3282  * Locate the free space cache EXTENT_DATA in root tree leaf and delete the
3317 		return -ENOENT;  in delete_v1_space_cache()
3333 	const u32 blocksize = rc->extent_root->fs_info->nodesize;  in add_data_references()
3338 	ctx.bytenr = extent_key->objectid;  in add_data_references()
3340 	ctx.fs_info = rc->extent_root->fs_info;  in add_data_references()
3351 		eb = read_tree_block(ctx.fs_info, ref_node->val, &check);  in add_data_references()
3356 		ret = delete_v1_space_cache(eb, rc->block_group,  in add_data_references()
3357 					    extent_key->objectid);  in add_data_references()
3361 		ret = __add_tree_block(rc, ref_node->val, blocksize, blocks);  in add_data_references()
3378 	struct btrfs_fs_info *fs_info = rc->extent_root->fs_info;  in find_next_extent()
3384 	last = rc->block_group->start + rc->block_group->length;  in find_next_extent()
3389 		if (rc->search_start >= last) {  in find_next_extent()
3394 		key.objectid = rc->search_start;  in find_next_extent()
3398 		path->search_commit_root = 1;  in find_next_extent()
3399 		path->skip_locking = 1;  in find_next_extent()
3400 		ret = btrfs_search_slot(NULL, rc->extent_root, &key, path,  in find_next_extent()
3405 		leaf = path->nodes[0];  in find_next_extent()
3406 		if (path->slots[0] >= btrfs_header_nritems(leaf)) {  in find_next_extent()
3407 			ret = btrfs_next_leaf(rc->extent_root, path);  in find_next_extent()
3410 			leaf = path->nodes[0];  in find_next_extent()
3413 		btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);  in find_next_extent()
3421 			path->slots[0]++;  in find_next_extent()
3426 		    key.objectid + key.offset <= rc->search_start) {  in find_next_extent()
3427 			path->slots[0]++;  in find_next_extent()
3432 		    key.objectid + fs_info->nodesize <=  in find_next_extent()
3433 		    rc->search_start) {  in find_next_extent()
3434 			path->slots[0]++;  in find_next_extent()
3438 		block_found = find_first_extent_bit(&rc->processed_blocks,  in find_next_extent()
3444 			rc->search_start = end + 1;  in find_next_extent()
3447 				rc->search_start = key.objectid + key.offset;  in find_next_extent()
3449 				rc->search_start = key.objectid +  in find_next_extent()
3450 					fs_info->nodesize;  in find_next_extent()
3461 	struct btrfs_fs_info *fs_info = rc->extent_root->fs_info;  in set_reloc_control()
3463 	mutex_lock(&fs_info->reloc_mutex);  in set_reloc_control()
3464 	fs_info->reloc_ctl = rc;  in set_reloc_control()
3465 	mutex_unlock(&fs_info->reloc_mutex);  in set_reloc_control()
3470 	struct btrfs_fs_info *fs_info = rc->extent_root->fs_info;  in unset_reloc_control()
3472 	mutex_lock(&fs_info->reloc_mutex);  in unset_reloc_control()
3473 	fs_info->reloc_ctl = NULL;  in unset_reloc_control()
3474 	mutex_unlock(&fs_info->reloc_mutex);  in unset_reloc_control()
3483 	rc->block_rsv = btrfs_alloc_block_rsv(rc->extent_root->fs_info,  in prepare_to_relocate()
3485 	if (!rc->block_rsv)  in prepare_to_relocate()
3486 		return -ENOMEM;  in prepare_to_relocate()
3488 	memset(&rc->cluster, 0, sizeof(rc->cluster));  in prepare_to_relocate()
3489 	rc->search_start = rc->block_group->start;  in prepare_to_relocate()
3490 	rc->extents_found = 0;  in prepare_to_relocate()
3491 	rc->nodes_relocated = 0;  in prepare_to_relocate()
3492 	rc->merging_rsv_size = 0;  in prepare_to_relocate()
3493 	rc->reserved_bytes = 0;  in prepare_to_relocate()
3494 	rc->block_rsv->size = rc->extent_root->fs_info->nodesize *  in prepare_to_relocate()
3496 	ret = btrfs_block_rsv_refill(rc->extent_root->fs_info,  in prepare_to_relocate()
3497 				     rc->block_rsv, rc->block_rsv->size,  in prepare_to_relocate()
3502 	rc->create_reloc_tree = true;  in prepare_to_relocate()
3505 	trans = btrfs_join_transaction(rc->extent_root);  in prepare_to_relocate()
3525 	struct btrfs_fs_info *fs_info = rc->extent_root->fs_info;  in relocate_block_group()
3538 		return -ENOMEM;  in relocate_block_group()
3539 	path->reada = READA_FORWARD;  in relocate_block_group()
3548 		rc->reserved_bytes = 0;  in relocate_block_group()
3549 		ret = btrfs_block_rsv_refill(fs_info, rc->block_rsv,  in relocate_block_group()
3550 					     rc->block_rsv->size,  in relocate_block_group()
3557 		trans = btrfs_start_transaction(rc->extent_root, 0);  in relocate_block_group()
3564 		if (rc->backref_cache.last_trans != trans->transid)  in relocate_block_group()
3565 			btrfs_backref_release_cache(&rc->backref_cache);  in relocate_block_group()
3566 		rc->backref_cache.last_trans = trans->transid;  in relocate_block_group()
3574 		rc->extents_found++;  in relocate_block_group()
3576 		ei = btrfs_item_ptr(path->nodes[0], path->slots[0],  in relocate_block_group()
3578 		flags = btrfs_extent_flags(path->nodes[0], ei);  in relocate_block_group()
3582 		 * need to note the owner on the reloc data root so that when  in relocate_block_group()
3584 		 * correct eventual owner (rather than the reloc data root).  in relocate_block_group()
3587 			struct btrfs_root *root = BTRFS_I(rc->data_inode)->root;  in relocate_block_group()  local
3589 								 path->nodes[0],  in relocate_block_group()
3590 								 path->slots[0]);  in relocate_block_group()
3592 			root->relocation_src_root = owning_root_id;  in relocate_block_group()
3597 		} else if (rc->stage == UPDATE_DATA_PTRS &&  in relocate_block_group()
3612 				if (ret != -EAGAIN) {  in relocate_block_group()
3616 				rc->extents_found--;  in relocate_block_group()
3617 				rc->search_start = key.objectid;  in relocate_block_group()
3625 		if (rc->stage == MOVE_DATA_EXTENTS &&  in relocate_block_group()
3627 			rc->found_file_extent = true;  in relocate_block_group()
3635 			err = -ECANCELED;  in relocate_block_group()
3639 	if (trans && progress && err == -ENOSPC) {  in relocate_block_group()
3640 		ret = btrfs_force_chunk_alloc(trans, rc->block_group->flags);  in relocate_block_group()
3649 	clear_extent_bits(&rc->processed_blocks, 0, (u64)-1, EXTENT_DIRTY);  in relocate_block_group()
3662 	rc->create_reloc_tree = false;  in relocate_block_group()
3665 	btrfs_backref_release_cache(&rc->backref_cache);  in relocate_block_group()
3666 	btrfs_block_rsv_release(fs_info, rc->block_rsv, (u64)-1, NULL);  in relocate_block_group()
3680 	rc->merge_reloc_tree = false;  in relocate_block_group()
3682 	btrfs_block_rsv_release(fs_info, rc->block_rsv, (u64)-1, NULL);  in relocate_block_group()
3685 	trans = btrfs_join_transaction(rc->extent_root);  in relocate_block_group()
3697 	btrfs_free_block_rsv(fs_info, rc->block_rsv);  in relocate_block_group()
3703 				 struct btrfs_root *root, u64 objectid)  in __insert_orphan_inode()  argument
3712 		return -ENOMEM;  in __insert_orphan_inode()
3714 	ret = btrfs_insert_empty_inode(trans, root, path, objectid);  in __insert_orphan_inode()
3718 	leaf = path->nodes[0];  in __insert_orphan_inode()
3719 	item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_inode_item);  in __insert_orphan_inode()
3732 				struct btrfs_root *root, u64 objectid)  in delete_orphan_inode()  argument
3740 		ret = -ENOMEM;  in delete_orphan_inode()
3747 	ret = btrfs_search_slot(trans, root, &key, path, -1, 1);  in delete_orphan_inode()
3750 			ret = -ENOENT;  in delete_orphan_inode()
3753 	ret = btrfs_del_item(trans, root, path);  in delete_orphan_inode()
3767 	struct btrfs_fs_info *fs_info = group->fs_info;  in create_reloc_inode()
3770 	struct btrfs_root *root;  in create_reloc_inode()  local
3774 	root = btrfs_grab_root(fs_info->data_reloc_root);  in create_reloc_inode()
3775 	trans = btrfs_start_transaction(root, 6);  in create_reloc_inode()
3777 		btrfs_put_root(root);  in create_reloc_inode()
3781 	ret = btrfs_get_free_objectid(root, &objectid);  in create_reloc_inode()
3785 	ret = __insert_orphan_inode(trans, root, objectid);  in create_reloc_inode()
3789 	inode = btrfs_iget(objectid, root);  in create_reloc_inode()
3791 		delete_orphan_inode(trans, root, objectid);  in create_reloc_inode()
3796 	inode->reloc_block_group_start = group->start;  in create_reloc_inode()
3800 	btrfs_put_root(root);  in create_reloc_inode()
3805 			iput(&inode->vfs_inode);  in create_reloc_inode()
3808 	return &inode->vfs_inode;  in create_reloc_inode()
3817  *   -EINPROGRESS  operation is already in progress, that's probably a bug
3818  *   -ECANCELED    cancellation request was set before the operation started
3822 	if (test_and_set_bit(BTRFS_FS_RELOC_RUNNING, &fs_info->flags)) {  in reloc_chunk_start()
3825 		return -EINPROGRESS;  in reloc_chunk_start()
3828 	if (atomic_read(&fs_info->reloc_cancel_req) > 0) {  in reloc_chunk_start()
3834 		atomic_set(&fs_info->reloc_cancel_req, 0);  in reloc_chunk_start()
3835 		return -ECANCELED;  in reloc_chunk_start()
3846 	if (atomic_read(&fs_info->reloc_cancel_req) > 0)  in reloc_chunk_end()
3848 	clear_and_wake_up_bit(BTRFS_FS_RELOC_RUNNING, &fs_info->flags);  in reloc_chunk_end()
3849 	atomic_set(&fs_info->reloc_cancel_req, 0);  in reloc_chunk_end()
3860 	INIT_LIST_HEAD(&rc->reloc_roots);  in alloc_reloc_control()
3861 	INIT_LIST_HEAD(&rc->dirty_subvol_roots);  in alloc_reloc_control()
3862 	btrfs_backref_init_cache(fs_info, &rc->backref_cache, true);  in alloc_reloc_control()
3863 	rc->reloc_root_tree.rb_root = RB_ROOT;  in alloc_reloc_control()
3864 	spin_lock_init(&rc->reloc_root_tree.lock);  in alloc_reloc_control()
3865 	extent_io_tree_init(fs_info, &rc->processed_blocks, IO_TREE_RELOC_BLOCKS);  in alloc_reloc_control()
3871 	struct mapping_node *node, *tmp;  in free_reloc_control()  local
3873 	free_reloc_roots(&rc->reloc_roots);  in free_reloc_control()
3874 	rbtree_postorder_for_each_entry_safe(node, tmp,  in free_reloc_control()
3875 			&rc->reloc_root_tree.rb_root, rb_node)  in free_reloc_control()
3876 		kfree(node);  in free_reloc_control()
3888 	btrfs_describe_block_groups(block_group->flags, buf, sizeof(buf));  in describe_relocation()
3890 	btrfs_info(block_group->fs_info, "relocating block group %llu flags %s",  in describe_relocation()
3891 		   block_group->start, buf);  in describe_relocation()
3918 	 * This only gets set if we had a half-deleted snapshot on mount.  We  in btrfs_relocate_block_group()
3922 	ret = wait_on_bit(&fs_info->flags, BTRFS_FS_UNFINISHED_DROPS, TASK_INTERRUPTIBLE);  in btrfs_relocate_block_group()
3928 		return -EINTR;  in btrfs_relocate_block_group()
3932 		return -ENOENT;  in btrfs_relocate_block_group()
3942 	if (bg->flags & BTRFS_BLOCK_GROUP_DATA)  in btrfs_relocate_block_group()
3943 		ASSERT(sb_write_started(fs_info->sb));  in btrfs_relocate_block_group()
3947 		return -ETXTBSY;  in btrfs_relocate_block_group()
3953 		return -ENOMEM;  in btrfs_relocate_block_group()
3962 	rc->extent_root = extent_root;  in btrfs_relocate_block_group()
3963 	rc->block_group = bg;  in btrfs_relocate_block_group()
3965 	ret = btrfs_inc_block_group_ro(rc->block_group, true);  in btrfs_relocate_block_group()
3974 		err = -ENOMEM;  in btrfs_relocate_block_group()
3978 	inode = lookup_free_space_inode(rc->block_group, path);  in btrfs_relocate_block_group()
3982 		ret = delete_block_group_cache(rc->block_group, inode, 0);  in btrfs_relocate_block_group()
3986 	if (ret && ret != -ENOENT) {  in btrfs_relocate_block_group()
3991 	rc->data_inode = create_reloc_inode(rc->block_group);  in btrfs_relocate_block_group()
3992 	if (IS_ERR(rc->data_inode)) {  in btrfs_relocate_block_group()
3993 		err = PTR_ERR(rc->data_inode);  in btrfs_relocate_block_group()
3994 		rc->data_inode = NULL;  in btrfs_relocate_block_group()
3998 	describe_relocation(rc->block_group);  in btrfs_relocate_block_group()
4000 	btrfs_wait_block_group_reservations(rc->block_group);  in btrfs_relocate_block_group()
4001 	btrfs_wait_nocow_writers(rc->block_group);  in btrfs_relocate_block_group()
4002 	btrfs_wait_ordered_roots(fs_info, U64_MAX, rc->block_group);  in btrfs_relocate_block_group()
4004 	ret = btrfs_zone_finish(rc->block_group);  in btrfs_relocate_block_group()
4005 	WARN_ON(ret && ret != -EAGAIN);  in btrfs_relocate_block_group()
4010 		mutex_lock(&fs_info->cleaner_mutex);  in btrfs_relocate_block_group()
4012 		mutex_unlock(&fs_info->cleaner_mutex);  in btrfs_relocate_block_group()
4016 		finishes_stage = rc->stage;  in btrfs_relocate_block_group()
4021 		 * BUG_ON(rc->stage == UPDATE_DATA_PTRS) in  in btrfs_relocate_block_group()
4026 		if (rc->stage == MOVE_DATA_EXTENTS && rc->found_file_extent) {  in btrfs_relocate_block_group()
4027 			ret = btrfs_wait_ordered_range(BTRFS_I(rc->data_inode), 0,  in btrfs_relocate_block_group()
4028 						       (u64)-1);  in btrfs_relocate_block_group()
4031 			invalidate_mapping_pages(rc->data_inode->i_mapping,  in btrfs_relocate_block_group()
4032 						 0, -1);  in btrfs_relocate_block_group()
4033 			rc->stage = UPDATE_DATA_PTRS;  in btrfs_relocate_block_group()
4039 		if (rc->extents_found == 0)  in btrfs_relocate_block_group()
4043 			   rc->extents_found, stage_to_string(finishes_stage));  in btrfs_relocate_block_group()
4046 	WARN_ON(rc->block_group->pinned > 0);  in btrfs_relocate_block_group()
4047 	WARN_ON(rc->block_group->reserved > 0);  in btrfs_relocate_block_group()
4048 	WARN_ON(rc->block_group->used > 0);  in btrfs_relocate_block_group()
4051 		btrfs_dec_block_group_ro(rc->block_group);  in btrfs_relocate_block_group()
4052 	iput(rc->data_inode);  in btrfs_relocate_block_group()
4060 static noinline_for_stack int mark_garbage_root(struct btrfs_root *root)  in mark_garbage_root()  argument
4062 	struct btrfs_fs_info *fs_info = root->fs_info;  in mark_garbage_root()
4066 	trans = btrfs_start_transaction(fs_info->tree_root, 0);  in mark_garbage_root()
4070 	memset(&root->root_item.drop_progress, 0,  in mark_garbage_root()
4071 		sizeof(root->root_item.drop_progress));  in mark_garbage_root()
4072 	btrfs_set_root_drop_level(&root->root_item, 0);  in mark_garbage_root()
4073 	btrfs_set_root_refs(&root->root_item, 0);  in mark_garbage_root()
4074 	ret = btrfs_update_root(trans, fs_info->tree_root,  in mark_garbage_root()
4075 				&root->root_key, &root->root_item);  in mark_garbage_root()
4104 		return -ENOMEM;  in btrfs_recover_relocation()
4105 	path->reada = READA_BACK;  in btrfs_recover_relocation()
4109 	key.offset = (u64)-1;  in btrfs_recover_relocation()
4112 		ret = btrfs_search_slot(NULL, fs_info->tree_root, &key,  in btrfs_recover_relocation()
4117 			if (path->slots[0] == 0)  in btrfs_recover_relocation()
4119 			path->slots[0]--;  in btrfs_recover_relocation()
4122 		leaf = path->nodes[0];  in btrfs_recover_relocation()
4123 		btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);  in btrfs_recover_relocation()
4130 		reloc_root = btrfs_read_tree_root(fs_info->tree_root, &key);  in btrfs_recover_relocation()
4136 		set_bit(BTRFS_ROOT_SHAREABLE, &reloc_root->state);  in btrfs_recover_relocation()
4137 		list_add(&reloc_root->root_list, &reloc_roots);  in btrfs_recover_relocation()
4139 		if (btrfs_root_refs(&reloc_root->root_item) > 0) {  in btrfs_recover_relocation()
4141 					reloc_root->root_key.offset, false);  in btrfs_recover_relocation()
4144 				if (ret != -ENOENT)  in btrfs_recover_relocation()
4158 		key.offset--;  in btrfs_recover_relocation()
4167 		ret = -ENOMEM;  in btrfs_recover_relocation()
4175 	rc->extent_root = btrfs_extent_root(fs_info, 0);  in btrfs_recover_relocation()
4179 	trans = btrfs_join_transaction(rc->extent_root);  in btrfs_recover_relocation()
4185 	rc->merge_reloc_tree = true;  in btrfs_recover_relocation()
4190 		list_del(&reloc_root->root_list);  in btrfs_recover_relocation()
4192 		if (btrfs_root_refs(&reloc_root->root_item) == 0) {  in btrfs_recover_relocation()
4193 			list_add_tail(&reloc_root->root_list,  in btrfs_recover_relocation()
4194 				      &rc->reloc_roots);  in btrfs_recover_relocation()
4198 		fs_root = btrfs_get_fs_root(fs_info, reloc_root->root_key.offset,  in btrfs_recover_relocation()
4202 			list_add_tail(&reloc_root->root_list, &reloc_roots);  in btrfs_recover_relocation()
4208 		ASSERT(ret != -EEXIST);  in btrfs_recover_relocation()
4210 			list_add_tail(&reloc_root->root_list, &reloc_roots);  in btrfs_recover_relocation()
4215 		fs_root->reloc_root = btrfs_grab_root(reloc_root);  in btrfs_recover_relocation()
4227 	trans = btrfs_join_transaction(rc->extent_root);  in btrfs_recover_relocation()
4249 		fs_root = btrfs_grab_root(fs_info->data_reloc_root);  in btrfs_recover_relocation()
4261  * it also saves CPU time to re-calculate the checksum.
4265 	struct btrfs_inode *inode = ordered->inode;  in btrfs_reloc_clone_csums()
4266 	struct btrfs_fs_info *fs_info = inode->root->fs_info;  in btrfs_reloc_clone_csums()
4267 	u64 disk_bytenr = ordered->file_offset + inode->reloc_block_group_start;  in btrfs_reloc_clone_csums()
4273 				      disk_bytenr + ordered->num_bytes - 1,  in btrfs_reloc_clone_csums()
4284 		list_del_init(&sums->list);  in btrfs_reloc_clone_csums()
4298 		sums->logical = ordered->disk_bytenr + sums->logical - disk_bytenr;  in btrfs_reloc_clone_csums()
4306 			  struct btrfs_root *root,  in btrfs_reloc_cow_block()  argument
4310 	struct btrfs_fs_info *fs_info = root->fs_info;  in btrfs_reloc_cow_block()
4312 	struct btrfs_backref_node *node;  in btrfs_reloc_cow_block()  local
4317 	rc = fs_info->reloc_ctl;  in btrfs_reloc_cow_block()
4321 	BUG_ON(rc->stage == UPDATE_DATA_PTRS && btrfs_is_data_reloc_root(root));  in btrfs_reloc_cow_block()
4325 	    btrfs_root_last_snapshot(&root->root_item))  in btrfs_reloc_cow_block()
4328 	if (btrfs_root_id(root) == BTRFS_TREE_RELOC_OBJECTID && rc->create_reloc_tree) {  in btrfs_reloc_cow_block()
4331 		node = rc->backref_cache.path[level];  in btrfs_reloc_cow_block()
4334 		 * If node->bytenr != buf->start and node->new_bytenr !=  in btrfs_reloc_cow_block()
4335 		 * buf->start then we've got the wrong backref node for what we  in btrfs_reloc_cow_block()
4338 		if (unlikely(node->bytenr != buf->start && node->new_bytenr != buf->start)) {  in btrfs_reloc_cow_block()
4341 				  buf->start, node->bytenr, node->new_bytenr);  in btrfs_reloc_cow_block()
4342 			return -EUCLEAN;  in btrfs_reloc_cow_block()
4345 		btrfs_backref_drop_node_buffer(node);  in btrfs_reloc_cow_block()
4346 		atomic_inc(&cow->refs);  in btrfs_reloc_cow_block()
4347 		node->eb = cow;  in btrfs_reloc_cow_block()
4348 		node->new_bytenr = cow->start;  in btrfs_reloc_cow_block()
4350 		if (!node->pending) {  in btrfs_reloc_cow_block()
4351 			list_move_tail(&node->list,  in btrfs_reloc_cow_block()
4352 				       &rc->backref_cache.pending[level]);  in btrfs_reloc_cow_block()
4353 			node->pending = 1;  in btrfs_reloc_cow_block()
4357 			mark_block_processed(rc, node);  in btrfs_reloc_cow_block()
4360 			rc->nodes_relocated += buf->len;  in btrfs_reloc_cow_block()
4363 	if (level == 0 && first_cow && rc->stage == UPDATE_DATA_PTRS)  in btrfs_reloc_cow_block()
4364 		ret = replace_file_extents(trans, rc, root, cow);  in btrfs_reloc_cow_block()
4375 	struct btrfs_root *root = pending->root;  in btrfs_reloc_pre_snapshot()  local
4376 	struct reloc_control *rc = root->fs_info->reloc_ctl;  in btrfs_reloc_pre_snapshot()
4378 	if (!rc || !have_reloc_root(root))  in btrfs_reloc_pre_snapshot()
4381 	if (!rc->merge_reloc_tree)  in btrfs_reloc_pre_snapshot()
4384 	root = root->reloc_root;  in btrfs_reloc_pre_snapshot()
4385 	BUG_ON(btrfs_root_refs(&root->root_item) == 0);  in btrfs_reloc_pre_snapshot()
4396 	*bytes_to_reserve += rc->nodes_relocated;  in btrfs_reloc_pre_snapshot()
4401  * and create reloc root for the newly created snapshot
4404  * references held on the reloc_root, one for root->reloc_root and one for
4405  * rc->reloc_roots.
4410 	struct btrfs_root *root = pending->root;  in btrfs_reloc_post_snapshot()  local
4413 	struct reloc_control *rc = root->fs_info->reloc_ctl;  in btrfs_reloc_post_snapshot()
4416 	if (!rc || !have_reloc_root(root))  in btrfs_reloc_post_snapshot()
4419 	rc = root->fs_info->reloc_ctl;  in btrfs_reloc_post_snapshot()
4420 	rc->merging_rsv_size += rc->nodes_relocated;  in btrfs_reloc_post_snapshot()
4422 	if (rc->merge_reloc_tree) {  in btrfs_reloc_post_snapshot()
4423 		ret = btrfs_block_rsv_migrate(&pending->block_rsv,  in btrfs_reloc_post_snapshot()
4424 					      rc->block_rsv,  in btrfs_reloc_post_snapshot()
4425 					      rc->nodes_relocated, true);  in btrfs_reloc_post_snapshot()
4430 	new_root = pending->snap;  in btrfs_reloc_post_snapshot()
4431 	reloc_root = create_reloc_root(trans, root->reloc_root, btrfs_root_id(new_root));  in btrfs_reloc_post_snapshot()
4436 	ASSERT(ret != -EEXIST);  in btrfs_reloc_post_snapshot()
4442 	new_root->reloc_root = btrfs_grab_root(reloc_root);  in btrfs_reloc_post_snapshot()
4455 	lockdep_assert_held(&fs_info->reloc_mutex);  in btrfs_get_reloc_bg_bytenr()
4457 	if (fs_info->reloc_ctl && fs_info->reloc_ctl->block_group)  in btrfs_get_reloc_bg_bytenr()
4458 		logical = fs_info->reloc_ctl->block_group->start;  in btrfs_get_reloc_bg_bytenr()