free-space-cache.c (3c09ec59cdea5b132212d97154d625fd34e436dd) free-space-cache.c (5297199a8bca12b8b96afcbf2341605efb6005de)
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2008 Red Hat. All rights reserved.
4 */
5
6#include <linux/pagemap.h>
7#include <linux/sched.h>
8#include <linux/sched/signal.h>
9#include <linux/slab.h>
10#include <linux/math64.h>
11#include <linux/ratelimit.h>
12#include <linux/error-injection.h>
13#include <linux/sched/mm.h>
14#include "ctree.h"
15#include "free-space-cache.h"
16#include "transaction.h"
17#include "disk-io.h"
18#include "extent_io.h"
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2008 Red Hat. All rights reserved.
4 */
5
6#include <linux/pagemap.h>
7#include <linux/sched.h>
8#include <linux/sched/signal.h>
9#include <linux/slab.h>
10#include <linux/math64.h>
11#include <linux/ratelimit.h>
12#include <linux/error-injection.h>
13#include <linux/sched/mm.h>
14#include "ctree.h"
15#include "free-space-cache.h"
16#include "transaction.h"
17#include "disk-io.h"
18#include "extent_io.h"
19#include "inode-map.h"
20#include "volumes.h"
21#include "space-info.h"
22#include "delalloc-space.h"
23#include "block-group.h"
24#include "discard.h"
25
26#define BITS_PER_BITMAP (PAGE_SIZE * 8UL)
27#define MAX_CACHE_BYTES_PER_GIG SZ_64K
28#define FORCE_EXTENT_THRESHOLD SZ_1M
29
30struct btrfs_trim_range {
31 u64 start;
32 u64 bytes;
33 struct list_head list;
34};
35
19#include "volumes.h"
20#include "space-info.h"
21#include "delalloc-space.h"
22#include "block-group.h"
23#include "discard.h"
24
25#define BITS_PER_BITMAP (PAGE_SIZE * 8UL)
26#define MAX_CACHE_BYTES_PER_GIG SZ_64K
27#define FORCE_EXTENT_THRESHOLD SZ_1M
28
29struct btrfs_trim_range {
30 u64 start;
31 u64 bytes;
32 struct list_head list;
33};
34
36static int count_bitmap_extents(struct btrfs_free_space_ctl *ctl,
37 struct btrfs_free_space *bitmap_info);
38static int link_free_space(struct btrfs_free_space_ctl *ctl,
39 struct btrfs_free_space *info);
40static void unlink_free_space(struct btrfs_free_space_ctl *ctl,
41 struct btrfs_free_space *info);
35static int link_free_space(struct btrfs_free_space_ctl *ctl,
36 struct btrfs_free_space *info);
37static void unlink_free_space(struct btrfs_free_space_ctl *ctl,
38 struct btrfs_free_space *info);
42static int btrfs_wait_cache_io_root(struct btrfs_root *root,
43 struct btrfs_trans_handle *trans,
44 struct btrfs_io_ctl *io_ctl,
45 struct btrfs_path *path);
39static int search_bitmap(struct btrfs_free_space_ctl *ctl,
40 struct btrfs_free_space *bitmap_info, u64 *offset,
41 u64 *bytes, bool for_alloc);
42static void free_bitmap(struct btrfs_free_space_ctl *ctl,
43 struct btrfs_free_space *bitmap_info);
44static void bitmap_clear_bits(struct btrfs_free_space_ctl *ctl,
45 struct btrfs_free_space *info, u64 offset,
46 u64 bytes);
46
47static struct inode *__lookup_free_space_inode(struct btrfs_root *root,
48 struct btrfs_path *path,
49 u64 offset)
50{
51 struct btrfs_fs_info *fs_info = root->fs_info;
52 struct btrfs_key key;
53 struct btrfs_key location;

--- 208 unchanged lines hidden (view full) ---

262
263 btrfs_i_size_write(BTRFS_I(inode), 0);
264 truncate_pagecache(inode, 0);
265
266 /*
267 * We skip the throttling logic for free space cache inodes, so we don't
268 * need to check for -EAGAIN.
269 */
47
48static struct inode *__lookup_free_space_inode(struct btrfs_root *root,
49 struct btrfs_path *path,
50 u64 offset)
51{
52 struct btrfs_fs_info *fs_info = root->fs_info;
53 struct btrfs_key key;
54 struct btrfs_key location;

--- 208 unchanged lines hidden (view full) ---

263
264 btrfs_i_size_write(BTRFS_I(inode), 0);
265 truncate_pagecache(inode, 0);
266
267 /*
268 * We skip the throttling logic for free space cache inodes, so we don't
269 * need to check for -EAGAIN.
270 */
270 ret = btrfs_truncate_inode_items(trans, root, inode,
271 ret = btrfs_truncate_inode_items(trans, root, BTRFS_I(inode),
271 0, BTRFS_EXTENT_DATA_KEY);
272 if (ret)
273 goto fail;
274
272 0, BTRFS_EXTENT_DATA_KEY);
273 if (ret)
274 goto fail;
275
275 ret = btrfs_update_inode(trans, root, inode);
276 ret = btrfs_update_inode(trans, root, BTRFS_I(inode));
276
277fail:
278 if (locked)
279 mutex_unlock(&trans->transaction->cache_write_mutex);
280 if (ret)
281 btrfs_abort_transaction(trans, ret);
282
283 return ret;

--- 336 unchanged lines hidden (view full) ---

620 return ret;
621
622 copy_page(entry->bitmap, io_ctl->cur);
623 io_ctl_unmap_page(io_ctl);
624
625 return 0;
626}
627
277
278fail:
279 if (locked)
280 mutex_unlock(&trans->transaction->cache_write_mutex);
281 if (ret)
282 btrfs_abort_transaction(trans, ret);
283
284 return ret;

--- 336 unchanged lines hidden (view full) ---

621 return ret;
622
623 copy_page(entry->bitmap, io_ctl->cur);
624 io_ctl_unmap_page(io_ctl);
625
626 return 0;
627}
628
628/*
629 * Since we attach pinned extents after the fact we can have contiguous sections
630 * of free space that are split up in entries. This poses a problem with the
631 * tree logging stuff since it could have allocated across what appears to be 2
632 * entries since we would have merged the entries when adding the pinned extents
633 * back to the free space cache. So run through the space cache that we just
634 * loaded and merge contiguous entries. This will make the log replay stuff not
635 * blow up and it will make for nicer allocator behavior.
636 */
637static void merge_space_tree(struct btrfs_free_space_ctl *ctl)
638{
639 struct btrfs_free_space *e, *prev = NULL;
640 struct rb_node *n;
641
642again:
643 spin_lock(&ctl->tree_lock);
644 for (n = rb_first(&ctl->free_space_offset); n; n = rb_next(n)) {
645 e = rb_entry(n, struct btrfs_free_space, offset_index);
646 if (!prev)
647 goto next;
648 if (e->bitmap || prev->bitmap)
649 goto next;
650 if (prev->offset + prev->bytes == e->offset) {
651 unlink_free_space(ctl, prev);
652 unlink_free_space(ctl, e);
653 prev->bytes += e->bytes;
654 kmem_cache_free(btrfs_free_space_cachep, e);
655 link_free_space(ctl, prev);
656 prev = NULL;
657 spin_unlock(&ctl->tree_lock);
658 goto again;
659 }
660next:
661 prev = e;
662 }
663 spin_unlock(&ctl->tree_lock);
664}
665
666static int __load_free_space_cache(struct btrfs_root *root, struct inode *inode,
667 struct btrfs_free_space_ctl *ctl,
668 struct btrfs_path *path, u64 offset)
669{
670 struct btrfs_fs_info *fs_info = root->fs_info;
671 struct btrfs_free_space_header *header;
672 struct extent_buffer *leaf;
673 struct btrfs_io_ctl io_ctl;

--- 74 unchanged lines hidden (view full) ---

748 goto free_cache;
749
750 ret = io_ctl_read_entry(&io_ctl, e, &type);
751 if (ret) {
752 kmem_cache_free(btrfs_free_space_cachep, e);
753 goto free_cache;
754 }
755
629static int __load_free_space_cache(struct btrfs_root *root, struct inode *inode,
630 struct btrfs_free_space_ctl *ctl,
631 struct btrfs_path *path, u64 offset)
632{
633 struct btrfs_fs_info *fs_info = root->fs_info;
634 struct btrfs_free_space_header *header;
635 struct extent_buffer *leaf;
636 struct btrfs_io_ctl io_ctl;

--- 74 unchanged lines hidden (view full) ---

711 goto free_cache;
712
713 ret = io_ctl_read_entry(&io_ctl, e, &type);
714 if (ret) {
715 kmem_cache_free(btrfs_free_space_cachep, e);
716 goto free_cache;
717 }
718
756 /*
757 * Sync discard ensures that the free space cache is always
758 * trimmed. So when reading this in, the state should reflect
759 * that. We also do this for async as a stop gap for lack of
760 * persistence.
761 */
762 if (btrfs_test_opt(fs_info, DISCARD_SYNC) ||
763 btrfs_test_opt(fs_info, DISCARD_ASYNC))
764 e->trim_state = BTRFS_TRIM_STATE_TRIMMED;
765
766 if (!e->bytes) {
767 kmem_cache_free(btrfs_free_space_cachep, e);
768 goto free_cache;
769 }
770
771 if (type == BTRFS_FREE_SPACE_EXTENT) {
772 spin_lock(&ctl->tree_lock);
773 ret = link_free_space(ctl, e);

--- 37 unchanged lines hidden (view full) ---

811 * We add the bitmaps at the end of the entries in order that
812 * the bitmap entries are added to the cache.
813 */
814 list_for_each_entry_safe(e, n, &bitmaps, list) {
815 list_del_init(&e->list);
816 ret = io_ctl_read_bitmap(&io_ctl, e);
817 if (ret)
818 goto free_cache;
719 if (!e->bytes) {
720 kmem_cache_free(btrfs_free_space_cachep, e);
721 goto free_cache;
722 }
723
724 if (type == BTRFS_FREE_SPACE_EXTENT) {
725 spin_lock(&ctl->tree_lock);
726 ret = link_free_space(ctl, e);

--- 37 unchanged lines hidden (view full) ---

764 * We add the bitmaps at the end of the entries in order that
765 * the bitmap entries are added to the cache.
766 */
767 list_for_each_entry_safe(e, n, &bitmaps, list) {
768 list_del_init(&e->list);
769 ret = io_ctl_read_bitmap(&io_ctl, e);
770 if (ret)
771 goto free_cache;
819 e->bitmap_extents = count_bitmap_extents(ctl, e);
820 if (!btrfs_free_space_trimmed(e)) {
821 ctl->discardable_extents[BTRFS_STAT_CURR] +=
822 e->bitmap_extents;
823 ctl->discardable_bytes[BTRFS_STAT_CURR] += e->bytes;
824 }
825 }
826
827 io_ctl_drop_pages(&io_ctl);
772 }
773
774 io_ctl_drop_pages(&io_ctl);
828 merge_space_tree(ctl);
829 ret = 1;
830out:
775 ret = 1;
776out:
831 btrfs_discard_update_discardable(ctl->private, ctl);
832 io_ctl_free(&io_ctl);
833 return ret;
834free_cache:
835 io_ctl_drop_pages(&io_ctl);
836 __btrfs_remove_free_space_cache(ctl);
837 goto out;
838}
839
777 io_ctl_free(&io_ctl);
778 return ret;
779free_cache:
780 io_ctl_drop_pages(&io_ctl);
781 __btrfs_remove_free_space_cache(ctl);
782 goto out;
783}
784
785static int copy_free_space_cache(struct btrfs_block_group *block_group,
786 struct btrfs_free_space_ctl *ctl)
787{
788 struct btrfs_free_space *info;
789 struct rb_node *n;
790 int ret = 0;
791
792 while (!ret && (n = rb_first(&ctl->free_space_offset)) != NULL) {
793 info = rb_entry(n, struct btrfs_free_space, offset_index);
794 if (!info->bitmap) {
795 unlink_free_space(ctl, info);
796 ret = btrfs_add_free_space(block_group, info->offset,
797 info->bytes);
798 kmem_cache_free(btrfs_free_space_cachep, info);
799 } else {
800 u64 offset = info->offset;
801 u64 bytes = ctl->unit;
802
803 while (search_bitmap(ctl, info, &offset, &bytes,
804 false) == 0) {
805 ret = btrfs_add_free_space(block_group, offset,
806 bytes);
807 if (ret)
808 break;
809 bitmap_clear_bits(ctl, info, offset, bytes);
810 offset = info->offset;
811 bytes = ctl->unit;
812 }
813 free_bitmap(ctl, info);
814 }
815 cond_resched();
816 }
817 return ret;
818}
819
840int load_free_space_cache(struct btrfs_block_group *block_group)
841{
842 struct btrfs_fs_info *fs_info = block_group->fs_info;
843 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;
820int load_free_space_cache(struct btrfs_block_group *block_group)
821{
822 struct btrfs_fs_info *fs_info = block_group->fs_info;
823 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;
824 struct btrfs_free_space_ctl tmp_ctl = {};
844 struct inode *inode;
845 struct btrfs_path *path;
846 int ret = 0;
847 bool matched;
848 u64 used = block_group->used;
849
850 /*
825 struct inode *inode;
826 struct btrfs_path *path;
827 int ret = 0;
828 bool matched;
829 u64 used = block_group->used;
830
831 /*
832 * Because we could potentially discard our loaded free space, we want
833 * to load everything into a temporary structure first, and then if it's
834 * valid copy it all into the actual free space ctl.
835 */
836 btrfs_init_free_space_ctl(block_group, &tmp_ctl);
837
838 /*
851 * If this block group has been marked to be cleared for one reason or
852 * another then we can't trust the on disk cache, so just return.
853 */
854 spin_lock(&block_group->lock);
855 if (block_group->disk_cache_state != BTRFS_DC_WRITTEN) {
856 spin_unlock(&block_group->lock);
857 return 0;
858 }

--- 34 unchanged lines hidden (view full) ---

893 spin_lock(&block_group->lock);
894 if (block_group->disk_cache_state != BTRFS_DC_WRITTEN) {
895 spin_unlock(&block_group->lock);
896 btrfs_free_path(path);
897 goto out;
898 }
899 spin_unlock(&block_group->lock);
900
839 * If this block group has been marked to be cleared for one reason or
840 * another then we can't trust the on disk cache, so just return.
841 */
842 spin_lock(&block_group->lock);
843 if (block_group->disk_cache_state != BTRFS_DC_WRITTEN) {
844 spin_unlock(&block_group->lock);
845 return 0;
846 }

--- 34 unchanged lines hidden (view full) ---

881 spin_lock(&block_group->lock);
882 if (block_group->disk_cache_state != BTRFS_DC_WRITTEN) {
883 spin_unlock(&block_group->lock);
884 btrfs_free_path(path);
885 goto out;
886 }
887 spin_unlock(&block_group->lock);
888
901 ret = __load_free_space_cache(fs_info->tree_root, inode, ctl,
889 ret = __load_free_space_cache(fs_info->tree_root, inode, &tmp_ctl,
902 path, block_group->start);
903 btrfs_free_path(path);
904 if (ret <= 0)
905 goto out;
906
890 path, block_group->start);
891 btrfs_free_path(path);
892 if (ret <= 0)
893 goto out;
894
907 spin_lock(&ctl->tree_lock);
908 matched = (ctl->free_space == (block_group->length - used -
909 block_group->bytes_super));
910 spin_unlock(&ctl->tree_lock);
895 matched = (tmp_ctl.free_space == (block_group->length - used -
896 block_group->bytes_super));
911
897
912 if (!matched) {
913 __btrfs_remove_free_space_cache(ctl);
898 if (matched) {
899 ret = copy_free_space_cache(block_group, &tmp_ctl);
900 /*
901 * ret == 1 means we successfully loaded the free space cache,
902 * so we need to re-set it here.
903 */
904 if (ret == 0)
905 ret = 1;
906 } else {
907 __btrfs_remove_free_space_cache(&tmp_ctl);
914 btrfs_warn(fs_info,
915 "block group %llu has wrong amount of free space",
916 block_group->start);
917 ret = -1;
918 }
919out:
920 if (ret < 0) {
921 /* This cache is bogus, make sure it gets cleared */
922 spin_lock(&block_group->lock);
923 block_group->disk_cache_state = BTRFS_DC_CLEAR;
924 spin_unlock(&block_group->lock);
925 ret = 0;
926
927 btrfs_warn(fs_info,
928 "failed to load free space cache for block group %llu, rebuilding it now",
929 block_group->start);
930 }
931
908 btrfs_warn(fs_info,
909 "block group %llu has wrong amount of free space",
910 block_group->start);
911 ret = -1;
912 }
913out:
914 if (ret < 0) {
915 /* This cache is bogus, make sure it gets cleared */
916 spin_lock(&block_group->lock);
917 block_group->disk_cache_state = BTRFS_DC_CLEAR;
918 spin_unlock(&block_group->lock);
919 ret = 0;
920
921 btrfs_warn(fs_info,
922 "failed to load free space cache for block group %llu, rebuilding it now",
923 block_group->start);
924 }
925
926 spin_lock(&ctl->tree_lock);
927 btrfs_discard_update_discardable(block_group);
928 spin_unlock(&ctl->tree_lock);
932 iput(inode);
933 return ret;
934}
935
936static noinline_for_stack
937int write_cache_extent_entries(struct btrfs_io_ctl *io_ctl,
938 struct btrfs_free_space_ctl *ctl,
939 struct btrfs_block_group *block_group,

--- 246 unchanged lines hidden (view full) ---

1186 if (ret) {
1187 invalidate_inode_pages2(inode->i_mapping);
1188 BTRFS_I(inode)->generation = 0;
1189 if (block_group)
1190 btrfs_debug(root->fs_info,
1191 "failed to write free space cache for block group %llu error %d",
1192 block_group->start, ret);
1193 }
929 iput(inode);
930 return ret;
931}
932
933static noinline_for_stack
934int write_cache_extent_entries(struct btrfs_io_ctl *io_ctl,
935 struct btrfs_free_space_ctl *ctl,
936 struct btrfs_block_group *block_group,

--- 246 unchanged lines hidden (view full) ---

1183 if (ret) {
1184 invalidate_inode_pages2(inode->i_mapping);
1185 BTRFS_I(inode)->generation = 0;
1186 if (block_group)
1187 btrfs_debug(root->fs_info,
1188 "failed to write free space cache for block group %llu error %d",
1189 block_group->start, ret);
1190 }
1194 btrfs_update_inode(trans, root, inode);
1191 btrfs_update_inode(trans, root, BTRFS_I(inode));
1195
1196 if (block_group) {
1197 /* the dirty list is protected by the dirty_bgs_lock */
1198 spin_lock(&trans->transaction->dirty_bgs_lock);
1199
1200 /* the disk_cache_state is protected by the block group lock */
1201 spin_lock(&block_group->lock);
1202

--- 12 unchanged lines hidden (view full) ---

1215 io_ctl->inode = NULL;
1216 iput(inode);
1217 }
1218
1219 return ret;
1220
1221}
1222
1192
1193 if (block_group) {
1194 /* the dirty list is protected by the dirty_bgs_lock */
1195 spin_lock(&trans->transaction->dirty_bgs_lock);
1196
1197 /* the disk_cache_state is protected by the block group lock */
1198 spin_lock(&block_group->lock);
1199

--- 12 unchanged lines hidden (view full) ---

1212 io_ctl->inode = NULL;
1213 iput(inode);
1214 }
1215
1216 return ret;
1217
1218}
1219
1223static int btrfs_wait_cache_io_root(struct btrfs_root *root,
1224 struct btrfs_trans_handle *trans,
1225 struct btrfs_io_ctl *io_ctl,
1226 struct btrfs_path *path)
1227{
1228 return __btrfs_wait_cache_io(root, trans, NULL, io_ctl, path, 0);
1229}
1230
1231int btrfs_wait_cache_io(struct btrfs_trans_handle *trans,
1232 struct btrfs_block_group *block_group,
1233 struct btrfs_path *path)
1234{
1235 return __btrfs_wait_cache_io(block_group->fs_info->tree_root, trans,
1236 block_group, &block_group->io_ctl,
1237 path, block_group->start);
1238}

--- 88 unchanged lines hidden (view full) ---

1327 goto out_nospc;
1328
1329 /* Zero out the rest of the pages just to make sure */
1330 io_ctl_zero_remaining_pages(io_ctl);
1331
1332 /* Everything is written out, now we dirty the pages in the file. */
1333 ret = btrfs_dirty_pages(BTRFS_I(inode), io_ctl->pages,
1334 io_ctl->num_pages, 0, i_size_read(inode),
1220int btrfs_wait_cache_io(struct btrfs_trans_handle *trans,
1221 struct btrfs_block_group *block_group,
1222 struct btrfs_path *path)
1223{
1224 return __btrfs_wait_cache_io(block_group->fs_info->tree_root, trans,
1225 block_group, &block_group->io_ctl,
1226 path, block_group->start);
1227}

--- 88 unchanged lines hidden (view full) ---

1316 goto out_nospc;
1317
1318 /* Zero out the rest of the pages just to make sure */
1319 io_ctl_zero_remaining_pages(io_ctl);
1320
1321 /* Everything is written out, now we dirty the pages in the file. */
1322 ret = btrfs_dirty_pages(BTRFS_I(inode), io_ctl->pages,
1323 io_ctl->num_pages, 0, i_size_read(inode),
1335 &cached_state);
1324 &cached_state, false);
1336 if (ret)
1337 goto out_nospc;
1338
1339 if (block_group && (block_group->flags & BTRFS_BLOCK_GROUP_DATA))
1340 up_write(&block_group->data_rwsem);
1341 /*
1342 * Release the pages and unlock the extent, we will flush
1343 * them out later

--- 32 unchanged lines hidden (view full) ---

1376
1377out:
1378 io_ctl->inode = NULL;
1379 io_ctl_free(io_ctl);
1380 if (ret) {
1381 invalidate_inode_pages2(inode->i_mapping);
1382 BTRFS_I(inode)->generation = 0;
1383 }
1325 if (ret)
1326 goto out_nospc;
1327
1328 if (block_group && (block_group->flags & BTRFS_BLOCK_GROUP_DATA))
1329 up_write(&block_group->data_rwsem);
1330 /*
1331 * Release the pages and unlock the extent, we will flush
1332 * them out later

--- 32 unchanged lines hidden (view full) ---

1365
1366out:
1367 io_ctl->inode = NULL;
1368 io_ctl_free(io_ctl);
1369 if (ret) {
1370 invalidate_inode_pages2(inode->i_mapping);
1371 BTRFS_I(inode)->generation = 0;
1372 }
1384 btrfs_update_inode(trans, root, inode);
1373 btrfs_update_inode(trans, root, BTRFS_I(inode));
1385 if (must_iput)
1386 iput(inode);
1387 return ret;
1388}
1389
1390int btrfs_write_out_cache(struct btrfs_trans_handle *trans,
1391 struct btrfs_block_group *block_group,
1392 struct btrfs_path *path)

--- 514 unchanged lines hidden (view full) ---

1907 *offset = tmp;
1908 *bytes = entry->bytes - align_off;
1909 return entry;
1910 }
1911out:
1912 return NULL;
1913}
1914
1374 if (must_iput)
1375 iput(inode);
1376 return ret;
1377}
1378
1379int btrfs_write_out_cache(struct btrfs_trans_handle *trans,
1380 struct btrfs_block_group *block_group,
1381 struct btrfs_path *path)

--- 514 unchanged lines hidden (view full) ---

1896 *offset = tmp;
1897 *bytes = entry->bytes - align_off;
1898 return entry;
1899 }
1900out:
1901 return NULL;
1902}
1903
1915static int count_bitmap_extents(struct btrfs_free_space_ctl *ctl,
1916 struct btrfs_free_space *bitmap_info)
1917{
1918 struct btrfs_block_group *block_group = ctl->private;
1919 u64 bytes = bitmap_info->bytes;
1920 unsigned int rs, re;
1921 int count = 0;
1922
1923 if (!block_group || !bytes)
1924 return count;
1925
1926 bitmap_for_each_set_region(bitmap_info->bitmap, rs, re, 0,
1927 BITS_PER_BITMAP) {
1928 bytes -= (rs - re) * ctl->unit;
1929 count++;
1930
1931 if (!bytes)
1932 break;
1933 }
1934
1935 return count;
1936}
1937
1938static void add_new_bitmap(struct btrfs_free_space_ctl *ctl,
1939 struct btrfs_free_space *info, u64 offset)
1940{
1941 info->offset = offset_to_bitmap(ctl, offset);
1942 info->bytes = 0;
1943 info->bitmap_extents = 0;
1944 INIT_LIST_HEAD(&info->list);
1945 link_free_space(ctl, info);

--- 557 unchanged lines hidden (view full) ---

2503 steal_from_bitmap(ctl, info, true);
2504
2505 filter_bytes = max(filter_bytes, info->bytes);
2506
2507 ret = link_free_space(ctl, info);
2508 if (ret)
2509 kmem_cache_free(btrfs_free_space_cachep, info);
2510out:
1904static void add_new_bitmap(struct btrfs_free_space_ctl *ctl,
1905 struct btrfs_free_space *info, u64 offset)
1906{
1907 info->offset = offset_to_bitmap(ctl, offset);
1908 info->bytes = 0;
1909 info->bitmap_extents = 0;
1910 INIT_LIST_HEAD(&info->list);
1911 link_free_space(ctl, info);

--- 557 unchanged lines hidden (view full) ---

2469 steal_from_bitmap(ctl, info, true);
2470
2471 filter_bytes = max(filter_bytes, info->bytes);
2472
2473 ret = link_free_space(ctl, info);
2474 if (ret)
2475 kmem_cache_free(btrfs_free_space_cachep, info);
2476out:
2511 btrfs_discard_update_discardable(block_group, ctl);
2477 btrfs_discard_update_discardable(block_group);
2512 spin_unlock(&ctl->tree_lock);
2513
2514 if (ret) {
2515 btrfs_crit(fs_info, "unable to add free space :%d", ret);
2516 ASSERT(ret != -EEXIST);
2517 }
2518
2519 if (trim_state != BTRFS_TRIM_STATE_TRIMMED) {

--- 118 unchanged lines hidden (view full) ---

2638 }
2639
2640 ret = remove_from_bitmap(ctl, info, &offset, &bytes);
2641 if (ret == -EAGAIN) {
2642 re_search = true;
2643 goto again;
2644 }
2645out_lock:
2478 spin_unlock(&ctl->tree_lock);
2479
2480 if (ret) {
2481 btrfs_crit(fs_info, "unable to add free space :%d", ret);
2482 ASSERT(ret != -EEXIST);
2483 }
2484
2485 if (trim_state != BTRFS_TRIM_STATE_TRIMMED) {

--- 118 unchanged lines hidden (view full) ---

2604 }
2605
2606 ret = remove_from_bitmap(ctl, info, &offset, &bytes);
2607 if (ret == -EAGAIN) {
2608 re_search = true;
2609 goto again;
2610 }
2611out_lock:
2646 btrfs_discard_update_discardable(block_group, ctl);
2612 btrfs_discard_update_discardable(block_group);
2647 spin_unlock(&ctl->tree_lock);
2648out:
2649 return ret;
2650}
2651
2652void btrfs_dump_free_space(struct btrfs_block_group *block_group,
2653 u64 bytes)
2654{

--- 14 unchanged lines hidden (view full) ---

2669 }
2670 spin_unlock(&ctl->tree_lock);
2671 btrfs_info(fs_info, "block group has cluster?: %s",
2672 list_empty(&block_group->cluster_list) ? "no" : "yes");
2673 btrfs_info(fs_info,
2674 "%d blocks of free space at or bigger than bytes is", count);
2675}
2676
2613 spin_unlock(&ctl->tree_lock);
2614out:
2615 return ret;
2616}
2617
2618void btrfs_dump_free_space(struct btrfs_block_group *block_group,
2619 u64 bytes)
2620{

--- 14 unchanged lines hidden (view full) ---

2635 }
2636 spin_unlock(&ctl->tree_lock);
2637 btrfs_info(fs_info, "block group has cluster?: %s",
2638 list_empty(&block_group->cluster_list) ? "no" : "yes");
2639 btrfs_info(fs_info,
2640 "%d blocks of free space at or bigger than bytes is", count);
2641}
2642
2677void btrfs_init_free_space_ctl(struct btrfs_block_group *block_group)
2643void btrfs_init_free_space_ctl(struct btrfs_block_group *block_group,
2644 struct btrfs_free_space_ctl *ctl)
2678{
2679 struct btrfs_fs_info *fs_info = block_group->fs_info;
2645{
2646 struct btrfs_fs_info *fs_info = block_group->fs_info;
2680 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;
2681
2682 spin_lock_init(&ctl->tree_lock);
2683 ctl->unit = fs_info->sectorsize;
2684 ctl->start = block_group->start;
2685 ctl->private = block_group;
2686 ctl->op = &free_space_op;
2687 INIT_LIST_HEAD(&ctl->trimming_ranges);
2688 mutex_init(&ctl->cache_writeout_mutex);

--- 85 unchanged lines hidden (view full) ---

2774 }
2775}
2776
2777void __btrfs_remove_free_space_cache(struct btrfs_free_space_ctl *ctl)
2778{
2779 spin_lock(&ctl->tree_lock);
2780 __btrfs_remove_free_space_cache_locked(ctl);
2781 if (ctl->private)
2647
2648 spin_lock_init(&ctl->tree_lock);
2649 ctl->unit = fs_info->sectorsize;
2650 ctl->start = block_group->start;
2651 ctl->private = block_group;
2652 ctl->op = &free_space_op;
2653 INIT_LIST_HEAD(&ctl->trimming_ranges);
2654 mutex_init(&ctl->cache_writeout_mutex);

--- 85 unchanged lines hidden (view full) ---

2740 }
2741}
2742
2743void __btrfs_remove_free_space_cache(struct btrfs_free_space_ctl *ctl)
2744{
2745 spin_lock(&ctl->tree_lock);
2746 __btrfs_remove_free_space_cache_locked(ctl);
2747 if (ctl->private)
2782 btrfs_discard_update_discardable(ctl->private, ctl);
2748 btrfs_discard_update_discardable(ctl->private);
2783 spin_unlock(&ctl->tree_lock);
2784}
2785
2786void btrfs_remove_free_space_cache(struct btrfs_block_group *block_group)
2787{
2788 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;
2789 struct btrfs_free_cluster *cluster;
2790 struct list_head *head;

--- 5 unchanged lines hidden (view full) ---

2796 block_group_list);
2797
2798 WARN_ON(cluster->block_group != block_group);
2799 __btrfs_return_cluster_to_free_space(block_group, cluster);
2800
2801 cond_resched_lock(&ctl->tree_lock);
2802 }
2803 __btrfs_remove_free_space_cache_locked(ctl);
2749 spin_unlock(&ctl->tree_lock);
2750}
2751
2752void btrfs_remove_free_space_cache(struct btrfs_block_group *block_group)
2753{
2754 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;
2755 struct btrfs_free_cluster *cluster;
2756 struct list_head *head;

--- 5 unchanged lines hidden (view full) ---

2762 block_group_list);
2763
2764 WARN_ON(cluster->block_group != block_group);
2765 __btrfs_return_cluster_to_free_space(block_group, cluster);
2766
2767 cond_resched_lock(&ctl->tree_lock);
2768 }
2769 __btrfs_remove_free_space_cache_locked(ctl);
2804 btrfs_discard_update_discardable(block_group, ctl);
2770 btrfs_discard_update_discardable(block_group);
2805 spin_unlock(&ctl->tree_lock);
2806
2807}
2808
2809/**
2810 * btrfs_is_free_space_trimmed - see if everything is trimmed
2811 * @block_group: block_group of interest
2812 *

--- 67 unchanged lines hidden (view full) ---

2880
2881 entry->bytes -= bytes + align_gap_len;
2882 if (!entry->bytes)
2883 kmem_cache_free(btrfs_free_space_cachep, entry);
2884 else
2885 link_free_space(ctl, entry);
2886 }
2887out:
2771 spin_unlock(&ctl->tree_lock);
2772
2773}
2774
2775/**
2776 * btrfs_is_free_space_trimmed - see if everything is trimmed
2777 * @block_group: block_group of interest
2778 *

--- 67 unchanged lines hidden (view full) ---

2846
2847 entry->bytes -= bytes + align_gap_len;
2848 if (!entry->bytes)
2849 kmem_cache_free(btrfs_free_space_cachep, entry);
2850 else
2851 link_free_space(ctl, entry);
2852 }
2853out:
2888 btrfs_discard_update_discardable(block_group, ctl);
2854 btrfs_discard_update_discardable(block_group);
2889 spin_unlock(&ctl->tree_lock);
2890
2891 if (align_gap_len)
2892 __btrfs_add_free_space(block_group->fs_info, ctl,
2893 align_gap, align_gap_len,
2894 align_gap_trim_state);
2895 return ret;
2896}

--- 926 unchanged lines hidden (view full) ---

3823 ret = trim_bitmaps(block_group, trimmed, start, end, minlen, maxlen,
3824 async);
3825
3826 btrfs_unfreeze_block_group(block_group);
3827
3828 return ret;
3829}
3830
2855 spin_unlock(&ctl->tree_lock);
2856
2857 if (align_gap_len)
2858 __btrfs_add_free_space(block_group->fs_info, ctl,
2859 align_gap, align_gap_len,
2860 align_gap_trim_state);
2861 return ret;
2862}

--- 926 unchanged lines hidden (view full) ---

3789 ret = trim_bitmaps(block_group, trimmed, start, end, minlen, maxlen,
3790 async);
3791
3792 btrfs_unfreeze_block_group(block_group);
3793
3794 return ret;
3795}
3796
3831/*
3832 * Find the left-most item in the cache tree, and then return the
3833 * smallest inode number in the item.
3834 *
3835 * Note: the returned inode number may not be the smallest one in
3836 * the tree, if the left-most item is a bitmap.
3837 */
3838u64 btrfs_find_ino_for_alloc(struct btrfs_root *fs_root)
3839{
3840 struct btrfs_free_space_ctl *ctl = fs_root->free_ino_ctl;
3841 struct btrfs_free_space *entry = NULL;
3842 u64 ino = 0;
3843
3844 spin_lock(&ctl->tree_lock);
3845
3846 if (RB_EMPTY_ROOT(&ctl->free_space_offset))
3847 goto out;
3848
3849 entry = rb_entry(rb_first(&ctl->free_space_offset),
3850 struct btrfs_free_space, offset_index);
3851
3852 if (!entry->bitmap) {
3853 ino = entry->offset;
3854
3855 unlink_free_space(ctl, entry);
3856 entry->offset++;
3857 entry->bytes--;
3858 if (!entry->bytes)
3859 kmem_cache_free(btrfs_free_space_cachep, entry);
3860 else
3861 link_free_space(ctl, entry);
3862 } else {
3863 u64 offset = 0;
3864 u64 count = 1;
3865 int ret;
3866
3867 ret = search_bitmap(ctl, entry, &offset, &count, true);
3868 /* Logic error; Should be empty if it can't find anything */
3869 ASSERT(!ret);
3870
3871 ino = offset;
3872 bitmap_clear_bits(ctl, entry, offset, 1);
3873 if (entry->bytes == 0)
3874 free_bitmap(ctl, entry);
3875 }
3876out:
3877 spin_unlock(&ctl->tree_lock);
3878
3879 return ino;
3880}
3881
3882struct inode *lookup_free_ino_inode(struct btrfs_root *root,
3883 struct btrfs_path *path)
3884{
3885 struct inode *inode = NULL;
3886
3887 spin_lock(&root->ino_cache_lock);
3888 if (root->ino_cache_inode)
3889 inode = igrab(root->ino_cache_inode);
3890 spin_unlock(&root->ino_cache_lock);
3891 if (inode)
3892 return inode;
3893
3894 inode = __lookup_free_space_inode(root, path, 0);
3895 if (IS_ERR(inode))
3896 return inode;
3897
3898 spin_lock(&root->ino_cache_lock);
3899 if (!btrfs_fs_closing(root->fs_info))
3900 root->ino_cache_inode = igrab(inode);
3901 spin_unlock(&root->ino_cache_lock);
3902
3903 return inode;
3904}
3905
3906int create_free_ino_inode(struct btrfs_root *root,
3907 struct btrfs_trans_handle *trans,
3908 struct btrfs_path *path)
3909{
3910 return __create_free_space_inode(root, trans, path,
3911 BTRFS_FREE_INO_OBJECTID, 0);
3912}
3913
3914int load_free_ino_cache(struct btrfs_fs_info *fs_info, struct btrfs_root *root)
3915{
3916 struct btrfs_free_space_ctl *ctl = root->free_ino_ctl;
3917 struct btrfs_path *path;
3918 struct inode *inode;
3919 int ret = 0;
3920 u64 root_gen = btrfs_root_generation(&root->root_item);
3921
3922 if (!btrfs_test_opt(fs_info, INODE_MAP_CACHE))
3923 return 0;
3924
3925 /*
3926 * If we're unmounting then just return, since this does a search on the
3927 * normal root and not the commit root and we could deadlock.
3928 */
3929 if (btrfs_fs_closing(fs_info))
3930 return 0;
3931
3932 path = btrfs_alloc_path();
3933 if (!path)
3934 return 0;
3935
3936 inode = lookup_free_ino_inode(root, path);
3937 if (IS_ERR(inode))
3938 goto out;
3939
3940 if (root_gen != BTRFS_I(inode)->generation)
3941 goto out_put;
3942
3943 ret = __load_free_space_cache(root, inode, ctl, path, 0);
3944
3945 if (ret < 0)
3946 btrfs_err(fs_info,
3947 "failed to load free ino cache for root %llu",
3948 root->root_key.objectid);
3949out_put:
3950 iput(inode);
3951out:
3952 btrfs_free_path(path);
3953 return ret;
3954}
3955
3956int btrfs_write_out_ino_cache(struct btrfs_root *root,
3957 struct btrfs_trans_handle *trans,
3958 struct btrfs_path *path,
3959 struct inode *inode)
3960{
3961 struct btrfs_fs_info *fs_info = root->fs_info;
3962 struct btrfs_free_space_ctl *ctl = root->free_ino_ctl;
3963 int ret;
3964 struct btrfs_io_ctl io_ctl;
3965 bool release_metadata = true;
3966
3967 if (!btrfs_test_opt(fs_info, INODE_MAP_CACHE))
3968 return 0;
3969
3970 memset(&io_ctl, 0, sizeof(io_ctl));
3971 ret = __btrfs_write_out_cache(root, inode, ctl, NULL, &io_ctl, trans);
3972 if (!ret) {
3973 /*
3974 * At this point writepages() didn't error out, so our metadata
3975 * reservation is released when the writeback finishes, at
3976 * inode.c:btrfs_finish_ordered_io(), regardless of it finishing
3977 * with or without an error.
3978 */
3979 release_metadata = false;
3980 ret = btrfs_wait_cache_io_root(root, trans, &io_ctl, path);
3981 }
3982
3983 if (ret) {
3984 if (release_metadata)
3985 btrfs_delalloc_release_metadata(BTRFS_I(inode),
3986 inode->i_size, true);
3987 btrfs_debug(fs_info,
3988 "failed to write free ino cache for root %llu error %d",
3989 root->root_key.objectid, ret);
3990 }
3991
3992 return ret;
3993}
3994
3995#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
3996/*
3997 * Use this if you need to make a bitmap or extent entry specifically, it
3998 * doesn't do any of the merging that add_free_space does, this acts a lot like
3999 * how the free space cache loading stuff works, so you can get really weird
4000 * configurations.
4001 */
4002int test_add_free_space_entry(struct btrfs_block_group *cache,

--- 149 unchanged lines hidden --- 		3797#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
3798/*
3799 * Use this if you need to make a bitmap or extent entry specifically, it
3800 * doesn't do any of the merging that add_free_space does, this acts a lot like
3801 * how the free space cache loading stuff works, so you can get really weird
3802 * configurations.
3803 */
3804int test_add_free_space_entry(struct btrfs_block_group *cache,

--- 149 unchanged lines hidden ---