extent_io.c (664b0bae0b87f69bc9deb098f5e0158b9cf18e04) extent_io.c (b93b016313b3ba8003c3b8bb71f569af91f19fc7)
1// SPDX-License-Identifier: GPL-2.0
2#include <linux/bitops.h>
3#include <linux/slab.h>
4#include <linux/bio.h>
5#include <linux/mm.h>
6#include <linux/pagemap.h>
7#include <linux/page-flags.h>
8#include <linux/spinlock.h>

--- 62 unchanged lines hidden (view full) ---

71 extent_state_in_tree(state),
72 refcount_read(&state->refs));
73 list_del(&state->leak_list);
74 kmem_cache_free(extent_state_cache, state);
75 }
76
77 while (!list_empty(&buffers)) {
78 eb = list_entry(buffers.next, struct extent_buffer, leak_list);
1// SPDX-License-Identifier: GPL-2.0
2#include <linux/bitops.h>
3#include <linux/slab.h>
4#include <linux/bio.h>
5#include <linux/mm.h>
6#include <linux/pagemap.h>
7#include <linux/page-flags.h>
8#include <linux/spinlock.h>

--- 62 unchanged lines hidden (view full) ---

71 extent_state_in_tree(state),
72 refcount_read(&state->refs));
73 list_del(&state->leak_list);
74 kmem_cache_free(extent_state_cache, state);
75 }
76
77 while (!list_empty(&buffers)) {
78 eb = list_entry(buffers.next, struct extent_buffer, leak_list);
79 pr_err("BTRFS: buffer leak start %llu len %lu refs %d\n",
80 eb->start, eb->len, atomic_read(&eb->refs));
79 pr_err("BTRFS: buffer leak start %llu len %lu refs %d bflags %lu\n",
80 eb->start, eb->len, atomic_read(&eb->refs), eb->bflags);
81 list_del(&eb->leak_list);
82 kmem_cache_free(extent_buffer_cache, eb);
83 }
84}
85
86#define btrfs_debug_check_extent_io_range(tree, start, end) \
87 __btrfs_debug_check_extent_io_range(__func__, (tree), (start), (end))
88static inline void __btrfs_debug_check_extent_io_range(const char *caller,

--- 25 unchanged lines hidden (view full) ---

114 * it still does the unlocking
115 */
116 unsigned int extent_locked:1;
117
118 /* tells the submit_bio code to use REQ_SYNC */
119 unsigned int sync_io:1;
120};
121
81 list_del(&eb->leak_list);
82 kmem_cache_free(extent_buffer_cache, eb);
83 }
84}
85
86#define btrfs_debug_check_extent_io_range(tree, start, end) \
87 __btrfs_debug_check_extent_io_range(__func__, (tree), (start), (end))
88static inline void __btrfs_debug_check_extent_io_range(const char *caller,

--- 25 unchanged lines hidden (view full) ---

114 * it still does the unlocking
115 */
116 unsigned int extent_locked:1;
117
118 /* tells the submit_bio code to use REQ_SYNC */
119 unsigned int sync_io:1;
120};
121
122static void add_extent_changeset(struct extent_state *state, unsigned bits,
122static int add_extent_changeset(struct extent_state *state, unsigned bits,
123 struct extent_changeset *changeset,
124 int set)
125{
126 int ret;
127
128 if (!changeset)
123 struct extent_changeset *changeset,
124 int set)
125{
126 int ret;
127
128 if (!changeset)
129 return;
129 return 0;
130 if (set && (state->state & bits) == bits)
130 if (set && (state->state & bits) == bits)
131 return;
131 return 0;
132 if (!set && (state->state & bits) == 0)
132 if (!set && (state->state & bits) == 0)
133 return;
133 return 0;
134 changeset->bytes_changed += state->end - state->start + 1;
135 ret = ulist_add(&changeset->range_changed, state->start, state->end,
136 GFP_ATOMIC);
134 changeset->bytes_changed += state->end - state->start + 1;
135 ret = ulist_add(&changeset->range_changed, state->start, state->end,
136 GFP_ATOMIC);
137 /* ENOMEM */
138 BUG_ON(ret < 0);
137 return ret;
139}
140
141static void flush_write_bio(struct extent_page_data *epd);
142
143static inline struct btrfs_fs_info *
144tree_fs_info(struct extent_io_tree *tree)
145{
146 if (tree->ops)

--- 35 unchanged lines hidden (view full) ---

182 extent_buffer_cache = NULL;
183
184free_state_cache:
185 kmem_cache_destroy(extent_state_cache);
186 extent_state_cache = NULL;
187 return -ENOMEM;
188}
189
138}
139
140static void flush_write_bio(struct extent_page_data *epd);
141
142static inline struct btrfs_fs_info *
143tree_fs_info(struct extent_io_tree *tree)
144{
145 if (tree->ops)

--- 35 unchanged lines hidden (view full) ---

181 extent_buffer_cache = NULL;
182
183free_state_cache:
184 kmem_cache_destroy(extent_state_cache);
185 extent_state_cache = NULL;
186 return -ENOMEM;
187}
188
190void extent_io_exit(void)
189void __cold extent_io_exit(void)
191{
192 btrfs_leak_debug_check();
193
194 /*
195 * Make sure all delayed rcu free are flushed before we
196 * destroy caches.
197 */
198 rcu_barrier();

--- 323 unchanged lines hidden (view full) ---

522 */
523static struct extent_state *clear_state_bit(struct extent_io_tree *tree,
524 struct extent_state *state,
525 unsigned *bits, int wake,
526 struct extent_changeset *changeset)
527{
528 struct extent_state *next;
529 unsigned bits_to_clear = *bits & ~EXTENT_CTLBITS;
190{
191 btrfs_leak_debug_check();
192
193 /*
194 * Make sure all delayed rcu free are flushed before we
195 * destroy caches.
196 */
197 rcu_barrier();

--- 323 unchanged lines hidden (view full) ---

521 */
522static struct extent_state *clear_state_bit(struct extent_io_tree *tree,
523 struct extent_state *state,
524 unsigned *bits, int wake,
525 struct extent_changeset *changeset)
526{
527 struct extent_state *next;
528 unsigned bits_to_clear = *bits & ~EXTENT_CTLBITS;
529 int ret;
530
531 if ((bits_to_clear & EXTENT_DIRTY) && (state->state & EXTENT_DIRTY)) {
532 u64 range = state->end - state->start + 1;
533 WARN_ON(range > tree->dirty_bytes);
534 tree->dirty_bytes -= range;
535 }
536 clear_state_cb(tree, state, bits);
530
531 if ((bits_to_clear & EXTENT_DIRTY) && (state->state & EXTENT_DIRTY)) {
532 u64 range = state->end - state->start + 1;
533 WARN_ON(range > tree->dirty_bytes);
534 tree->dirty_bytes -= range;
535 }
536 clear_state_cb(tree, state, bits);
537 add_extent_changeset(state, bits_to_clear, changeset, 0);
537 ret = add_extent_changeset(state, bits_to_clear, changeset, 0);
538 BUG_ON(ret < 0);
538 state->state &= ~bits_to_clear;
539 if (wake)
540 wake_up(&state->wq);
541 if (state->state == 0) {
542 next = next_state(state);
543 if (extent_state_in_tree(state)) {
544 rb_erase(&state->rb_node, &tree->state);
545 RB_CLEAR_NODE(&state->rb_node);

--- 254 unchanged lines hidden (view full) ---

800 spin_unlock(&tree->lock);
801}
802
803static void set_state_bits(struct extent_io_tree *tree,
804 struct extent_state *state,
805 unsigned *bits, struct extent_changeset *changeset)
806{
807 unsigned bits_to_set = *bits & ~EXTENT_CTLBITS;
539 state->state &= ~bits_to_clear;
540 if (wake)
541 wake_up(&state->wq);
542 if (state->state == 0) {
543 next = next_state(state);
544 if (extent_state_in_tree(state)) {
545 rb_erase(&state->rb_node, &tree->state);
546 RB_CLEAR_NODE(&state->rb_node);

--- 254 unchanged lines hidden (view full) ---

801 spin_unlock(&tree->lock);
802}
803
804static void set_state_bits(struct extent_io_tree *tree,
805 struct extent_state *state,
806 unsigned *bits, struct extent_changeset *changeset)
807{
808 unsigned bits_to_set = *bits & ~EXTENT_CTLBITS;
809 int ret;
808
809 set_state_cb(tree, state, bits);
810 if ((bits_to_set & EXTENT_DIRTY) && !(state->state & EXTENT_DIRTY)) {
811 u64 range = state->end - state->start + 1;
812 tree->dirty_bytes += range;
813 }
810
811 set_state_cb(tree, state, bits);
812 if ((bits_to_set & EXTENT_DIRTY) && !(state->state & EXTENT_DIRTY)) {
813 u64 range = state->end - state->start + 1;
814 tree->dirty_bytes += range;
815 }
814 add_extent_changeset(state, bits_to_set, changeset, 1);
816 ret = add_extent_changeset(state, bits_to_set, changeset, 1);
817 BUG_ON(ret < 0);
815 state->state |= bits_to_set;
816}
817
818static void cache_state_if_flags(struct extent_state *state,
819 struct extent_state **cached_ptr,
820 unsigned flags)
821{
822 if (cached_ptr && !(*cached_ptr)) {

--- 1916 unchanged lines hidden (view full) ---

2739 ret = tree->ops->submit_bio_hook(tree->private_data, bio,
2740 mirror_num, bio_flags, start);
2741 else
2742 btrfsic_submit_bio(bio);
2743
2744 return blk_status_to_errno(ret);
2745}
2746
818 state->state |= bits_to_set;
819}
820
821static void cache_state_if_flags(struct extent_state *state,
822 struct extent_state **cached_ptr,
823 unsigned flags)
824{
825 if (cached_ptr && !(*cached_ptr)) {

--- 1916 unchanged lines hidden (view full) ---

2742 ret = tree->ops->submit_bio_hook(tree->private_data, bio,
2743 mirror_num, bio_flags, start);
2744 else
2745 btrfsic_submit_bio(bio);
2746
2747 return blk_status_to_errno(ret);
2748}
2749
2747static int merge_bio(struct extent_io_tree *tree, struct page *page,
2748 unsigned long offset, size_t size, struct bio *bio,
2749 unsigned long bio_flags)
2750{
2751 int ret = 0;
2752 if (tree->ops)
2753 ret = tree->ops->merge_bio_hook(page, offset, size, bio,
2754 bio_flags);
2755 return ret;
2756
2757}
2758
2759/*
2760 * @opf: bio REQ_OP_* and REQ_* flags as one value
2750/*
2751 * @opf: bio REQ_OP_* and REQ_* flags as one value
2752 * @tree: tree so we can call our merge_bio hook
2753 * @wbc: optional writeback control for io accounting
2754 * @page: page to add to the bio
2755 * @pg_offset: offset of the new bio or to check whether we are adding
2756 * a contiguous page to the previous one
2757 * @size: portion of page that we want to write
2758 * @offset: starting offset in the page
2759 * @bdev: attach newly created bios to this bdev
2760 * @bio_ret: must be valid pointer, newly allocated bio will be stored there
2761 * @end_io_func: end_io callback for new bio
2762 * @mirror_num: desired mirror to read/write
2763 * @prev_bio_flags: flags of previous bio to see if we can merge the current one
2764 * @bio_flags: flags of the current bio to see if we can merge them
2761 */
2762static int submit_extent_page(unsigned int opf, struct extent_io_tree *tree,
2763 struct writeback_control *wbc,
2764 struct page *page, u64 offset,
2765 size_t size, unsigned long pg_offset,
2766 struct block_device *bdev,
2767 struct bio **bio_ret,
2768 bio_end_io_t end_io_func,
2769 int mirror_num,
2770 unsigned long prev_bio_flags,
2771 unsigned long bio_flags,
2772 bool force_bio_submit)
2773{
2774 int ret = 0;
2775 struct bio *bio;
2765 */
2766static int submit_extent_page(unsigned int opf, struct extent_io_tree *tree,
2767 struct writeback_control *wbc,
2768 struct page *page, u64 offset,
2769 size_t size, unsigned long pg_offset,
2770 struct block_device *bdev,
2771 struct bio **bio_ret,
2772 bio_end_io_t end_io_func,
2773 int mirror_num,
2774 unsigned long prev_bio_flags,
2775 unsigned long bio_flags,
2776 bool force_bio_submit)
2777{
2778 int ret = 0;
2779 struct bio *bio;
2776 int contig = 0;
2777 int old_compressed = prev_bio_flags & EXTENT_BIO_COMPRESSED;
2778 size_t page_size = min_t(size_t, size, PAGE_SIZE);
2779 sector_t sector = offset >> 9;
2780
2780 size_t page_size = min_t(size_t, size, PAGE_SIZE);
2781 sector_t sector = offset >> 9;
2782
2781 if (bio_ret && *bio_ret) {
2783 ASSERT(bio_ret);
2784
2785 if (*bio_ret) {
2786 bool contig;
2787 bool can_merge = true;
2788
2782 bio = *bio_ret;
2789 bio = *bio_ret;
2783 if (old_compressed)
2790 if (prev_bio_flags & EXTENT_BIO_COMPRESSED)
2784 contig = bio->bi_iter.bi_sector == sector;
2785 else
2786 contig = bio_end_sector(bio) == sector;
2787
2791 contig = bio->bi_iter.bi_sector == sector;
2792 else
2793 contig = bio_end_sector(bio) == sector;
2794
2788 if (prev_bio_flags != bio_flags || !contig ||
2795 if (tree->ops && tree->ops->merge_bio_hook(page, offset,
2796 page_size, bio, bio_flags))
2797 can_merge = false;
2798
2799 if (prev_bio_flags != bio_flags || !contig || !can_merge ||
2789 force_bio_submit ||
2800 force_bio_submit ||
2790 merge_bio(tree, page, pg_offset, page_size, bio, bio_flags) ||
2791 bio_add_page(bio, page, page_size, pg_offset) < page_size) {
2792 ret = submit_one_bio(bio, mirror_num, prev_bio_flags);
2793 if (ret < 0) {
2794 *bio_ret = NULL;
2795 return ret;
2796 }
2797 bio = NULL;
2798 } else {

--- 9 unchanged lines hidden (view full) ---

2808 bio->bi_private = tree;
2809 bio->bi_write_hint = page->mapping->host->i_write_hint;
2810 bio->bi_opf = opf;
2811 if (wbc) {
2812 wbc_init_bio(wbc, bio);
2813 wbc_account_io(wbc, page, page_size);
2814 }
2815
2801 bio_add_page(bio, page, page_size, pg_offset) < page_size) {
2802 ret = submit_one_bio(bio, mirror_num, prev_bio_flags);
2803 if (ret < 0) {
2804 *bio_ret = NULL;
2805 return ret;
2806 }
2807 bio = NULL;
2808 } else {

--- 9 unchanged lines hidden (view full) ---

2818 bio->bi_private = tree;
2819 bio->bi_write_hint = page->mapping->host->i_write_hint;
2820 bio->bi_opf = opf;
2821 if (wbc) {
2822 wbc_init_bio(wbc, bio);
2823 wbc_account_io(wbc, page, page_size);
2824 }
2825
2816 if (bio_ret)
2817 *bio_ret = bio;
2818 else
2819 ret = submit_one_bio(bio, mirror_num, bio_flags);
2826 *bio_ret = bio;
2820
2821 return ret;
2822}
2823
2824static void attach_extent_buffer_page(struct extent_buffer *eb,
2825 struct page *page)
2826{
2827 if (!PagePrivate(page)) {

--- 53 unchanged lines hidden (view full) ---

2881 get_extent_t *get_extent,
2882 struct extent_map **em_cached,
2883 struct bio **bio, int mirror_num,
2884 unsigned long *bio_flags, unsigned int read_flags,
2885 u64 *prev_em_start)
2886{
2887 struct inode *inode = page->mapping->host;
2888 u64 start = page_offset(page);
2827
2828 return ret;
2829}
2830
2831static void attach_extent_buffer_page(struct extent_buffer *eb,
2832 struct page *page)
2833{
2834 if (!PagePrivate(page)) {

--- 53 unchanged lines hidden (view full) ---

2888 get_extent_t *get_extent,
2889 struct extent_map **em_cached,
2890 struct bio **bio, int mirror_num,
2891 unsigned long *bio_flags, unsigned int read_flags,
2892 u64 *prev_em_start)
2893{
2894 struct inode *inode = page->mapping->host;
2895 u64 start = page_offset(page);
2889 u64 page_end = start + PAGE_SIZE - 1;
2890 u64 end;
2896 const u64 end = start + PAGE_SIZE - 1;
2891 u64 cur = start;
2892 u64 extent_offset;
2893 u64 last_byte = i_size_read(inode);
2894 u64 block_start;
2895 u64 cur_end;
2896 struct extent_map *em;
2897 struct block_device *bdev;
2898 int ret = 0;
2899 int nr = 0;
2900 size_t pg_offset = 0;
2901 size_t iosize;
2902 size_t disk_io_size;
2903 size_t blocksize = inode->i_sb->s_blocksize;
2904 unsigned long this_bio_flag = 0;
2905
2906 set_page_extent_mapped(page);
2907
2897 u64 cur = start;
2898 u64 extent_offset;
2899 u64 last_byte = i_size_read(inode);
2900 u64 block_start;
2901 u64 cur_end;
2902 struct extent_map *em;
2903 struct block_device *bdev;
2904 int ret = 0;
2905 int nr = 0;
2906 size_t pg_offset = 0;
2907 size_t iosize;
2908 size_t disk_io_size;
2909 size_t blocksize = inode->i_sb->s_blocksize;
2910 unsigned long this_bio_flag = 0;
2911
2912 set_page_extent_mapped(page);
2913
2908 end = page_end;
2909 if (!PageUptodate(page)) {
2910 if (cleancache_get_page(page) == 0) {
2911 BUG_ON(blocksize != PAGE_SIZE);
2912 unlock_extent(tree, start, end);
2913 goto out;
2914 }
2915 }
2916

--- 1036 unchanged lines hidden (view full) ---

3953 unsigned i;
3954
3955 scanned = 1;
3956 for (i = 0; i < nr_pages; i++) {
3957 struct page *page = pvec.pages[i];
3958
3959 done_index = page->index;
3960 /*
2914 if (!PageUptodate(page)) {
2915 if (cleancache_get_page(page) == 0) {
2916 BUG_ON(blocksize != PAGE_SIZE);
2917 unlock_extent(tree, start, end);
2918 goto out;
2919 }
2920 }
2921

--- 1036 unchanged lines hidden (view full) ---

3958 unsigned i;
3959
3960 scanned = 1;
3961 for (i = 0; i < nr_pages; i++) {
3962 struct page *page = pvec.pages[i];
3963
3964 done_index = page->index;
3965 /*
3961 * At this point we hold neither mapping->tree_lock nor
3962 * lock on the page itself: the page may be truncated or
3963 * invalidated (changing page->mapping to NULL), or even
3964 * swizzled back from swapper_space to tmpfs file
3965 * mapping
3966 * At this point we hold neither the i_pages lock nor
3967 * the page lock: the page may be truncated or
3968 * invalidated (changing page->mapping to NULL),
3969 * or even swizzled back from swapper_space to
3970 * tmpfs file mapping
3966 */
3967 if (!trylock_page(page)) {
3968 flush_write_bio(epd);
3969 lock_page(page);
3970 }
3971
3972 if (unlikely(page->mapping != mapping)) {
3973 unlock_page(page);

--- 1190 unchanged lines hidden (view full) ---

5164 page = eb->pages[i];
5165 if (!PageDirty(page))
5166 continue;
5167
5168 lock_page(page);
5169 WARN_ON(!PagePrivate(page));
5170
5171 clear_page_dirty_for_io(page);
3971 */
3972 if (!trylock_page(page)) {
3973 flush_write_bio(epd);
3974 lock_page(page);
3975 }
3976
3977 if (unlikely(page->mapping != mapping)) {
3978 unlock_page(page);

--- 1190 unchanged lines hidden (view full) ---

5169 page = eb->pages[i];
5170 if (!PageDirty(page))
5171 continue;
5172
5173 lock_page(page);
5174 WARN_ON(!PagePrivate(page));
5175
5176 clear_page_dirty_for_io(page);
5172 spin_lock_irq(&page->mapping->tree_lock);
5177 xa_lock_irq(&page->mapping->i_pages);
5173 if (!PageDirty(page)) {
5178 if (!PageDirty(page)) {
5174 radix_tree_tag_clear(&page->mapping->page_tree,
5179 radix_tree_tag_clear(&page->mapping->i_pages,
5175 page_index(page),
5176 PAGECACHE_TAG_DIRTY);
5177 }
5180 page_index(page),
5181 PAGECACHE_TAG_DIRTY);
5182 }
5178 spin_unlock_irq(&page->mapping->tree_lock);
5183 xa_unlock_irq(&page->mapping->i_pages);
5179 ClearPageError(page);
5180 unlock_page(page);
5181 }
5182 WARN_ON(atomic_read(&eb->refs) == 0);
5183}
5184
5185int set_extent_buffer_dirty(struct extent_buffer *eb)
5186{

--- 38 unchanged lines hidden (view full) ---

5225 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
5226 num_pages = num_extent_pages(eb->start, eb->len);
5227 for (i = 0; i < num_pages; i++) {
5228 page = eb->pages[i];
5229 SetPageUptodate(page);
5230 }
5231}
5232
5184 ClearPageError(page);
5185 unlock_page(page);
5186 }
5187 WARN_ON(atomic_read(&eb->refs) == 0);
5188}
5189
5190int set_extent_buffer_dirty(struct extent_buffer *eb)
5191{

--- 38 unchanged lines hidden (view full) ---

5230 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
5231 num_pages = num_extent_pages(eb->start, eb->len);
5232 for (i = 0; i < num_pages; i++) {
5233 page = eb->pages[i];
5234 SetPageUptodate(page);
5235 }
5236}
5237
5233int extent_buffer_uptodate(struct extent_buffer *eb)
5234{
5235 return test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
5236}
5237
5238int read_extent_buffer_pages(struct extent_io_tree *tree,
5239 struct extent_buffer *eb, int wait, int mirror_num)
5240{
5241 unsigned long i;
5242 struct page *page;
5243 int err;
5244 int ret = 0;
5245 int locked_pages = 0;

--- 719 unchanged lines hidden --- 		5238int read_extent_buffer_pages(struct extent_io_tree *tree,
5239 struct extent_buffer *eb, int wait, int mirror_num)
5240{
5241 unsigned long i;
5242 struct page *page;
5243 int err;
5244 int ret = 0;
5245 int locked_pages = 0;

--- 719 unchanged lines hidden ---