1c1d7c514SDavid Sterba // SPDX-License-Identifier: GPL-2.0
253b381b3SDavid Woodhouse /*
353b381b3SDavid Woodhouse * Copyright (C) 2012 Fusion-io All rights reserved.
453b381b3SDavid Woodhouse * Copyright (C) 2012 Intel Corp. All rights reserved.
553b381b3SDavid Woodhouse */
6c1d7c514SDavid Sterba
753b381b3SDavid Woodhouse #include <linux/sched.h>
853b381b3SDavid Woodhouse #include <linux/bio.h>
953b381b3SDavid Woodhouse #include <linux/slab.h>
1053b381b3SDavid Woodhouse #include <linux/blkdev.h>
1153b381b3SDavid Woodhouse #include <linux/raid/pq.h>
1253b381b3SDavid Woodhouse #include <linux/hash.h>
1353b381b3SDavid Woodhouse #include <linux/list_sort.h>
1453b381b3SDavid Woodhouse #include <linux/raid/xor.h>
15818e010bSDavid Sterba #include <linux/mm.h>
169b569ea0SJosef Bacik #include "messages.h"
1753b381b3SDavid Woodhouse #include "ctree.h"
1853b381b3SDavid Woodhouse #include "disk-io.h"
1953b381b3SDavid Woodhouse #include "volumes.h"
2053b381b3SDavid Woodhouse #include "raid56.h"
2153b381b3SDavid Woodhouse #include "async-thread.h"
22c5a41562SQu Wenruo #include "file-item.h"
237a315072SQu Wenruo #include "btrfs_inode.h"
2453b381b3SDavid Woodhouse
2553b381b3SDavid Woodhouse /* set when additional merges to this rbio are not allowed */
2653b381b3SDavid Woodhouse #define RBIO_RMW_LOCKED_BIT 1
2753b381b3SDavid Woodhouse
284ae10b3aSChris Mason /*
294ae10b3aSChris Mason * set when this rbio is sitting in the hash, but it is just a cache
304ae10b3aSChris Mason * of past RMW
314ae10b3aSChris Mason */
324ae10b3aSChris Mason #define RBIO_CACHE_BIT 2
334ae10b3aSChris Mason
344ae10b3aSChris Mason /*
354ae10b3aSChris Mason * set when it is safe to trust the stripe_pages for caching
364ae10b3aSChris Mason */
374ae10b3aSChris Mason #define RBIO_CACHE_READY_BIT 3
384ae10b3aSChris Mason
394ae10b3aSChris Mason #define RBIO_CACHE_SIZE 1024
404ae10b3aSChris Mason
418a953348SDavid Sterba #define BTRFS_STRIPE_HASH_TABLE_BITS 11
428a953348SDavid Sterba
dump_bioc(const struct btrfs_fs_info * fs_info,const struct btrfs_io_context * bioc)43bbbee460SQu Wenruo static void dump_bioc(const struct btrfs_fs_info *fs_info, const struct btrfs_io_context *bioc)
44bbbee460SQu Wenruo {
45bbbee460SQu Wenruo if (unlikely(!bioc)) {
46bbbee460SQu Wenruo btrfs_crit(fs_info, "bioc=NULL");
47bbbee460SQu Wenruo return;
48bbbee460SQu Wenruo }
49bbbee460SQu Wenruo btrfs_crit(fs_info,
50bbbee460SQu Wenruo "bioc logical=%llu full_stripe=%llu size=%llu map_type=0x%llx mirror=%u replace_nr_stripes=%u replace_stripe_src=%d num_stripes=%u",
51bbbee460SQu Wenruo bioc->logical, bioc->full_stripe_logical, bioc->size,
52bbbee460SQu Wenruo bioc->map_type, bioc->mirror_num, bioc->replace_nr_stripes,
53bbbee460SQu Wenruo bioc->replace_stripe_src, bioc->num_stripes);
54bbbee460SQu Wenruo for (int i = 0; i < bioc->num_stripes; i++) {
55bbbee460SQu Wenruo btrfs_crit(fs_info, " nr=%d devid=%llu physical=%llu",
56bbbee460SQu Wenruo i, bioc->stripes[i].dev->devid,
57bbbee460SQu Wenruo bioc->stripes[i].physical);
58bbbee460SQu Wenruo }
59bbbee460SQu Wenruo }
60bbbee460SQu Wenruo
btrfs_dump_rbio(const struct btrfs_fs_info * fs_info,const struct btrfs_raid_bio * rbio)61bbbee460SQu Wenruo static void btrfs_dump_rbio(const struct btrfs_fs_info *fs_info,
62bbbee460SQu Wenruo const struct btrfs_raid_bio *rbio)
63bbbee460SQu Wenruo {
64bbbee460SQu Wenruo if (!IS_ENABLED(CONFIG_BTRFS_ASSERT))
65bbbee460SQu Wenruo return;
66bbbee460SQu Wenruo
67bbbee460SQu Wenruo dump_bioc(fs_info, rbio->bioc);
68bbbee460SQu Wenruo btrfs_crit(fs_info,
69bbbee460SQu Wenruo "rbio flags=0x%lx nr_sectors=%u nr_data=%u real_stripes=%u stripe_nsectors=%u scrubp=%u dbitmap=0x%lx",
70bbbee460SQu Wenruo rbio->flags, rbio->nr_sectors, rbio->nr_data,
71bbbee460SQu Wenruo rbio->real_stripes, rbio->stripe_nsectors,
72bbbee460SQu Wenruo rbio->scrubp, rbio->dbitmap);
73bbbee460SQu Wenruo }
74bbbee460SQu Wenruo
75bbbee460SQu Wenruo #define ASSERT_RBIO(expr, rbio) \
76bbbee460SQu Wenruo ({ \
77bbbee460SQu Wenruo if (IS_ENABLED(CONFIG_BTRFS_ASSERT) && unlikely(!(expr))) { \
78bbbee460SQu Wenruo const struct btrfs_fs_info *__fs_info = (rbio)->bioc ? \
79bbbee460SQu Wenruo (rbio)->bioc->fs_info : NULL; \
80bbbee460SQu Wenruo \
81bbbee460SQu Wenruo btrfs_dump_rbio(__fs_info, (rbio)); \
82bbbee460SQu Wenruo } \
83bbbee460SQu Wenruo ASSERT((expr)); \
84bbbee460SQu Wenruo })
85bbbee460SQu Wenruo
86bbbee460SQu Wenruo #define ASSERT_RBIO_STRIPE(expr, rbio, stripe_nr) \
87bbbee460SQu Wenruo ({ \
88bbbee460SQu Wenruo if (IS_ENABLED(CONFIG_BTRFS_ASSERT) && unlikely(!(expr))) { \
89bbbee460SQu Wenruo const struct btrfs_fs_info *__fs_info = (rbio)->bioc ? \
90bbbee460SQu Wenruo (rbio)->bioc->fs_info : NULL; \
91bbbee460SQu Wenruo \
92bbbee460SQu Wenruo btrfs_dump_rbio(__fs_info, (rbio)); \
93bbbee460SQu Wenruo btrfs_crit(__fs_info, "stripe_nr=%d", (stripe_nr)); \
94bbbee460SQu Wenruo } \
95bbbee460SQu Wenruo ASSERT((expr)); \
96bbbee460SQu Wenruo })
97bbbee460SQu Wenruo
98bbbee460SQu Wenruo #define ASSERT_RBIO_SECTOR(expr, rbio, sector_nr) \
99bbbee460SQu Wenruo ({ \
100bbbee460SQu Wenruo if (IS_ENABLED(CONFIG_BTRFS_ASSERT) && unlikely(!(expr))) { \
101bbbee460SQu Wenruo const struct btrfs_fs_info *__fs_info = (rbio)->bioc ? \
102bbbee460SQu Wenruo (rbio)->bioc->fs_info : NULL; \
103bbbee460SQu Wenruo \
104bbbee460SQu Wenruo btrfs_dump_rbio(__fs_info, (rbio)); \
105bbbee460SQu Wenruo btrfs_crit(__fs_info, "sector_nr=%d", (sector_nr)); \
106bbbee460SQu Wenruo } \
107bbbee460SQu Wenruo ASSERT((expr)); \
108bbbee460SQu Wenruo })
109bbbee460SQu Wenruo
110bbbee460SQu Wenruo #define ASSERT_RBIO_LOGICAL(expr, rbio, logical) \
111bbbee460SQu Wenruo ({ \
112bbbee460SQu Wenruo if (IS_ENABLED(CONFIG_BTRFS_ASSERT) && unlikely(!(expr))) { \
113bbbee460SQu Wenruo const struct btrfs_fs_info *__fs_info = (rbio)->bioc ? \
114bbbee460SQu Wenruo (rbio)->bioc->fs_info : NULL; \
115bbbee460SQu Wenruo \
116bbbee460SQu Wenruo btrfs_dump_rbio(__fs_info, (rbio)); \
117bbbee460SQu Wenruo btrfs_crit(__fs_info, "logical=%llu", (logical)); \
118bbbee460SQu Wenruo } \
119bbbee460SQu Wenruo ASSERT((expr)); \
120bbbee460SQu Wenruo })
121bbbee460SQu Wenruo
1228a953348SDavid Sterba /* Used by the raid56 code to lock stripes for read/modify/write */
1238a953348SDavid Sterba struct btrfs_stripe_hash {
1248a953348SDavid Sterba struct list_head hash_list;
1258a953348SDavid Sterba spinlock_t lock;
1268a953348SDavid Sterba };
1278a953348SDavid Sterba
1288a953348SDavid Sterba /* Used by the raid56 code to lock stripes for read/modify/write */
1298a953348SDavid Sterba struct btrfs_stripe_hash_table {
1308a953348SDavid Sterba struct list_head stripe_cache;
1318a953348SDavid Sterba spinlock_t cache_lock;
1328a953348SDavid Sterba int cache_size;
1338a953348SDavid Sterba struct btrfs_stripe_hash table[];
1348a953348SDavid Sterba };
1358a953348SDavid Sterba
136eb357060SQu Wenruo /*
137eb357060SQu Wenruo * A bvec like structure to present a sector inside a page.
138eb357060SQu Wenruo *
139eb357060SQu Wenruo * Unlike bvec we don't need bvlen, as it's fixed to sectorsize.
140eb357060SQu Wenruo */
141eb357060SQu Wenruo struct sector_ptr {
142eb357060SQu Wenruo struct page *page;
14300425dd9SQu Wenruo unsigned int pgoff:24;
14400425dd9SQu Wenruo unsigned int uptodate:8;
145eb357060SQu Wenruo };
146eb357060SQu Wenruo
14793723095SQu Wenruo static void rmw_rbio_work(struct work_struct *work);
14893723095SQu Wenruo static void rmw_rbio_work_locked(struct work_struct *work);
14953b381b3SDavid Woodhouse static void index_rbio_pages(struct btrfs_raid_bio *rbio);
15053b381b3SDavid Woodhouse static int alloc_rbio_pages(struct btrfs_raid_bio *rbio);
15153b381b3SDavid Woodhouse
152486c737fSQu Wenruo static int finish_parity_scrub(struct btrfs_raid_bio *rbio);
1536bfd0133SQu Wenruo static void scrub_rbio_work_locked(struct work_struct *work);
1545a6ac9eaSMiao Xie
free_raid_bio_pointers(struct btrfs_raid_bio * rbio)155797d74b7SQu Wenruo static void free_raid_bio_pointers(struct btrfs_raid_bio *rbio)
156797d74b7SQu Wenruo {
1572942a50dSQu Wenruo bitmap_free(rbio->error_bitmap);
158797d74b7SQu Wenruo kfree(rbio->stripe_pages);
159797d74b7SQu Wenruo kfree(rbio->bio_sectors);
160797d74b7SQu Wenruo kfree(rbio->stripe_sectors);
161797d74b7SQu Wenruo kfree(rbio->finish_pointers);
162797d74b7SQu Wenruo }
163797d74b7SQu Wenruo
free_raid_bio(struct btrfs_raid_bio * rbio)164ff2b64a2SQu Wenruo static void free_raid_bio(struct btrfs_raid_bio *rbio)
165ff2b64a2SQu Wenruo {
166ff2b64a2SQu Wenruo int i;
167ff2b64a2SQu Wenruo
168ff2b64a2SQu Wenruo if (!refcount_dec_and_test(&rbio->refs))
169ff2b64a2SQu Wenruo return;
170ff2b64a2SQu Wenruo
171ff2b64a2SQu Wenruo WARN_ON(!list_empty(&rbio->stripe_cache));
172ff2b64a2SQu Wenruo WARN_ON(!list_empty(&rbio->hash_list));
173ff2b64a2SQu Wenruo WARN_ON(!bio_list_empty(&rbio->bio_list));
174ff2b64a2SQu Wenruo
175ff2b64a2SQu Wenruo for (i = 0; i < rbio->nr_pages; i++) {
176ff2b64a2SQu Wenruo if (rbio->stripe_pages[i]) {
177ff2b64a2SQu Wenruo __free_page(rbio->stripe_pages[i]);
178ff2b64a2SQu Wenruo rbio->stripe_pages[i] = NULL;
179ff2b64a2SQu Wenruo }
180ff2b64a2SQu Wenruo }
181ff2b64a2SQu Wenruo
182ff2b64a2SQu Wenruo btrfs_put_bioc(rbio->bioc);
183797d74b7SQu Wenruo free_raid_bio_pointers(rbio);
184ff2b64a2SQu Wenruo kfree(rbio);
185ff2b64a2SQu Wenruo }
186ff2b64a2SQu Wenruo
start_async_work(struct btrfs_raid_bio * rbio,work_func_t work_func)187385de0efSChristoph Hellwig static void start_async_work(struct btrfs_raid_bio *rbio, work_func_t work_func)
188ac638859SDavid Sterba {
189385de0efSChristoph Hellwig INIT_WORK(&rbio->work, work_func);
190385de0efSChristoph Hellwig queue_work(rbio->bioc->fs_info->rmw_workers, &rbio->work);
191ac638859SDavid Sterba }
192ac638859SDavid Sterba
19353b381b3SDavid Woodhouse /*
19453b381b3SDavid Woodhouse * the stripe hash table is used for locking, and to collect
19553b381b3SDavid Woodhouse * bios in hopes of making a full stripe
19653b381b3SDavid Woodhouse */
btrfs_alloc_stripe_hash_table(struct btrfs_fs_info * info)19753b381b3SDavid Woodhouse int btrfs_alloc_stripe_hash_table(struct btrfs_fs_info *info)
19853b381b3SDavid Woodhouse {
19953b381b3SDavid Woodhouse struct btrfs_stripe_hash_table *table;
20053b381b3SDavid Woodhouse struct btrfs_stripe_hash_table *x;
20153b381b3SDavid Woodhouse struct btrfs_stripe_hash *cur;
20253b381b3SDavid Woodhouse struct btrfs_stripe_hash *h;
20353b381b3SDavid Woodhouse int num_entries = 1 << BTRFS_STRIPE_HASH_TABLE_BITS;
20453b381b3SDavid Woodhouse int i;
20553b381b3SDavid Woodhouse
20653b381b3SDavid Woodhouse if (info->stripe_hash_table)
20753b381b3SDavid Woodhouse return 0;
20853b381b3SDavid Woodhouse
20983c8266aSDavid Sterba /*
21083c8266aSDavid Sterba * The table is large, starting with order 4 and can go as high as
21183c8266aSDavid Sterba * order 7 in case lock debugging is turned on.
21283c8266aSDavid Sterba *
21383c8266aSDavid Sterba * Try harder to allocate and fallback to vmalloc to lower the chance
21483c8266aSDavid Sterba * of a failing mount.
21583c8266aSDavid Sterba */
216ee787f95SDavid Sterba table = kvzalloc(struct_size(table, table, num_entries), GFP_KERNEL);
21753b381b3SDavid Woodhouse if (!table)
21853b381b3SDavid Woodhouse return -ENOMEM;
21953b381b3SDavid Woodhouse
2204ae10b3aSChris Mason spin_lock_init(&table->cache_lock);
2214ae10b3aSChris Mason INIT_LIST_HEAD(&table->stripe_cache);
2224ae10b3aSChris Mason
22353b381b3SDavid Woodhouse h = table->table;
22453b381b3SDavid Woodhouse
22553b381b3SDavid Woodhouse for (i = 0; i < num_entries; i++) {
22653b381b3SDavid Woodhouse cur = h + i;
22753b381b3SDavid Woodhouse INIT_LIST_HEAD(&cur->hash_list);
22853b381b3SDavid Woodhouse spin_lock_init(&cur->lock);
22953b381b3SDavid Woodhouse }
23053b381b3SDavid Woodhouse
23153b381b3SDavid Woodhouse x = cmpxchg(&info->stripe_hash_table, NULL, table);
232f749303bSWang Shilong kvfree(x);
23353b381b3SDavid Woodhouse return 0;
23453b381b3SDavid Woodhouse }
23553b381b3SDavid Woodhouse
23653b381b3SDavid Woodhouse /*
2374ae10b3aSChris Mason * caching an rbio means to copy anything from the
238ac26df8bSQu Wenruo * bio_sectors array into the stripe_pages array. We
2394ae10b3aSChris Mason * use the page uptodate bit in the stripe cache array
2404ae10b3aSChris Mason * to indicate if it has valid data
2414ae10b3aSChris Mason *
2424ae10b3aSChris Mason * once the caching is done, we set the cache ready
2434ae10b3aSChris Mason * bit.
2444ae10b3aSChris Mason */
cache_rbio_pages(struct btrfs_raid_bio * rbio)2454ae10b3aSChris Mason static void cache_rbio_pages(struct btrfs_raid_bio *rbio)
2464ae10b3aSChris Mason {
2474ae10b3aSChris Mason int i;
2484ae10b3aSChris Mason int ret;
2494ae10b3aSChris Mason
2504ae10b3aSChris Mason ret = alloc_rbio_pages(rbio);
2514ae10b3aSChris Mason if (ret)
2524ae10b3aSChris Mason return;
2534ae10b3aSChris Mason
25400425dd9SQu Wenruo for (i = 0; i < rbio->nr_sectors; i++) {
25500425dd9SQu Wenruo /* Some range not covered by bio (partial write), skip it */
25688074c8bSQu Wenruo if (!rbio->bio_sectors[i].page) {
25788074c8bSQu Wenruo /*
25888074c8bSQu Wenruo * Even if the sector is not covered by bio, if it is
25988074c8bSQu Wenruo * a data sector it should still be uptodate as it is
26088074c8bSQu Wenruo * read from disk.
26188074c8bSQu Wenruo */
26288074c8bSQu Wenruo if (i < rbio->nr_data * rbio->stripe_nsectors)
26388074c8bSQu Wenruo ASSERT(rbio->stripe_sectors[i].uptodate);
26400425dd9SQu Wenruo continue;
26588074c8bSQu Wenruo }
26600425dd9SQu Wenruo
26700425dd9SQu Wenruo ASSERT(rbio->stripe_sectors[i].page);
26800425dd9SQu Wenruo memcpy_page(rbio->stripe_sectors[i].page,
26900425dd9SQu Wenruo rbio->stripe_sectors[i].pgoff,
27000425dd9SQu Wenruo rbio->bio_sectors[i].page,
27100425dd9SQu Wenruo rbio->bio_sectors[i].pgoff,
27200425dd9SQu Wenruo rbio->bioc->fs_info->sectorsize);
27300425dd9SQu Wenruo rbio->stripe_sectors[i].uptodate = 1;
27400425dd9SQu Wenruo }
2754ae10b3aSChris Mason set_bit(RBIO_CACHE_READY_BIT, &rbio->flags);
2764ae10b3aSChris Mason }
2774ae10b3aSChris Mason
2784ae10b3aSChris Mason /*
27953b381b3SDavid Woodhouse * we hash on the first logical address of the stripe
28053b381b3SDavid Woodhouse */
rbio_bucket(struct btrfs_raid_bio * rbio)28153b381b3SDavid Woodhouse static int rbio_bucket(struct btrfs_raid_bio *rbio)
28253b381b3SDavid Woodhouse {
28318d758a2SQu Wenruo u64 num = rbio->bioc->full_stripe_logical;
28453b381b3SDavid Woodhouse
28553b381b3SDavid Woodhouse /*
28653b381b3SDavid Woodhouse * we shift down quite a bit. We're using byte
28753b381b3SDavid Woodhouse * addressing, and most of the lower bits are zeros.
28853b381b3SDavid Woodhouse * This tends to upset hash_64, and it consistently
28953b381b3SDavid Woodhouse * returns just one or two different values.
29053b381b3SDavid Woodhouse *
29153b381b3SDavid Woodhouse * shifting off the lower bits fixes things.
29253b381b3SDavid Woodhouse */
29353b381b3SDavid Woodhouse return hash_64(num >> 16, BTRFS_STRIPE_HASH_TABLE_BITS);
29453b381b3SDavid Woodhouse }
29553b381b3SDavid Woodhouse
full_page_sectors_uptodate(struct btrfs_raid_bio * rbio,unsigned int page_nr)296d4e28d9bSQu Wenruo static bool full_page_sectors_uptodate(struct btrfs_raid_bio *rbio,
297d4e28d9bSQu Wenruo unsigned int page_nr)
298d4e28d9bSQu Wenruo {
299d4e28d9bSQu Wenruo const u32 sectorsize = rbio->bioc->fs_info->sectorsize;
300d4e28d9bSQu Wenruo const u32 sectors_per_page = PAGE_SIZE / sectorsize;
301d4e28d9bSQu Wenruo int i;
302d4e28d9bSQu Wenruo
303d4e28d9bSQu Wenruo ASSERT(page_nr < rbio->nr_pages);
304d4e28d9bSQu Wenruo
305d4e28d9bSQu Wenruo for (i = sectors_per_page * page_nr;
306d4e28d9bSQu Wenruo i < sectors_per_page * page_nr + sectors_per_page;
307d4e28d9bSQu Wenruo i++) {
308d4e28d9bSQu Wenruo if (!rbio->stripe_sectors[i].uptodate)
309d4e28d9bSQu Wenruo return false;
310d4e28d9bSQu Wenruo }
311d4e28d9bSQu Wenruo return true;
312d4e28d9bSQu Wenruo }
313d4e28d9bSQu Wenruo
31453b381b3SDavid Woodhouse /*
315eb357060SQu Wenruo * Update the stripe_sectors[] array to use correct page and pgoff
316eb357060SQu Wenruo *
317eb357060SQu Wenruo * Should be called every time any page pointer in stripes_pages[] got modified.
318eb357060SQu Wenruo */
index_stripe_sectors(struct btrfs_raid_bio * rbio)319eb357060SQu Wenruo static void index_stripe_sectors(struct btrfs_raid_bio *rbio)
320eb357060SQu Wenruo {
321eb357060SQu Wenruo const u32 sectorsize = rbio->bioc->fs_info->sectorsize;
322eb357060SQu Wenruo u32 offset;
323eb357060SQu Wenruo int i;
324eb357060SQu Wenruo
325eb357060SQu Wenruo for (i = 0, offset = 0; i < rbio->nr_sectors; i++, offset += sectorsize) {
326eb357060SQu Wenruo int page_index = offset >> PAGE_SHIFT;
327eb357060SQu Wenruo
328eb357060SQu Wenruo ASSERT(page_index < rbio->nr_pages);
329eb357060SQu Wenruo rbio->stripe_sectors[i].page = rbio->stripe_pages[page_index];
330eb357060SQu Wenruo rbio->stripe_sectors[i].pgoff = offset_in_page(offset);
331eb357060SQu Wenruo }
332eb357060SQu Wenruo }
333eb357060SQu Wenruo
steal_rbio_page(struct btrfs_raid_bio * src,struct btrfs_raid_bio * dest,int page_nr)3344d100466SQu Wenruo static void steal_rbio_page(struct btrfs_raid_bio *src,
3354d100466SQu Wenruo struct btrfs_raid_bio *dest, int page_nr)
3364d100466SQu Wenruo {
3374d100466SQu Wenruo const u32 sectorsize = src->bioc->fs_info->sectorsize;
3384d100466SQu Wenruo const u32 sectors_per_page = PAGE_SIZE / sectorsize;
3394d100466SQu Wenruo int i;
3404d100466SQu Wenruo
3414d100466SQu Wenruo if (dest->stripe_pages[page_nr])
3424d100466SQu Wenruo __free_page(dest->stripe_pages[page_nr]);
3434d100466SQu Wenruo dest->stripe_pages[page_nr] = src->stripe_pages[page_nr];
3444d100466SQu Wenruo src->stripe_pages[page_nr] = NULL;
3454d100466SQu Wenruo
3464d100466SQu Wenruo /* Also update the sector->uptodate bits. */
3474d100466SQu Wenruo for (i = sectors_per_page * page_nr;
3484d100466SQu Wenruo i < sectors_per_page * page_nr + sectors_per_page; i++)
3494d100466SQu Wenruo dest->stripe_sectors[i].uptodate = true;
3504d100466SQu Wenruo }
3514d100466SQu Wenruo
is_data_stripe_page(struct btrfs_raid_bio * rbio,int page_nr)35288074c8bSQu Wenruo static bool is_data_stripe_page(struct btrfs_raid_bio *rbio, int page_nr)
35388074c8bSQu Wenruo {
35488074c8bSQu Wenruo const int sector_nr = (page_nr << PAGE_SHIFT) >>
35588074c8bSQu Wenruo rbio->bioc->fs_info->sectorsize_bits;
35688074c8bSQu Wenruo
35788074c8bSQu Wenruo /*
35888074c8bSQu Wenruo * We have ensured PAGE_SIZE is aligned with sectorsize, thus
35988074c8bSQu Wenruo * we won't have a page which is half data half parity.
36088074c8bSQu Wenruo *
36188074c8bSQu Wenruo * Thus if the first sector of the page belongs to data stripes, then
36288074c8bSQu Wenruo * the full page belongs to data stripes.
36388074c8bSQu Wenruo */
36488074c8bSQu Wenruo return (sector_nr < rbio->nr_data * rbio->stripe_nsectors);
36588074c8bSQu Wenruo }
36688074c8bSQu Wenruo
367eb357060SQu Wenruo /*
368d4e28d9bSQu Wenruo * Stealing an rbio means taking all the uptodate pages from the stripe array
369d4e28d9bSQu Wenruo * in the source rbio and putting them into the destination rbio.
370d4e28d9bSQu Wenruo *
371d4e28d9bSQu Wenruo * This will also update the involved stripe_sectors[] which are referring to
372d4e28d9bSQu Wenruo * the old pages.
3734ae10b3aSChris Mason */
steal_rbio(struct btrfs_raid_bio * src,struct btrfs_raid_bio * dest)3744ae10b3aSChris Mason static void steal_rbio(struct btrfs_raid_bio *src, struct btrfs_raid_bio *dest)
3754ae10b3aSChris Mason {
3764ae10b3aSChris Mason int i;
3774ae10b3aSChris Mason
3784ae10b3aSChris Mason if (!test_bit(RBIO_CACHE_READY_BIT, &src->flags))
3794ae10b3aSChris Mason return;
3804ae10b3aSChris Mason
3814ae10b3aSChris Mason for (i = 0; i < dest->nr_pages; i++) {
38288074c8bSQu Wenruo struct page *p = src->stripe_pages[i];
38388074c8bSQu Wenruo
38488074c8bSQu Wenruo /*
38588074c8bSQu Wenruo * We don't need to steal P/Q pages as they will always be
38688074c8bSQu Wenruo * regenerated for RMW or full write anyway.
38788074c8bSQu Wenruo */
38888074c8bSQu Wenruo if (!is_data_stripe_page(src, i))
3894ae10b3aSChris Mason continue;
3904ae10b3aSChris Mason
39188074c8bSQu Wenruo /*
39288074c8bSQu Wenruo * If @src already has RBIO_CACHE_READY_BIT, it should have
39388074c8bSQu Wenruo * all data stripe pages present and uptodate.
39488074c8bSQu Wenruo */
39588074c8bSQu Wenruo ASSERT(p);
39688074c8bSQu Wenruo ASSERT(full_page_sectors_uptodate(src, i));
3974d100466SQu Wenruo steal_rbio_page(src, dest, i);
3984ae10b3aSChris Mason }
399eb357060SQu Wenruo index_stripe_sectors(dest);
400eb357060SQu Wenruo index_stripe_sectors(src);
4014ae10b3aSChris Mason }
4024ae10b3aSChris Mason
4034ae10b3aSChris Mason /*
40453b381b3SDavid Woodhouse * merging means we take the bio_list from the victim and
40553b381b3SDavid Woodhouse * splice it into the destination. The victim should
40653b381b3SDavid Woodhouse * be discarded afterwards.
40753b381b3SDavid Woodhouse *
40853b381b3SDavid Woodhouse * must be called with dest->rbio_list_lock held
40953b381b3SDavid Woodhouse */
merge_rbio(struct btrfs_raid_bio * dest,struct btrfs_raid_bio * victim)41053b381b3SDavid Woodhouse static void merge_rbio(struct btrfs_raid_bio *dest,
41153b381b3SDavid Woodhouse struct btrfs_raid_bio *victim)
41253b381b3SDavid Woodhouse {
413fa1af65bSChristoph Hellwig bio_list_merge_init(&dest->bio_list, &victim->bio_list);
41453b381b3SDavid Woodhouse dest->bio_list_bytes += victim->bio_list_bytes;
415bd8f7e62SQu Wenruo /* Also inherit the bitmaps from @victim. */
416bd8f7e62SQu Wenruo bitmap_or(&dest->dbitmap, &victim->dbitmap, &dest->dbitmap,
417bd8f7e62SQu Wenruo dest->stripe_nsectors);
41853b381b3SDavid Woodhouse }
41953b381b3SDavid Woodhouse
42053b381b3SDavid Woodhouse /*
4214ae10b3aSChris Mason * used to prune items that are in the cache. The caller
4224ae10b3aSChris Mason * must hold the hash table lock.
4234ae10b3aSChris Mason */
__remove_rbio_from_cache(struct btrfs_raid_bio * rbio)4244ae10b3aSChris Mason static void __remove_rbio_from_cache(struct btrfs_raid_bio *rbio)
4254ae10b3aSChris Mason {
4264ae10b3aSChris Mason int bucket = rbio_bucket(rbio);
4274ae10b3aSChris Mason struct btrfs_stripe_hash_table *table;
4284ae10b3aSChris Mason struct btrfs_stripe_hash *h;
4294ae10b3aSChris Mason int freeit = 0;
4304ae10b3aSChris Mason
4314ae10b3aSChris Mason /*
4324ae10b3aSChris Mason * check the bit again under the hash table lock.
4334ae10b3aSChris Mason */
4344ae10b3aSChris Mason if (!test_bit(RBIO_CACHE_BIT, &rbio->flags))
4354ae10b3aSChris Mason return;
4364ae10b3aSChris Mason
4376a258d72SQu Wenruo table = rbio->bioc->fs_info->stripe_hash_table;
4384ae10b3aSChris Mason h = table->table + bucket;
4394ae10b3aSChris Mason
4404ae10b3aSChris Mason /* hold the lock for the bucket because we may be
4414ae10b3aSChris Mason * removing it from the hash table
4424ae10b3aSChris Mason */
4434ae10b3aSChris Mason spin_lock(&h->lock);
4444ae10b3aSChris Mason
4454ae10b3aSChris Mason /*
4464ae10b3aSChris Mason * hold the lock for the bio list because we need
4474ae10b3aSChris Mason * to make sure the bio list is empty
4484ae10b3aSChris Mason */
4494ae10b3aSChris Mason spin_lock(&rbio->bio_list_lock);
4504ae10b3aSChris Mason
4514ae10b3aSChris Mason if (test_and_clear_bit(RBIO_CACHE_BIT, &rbio->flags)) {
4524ae10b3aSChris Mason list_del_init(&rbio->stripe_cache);
4534ae10b3aSChris Mason table->cache_size -= 1;
4544ae10b3aSChris Mason freeit = 1;
4554ae10b3aSChris Mason
4564ae10b3aSChris Mason /* if the bio list isn't empty, this rbio is
4574ae10b3aSChris Mason * still involved in an IO. We take it out
4584ae10b3aSChris Mason * of the cache list, and drop the ref that
4594ae10b3aSChris Mason * was held for the list.
4604ae10b3aSChris Mason *
4614ae10b3aSChris Mason * If the bio_list was empty, we also remove
4624ae10b3aSChris Mason * the rbio from the hash_table, and drop
4634ae10b3aSChris Mason * the corresponding ref
4644ae10b3aSChris Mason */
4654ae10b3aSChris Mason if (bio_list_empty(&rbio->bio_list)) {
4664ae10b3aSChris Mason if (!list_empty(&rbio->hash_list)) {
4674ae10b3aSChris Mason list_del_init(&rbio->hash_list);
468dec95574SElena Reshetova refcount_dec(&rbio->refs);
4694ae10b3aSChris Mason BUG_ON(!list_empty(&rbio->plug_list));
4704ae10b3aSChris Mason }
4714ae10b3aSChris Mason }
4724ae10b3aSChris Mason }
4734ae10b3aSChris Mason
4744ae10b3aSChris Mason spin_unlock(&rbio->bio_list_lock);
4754ae10b3aSChris Mason spin_unlock(&h->lock);
4764ae10b3aSChris Mason
4774ae10b3aSChris Mason if (freeit)
478ff2b64a2SQu Wenruo free_raid_bio(rbio);
4794ae10b3aSChris Mason }
4804ae10b3aSChris Mason
4814ae10b3aSChris Mason /*
4824ae10b3aSChris Mason * prune a given rbio from the cache
4834ae10b3aSChris Mason */
remove_rbio_from_cache(struct btrfs_raid_bio * rbio)4844ae10b3aSChris Mason static void remove_rbio_from_cache(struct btrfs_raid_bio *rbio)
4854ae10b3aSChris Mason {
4864ae10b3aSChris Mason struct btrfs_stripe_hash_table *table;
4874ae10b3aSChris Mason
4884ae10b3aSChris Mason if (!test_bit(RBIO_CACHE_BIT, &rbio->flags))
4894ae10b3aSChris Mason return;
4904ae10b3aSChris Mason
4916a258d72SQu Wenruo table = rbio->bioc->fs_info->stripe_hash_table;
4924ae10b3aSChris Mason
49374cc3600SChristoph Hellwig spin_lock(&table->cache_lock);
4944ae10b3aSChris Mason __remove_rbio_from_cache(rbio);
49574cc3600SChristoph Hellwig spin_unlock(&table->cache_lock);
4964ae10b3aSChris Mason }
4974ae10b3aSChris Mason
4984ae10b3aSChris Mason /*
4994ae10b3aSChris Mason * remove everything in the cache
5004ae10b3aSChris Mason */
btrfs_clear_rbio_cache(struct btrfs_fs_info * info)50148a3b636SEric Sandeen static void btrfs_clear_rbio_cache(struct btrfs_fs_info *info)
5024ae10b3aSChris Mason {
5034ae10b3aSChris Mason struct btrfs_stripe_hash_table *table;
5044ae10b3aSChris Mason struct btrfs_raid_bio *rbio;
5054ae10b3aSChris Mason
5064ae10b3aSChris Mason table = info->stripe_hash_table;
5074ae10b3aSChris Mason
50874cc3600SChristoph Hellwig spin_lock(&table->cache_lock);
5094ae10b3aSChris Mason while (!list_empty(&table->stripe_cache)) {
5104ae10b3aSChris Mason rbio = list_entry(table->stripe_cache.next,
5114ae10b3aSChris Mason struct btrfs_raid_bio,
5124ae10b3aSChris Mason stripe_cache);
5134ae10b3aSChris Mason __remove_rbio_from_cache(rbio);
5144ae10b3aSChris Mason }
51574cc3600SChristoph Hellwig spin_unlock(&table->cache_lock);
5164ae10b3aSChris Mason }
5174ae10b3aSChris Mason
5184ae10b3aSChris Mason /*
5194ae10b3aSChris Mason * remove all cached entries and free the hash table
5204ae10b3aSChris Mason * used by unmount
52153b381b3SDavid Woodhouse */
btrfs_free_stripe_hash_table(struct btrfs_fs_info * info)52253b381b3SDavid Woodhouse void btrfs_free_stripe_hash_table(struct btrfs_fs_info *info)
52353b381b3SDavid Woodhouse {
52453b381b3SDavid Woodhouse if (!info->stripe_hash_table)
52553b381b3SDavid Woodhouse return;
5264ae10b3aSChris Mason btrfs_clear_rbio_cache(info);
527f749303bSWang Shilong kvfree(info->stripe_hash_table);
52853b381b3SDavid Woodhouse info->stripe_hash_table = NULL;
52953b381b3SDavid Woodhouse }
53053b381b3SDavid Woodhouse
53153b381b3SDavid Woodhouse /*
5324ae10b3aSChris Mason * insert an rbio into the stripe cache. It
5334ae10b3aSChris Mason * must have already been prepared by calling
5344ae10b3aSChris Mason * cache_rbio_pages
5354ae10b3aSChris Mason *
5364ae10b3aSChris Mason * If this rbio was already cached, it gets
5374ae10b3aSChris Mason * moved to the front of the lru.
5384ae10b3aSChris Mason *
5394ae10b3aSChris Mason * If the size of the rbio cache is too big, we
5404ae10b3aSChris Mason * prune an item.
5414ae10b3aSChris Mason */
cache_rbio(struct btrfs_raid_bio * rbio)5424ae10b3aSChris Mason static void cache_rbio(struct btrfs_raid_bio *rbio)
5434ae10b3aSChris Mason {
5444ae10b3aSChris Mason struct btrfs_stripe_hash_table *table;
5454ae10b3aSChris Mason
5464ae10b3aSChris Mason if (!test_bit(RBIO_CACHE_READY_BIT, &rbio->flags))
5474ae10b3aSChris Mason return;
5484ae10b3aSChris Mason
5496a258d72SQu Wenruo table = rbio->bioc->fs_info->stripe_hash_table;
5504ae10b3aSChris Mason
55174cc3600SChristoph Hellwig spin_lock(&table->cache_lock);
5524ae10b3aSChris Mason spin_lock(&rbio->bio_list_lock);
5534ae10b3aSChris Mason
5544ae10b3aSChris Mason /* bump our ref if we were not in the list before */
5554ae10b3aSChris Mason if (!test_and_set_bit(RBIO_CACHE_BIT, &rbio->flags))
556dec95574SElena Reshetova refcount_inc(&rbio->refs);
5574ae10b3aSChris Mason
5584ae10b3aSChris Mason if (!list_empty(&rbio->stripe_cache)){
5594ae10b3aSChris Mason list_move(&rbio->stripe_cache, &table->stripe_cache);
5604ae10b3aSChris Mason } else {
5614ae10b3aSChris Mason list_add(&rbio->stripe_cache, &table->stripe_cache);
5624ae10b3aSChris Mason table->cache_size += 1;
5634ae10b3aSChris Mason }
5644ae10b3aSChris Mason
5654ae10b3aSChris Mason spin_unlock(&rbio->bio_list_lock);
5664ae10b3aSChris Mason
5674ae10b3aSChris Mason if (table->cache_size > RBIO_CACHE_SIZE) {
5684ae10b3aSChris Mason struct btrfs_raid_bio *found;
5694ae10b3aSChris Mason
5704ae10b3aSChris Mason found = list_entry(table->stripe_cache.prev,
5714ae10b3aSChris Mason struct btrfs_raid_bio,
5724ae10b3aSChris Mason stripe_cache);
5734ae10b3aSChris Mason
5744ae10b3aSChris Mason if (found != rbio)
5754ae10b3aSChris Mason __remove_rbio_from_cache(found);
5764ae10b3aSChris Mason }
5774ae10b3aSChris Mason
57874cc3600SChristoph Hellwig spin_unlock(&table->cache_lock);
5794ae10b3aSChris Mason }
5804ae10b3aSChris Mason
5814ae10b3aSChris Mason /*
58253b381b3SDavid Woodhouse * helper function to run the xor_blocks api. It is only
58353b381b3SDavid Woodhouse * able to do MAX_XOR_BLOCKS at a time, so we need to
58453b381b3SDavid Woodhouse * loop through.
58553b381b3SDavid Woodhouse */
run_xor(void ** pages,int src_cnt,ssize_t len)58653b381b3SDavid Woodhouse static void run_xor(void **pages, int src_cnt, ssize_t len)
58753b381b3SDavid Woodhouse {
58853b381b3SDavid Woodhouse int src_off = 0;
58953b381b3SDavid Woodhouse int xor_src_cnt = 0;
59053b381b3SDavid Woodhouse void *dest = pages[src_cnt];
59153b381b3SDavid Woodhouse
59253b381b3SDavid Woodhouse while(src_cnt > 0) {
59353b381b3SDavid Woodhouse xor_src_cnt = min(src_cnt, MAX_XOR_BLOCKS);
59453b381b3SDavid Woodhouse xor_blocks(xor_src_cnt, len, dest, pages + src_off);
59553b381b3SDavid Woodhouse
59653b381b3SDavid Woodhouse src_cnt -= xor_src_cnt;
59753b381b3SDavid Woodhouse src_off += xor_src_cnt;
59853b381b3SDavid Woodhouse }
59953b381b3SDavid Woodhouse }
60053b381b3SDavid Woodhouse
60153b381b3SDavid Woodhouse /*
602176571a1SDavid Sterba * Returns true if the bio list inside this rbio covers an entire stripe (no
603176571a1SDavid Sterba * rmw required).
60453b381b3SDavid Woodhouse */
rbio_is_full(struct btrfs_raid_bio * rbio)60553b381b3SDavid Woodhouse static int rbio_is_full(struct btrfs_raid_bio *rbio)
60653b381b3SDavid Woodhouse {
607176571a1SDavid Sterba unsigned long size = rbio->bio_list_bytes;
608176571a1SDavid Sterba int ret = 1;
60953b381b3SDavid Woodhouse
61074cc3600SChristoph Hellwig spin_lock(&rbio->bio_list_lock);
611ff18a4afSChristoph Hellwig if (size != rbio->nr_data * BTRFS_STRIPE_LEN)
612176571a1SDavid Sterba ret = 0;
613ff18a4afSChristoph Hellwig BUG_ON(size > rbio->nr_data * BTRFS_STRIPE_LEN);
61474cc3600SChristoph Hellwig spin_unlock(&rbio->bio_list_lock);
615176571a1SDavid Sterba
61653b381b3SDavid Woodhouse return ret;
61753b381b3SDavid Woodhouse }
61853b381b3SDavid Woodhouse
61953b381b3SDavid Woodhouse /*
62053b381b3SDavid Woodhouse * returns 1 if it is safe to merge two rbios together.
62153b381b3SDavid Woodhouse * The merging is safe if the two rbios correspond to
62253b381b3SDavid Woodhouse * the same stripe and if they are both going in the same
62353b381b3SDavid Woodhouse * direction (read vs write), and if neither one is
62453b381b3SDavid Woodhouse * locked for final IO
62553b381b3SDavid Woodhouse *
62653b381b3SDavid Woodhouse * The caller is responsible for locking such that
62753b381b3SDavid Woodhouse * rmw_locked is safe to test
62853b381b3SDavid Woodhouse */
rbio_can_merge(struct btrfs_raid_bio * last,struct btrfs_raid_bio * cur)62953b381b3SDavid Woodhouse static int rbio_can_merge(struct btrfs_raid_bio *last,
63053b381b3SDavid Woodhouse struct btrfs_raid_bio *cur)
63153b381b3SDavid Woodhouse {
63253b381b3SDavid Woodhouse if (test_bit(RBIO_RMW_LOCKED_BIT, &last->flags) ||
63353b381b3SDavid Woodhouse test_bit(RBIO_RMW_LOCKED_BIT, &cur->flags))
63453b381b3SDavid Woodhouse return 0;
63553b381b3SDavid Woodhouse
6364ae10b3aSChris Mason /*
6374ae10b3aSChris Mason * we can't merge with cached rbios, since the
6384ae10b3aSChris Mason * idea is that when we merge the destination
6394ae10b3aSChris Mason * rbio is going to run our IO for us. We can
64001327610SNicholas D Steeves * steal from cached rbios though, other functions
6414ae10b3aSChris Mason * handle that.
6424ae10b3aSChris Mason */
6434ae10b3aSChris Mason if (test_bit(RBIO_CACHE_BIT, &last->flags) ||
6444ae10b3aSChris Mason test_bit(RBIO_CACHE_BIT, &cur->flags))
6454ae10b3aSChris Mason return 0;
6464ae10b3aSChris Mason
64718d758a2SQu Wenruo if (last->bioc->full_stripe_logical != cur->bioc->full_stripe_logical)
64853b381b3SDavid Woodhouse return 0;
64953b381b3SDavid Woodhouse
6505a6ac9eaSMiao Xie /* we can't merge with different operations */
6515a6ac9eaSMiao Xie if (last->operation != cur->operation)
65253b381b3SDavid Woodhouse return 0;
6535a6ac9eaSMiao Xie /*
6545a6ac9eaSMiao Xie * We've need read the full stripe from the drive.
6555a6ac9eaSMiao Xie * check and repair the parity and write the new results.
6565a6ac9eaSMiao Xie *
6575a6ac9eaSMiao Xie * We're not allowed to add any new bios to the
6585a6ac9eaSMiao Xie * bio list here, anyone else that wants to
6595a6ac9eaSMiao Xie * change this stripe needs to do their own rmw.
6605a6ac9eaSMiao Xie */
661db34be19SLiu Bo if (last->operation == BTRFS_RBIO_PARITY_SCRUB)
6625a6ac9eaSMiao Xie return 0;
66353b381b3SDavid Woodhouse
6643a3c7a7fSQu Wenruo if (last->operation == BTRFS_RBIO_READ_REBUILD)
665b4ee1782SOmar Sandoval return 0;
666b4ee1782SOmar Sandoval
66753b381b3SDavid Woodhouse return 1;
66853b381b3SDavid Woodhouse }
66953b381b3SDavid Woodhouse
rbio_stripe_sector_index(const struct btrfs_raid_bio * rbio,unsigned int stripe_nr,unsigned int sector_nr)6703e77605dSQu Wenruo static unsigned int rbio_stripe_sector_index(const struct btrfs_raid_bio *rbio,
6713e77605dSQu Wenruo unsigned int stripe_nr,
6723e77605dSQu Wenruo unsigned int sector_nr)
6733e77605dSQu Wenruo {
674bbbee460SQu Wenruo ASSERT_RBIO_STRIPE(stripe_nr < rbio->real_stripes, rbio, stripe_nr);
675bbbee460SQu Wenruo ASSERT_RBIO_SECTOR(sector_nr < rbio->stripe_nsectors, rbio, sector_nr);
6763e77605dSQu Wenruo
6773e77605dSQu Wenruo return stripe_nr * rbio->stripe_nsectors + sector_nr;
6783e77605dSQu Wenruo }
6793e77605dSQu Wenruo
6803e77605dSQu Wenruo /* Return a sector from rbio->stripe_sectors, not from the bio list */
rbio_stripe_sector(const struct btrfs_raid_bio * rbio,unsigned int stripe_nr,unsigned int sector_nr)6813e77605dSQu Wenruo static struct sector_ptr *rbio_stripe_sector(const struct btrfs_raid_bio *rbio,
6823e77605dSQu Wenruo unsigned int stripe_nr,
6833e77605dSQu Wenruo unsigned int sector_nr)
6843e77605dSQu Wenruo {
6853e77605dSQu Wenruo return &rbio->stripe_sectors[rbio_stripe_sector_index(rbio, stripe_nr,
6863e77605dSQu Wenruo sector_nr)];
6873e77605dSQu Wenruo }
6883e77605dSQu Wenruo
6891145059aSQu Wenruo /* Grab a sector inside P stripe */
rbio_pstripe_sector(const struct btrfs_raid_bio * rbio,unsigned int sector_nr)6901145059aSQu Wenruo static struct sector_ptr *rbio_pstripe_sector(const struct btrfs_raid_bio *rbio,
6911145059aSQu Wenruo unsigned int sector_nr)
692b7178a5fSZhao Lei {
6931145059aSQu Wenruo return rbio_stripe_sector(rbio, rbio->nr_data, sector_nr);
694b7178a5fSZhao Lei }
695b7178a5fSZhao Lei
6961145059aSQu Wenruo /* Grab a sector inside Q stripe, return NULL if not RAID6 */
rbio_qstripe_sector(const struct btrfs_raid_bio * rbio,unsigned int sector_nr)6971145059aSQu Wenruo static struct sector_ptr *rbio_qstripe_sector(const struct btrfs_raid_bio *rbio,
6981145059aSQu Wenruo unsigned int sector_nr)
69953b381b3SDavid Woodhouse {
7002c8cdd6eSMiao Xie if (rbio->nr_data + 1 == rbio->real_stripes)
70153b381b3SDavid Woodhouse return NULL;
7021145059aSQu Wenruo return rbio_stripe_sector(rbio, rbio->nr_data + 1, sector_nr);
7031145059aSQu Wenruo }
7041145059aSQu Wenruo
70553b381b3SDavid Woodhouse /*
70653b381b3SDavid Woodhouse * The first stripe in the table for a logical address
70753b381b3SDavid Woodhouse * has the lock. rbios are added in one of three ways:
70853b381b3SDavid Woodhouse *
70953b381b3SDavid Woodhouse * 1) Nobody has the stripe locked yet. The rbio is given
71053b381b3SDavid Woodhouse * the lock and 0 is returned. The caller must start the IO
71153b381b3SDavid Woodhouse * themselves.
71253b381b3SDavid Woodhouse *
71353b381b3SDavid Woodhouse * 2) Someone has the stripe locked, but we're able to merge
71453b381b3SDavid Woodhouse * with the lock owner. The rbio is freed and the IO will
71553b381b3SDavid Woodhouse * start automatically along with the existing rbio. 1 is returned.
71653b381b3SDavid Woodhouse *
71753b381b3SDavid Woodhouse * 3) Someone has the stripe locked, but we're not able to merge.
71853b381b3SDavid Woodhouse * The rbio is added to the lock owner's plug list, or merged into
71953b381b3SDavid Woodhouse * an rbio already on the plug list. When the lock owner unlocks,
72053b381b3SDavid Woodhouse * the next rbio on the list is run and the IO is started automatically.
72153b381b3SDavid Woodhouse * 1 is returned
72253b381b3SDavid Woodhouse *
72353b381b3SDavid Woodhouse * If we return 0, the caller still owns the rbio and must continue with
72453b381b3SDavid Woodhouse * IO submission. If we return 1, the caller must assume the rbio has
72553b381b3SDavid Woodhouse * already been freed.
72653b381b3SDavid Woodhouse */
lock_stripe_add(struct btrfs_raid_bio * rbio)72753b381b3SDavid Woodhouse static noinline int lock_stripe_add(struct btrfs_raid_bio *rbio)
72853b381b3SDavid Woodhouse {
729721860d5SJohannes Thumshirn struct btrfs_stripe_hash *h;
73053b381b3SDavid Woodhouse struct btrfs_raid_bio *cur;
73153b381b3SDavid Woodhouse struct btrfs_raid_bio *pending;
73253b381b3SDavid Woodhouse struct btrfs_raid_bio *freeit = NULL;
7334ae10b3aSChris Mason struct btrfs_raid_bio *cache_drop = NULL;
73453b381b3SDavid Woodhouse int ret = 0;
73553b381b3SDavid Woodhouse
7366a258d72SQu Wenruo h = rbio->bioc->fs_info->stripe_hash_table->table + rbio_bucket(rbio);
737721860d5SJohannes Thumshirn
73874cc3600SChristoph Hellwig spin_lock(&h->lock);
73953b381b3SDavid Woodhouse list_for_each_entry(cur, &h->hash_list, hash_list) {
74018d758a2SQu Wenruo if (cur->bioc->full_stripe_logical != rbio->bioc->full_stripe_logical)
7419d6cb1b0SJohannes Thumshirn continue;
7429d6cb1b0SJohannes Thumshirn
74353b381b3SDavid Woodhouse spin_lock(&cur->bio_list_lock);
74453b381b3SDavid Woodhouse
7459d6cb1b0SJohannes Thumshirn /* Can we steal this cached rbio's pages? */
7464ae10b3aSChris Mason if (bio_list_empty(&cur->bio_list) &&
7474ae10b3aSChris Mason list_empty(&cur->plug_list) &&
7484ae10b3aSChris Mason test_bit(RBIO_CACHE_BIT, &cur->flags) &&
7494ae10b3aSChris Mason !test_bit(RBIO_RMW_LOCKED_BIT, &cur->flags)) {
7504ae10b3aSChris Mason list_del_init(&cur->hash_list);
751dec95574SElena Reshetova refcount_dec(&cur->refs);
7524ae10b3aSChris Mason
7534ae10b3aSChris Mason steal_rbio(cur, rbio);
7544ae10b3aSChris Mason cache_drop = cur;
7554ae10b3aSChris Mason spin_unlock(&cur->bio_list_lock);
7564ae10b3aSChris Mason
7574ae10b3aSChris Mason goto lockit;
7584ae10b3aSChris Mason }
7594ae10b3aSChris Mason
7609d6cb1b0SJohannes Thumshirn /* Can we merge into the lock owner? */
76153b381b3SDavid Woodhouse if (rbio_can_merge(cur, rbio)) {
76253b381b3SDavid Woodhouse merge_rbio(cur, rbio);
76353b381b3SDavid Woodhouse spin_unlock(&cur->bio_list_lock);
76453b381b3SDavid Woodhouse freeit = rbio;
76553b381b3SDavid Woodhouse ret = 1;
76653b381b3SDavid Woodhouse goto out;
76753b381b3SDavid Woodhouse }
76853b381b3SDavid Woodhouse
7694ae10b3aSChris Mason
77053b381b3SDavid Woodhouse /*
7719d6cb1b0SJohannes Thumshirn * We couldn't merge with the running rbio, see if we can merge
7729d6cb1b0SJohannes Thumshirn * with the pending ones. We don't have to check for rmw_locked
7739d6cb1b0SJohannes Thumshirn * because there is no way they are inside finish_rmw right now
77453b381b3SDavid Woodhouse */
7759d6cb1b0SJohannes Thumshirn list_for_each_entry(pending, &cur->plug_list, plug_list) {
77653b381b3SDavid Woodhouse if (rbio_can_merge(pending, rbio)) {
77753b381b3SDavid Woodhouse merge_rbio(pending, rbio);
77853b381b3SDavid Woodhouse spin_unlock(&cur->bio_list_lock);
77953b381b3SDavid Woodhouse freeit = rbio;
78053b381b3SDavid Woodhouse ret = 1;
78153b381b3SDavid Woodhouse goto out;
78253b381b3SDavid Woodhouse }
78353b381b3SDavid Woodhouse }
78453b381b3SDavid Woodhouse
7859d6cb1b0SJohannes Thumshirn /*
7869d6cb1b0SJohannes Thumshirn * No merging, put us on the tail of the plug list, our rbio
7879d6cb1b0SJohannes Thumshirn * will be started with the currently running rbio unlocks
78853b381b3SDavid Woodhouse */
78953b381b3SDavid Woodhouse list_add_tail(&rbio->plug_list, &cur->plug_list);
79053b381b3SDavid Woodhouse spin_unlock(&cur->bio_list_lock);
79153b381b3SDavid Woodhouse ret = 1;
79253b381b3SDavid Woodhouse goto out;
79353b381b3SDavid Woodhouse }
7944ae10b3aSChris Mason lockit:
795dec95574SElena Reshetova refcount_inc(&rbio->refs);
79653b381b3SDavid Woodhouse list_add(&rbio->hash_list, &h->hash_list);
79753b381b3SDavid Woodhouse out:
79874cc3600SChristoph Hellwig spin_unlock(&h->lock);
7994ae10b3aSChris Mason if (cache_drop)
8004ae10b3aSChris Mason remove_rbio_from_cache(cache_drop);
80153b381b3SDavid Woodhouse if (freeit)
802ff2b64a2SQu Wenruo free_raid_bio(freeit);
80353b381b3SDavid Woodhouse return ret;
80453b381b3SDavid Woodhouse }
80553b381b3SDavid Woodhouse
806d817ce35SQu Wenruo static void recover_rbio_work_locked(struct work_struct *work);
807d817ce35SQu Wenruo
80853b381b3SDavid Woodhouse /*
80953b381b3SDavid Woodhouse * called as rmw or parity rebuild is completed. If the plug list has more
81053b381b3SDavid Woodhouse * rbios waiting for this stripe, the next one on the list will be started
81153b381b3SDavid Woodhouse */
unlock_stripe(struct btrfs_raid_bio * rbio)81253b381b3SDavid Woodhouse static noinline void unlock_stripe(struct btrfs_raid_bio *rbio)
81353b381b3SDavid Woodhouse {
81453b381b3SDavid Woodhouse int bucket;
81553b381b3SDavid Woodhouse struct btrfs_stripe_hash *h;
8164ae10b3aSChris Mason int keep_cache = 0;
81753b381b3SDavid Woodhouse
81853b381b3SDavid Woodhouse bucket = rbio_bucket(rbio);
8196a258d72SQu Wenruo h = rbio->bioc->fs_info->stripe_hash_table->table + bucket;
82053b381b3SDavid Woodhouse
8214ae10b3aSChris Mason if (list_empty(&rbio->plug_list))
8224ae10b3aSChris Mason cache_rbio(rbio);
8234ae10b3aSChris Mason
82474cc3600SChristoph Hellwig spin_lock(&h->lock);
82553b381b3SDavid Woodhouse spin_lock(&rbio->bio_list_lock);
82653b381b3SDavid Woodhouse
82753b381b3SDavid Woodhouse if (!list_empty(&rbio->hash_list)) {
8284ae10b3aSChris Mason /*
8294ae10b3aSChris Mason * if we're still cached and there is no other IO
8304ae10b3aSChris Mason * to perform, just leave this rbio here for others
8314ae10b3aSChris Mason * to steal from later
8324ae10b3aSChris Mason */
8334ae10b3aSChris Mason if (list_empty(&rbio->plug_list) &&
8344ae10b3aSChris Mason test_bit(RBIO_CACHE_BIT, &rbio->flags)) {
8354ae10b3aSChris Mason keep_cache = 1;
8364ae10b3aSChris Mason clear_bit(RBIO_RMW_LOCKED_BIT, &rbio->flags);
8374ae10b3aSChris Mason BUG_ON(!bio_list_empty(&rbio->bio_list));
8384ae10b3aSChris Mason goto done;
8394ae10b3aSChris Mason }
84053b381b3SDavid Woodhouse
84153b381b3SDavid Woodhouse list_del_init(&rbio->hash_list);
842dec95574SElena Reshetova refcount_dec(&rbio->refs);
84353b381b3SDavid Woodhouse
84453b381b3SDavid Woodhouse /*
84553b381b3SDavid Woodhouse * we use the plug list to hold all the rbios
84653b381b3SDavid Woodhouse * waiting for the chance to lock this stripe.
84753b381b3SDavid Woodhouse * hand the lock over to one of them.
84853b381b3SDavid Woodhouse */
84953b381b3SDavid Woodhouse if (!list_empty(&rbio->plug_list)) {
85053b381b3SDavid Woodhouse struct btrfs_raid_bio *next;
85153b381b3SDavid Woodhouse struct list_head *head = rbio->plug_list.next;
85253b381b3SDavid Woodhouse
85353b381b3SDavid Woodhouse next = list_entry(head, struct btrfs_raid_bio,
85453b381b3SDavid Woodhouse plug_list);
85553b381b3SDavid Woodhouse
85653b381b3SDavid Woodhouse list_del_init(&rbio->plug_list);
85753b381b3SDavid Woodhouse
85853b381b3SDavid Woodhouse list_add(&next->hash_list, &h->hash_list);
859dec95574SElena Reshetova refcount_inc(&next->refs);
86053b381b3SDavid Woodhouse spin_unlock(&rbio->bio_list_lock);
86174cc3600SChristoph Hellwig spin_unlock(&h->lock);
86253b381b3SDavid Woodhouse
8633a3c7a7fSQu Wenruo if (next->operation == BTRFS_RBIO_READ_REBUILD) {
864d817ce35SQu Wenruo start_async_work(next, recover_rbio_work_locked);
865b4ee1782SOmar Sandoval } else if (next->operation == BTRFS_RBIO_WRITE) {
8664ae10b3aSChris Mason steal_rbio(rbio, next);
86793723095SQu Wenruo start_async_work(next, rmw_rbio_work_locked);
8685a6ac9eaSMiao Xie } else if (next->operation == BTRFS_RBIO_PARITY_SCRUB) {
8695a6ac9eaSMiao Xie steal_rbio(rbio, next);
8706bfd0133SQu Wenruo start_async_work(next, scrub_rbio_work_locked);
8714ae10b3aSChris Mason }
87253b381b3SDavid Woodhouse
87353b381b3SDavid Woodhouse goto done_nolock;
87453b381b3SDavid Woodhouse }
87553b381b3SDavid Woodhouse }
8764ae10b3aSChris Mason done:
87753b381b3SDavid Woodhouse spin_unlock(&rbio->bio_list_lock);
87874cc3600SChristoph Hellwig spin_unlock(&h->lock);
87953b381b3SDavid Woodhouse
88053b381b3SDavid Woodhouse done_nolock:
8814ae10b3aSChris Mason if (!keep_cache)
8824ae10b3aSChris Mason remove_rbio_from_cache(rbio);
88353b381b3SDavid Woodhouse }
88453b381b3SDavid Woodhouse
rbio_endio_bio_list(struct bio * cur,blk_status_t err)8857583d8d0SLiu Bo static void rbio_endio_bio_list(struct bio *cur, blk_status_t err)
88653b381b3SDavid Woodhouse {
8877583d8d0SLiu Bo struct bio *next;
8887583d8d0SLiu Bo
8897583d8d0SLiu Bo while (cur) {
8907583d8d0SLiu Bo next = cur->bi_next;
8917583d8d0SLiu Bo cur->bi_next = NULL;
8927583d8d0SLiu Bo cur->bi_status = err;
8937583d8d0SLiu Bo bio_endio(cur);
8947583d8d0SLiu Bo cur = next;
8957583d8d0SLiu Bo }
89653b381b3SDavid Woodhouse }
89753b381b3SDavid Woodhouse
89853b381b3SDavid Woodhouse /*
89953b381b3SDavid Woodhouse * this frees the rbio and runs through all the bios in the
90053b381b3SDavid Woodhouse * bio_list and calls end_io on them
90153b381b3SDavid Woodhouse */
rbio_orig_end_io(struct btrfs_raid_bio * rbio,blk_status_t err)9024e4cbee9SChristoph Hellwig static void rbio_orig_end_io(struct btrfs_raid_bio *rbio, blk_status_t err)
90353b381b3SDavid Woodhouse {
90453b381b3SDavid Woodhouse struct bio *cur = bio_list_get(&rbio->bio_list);
9057583d8d0SLiu Bo struct bio *extra;
9064245215dSMiao Xie
907c5a41562SQu Wenruo kfree(rbio->csum_buf);
908c5a41562SQu Wenruo bitmap_free(rbio->csum_bitmap);
909c5a41562SQu Wenruo rbio->csum_buf = NULL;
910c5a41562SQu Wenruo rbio->csum_bitmap = NULL;
911c5a41562SQu Wenruo
912bd8f7e62SQu Wenruo /*
913bd8f7e62SQu Wenruo * Clear the data bitmap, as the rbio may be cached for later usage.
914bd8f7e62SQu Wenruo * do this before before unlock_stripe() so there will be no new bio
915bd8f7e62SQu Wenruo * for this bio.
916bd8f7e62SQu Wenruo */
917bd8f7e62SQu Wenruo bitmap_clear(&rbio->dbitmap, 0, rbio->stripe_nsectors);
9184245215dSMiao Xie
9197583d8d0SLiu Bo /*
9207583d8d0SLiu Bo * At this moment, rbio->bio_list is empty, however since rbio does not
9217583d8d0SLiu Bo * always have RBIO_RMW_LOCKED_BIT set and rbio is still linked on the
9227583d8d0SLiu Bo * hash list, rbio may be merged with others so that rbio->bio_list
9237583d8d0SLiu Bo * becomes non-empty.
9247583d8d0SLiu Bo * Once unlock_stripe() is done, rbio->bio_list will not be updated any
9257583d8d0SLiu Bo * more and we can call bio_endio() on all queued bios.
9267583d8d0SLiu Bo */
9277583d8d0SLiu Bo unlock_stripe(rbio);
9287583d8d0SLiu Bo extra = bio_list_get(&rbio->bio_list);
929ff2b64a2SQu Wenruo free_raid_bio(rbio);
93053b381b3SDavid Woodhouse
9317583d8d0SLiu Bo rbio_endio_bio_list(cur, err);
9327583d8d0SLiu Bo if (extra)
9337583d8d0SLiu Bo rbio_endio_bio_list(extra, err);
93453b381b3SDavid Woodhouse }
93553b381b3SDavid Woodhouse
93653b381b3SDavid Woodhouse /*
93743dd529aSDavid Sterba * Get a sector pointer specified by its @stripe_nr and @sector_nr.
9383e77605dSQu Wenruo *
9393e77605dSQu Wenruo * @rbio: The raid bio
9403e77605dSQu Wenruo * @stripe_nr: Stripe number, valid range [0, real_stripe)
9413e77605dSQu Wenruo * @sector_nr: Sector number inside the stripe,
9423e77605dSQu Wenruo * valid range [0, stripe_nsectors)
9433e77605dSQu Wenruo * @bio_list_only: Whether to use sectors inside the bio list only.
9443e77605dSQu Wenruo *
9453e77605dSQu Wenruo * The read/modify/write code wants to reuse the original bio page as much
9463e77605dSQu Wenruo * as possible, and only use stripe_sectors as fallback.
9473e77605dSQu Wenruo */
sector_in_rbio(struct btrfs_raid_bio * rbio,int stripe_nr,int sector_nr,bool bio_list_only)9483e77605dSQu Wenruo static struct sector_ptr *sector_in_rbio(struct btrfs_raid_bio *rbio,
9493e77605dSQu Wenruo int stripe_nr, int sector_nr,
9503e77605dSQu Wenruo bool bio_list_only)
9513e77605dSQu Wenruo {
9523e77605dSQu Wenruo struct sector_ptr *sector;
9533e77605dSQu Wenruo int index;
9543e77605dSQu Wenruo
955bbbee460SQu Wenruo ASSERT_RBIO_STRIPE(stripe_nr >= 0 && stripe_nr < rbio->real_stripes,
956bbbee460SQu Wenruo rbio, stripe_nr);
957bbbee460SQu Wenruo ASSERT_RBIO_SECTOR(sector_nr >= 0 && sector_nr < rbio->stripe_nsectors,
958bbbee460SQu Wenruo rbio, sector_nr);
9593e77605dSQu Wenruo
9603e77605dSQu Wenruo index = stripe_nr * rbio->stripe_nsectors + sector_nr;
9613e77605dSQu Wenruo ASSERT(index >= 0 && index < rbio->nr_sectors);
9623e77605dSQu Wenruo
96374cc3600SChristoph Hellwig spin_lock(&rbio->bio_list_lock);
9643e77605dSQu Wenruo sector = &rbio->bio_sectors[index];
9653e77605dSQu Wenruo if (sector->page || bio_list_only) {
9663e77605dSQu Wenruo /* Don't return sector without a valid page pointer */
9673e77605dSQu Wenruo if (!sector->page)
9683e77605dSQu Wenruo sector = NULL;
96974cc3600SChristoph Hellwig spin_unlock(&rbio->bio_list_lock);
9703e77605dSQu Wenruo return sector;
9713e77605dSQu Wenruo }
97274cc3600SChristoph Hellwig spin_unlock(&rbio->bio_list_lock);
9733e77605dSQu Wenruo
9743e77605dSQu Wenruo return &rbio->stripe_sectors[index];
9753e77605dSQu Wenruo }
9763e77605dSQu Wenruo
97753b381b3SDavid Woodhouse /*
97853b381b3SDavid Woodhouse * allocation and initial setup for the btrfs_raid_bio. Not
97953b381b3SDavid Woodhouse * this does not allocate any pages for rbio->pages.
98053b381b3SDavid Woodhouse */
alloc_rbio(struct btrfs_fs_info * fs_info,struct btrfs_io_context * bioc)9812ff7e61eSJeff Mahoney static struct btrfs_raid_bio *alloc_rbio(struct btrfs_fs_info *fs_info,
982ff18a4afSChristoph Hellwig struct btrfs_io_context *bioc)
98353b381b3SDavid Woodhouse {
9841faf3885SQu Wenruo const unsigned int real_stripes = bioc->num_stripes - bioc->replace_nr_stripes;
985ff18a4afSChristoph Hellwig const unsigned int stripe_npages = BTRFS_STRIPE_LEN >> PAGE_SHIFT;
986843de58bSQu Wenruo const unsigned int num_pages = stripe_npages * real_stripes;
987ff18a4afSChristoph Hellwig const unsigned int stripe_nsectors =
988ff18a4afSChristoph Hellwig BTRFS_STRIPE_LEN >> fs_info->sectorsize_bits;
98994efbe19SQu Wenruo const unsigned int num_sectors = stripe_nsectors * real_stripes;
99053b381b3SDavid Woodhouse struct btrfs_raid_bio *rbio;
99153b381b3SDavid Woodhouse
99294efbe19SQu Wenruo /* PAGE_SIZE must also be aligned to sectorsize for subpage support */
99394efbe19SQu Wenruo ASSERT(IS_ALIGNED(PAGE_SIZE, fs_info->sectorsize));
994c67c68ebSQu Wenruo /*
995c67c68ebSQu Wenruo * Our current stripe len should be fixed to 64k thus stripe_nsectors
996c67c68ebSQu Wenruo * (at most 16) should be no larger than BITS_PER_LONG.
997c67c68ebSQu Wenruo */
998c67c68ebSQu Wenruo ASSERT(stripe_nsectors <= BITS_PER_LONG);
999843de58bSQu Wenruo
1000b2324e08SQu Wenruo /*
1001b2324e08SQu Wenruo * Real stripes must be between 2 (2 disks RAID5, aka RAID1) and 256
1002b2324e08SQu Wenruo * (limited by u8).
1003b2324e08SQu Wenruo */
1004b2324e08SQu Wenruo ASSERT(real_stripes >= 2);
1005b2324e08SQu Wenruo ASSERT(real_stripes <= U8_MAX);
1006b2324e08SQu Wenruo
1007797d74b7SQu Wenruo rbio = kzalloc(sizeof(*rbio), GFP_NOFS);
1008af8e2d1dSMiao Xie if (!rbio)
100953b381b3SDavid Woodhouse return ERR_PTR(-ENOMEM);
1010797d74b7SQu Wenruo rbio->stripe_pages = kcalloc(num_pages, sizeof(struct page *),
1011797d74b7SQu Wenruo GFP_NOFS);
1012797d74b7SQu Wenruo rbio->bio_sectors = kcalloc(num_sectors, sizeof(struct sector_ptr),
1013797d74b7SQu Wenruo GFP_NOFS);
1014797d74b7SQu Wenruo rbio->stripe_sectors = kcalloc(num_sectors, sizeof(struct sector_ptr),
1015797d74b7SQu Wenruo GFP_NOFS);
1016797d74b7SQu Wenruo rbio->finish_pointers = kcalloc(real_stripes, sizeof(void *), GFP_NOFS);
10172942a50dSQu Wenruo rbio->error_bitmap = bitmap_zalloc(num_sectors, GFP_NOFS);
1018797d74b7SQu Wenruo
1019797d74b7SQu Wenruo if (!rbio->stripe_pages || !rbio->bio_sectors || !rbio->stripe_sectors ||
10202942a50dSQu Wenruo !rbio->finish_pointers || !rbio->error_bitmap) {
1021797d74b7SQu Wenruo free_raid_bio_pointers(rbio);
1022797d74b7SQu Wenruo kfree(rbio);
1023797d74b7SQu Wenruo return ERR_PTR(-ENOMEM);
1024797d74b7SQu Wenruo }
102553b381b3SDavid Woodhouse
102653b381b3SDavid Woodhouse bio_list_init(&rbio->bio_list);
1027d817ce35SQu Wenruo init_waitqueue_head(&rbio->io_wait);
102853b381b3SDavid Woodhouse INIT_LIST_HEAD(&rbio->plug_list);
102953b381b3SDavid Woodhouse spin_lock_init(&rbio->bio_list_lock);
10304ae10b3aSChris Mason INIT_LIST_HEAD(&rbio->stripe_cache);
103153b381b3SDavid Woodhouse INIT_LIST_HEAD(&rbio->hash_list);
1032f1c29379SChristoph Hellwig btrfs_get_bioc(bioc);
10334c664611SQu Wenruo rbio->bioc = bioc;
103453b381b3SDavid Woodhouse rbio->nr_pages = num_pages;
103594efbe19SQu Wenruo rbio->nr_sectors = num_sectors;
10362c8cdd6eSMiao Xie rbio->real_stripes = real_stripes;
10375a6ac9eaSMiao Xie rbio->stripe_npages = stripe_npages;
103894efbe19SQu Wenruo rbio->stripe_nsectors = stripe_nsectors;
1039dec95574SElena Reshetova refcount_set(&rbio->refs, 1);
1040b89e1b01SMiao Xie atomic_set(&rbio->stripes_pending, 0);
104153b381b3SDavid Woodhouse
10420b30f719SQu Wenruo ASSERT(btrfs_nr_parity_stripes(bioc->map_type));
10430b30f719SQu Wenruo rbio->nr_data = real_stripes - btrfs_nr_parity_stripes(bioc->map_type);
1044b2324e08SQu Wenruo ASSERT(rbio->nr_data > 0);
104553b381b3SDavid Woodhouse
104653b381b3SDavid Woodhouse return rbio;
104753b381b3SDavid Woodhouse }
104853b381b3SDavid Woodhouse
104953b381b3SDavid Woodhouse /* allocate pages for all the stripes in the bio, including parity */
alloc_rbio_pages(struct btrfs_raid_bio * rbio)105053b381b3SDavid Woodhouse static int alloc_rbio_pages(struct btrfs_raid_bio *rbio)
105153b381b3SDavid Woodhouse {
1052eb357060SQu Wenruo int ret;
1053eb357060SQu Wenruo
1054*0fbf6cbdSQu Wenruo ret = btrfs_alloc_page_array(rbio->nr_pages, rbio->stripe_pages, false);
1055eb357060SQu Wenruo if (ret < 0)
1056eb357060SQu Wenruo return ret;
1057eb357060SQu Wenruo /* Mapping all sectors */
1058eb357060SQu Wenruo index_stripe_sectors(rbio);
1059eb357060SQu Wenruo return 0;
106053b381b3SDavid Woodhouse }
106153b381b3SDavid Woodhouse
1062b7178a5fSZhao Lei /* only allocate pages for p/q stripes */
alloc_rbio_parity_pages(struct btrfs_raid_bio * rbio)106353b381b3SDavid Woodhouse static int alloc_rbio_parity_pages(struct btrfs_raid_bio *rbio)
106453b381b3SDavid Woodhouse {
1065f77183dcSQu Wenruo const int data_pages = rbio->nr_data * rbio->stripe_npages;
1066eb357060SQu Wenruo int ret;
106753b381b3SDavid Woodhouse
1068eb357060SQu Wenruo ret = btrfs_alloc_page_array(rbio->nr_pages - data_pages,
1069*0fbf6cbdSQu Wenruo rbio->stripe_pages + data_pages, false);
1070eb357060SQu Wenruo if (ret < 0)
1071eb357060SQu Wenruo return ret;
1072eb357060SQu Wenruo
1073eb357060SQu Wenruo index_stripe_sectors(rbio);
1074eb357060SQu Wenruo return 0;
107553b381b3SDavid Woodhouse }
107653b381b3SDavid Woodhouse
107753b381b3SDavid Woodhouse /*
107867da05b3SColin Ian King * Return the total number of errors found in the vertical stripe of @sector_nr.
107975b47033SQu Wenruo *
108075b47033SQu Wenruo * @faila and @failb will also be updated to the first and second stripe
108175b47033SQu Wenruo * number of the errors.
108275b47033SQu Wenruo */
get_rbio_veritical_errors(struct btrfs_raid_bio * rbio,int sector_nr,int * faila,int * failb)108375b47033SQu Wenruo static int get_rbio_veritical_errors(struct btrfs_raid_bio *rbio, int sector_nr,
108475b47033SQu Wenruo int *faila, int *failb)
108575b47033SQu Wenruo {
108675b47033SQu Wenruo int stripe_nr;
108775b47033SQu Wenruo int found_errors = 0;
108875b47033SQu Wenruo
1089ad3daf1cSQu Wenruo if (faila || failb) {
1090ad3daf1cSQu Wenruo /*
1091ad3daf1cSQu Wenruo * Both @faila and @failb should be valid pointers if any of
1092ad3daf1cSQu Wenruo * them is specified.
1093ad3daf1cSQu Wenruo */
109475b47033SQu Wenruo ASSERT(faila && failb);
109575b47033SQu Wenruo *faila = -1;
109675b47033SQu Wenruo *failb = -1;
1097ad3daf1cSQu Wenruo }
109875b47033SQu Wenruo
109975b47033SQu Wenruo for (stripe_nr = 0; stripe_nr < rbio->real_stripes; stripe_nr++) {
110075b47033SQu Wenruo int total_sector_nr = stripe_nr * rbio->stripe_nsectors + sector_nr;
110175b47033SQu Wenruo
110275b47033SQu Wenruo if (test_bit(total_sector_nr, rbio->error_bitmap)) {
110375b47033SQu Wenruo found_errors++;
1104ad3daf1cSQu Wenruo if (faila) {
1105ad3daf1cSQu Wenruo /* Update faila and failb. */
110675b47033SQu Wenruo if (*faila < 0)
110775b47033SQu Wenruo *faila = stripe_nr;
110875b47033SQu Wenruo else if (*failb < 0)
110975b47033SQu Wenruo *failb = stripe_nr;
111075b47033SQu Wenruo }
111175b47033SQu Wenruo }
1112ad3daf1cSQu Wenruo }
111375b47033SQu Wenruo return found_errors;
111475b47033SQu Wenruo }
111575b47033SQu Wenruo
111675b47033SQu Wenruo /*
11173e77605dSQu Wenruo * Add a single sector @sector into our list of bios for IO.
11183e77605dSQu Wenruo *
11193e77605dSQu Wenruo * Return 0 if everything went well.
11203e77605dSQu Wenruo * Return <0 for error.
112153b381b3SDavid Woodhouse */
rbio_add_io_sector(struct btrfs_raid_bio * rbio,struct bio_list * bio_list,struct sector_ptr * sector,unsigned int stripe_nr,unsigned int sector_nr,enum req_op op)11223e77605dSQu Wenruo static int rbio_add_io_sector(struct btrfs_raid_bio *rbio,
112353b381b3SDavid Woodhouse struct bio_list *bio_list,
11243e77605dSQu Wenruo struct sector_ptr *sector,
11253e77605dSQu Wenruo unsigned int stripe_nr,
11263e77605dSQu Wenruo unsigned int sector_nr,
1127bf9486d6SBart Van Assche enum req_op op)
112853b381b3SDavid Woodhouse {
11293e77605dSQu Wenruo const u32 sectorsize = rbio->bioc->fs_info->sectorsize;
113053b381b3SDavid Woodhouse struct bio *last = bio_list->tail;
113153b381b3SDavid Woodhouse int ret;
113253b381b3SDavid Woodhouse struct bio *bio;
11334c664611SQu Wenruo struct btrfs_io_stripe *stripe;
113453b381b3SDavid Woodhouse u64 disk_start;
113553b381b3SDavid Woodhouse
11363e77605dSQu Wenruo /*
11373e77605dSQu Wenruo * Note: here stripe_nr has taken device replace into consideration,
11383e77605dSQu Wenruo * thus it can be larger than rbio->real_stripe.
11393e77605dSQu Wenruo * So here we check against bioc->num_stripes, not rbio->real_stripes.
11403e77605dSQu Wenruo */
1141bbbee460SQu Wenruo ASSERT_RBIO_STRIPE(stripe_nr >= 0 && stripe_nr < rbio->bioc->num_stripes,
1142bbbee460SQu Wenruo rbio, stripe_nr);
1143bbbee460SQu Wenruo ASSERT_RBIO_SECTOR(sector_nr >= 0 && sector_nr < rbio->stripe_nsectors,
1144bbbee460SQu Wenruo rbio, sector_nr);
11453e77605dSQu Wenruo ASSERT(sector->page);
11463e77605dSQu Wenruo
11474c664611SQu Wenruo stripe = &rbio->bioc->stripes[stripe_nr];
11483e77605dSQu Wenruo disk_start = stripe->physical + sector_nr * sectorsize;
114953b381b3SDavid Woodhouse
115053b381b3SDavid Woodhouse /* if the device is missing, just fail this stripe */
11512942a50dSQu Wenruo if (!stripe->dev->bdev) {
1152ad3daf1cSQu Wenruo int found_errors;
1153ad3daf1cSQu Wenruo
11542942a50dSQu Wenruo set_bit(stripe_nr * rbio->stripe_nsectors + sector_nr,
11552942a50dSQu Wenruo rbio->error_bitmap);
1156ad3daf1cSQu Wenruo
1157ad3daf1cSQu Wenruo /* Check if we have reached tolerance early. */
1158ad3daf1cSQu Wenruo found_errors = get_rbio_veritical_errors(rbio, sector_nr,
1159ad3daf1cSQu Wenruo NULL, NULL);
1160ad3daf1cSQu Wenruo if (found_errors > rbio->bioc->max_errors)
1161ad3daf1cSQu Wenruo return -EIO;
1162ad3daf1cSQu Wenruo return 0;
11632942a50dSQu Wenruo }
116453b381b3SDavid Woodhouse
116553b381b3SDavid Woodhouse /* see if we can add this page onto our existing bio */
116653b381b3SDavid Woodhouse if (last) {
1167adbe7e38SAnand Jain u64 last_end = last->bi_iter.bi_sector << SECTOR_SHIFT;
11684f024f37SKent Overstreet last_end += last->bi_iter.bi_size;
116953b381b3SDavid Woodhouse
117053b381b3SDavid Woodhouse /*
117153b381b3SDavid Woodhouse * we can't merge these if they are from different
117253b381b3SDavid Woodhouse * devices or if they are not contiguous
117353b381b3SDavid Woodhouse */
1174f90ae76aSNikolay Borisov if (last_end == disk_start && !last->bi_status &&
1175309dca30SChristoph Hellwig last->bi_bdev == stripe->dev->bdev) {
11763e77605dSQu Wenruo ret = bio_add_page(last, sector->page, sectorsize,
11773e77605dSQu Wenruo sector->pgoff);
11783e77605dSQu Wenruo if (ret == sectorsize)
117953b381b3SDavid Woodhouse return 0;
118053b381b3SDavid Woodhouse }
118153b381b3SDavid Woodhouse }
118253b381b3SDavid Woodhouse
118353b381b3SDavid Woodhouse /* put a new bio on the list */
1184ff18a4afSChristoph Hellwig bio = bio_alloc(stripe->dev->bdev,
1185ff18a4afSChristoph Hellwig max(BTRFS_STRIPE_LEN >> PAGE_SHIFT, 1),
1186bf9486d6SBart Van Assche op, GFP_NOFS);
118729e70be2SAnand Jain bio->bi_iter.bi_sector = disk_start >> SECTOR_SHIFT;
1188e01bf588SChristoph Hellwig bio->bi_private = rbio;
118953b381b3SDavid Woodhouse
1190cf32e41fSJohannes Thumshirn __bio_add_page(bio, sector->page, sectorsize, sector->pgoff);
119153b381b3SDavid Woodhouse bio_list_add(bio_list, bio);
119253b381b3SDavid Woodhouse return 0;
119353b381b3SDavid Woodhouse }
119453b381b3SDavid Woodhouse
index_one_bio(struct btrfs_raid_bio * rbio,struct bio * bio)119500425dd9SQu Wenruo static void index_one_bio(struct btrfs_raid_bio *rbio, struct bio *bio)
119600425dd9SQu Wenruo {
119700425dd9SQu Wenruo const u32 sectorsize = rbio->bioc->fs_info->sectorsize;
119800425dd9SQu Wenruo struct bio_vec bvec;
119900425dd9SQu Wenruo struct bvec_iter iter;
120000425dd9SQu Wenruo u32 offset = (bio->bi_iter.bi_sector << SECTOR_SHIFT) -
120118d758a2SQu Wenruo rbio->bioc->full_stripe_logical;
120200425dd9SQu Wenruo
120300425dd9SQu Wenruo bio_for_each_segment(bvec, bio, iter) {
120400425dd9SQu Wenruo u32 bvec_offset;
120500425dd9SQu Wenruo
120600425dd9SQu Wenruo for (bvec_offset = 0; bvec_offset < bvec.bv_len;
120700425dd9SQu Wenruo bvec_offset += sectorsize, offset += sectorsize) {
120800425dd9SQu Wenruo int index = offset / sectorsize;
120900425dd9SQu Wenruo struct sector_ptr *sector = &rbio->bio_sectors[index];
121000425dd9SQu Wenruo
121100425dd9SQu Wenruo sector->page = bvec.bv_page;
121200425dd9SQu Wenruo sector->pgoff = bvec.bv_offset + bvec_offset;
121300425dd9SQu Wenruo ASSERT(sector->pgoff < PAGE_SIZE);
121400425dd9SQu Wenruo }
121500425dd9SQu Wenruo }
121600425dd9SQu Wenruo }
121700425dd9SQu Wenruo
121853b381b3SDavid Woodhouse /*
121953b381b3SDavid Woodhouse * helper function to walk our bio list and populate the bio_pages array with
122053b381b3SDavid Woodhouse * the result. This seems expensive, but it is faster than constantly
122153b381b3SDavid Woodhouse * searching through the bio list as we setup the IO in finish_rmw or stripe
122253b381b3SDavid Woodhouse * reconstruction.
122353b381b3SDavid Woodhouse *
122453b381b3SDavid Woodhouse * This must be called before you trust the answers from page_in_rbio
122553b381b3SDavid Woodhouse */
index_rbio_pages(struct btrfs_raid_bio * rbio)122653b381b3SDavid Woodhouse static void index_rbio_pages(struct btrfs_raid_bio *rbio)
122753b381b3SDavid Woodhouse {
122853b381b3SDavid Woodhouse struct bio *bio;
122953b381b3SDavid Woodhouse
123074cc3600SChristoph Hellwig spin_lock(&rbio->bio_list_lock);
123100425dd9SQu Wenruo bio_list_for_each(bio, &rbio->bio_list)
123200425dd9SQu Wenruo index_one_bio(rbio, bio);
123300425dd9SQu Wenruo
123474cc3600SChristoph Hellwig spin_unlock(&rbio->bio_list_lock);
123553b381b3SDavid Woodhouse }
123653b381b3SDavid Woodhouse
bio_get_trace_info(struct btrfs_raid_bio * rbio,struct bio * bio,struct raid56_bio_trace_info * trace_info)1237b8bea09aSQu Wenruo static void bio_get_trace_info(struct btrfs_raid_bio *rbio, struct bio *bio,
1238b8bea09aSQu Wenruo struct raid56_bio_trace_info *trace_info)
1239b8bea09aSQu Wenruo {
1240b8bea09aSQu Wenruo const struct btrfs_io_context *bioc = rbio->bioc;
1241b8bea09aSQu Wenruo int i;
1242b8bea09aSQu Wenruo
1243b8bea09aSQu Wenruo ASSERT(bioc);
1244b8bea09aSQu Wenruo
1245b8bea09aSQu Wenruo /* We rely on bio->bi_bdev to find the stripe number. */
1246b8bea09aSQu Wenruo if (!bio->bi_bdev)
1247b8bea09aSQu Wenruo goto not_found;
1248b8bea09aSQu Wenruo
1249b8bea09aSQu Wenruo for (i = 0; i < bioc->num_stripes; i++) {
1250b8bea09aSQu Wenruo if (bio->bi_bdev != bioc->stripes[i].dev->bdev)
1251b8bea09aSQu Wenruo continue;
1252b8bea09aSQu Wenruo trace_info->stripe_nr = i;
1253b8bea09aSQu Wenruo trace_info->devid = bioc->stripes[i].dev->devid;
1254b8bea09aSQu Wenruo trace_info->offset = (bio->bi_iter.bi_sector << SECTOR_SHIFT) -
1255b8bea09aSQu Wenruo bioc->stripes[i].physical;
1256b8bea09aSQu Wenruo return;
1257b8bea09aSQu Wenruo }
1258b8bea09aSQu Wenruo
1259b8bea09aSQu Wenruo not_found:
1260b8bea09aSQu Wenruo trace_info->devid = -1;
1261b8bea09aSQu Wenruo trace_info->offset = -1;
1262b8bea09aSQu Wenruo trace_info->stripe_nr = -1;
1263b8bea09aSQu Wenruo }
1264b8bea09aSQu Wenruo
bio_list_put(struct bio_list * bio_list)1265801fcfc5SChristoph Hellwig static inline void bio_list_put(struct bio_list *bio_list)
1266801fcfc5SChristoph Hellwig {
1267801fcfc5SChristoph Hellwig struct bio *bio;
1268801fcfc5SChristoph Hellwig
1269801fcfc5SChristoph Hellwig while ((bio = bio_list_pop(bio_list)))
1270801fcfc5SChristoph Hellwig bio_put(bio);
1271801fcfc5SChristoph Hellwig }
1272801fcfc5SChristoph Hellwig
assert_rbio(struct btrfs_raid_bio * rbio)1273b2324e08SQu Wenruo static void assert_rbio(struct btrfs_raid_bio *rbio)
1274b2324e08SQu Wenruo {
1275b2324e08SQu Wenruo if (!IS_ENABLED(CONFIG_BTRFS_DEBUG) ||
1276b2324e08SQu Wenruo !IS_ENABLED(CONFIG_BTRFS_ASSERT))
1277b2324e08SQu Wenruo return;
1278b2324e08SQu Wenruo
1279b2324e08SQu Wenruo /*
1280b2324e08SQu Wenruo * At least two stripes (2 disks RAID5), and since real_stripes is U8,
1281b2324e08SQu Wenruo * we won't go beyond 256 disks anyway.
1282b2324e08SQu Wenruo */
1283bbbee460SQu Wenruo ASSERT_RBIO(rbio->real_stripes >= 2, rbio);
1284bbbee460SQu Wenruo ASSERT_RBIO(rbio->nr_data > 0, rbio);
1285b2324e08SQu Wenruo
1286b2324e08SQu Wenruo /*
1287b2324e08SQu Wenruo * This is another check to make sure nr data stripes is smaller
1288b2324e08SQu Wenruo * than total stripes.
1289b2324e08SQu Wenruo */
1290bbbee460SQu Wenruo ASSERT_RBIO(rbio->nr_data < rbio->real_stripes, rbio);
1291b2324e08SQu Wenruo }
1292b2324e08SQu Wenruo
129367da05b3SColin Ian King /* Generate PQ for one vertical stripe. */
generate_pq_vertical(struct btrfs_raid_bio * rbio,int sectornr)129430e3c897SQu Wenruo static void generate_pq_vertical(struct btrfs_raid_bio *rbio, int sectornr)
129530e3c897SQu Wenruo {
129630e3c897SQu Wenruo void **pointers = rbio->finish_pointers;
129730e3c897SQu Wenruo const u32 sectorsize = rbio->bioc->fs_info->sectorsize;
129830e3c897SQu Wenruo struct sector_ptr *sector;
129930e3c897SQu Wenruo int stripe;
130030e3c897SQu Wenruo const bool has_qstripe = rbio->bioc->map_type & BTRFS_BLOCK_GROUP_RAID6;
130130e3c897SQu Wenruo
130230e3c897SQu Wenruo /* First collect one sector from each data stripe */
130330e3c897SQu Wenruo for (stripe = 0; stripe < rbio->nr_data; stripe++) {
130430e3c897SQu Wenruo sector = sector_in_rbio(rbio, stripe, sectornr, 0);
130530e3c897SQu Wenruo pointers[stripe] = kmap_local_page(sector->page) +
130630e3c897SQu Wenruo sector->pgoff;
130730e3c897SQu Wenruo }
130830e3c897SQu Wenruo
130930e3c897SQu Wenruo /* Then add the parity stripe */
131030e3c897SQu Wenruo sector = rbio_pstripe_sector(rbio, sectornr);
131130e3c897SQu Wenruo sector->uptodate = 1;
131230e3c897SQu Wenruo pointers[stripe++] = kmap_local_page(sector->page) + sector->pgoff;
131330e3c897SQu Wenruo
131430e3c897SQu Wenruo if (has_qstripe) {
131530e3c897SQu Wenruo /*
131630e3c897SQu Wenruo * RAID6, add the qstripe and call the library function
131730e3c897SQu Wenruo * to fill in our p/q
131830e3c897SQu Wenruo */
131930e3c897SQu Wenruo sector = rbio_qstripe_sector(rbio, sectornr);
132030e3c897SQu Wenruo sector->uptodate = 1;
132130e3c897SQu Wenruo pointers[stripe++] = kmap_local_page(sector->page) +
132230e3c897SQu Wenruo sector->pgoff;
132330e3c897SQu Wenruo
1324b2324e08SQu Wenruo assert_rbio(rbio);
132530e3c897SQu Wenruo raid6_call.gen_syndrome(rbio->real_stripes, sectorsize,
132630e3c897SQu Wenruo pointers);
132730e3c897SQu Wenruo } else {
132830e3c897SQu Wenruo /* raid5 */
132930e3c897SQu Wenruo memcpy(pointers[rbio->nr_data], pointers[0], sectorsize);
133030e3c897SQu Wenruo run_xor(pointers + 1, rbio->nr_data - 1, sectorsize);
133130e3c897SQu Wenruo }
133230e3c897SQu Wenruo for (stripe = stripe - 1; stripe >= 0; stripe--)
133330e3c897SQu Wenruo kunmap_local(pointers[stripe]);
133430e3c897SQu Wenruo }
133530e3c897SQu Wenruo
rmw_assemble_write_bios(struct btrfs_raid_bio * rbio,struct bio_list * bio_list)13366486d21cSQu Wenruo static int rmw_assemble_write_bios(struct btrfs_raid_bio *rbio,
13376486d21cSQu Wenruo struct bio_list *bio_list)
13386486d21cSQu Wenruo {
13396486d21cSQu Wenruo /* The total sector number inside the full stripe. */
13406486d21cSQu Wenruo int total_sector_nr;
13416486d21cSQu Wenruo int sectornr;
13426486d21cSQu Wenruo int stripe;
13436486d21cSQu Wenruo int ret;
13446486d21cSQu Wenruo
13456486d21cSQu Wenruo ASSERT(bio_list_size(bio_list) == 0);
13466486d21cSQu Wenruo
13476486d21cSQu Wenruo /* We should have at least one data sector. */
13486486d21cSQu Wenruo ASSERT(bitmap_weight(&rbio->dbitmap, rbio->stripe_nsectors));
13496486d21cSQu Wenruo
13506486d21cSQu Wenruo /*
13515eb30ee2SQu Wenruo * Reset errors, as we may have errors inherited from from degraded
13525eb30ee2SQu Wenruo * write.
13535eb30ee2SQu Wenruo */
13542942a50dSQu Wenruo bitmap_clear(rbio->error_bitmap, 0, rbio->nr_sectors);
13555eb30ee2SQu Wenruo
13565eb30ee2SQu Wenruo /*
13576486d21cSQu Wenruo * Start assembly. Make bios for everything from the higher layers (the
13586486d21cSQu Wenruo * bio_list in our rbio) and our P/Q. Ignore everything else.
13596486d21cSQu Wenruo */
13606486d21cSQu Wenruo for (total_sector_nr = 0; total_sector_nr < rbio->nr_sectors;
13616486d21cSQu Wenruo total_sector_nr++) {
13626486d21cSQu Wenruo struct sector_ptr *sector;
13636486d21cSQu Wenruo
13646486d21cSQu Wenruo stripe = total_sector_nr / rbio->stripe_nsectors;
13656486d21cSQu Wenruo sectornr = total_sector_nr % rbio->stripe_nsectors;
13666486d21cSQu Wenruo
13676486d21cSQu Wenruo /* This vertical stripe has no data, skip it. */
13686486d21cSQu Wenruo if (!test_bit(sectornr, &rbio->dbitmap))
13696486d21cSQu Wenruo continue;
13706486d21cSQu Wenruo
13716486d21cSQu Wenruo if (stripe < rbio->nr_data) {
13726486d21cSQu Wenruo sector = sector_in_rbio(rbio, stripe, sectornr, 1);
13736486d21cSQu Wenruo if (!sector)
13746486d21cSQu Wenruo continue;
13756486d21cSQu Wenruo } else {
13766486d21cSQu Wenruo sector = rbio_stripe_sector(rbio, stripe, sectornr);
13776486d21cSQu Wenruo }
13786486d21cSQu Wenruo
13796486d21cSQu Wenruo ret = rbio_add_io_sector(rbio, bio_list, sector, stripe,
13806486d21cSQu Wenruo sectornr, REQ_OP_WRITE);
13816486d21cSQu Wenruo if (ret)
13826486d21cSQu Wenruo goto error;
13836486d21cSQu Wenruo }
13846486d21cSQu Wenruo
13851faf3885SQu Wenruo if (likely(!rbio->bioc->replace_nr_stripes))
13866486d21cSQu Wenruo return 0;
13876486d21cSQu Wenruo
13881faf3885SQu Wenruo /*
13891faf3885SQu Wenruo * Make a copy for the replace target device.
13901faf3885SQu Wenruo *
13911faf3885SQu Wenruo * Thus the source stripe number (in replace_stripe_src) should be valid.
13921faf3885SQu Wenruo */
13931faf3885SQu Wenruo ASSERT(rbio->bioc->replace_stripe_src >= 0);
13941faf3885SQu Wenruo
13956486d21cSQu Wenruo for (total_sector_nr = 0; total_sector_nr < rbio->nr_sectors;
13966486d21cSQu Wenruo total_sector_nr++) {
13976486d21cSQu Wenruo struct sector_ptr *sector;
13986486d21cSQu Wenruo
13996486d21cSQu Wenruo stripe = total_sector_nr / rbio->stripe_nsectors;
14006486d21cSQu Wenruo sectornr = total_sector_nr % rbio->stripe_nsectors;
14016486d21cSQu Wenruo
14021faf3885SQu Wenruo /*
14031faf3885SQu Wenruo * For RAID56, there is only one device that can be replaced,
14041faf3885SQu Wenruo * and replace_stripe_src[0] indicates the stripe number we
14051faf3885SQu Wenruo * need to copy from.
14061faf3885SQu Wenruo */
14071faf3885SQu Wenruo if (stripe != rbio->bioc->replace_stripe_src) {
14086486d21cSQu Wenruo /*
14096486d21cSQu Wenruo * We can skip the whole stripe completely, note
14106486d21cSQu Wenruo * total_sector_nr will be increased by one anyway.
14116486d21cSQu Wenruo */
14126486d21cSQu Wenruo ASSERT(sectornr == 0);
14136486d21cSQu Wenruo total_sector_nr += rbio->stripe_nsectors - 1;
14146486d21cSQu Wenruo continue;
14156486d21cSQu Wenruo }
14166486d21cSQu Wenruo
14176486d21cSQu Wenruo /* This vertical stripe has no data, skip it. */
14186486d21cSQu Wenruo if (!test_bit(sectornr, &rbio->dbitmap))
14196486d21cSQu Wenruo continue;
14206486d21cSQu Wenruo
14216486d21cSQu Wenruo if (stripe < rbio->nr_data) {
14226486d21cSQu Wenruo sector = sector_in_rbio(rbio, stripe, sectornr, 1);
14236486d21cSQu Wenruo if (!sector)
14246486d21cSQu Wenruo continue;
14256486d21cSQu Wenruo } else {
14266486d21cSQu Wenruo sector = rbio_stripe_sector(rbio, stripe, sectornr);
14276486d21cSQu Wenruo }
14286486d21cSQu Wenruo
14296486d21cSQu Wenruo ret = rbio_add_io_sector(rbio, bio_list, sector,
14301faf3885SQu Wenruo rbio->real_stripes,
14316486d21cSQu Wenruo sectornr, REQ_OP_WRITE);
14326486d21cSQu Wenruo if (ret)
14336486d21cSQu Wenruo goto error;
14346486d21cSQu Wenruo }
14356486d21cSQu Wenruo
14366486d21cSQu Wenruo return 0;
14376486d21cSQu Wenruo error:
1438801fcfc5SChristoph Hellwig bio_list_put(bio_list);
14396486d21cSQu Wenruo return -EIO;
14406486d21cSQu Wenruo }
14416486d21cSQu Wenruo
set_rbio_range_error(struct btrfs_raid_bio * rbio,struct bio * bio)14422942a50dSQu Wenruo static void set_rbio_range_error(struct btrfs_raid_bio *rbio, struct bio *bio)
14432942a50dSQu Wenruo {
14442942a50dSQu Wenruo struct btrfs_fs_info *fs_info = rbio->bioc->fs_info;
14452942a50dSQu Wenruo u32 offset = (bio->bi_iter.bi_sector << SECTOR_SHIFT) -
144618d758a2SQu Wenruo rbio->bioc->full_stripe_logical;
14472942a50dSQu Wenruo int total_nr_sector = offset >> fs_info->sectorsize_bits;
14482942a50dSQu Wenruo
14492942a50dSQu Wenruo ASSERT(total_nr_sector < rbio->nr_data * rbio->stripe_nsectors);
14502942a50dSQu Wenruo
14512942a50dSQu Wenruo bitmap_set(rbio->error_bitmap, total_nr_sector,
14522942a50dSQu Wenruo bio->bi_iter.bi_size >> fs_info->sectorsize_bits);
14532942a50dSQu Wenruo
14542942a50dSQu Wenruo /*
14552942a50dSQu Wenruo * Special handling for raid56_alloc_missing_rbio() used by
14562942a50dSQu Wenruo * scrub/replace. Unlike call path in raid56_parity_recover(), they
14572942a50dSQu Wenruo * pass an empty bio here. Thus we have to find out the missing device
14582942a50dSQu Wenruo * and mark the stripe error instead.
14592942a50dSQu Wenruo */
14602942a50dSQu Wenruo if (bio->bi_iter.bi_size == 0) {
14612942a50dSQu Wenruo bool found_missing = false;
14622942a50dSQu Wenruo int stripe_nr;
14632942a50dSQu Wenruo
14642942a50dSQu Wenruo for (stripe_nr = 0; stripe_nr < rbio->real_stripes; stripe_nr++) {
14652942a50dSQu Wenruo if (!rbio->bioc->stripes[stripe_nr].dev->bdev) {
14662942a50dSQu Wenruo found_missing = true;
14672942a50dSQu Wenruo bitmap_set(rbio->error_bitmap,
14682942a50dSQu Wenruo stripe_nr * rbio->stripe_nsectors,
14692942a50dSQu Wenruo rbio->stripe_nsectors);
14702942a50dSQu Wenruo }
14712942a50dSQu Wenruo }
14722942a50dSQu Wenruo ASSERT(found_missing);
14732942a50dSQu Wenruo }
14742942a50dSQu Wenruo }
14752942a50dSQu Wenruo
147653b381b3SDavid Woodhouse /*
147767da05b3SColin Ian King * For subpage case, we can no longer set page Up-to-date directly for
14785fdb7afcSQu Wenruo * stripe_pages[], thus we need to locate the sector.
14795fdb7afcSQu Wenruo */
find_stripe_sector(struct btrfs_raid_bio * rbio,struct page * page,unsigned int pgoff)14805fdb7afcSQu Wenruo static struct sector_ptr *find_stripe_sector(struct btrfs_raid_bio *rbio,
14815fdb7afcSQu Wenruo struct page *page,
14825fdb7afcSQu Wenruo unsigned int pgoff)
14835fdb7afcSQu Wenruo {
14845fdb7afcSQu Wenruo int i;
14855fdb7afcSQu Wenruo
14865fdb7afcSQu Wenruo for (i = 0; i < rbio->nr_sectors; i++) {
14875fdb7afcSQu Wenruo struct sector_ptr *sector = &rbio->stripe_sectors[i];
14885fdb7afcSQu Wenruo
14895fdb7afcSQu Wenruo if (sector->page == page && sector->pgoff == pgoff)
14905fdb7afcSQu Wenruo return sector;
14915fdb7afcSQu Wenruo }
14925fdb7afcSQu Wenruo return NULL;
14935fdb7afcSQu Wenruo }
14945fdb7afcSQu Wenruo
14955fdb7afcSQu Wenruo /*
149653b381b3SDavid Woodhouse * this sets each page in the bio uptodate. It should only be used on private
149753b381b3SDavid Woodhouse * rbio pages, nothing that comes in from the higher layers
149853b381b3SDavid Woodhouse */
set_bio_pages_uptodate(struct btrfs_raid_bio * rbio,struct bio * bio)14995fdb7afcSQu Wenruo static void set_bio_pages_uptodate(struct btrfs_raid_bio *rbio, struct bio *bio)
150053b381b3SDavid Woodhouse {
15015fdb7afcSQu Wenruo const u32 sectorsize = rbio->bioc->fs_info->sectorsize;
15020198e5b7SLiu Bo struct bio_vec *bvec;
15036dc4f100SMing Lei struct bvec_iter_all iter_all;
150453b381b3SDavid Woodhouse
15050198e5b7SLiu Bo ASSERT(!bio_flagged(bio, BIO_CLONED));
15066592e58cSFilipe Manana
15075fdb7afcSQu Wenruo bio_for_each_segment_all(bvec, bio, iter_all) {
15085fdb7afcSQu Wenruo struct sector_ptr *sector;
15095fdb7afcSQu Wenruo int pgoff;
15105fdb7afcSQu Wenruo
15115fdb7afcSQu Wenruo for (pgoff = bvec->bv_offset; pgoff - bvec->bv_offset < bvec->bv_len;
15125fdb7afcSQu Wenruo pgoff += sectorsize) {
15135fdb7afcSQu Wenruo sector = find_stripe_sector(rbio, bvec->bv_page, pgoff);
15145fdb7afcSQu Wenruo ASSERT(sector);
15155fdb7afcSQu Wenruo if (sector)
15165fdb7afcSQu Wenruo sector->uptodate = 1;
15175fdb7afcSQu Wenruo }
15185fdb7afcSQu Wenruo }
151953b381b3SDavid Woodhouse }
152053b381b3SDavid Woodhouse
get_bio_sector_nr(struct btrfs_raid_bio * rbio,struct bio * bio)15212942a50dSQu Wenruo static int get_bio_sector_nr(struct btrfs_raid_bio *rbio, struct bio *bio)
15222942a50dSQu Wenruo {
15232942a50dSQu Wenruo struct bio_vec *bv = bio_first_bvec_all(bio);
15242942a50dSQu Wenruo int i;
15252942a50dSQu Wenruo
15262942a50dSQu Wenruo for (i = 0; i < rbio->nr_sectors; i++) {
15272942a50dSQu Wenruo struct sector_ptr *sector;
15282942a50dSQu Wenruo
15292942a50dSQu Wenruo sector = &rbio->stripe_sectors[i];
15302942a50dSQu Wenruo if (sector->page == bv->bv_page && sector->pgoff == bv->bv_offset)
15312942a50dSQu Wenruo break;
15322942a50dSQu Wenruo sector = &rbio->bio_sectors[i];
15332942a50dSQu Wenruo if (sector->page == bv->bv_page && sector->pgoff == bv->bv_offset)
15342942a50dSQu Wenruo break;
15352942a50dSQu Wenruo }
15362942a50dSQu Wenruo ASSERT(i < rbio->nr_sectors);
15372942a50dSQu Wenruo return i;
15382942a50dSQu Wenruo }
15392942a50dSQu Wenruo
rbio_update_error_bitmap(struct btrfs_raid_bio * rbio,struct bio * bio)15402942a50dSQu Wenruo static void rbio_update_error_bitmap(struct btrfs_raid_bio *rbio, struct bio *bio)
15412942a50dSQu Wenruo {
15422942a50dSQu Wenruo int total_sector_nr = get_bio_sector_nr(rbio, bio);
15432942a50dSQu Wenruo u32 bio_size = 0;
15442942a50dSQu Wenruo struct bio_vec *bvec;
1545a9ad4d87SQu Wenruo int i;
15462942a50dSQu Wenruo
1547c9a43aafSQu Wenruo bio_for_each_bvec_all(bvec, bio, i)
15482942a50dSQu Wenruo bio_size += bvec->bv_len;
15492942a50dSQu Wenruo
1550a9ad4d87SQu Wenruo /*
1551a9ad4d87SQu Wenruo * Since we can have multiple bios touching the error_bitmap, we cannot
1552a9ad4d87SQu Wenruo * call bitmap_set() without protection.
1553a9ad4d87SQu Wenruo *
1554a9ad4d87SQu Wenruo * Instead use set_bit() for each bit, as set_bit() itself is atomic.
1555a9ad4d87SQu Wenruo */
1556a9ad4d87SQu Wenruo for (i = total_sector_nr; i < total_sector_nr +
1557a9ad4d87SQu Wenruo (bio_size >> rbio->bioc->fs_info->sectorsize_bits); i++)
1558a9ad4d87SQu Wenruo set_bit(i, rbio->error_bitmap);
15592942a50dSQu Wenruo }
15602942a50dSQu Wenruo
15617a315072SQu Wenruo /* Verify the data sectors at read time. */
verify_bio_data_sectors(struct btrfs_raid_bio * rbio,struct bio * bio)15627a315072SQu Wenruo static void verify_bio_data_sectors(struct btrfs_raid_bio *rbio,
15637a315072SQu Wenruo struct bio *bio)
15647a315072SQu Wenruo {
15657a315072SQu Wenruo struct btrfs_fs_info *fs_info = rbio->bioc->fs_info;
15667a315072SQu Wenruo int total_sector_nr = get_bio_sector_nr(rbio, bio);
15677a315072SQu Wenruo struct bio_vec *bvec;
15687a315072SQu Wenruo struct bvec_iter_all iter_all;
15697a315072SQu Wenruo
15707a315072SQu Wenruo /* No data csum for the whole stripe, no need to verify. */
15717a315072SQu Wenruo if (!rbio->csum_bitmap || !rbio->csum_buf)
15727a315072SQu Wenruo return;
15737a315072SQu Wenruo
15747a315072SQu Wenruo /* P/Q stripes, they have no data csum to verify against. */
15757a315072SQu Wenruo if (total_sector_nr >= rbio->nr_data * rbio->stripe_nsectors)
15767a315072SQu Wenruo return;
15777a315072SQu Wenruo
15787a315072SQu Wenruo bio_for_each_segment_all(bvec, bio, iter_all) {
15797a315072SQu Wenruo int bv_offset;
15807a315072SQu Wenruo
15817a315072SQu Wenruo for (bv_offset = bvec->bv_offset;
15827a315072SQu Wenruo bv_offset < bvec->bv_offset + bvec->bv_len;
15837a315072SQu Wenruo bv_offset += fs_info->sectorsize, total_sector_nr++) {
15847a315072SQu Wenruo u8 csum_buf[BTRFS_CSUM_SIZE];
15857a315072SQu Wenruo u8 *expected_csum = rbio->csum_buf +
15867a315072SQu Wenruo total_sector_nr * fs_info->csum_size;
15877a315072SQu Wenruo int ret;
15887a315072SQu Wenruo
15897a315072SQu Wenruo /* No csum for this sector, skip to the next sector. */
15907a315072SQu Wenruo if (!test_bit(total_sector_nr, rbio->csum_bitmap))
15917a315072SQu Wenruo continue;
15927a315072SQu Wenruo
15937a315072SQu Wenruo ret = btrfs_check_sector_csum(fs_info, bvec->bv_page,
15947a315072SQu Wenruo bv_offset, csum_buf, expected_csum);
15957a315072SQu Wenruo if (ret < 0)
15967a315072SQu Wenruo set_bit(total_sector_nr, rbio->error_bitmap);
15977a315072SQu Wenruo }
15987a315072SQu Wenruo }
15997a315072SQu Wenruo }
16007a315072SQu Wenruo
raid_wait_read_end_io(struct bio * bio)1601d817ce35SQu Wenruo static void raid_wait_read_end_io(struct bio *bio)
1602d817ce35SQu Wenruo {
1603d817ce35SQu Wenruo struct btrfs_raid_bio *rbio = bio->bi_private;
1604d817ce35SQu Wenruo
16057a315072SQu Wenruo if (bio->bi_status) {
16062942a50dSQu Wenruo rbio_update_error_bitmap(rbio, bio);
16077a315072SQu Wenruo } else {
1608d817ce35SQu Wenruo set_bio_pages_uptodate(rbio, bio);
16097a315072SQu Wenruo verify_bio_data_sectors(rbio, bio);
16107a315072SQu Wenruo }
1611d817ce35SQu Wenruo
1612d817ce35SQu Wenruo bio_put(bio);
1613d817ce35SQu Wenruo if (atomic_dec_and_test(&rbio->stripes_pending))
1614d817ce35SQu Wenruo wake_up(&rbio->io_wait);
1615d817ce35SQu Wenruo }
1616d817ce35SQu Wenruo
submit_read_wait_bio_list(struct btrfs_raid_bio * rbio,struct bio_list * bio_list)16171c76fb7bSChristoph Hellwig static void submit_read_wait_bio_list(struct btrfs_raid_bio *rbio,
1618d817ce35SQu Wenruo struct bio_list *bio_list)
1619d817ce35SQu Wenruo {
1620d817ce35SQu Wenruo struct bio *bio;
1621d817ce35SQu Wenruo
1622d817ce35SQu Wenruo atomic_set(&rbio->stripes_pending, bio_list_size(bio_list));
1623d817ce35SQu Wenruo while ((bio = bio_list_pop(bio_list))) {
1624d817ce35SQu Wenruo bio->bi_end_io = raid_wait_read_end_io;
1625d817ce35SQu Wenruo
1626dbb6ecb3SQu Wenruo if (trace_raid56_read_enabled()) {
1627d817ce35SQu Wenruo struct raid56_bio_trace_info trace_info = { 0 };
1628d817ce35SQu Wenruo
1629d817ce35SQu Wenruo bio_get_trace_info(rbio, bio, &trace_info);
1630dbb6ecb3SQu Wenruo trace_raid56_read(rbio, bio, &trace_info);
1631d817ce35SQu Wenruo }
1632d817ce35SQu Wenruo submit_bio(bio);
1633d817ce35SQu Wenruo }
16341c76fb7bSChristoph Hellwig
16351c76fb7bSChristoph Hellwig wait_event(rbio->io_wait, atomic_read(&rbio->stripes_pending) == 0);
1636d817ce35SQu Wenruo }
1637d817ce35SQu Wenruo
alloc_rbio_data_pages(struct btrfs_raid_bio * rbio)16385eb30ee2SQu Wenruo static int alloc_rbio_data_pages(struct btrfs_raid_bio *rbio)
16395eb30ee2SQu Wenruo {
16405eb30ee2SQu Wenruo const int data_pages = rbio->nr_data * rbio->stripe_npages;
16415eb30ee2SQu Wenruo int ret;
16425eb30ee2SQu Wenruo
1643*0fbf6cbdSQu Wenruo ret = btrfs_alloc_page_array(data_pages, rbio->stripe_pages, false);
16445eb30ee2SQu Wenruo if (ret < 0)
16455eb30ee2SQu Wenruo return ret;
16465eb30ee2SQu Wenruo
16475eb30ee2SQu Wenruo index_stripe_sectors(rbio);
16485eb30ee2SQu Wenruo return 0;
16495eb30ee2SQu Wenruo }
16505eb30ee2SQu Wenruo
1651509c27aaSQu Wenruo /*
16526ac0f488SChris Mason * We use plugging call backs to collect full stripes.
16536ac0f488SChris Mason * Any time we get a partial stripe write while plugged
16546ac0f488SChris Mason * we collect it into a list. When the unplug comes down,
16556ac0f488SChris Mason * we sort the list by logical block number and merge
16566ac0f488SChris Mason * everything we can into the same rbios
16576ac0f488SChris Mason */
16586ac0f488SChris Mason struct btrfs_plug_cb {
16596ac0f488SChris Mason struct blk_plug_cb cb;
16606ac0f488SChris Mason struct btrfs_fs_info *info;
16616ac0f488SChris Mason struct list_head rbio_list;
16626ac0f488SChris Mason };
16636ac0f488SChris Mason
16646ac0f488SChris Mason /*
16656ac0f488SChris Mason * rbios on the plug list are sorted for easier merging.
16666ac0f488SChris Mason */
plug_cmp(void * priv,const struct list_head * a,const struct list_head * b)16674f0f586bSSami Tolvanen static int plug_cmp(void *priv, const struct list_head *a,
16684f0f586bSSami Tolvanen const struct list_head *b)
16696ac0f488SChris Mason {
1670214cc184SDavid Sterba const struct btrfs_raid_bio *ra = container_of(a, struct btrfs_raid_bio,
16716ac0f488SChris Mason plug_list);
1672214cc184SDavid Sterba const struct btrfs_raid_bio *rb = container_of(b, struct btrfs_raid_bio,
16736ac0f488SChris Mason plug_list);
16744f024f37SKent Overstreet u64 a_sector = ra->bio_list.head->bi_iter.bi_sector;
16754f024f37SKent Overstreet u64 b_sector = rb->bio_list.head->bi_iter.bi_sector;
16766ac0f488SChris Mason
16776ac0f488SChris Mason if (a_sector < b_sector)
16786ac0f488SChris Mason return -1;
16796ac0f488SChris Mason if (a_sector > b_sector)
16806ac0f488SChris Mason return 1;
16816ac0f488SChris Mason return 0;
16826ac0f488SChris Mason }
16836ac0f488SChris Mason
raid_unplug(struct blk_plug_cb * cb,bool from_schedule)168493723095SQu Wenruo static void raid_unplug(struct blk_plug_cb *cb, bool from_schedule)
16856ac0f488SChris Mason {
168693723095SQu Wenruo struct btrfs_plug_cb *plug = container_of(cb, struct btrfs_plug_cb, cb);
16876ac0f488SChris Mason struct btrfs_raid_bio *cur;
16886ac0f488SChris Mason struct btrfs_raid_bio *last = NULL;
16896ac0f488SChris Mason
16906ac0f488SChris Mason list_sort(NULL, &plug->rbio_list, plug_cmp);
169193723095SQu Wenruo
16926ac0f488SChris Mason while (!list_empty(&plug->rbio_list)) {
16936ac0f488SChris Mason cur = list_entry(plug->rbio_list.next,
16946ac0f488SChris Mason struct btrfs_raid_bio, plug_list);
16956ac0f488SChris Mason list_del_init(&cur->plug_list);
16966ac0f488SChris Mason
16976ac0f488SChris Mason if (rbio_is_full(cur)) {
169893723095SQu Wenruo /* We have a full stripe, queue it down. */
169993723095SQu Wenruo start_async_work(cur, rmw_rbio_work);
17006ac0f488SChris Mason continue;
17016ac0f488SChris Mason }
17026ac0f488SChris Mason if (last) {
17036ac0f488SChris Mason if (rbio_can_merge(last, cur)) {
17046ac0f488SChris Mason merge_rbio(last, cur);
1705ff2b64a2SQu Wenruo free_raid_bio(cur);
17066ac0f488SChris Mason continue;
17076ac0f488SChris Mason }
170893723095SQu Wenruo start_async_work(last, rmw_rbio_work);
17096ac0f488SChris Mason }
17106ac0f488SChris Mason last = cur;
17116ac0f488SChris Mason }
171293723095SQu Wenruo if (last)
171393723095SQu Wenruo start_async_work(last, rmw_rbio_work);
17146ac0f488SChris Mason kfree(plug);
17156ac0f488SChris Mason }
17166ac0f488SChris Mason
1717bd8f7e62SQu Wenruo /* Add the original bio into rbio->bio_list, and update rbio::dbitmap. */
rbio_add_bio(struct btrfs_raid_bio * rbio,struct bio * orig_bio)1718bd8f7e62SQu Wenruo static void rbio_add_bio(struct btrfs_raid_bio *rbio, struct bio *orig_bio)
1719bd8f7e62SQu Wenruo {
1720bd8f7e62SQu Wenruo const struct btrfs_fs_info *fs_info = rbio->bioc->fs_info;
1721bd8f7e62SQu Wenruo const u64 orig_logical = orig_bio->bi_iter.bi_sector << SECTOR_SHIFT;
172218d758a2SQu Wenruo const u64 full_stripe_start = rbio->bioc->full_stripe_logical;
1723bd8f7e62SQu Wenruo const u32 orig_len = orig_bio->bi_iter.bi_size;
1724bd8f7e62SQu Wenruo const u32 sectorsize = fs_info->sectorsize;
1725bd8f7e62SQu Wenruo u64 cur_logical;
1726bd8f7e62SQu Wenruo
1727bbbee460SQu Wenruo ASSERT_RBIO_LOGICAL(orig_logical >= full_stripe_start &&
1728bd8f7e62SQu Wenruo orig_logical + orig_len <= full_stripe_start +
1729bbbee460SQu Wenruo rbio->nr_data * BTRFS_STRIPE_LEN,
1730bbbee460SQu Wenruo rbio, orig_logical);
1731bd8f7e62SQu Wenruo
1732bd8f7e62SQu Wenruo bio_list_add(&rbio->bio_list, orig_bio);
1733bd8f7e62SQu Wenruo rbio->bio_list_bytes += orig_bio->bi_iter.bi_size;
1734bd8f7e62SQu Wenruo
1735bd8f7e62SQu Wenruo /* Update the dbitmap. */
1736bd8f7e62SQu Wenruo for (cur_logical = orig_logical; cur_logical < orig_logical + orig_len;
1737bd8f7e62SQu Wenruo cur_logical += sectorsize) {
1738bd8f7e62SQu Wenruo int bit = ((u32)(cur_logical - full_stripe_start) >>
1739bd8f7e62SQu Wenruo fs_info->sectorsize_bits) % rbio->stripe_nsectors;
1740bd8f7e62SQu Wenruo
1741bd8f7e62SQu Wenruo set_bit(bit, &rbio->dbitmap);
1742bd8f7e62SQu Wenruo }
1743bd8f7e62SQu Wenruo }
1744bd8f7e62SQu Wenruo
17456ac0f488SChris Mason /*
174653b381b3SDavid Woodhouse * our main entry point for writes from the rest of the FS.
174753b381b3SDavid Woodhouse */
raid56_parity_write(struct bio * bio,struct btrfs_io_context * bioc)174831683f4aSChristoph Hellwig void raid56_parity_write(struct bio *bio, struct btrfs_io_context *bioc)
174953b381b3SDavid Woodhouse {
17506a258d72SQu Wenruo struct btrfs_fs_info *fs_info = bioc->fs_info;
175153b381b3SDavid Woodhouse struct btrfs_raid_bio *rbio;
17526ac0f488SChris Mason struct btrfs_plug_cb *plug = NULL;
17536ac0f488SChris Mason struct blk_plug_cb *cb;
175453b381b3SDavid Woodhouse
1755ff18a4afSChristoph Hellwig rbio = alloc_rbio(fs_info, bioc);
1756af8e2d1dSMiao Xie if (IS_ERR(rbio)) {
1757abb49e87SChristoph Hellwig bio->bi_status = errno_to_blk_status(PTR_ERR(rbio));
1758abb49e87SChristoph Hellwig bio_endio(bio);
1759abb49e87SChristoph Hellwig return;
1760af8e2d1dSMiao Xie }
17611b94b556SMiao Xie rbio->operation = BTRFS_RBIO_WRITE;
1762bd8f7e62SQu Wenruo rbio_add_bio(rbio, bio);
17636ac0f488SChris Mason
17646ac0f488SChris Mason /*
176593723095SQu Wenruo * Don't plug on full rbios, just get them out the door
17666ac0f488SChris Mason * as quickly as we can
17676ac0f488SChris Mason */
1768abb49e87SChristoph Hellwig if (!rbio_is_full(rbio)) {
176993723095SQu Wenruo cb = blk_check_plugged(raid_unplug, fs_info, sizeof(*plug));
17706ac0f488SChris Mason if (cb) {
17716ac0f488SChris Mason plug = container_of(cb, struct btrfs_plug_cb, cb);
17726ac0f488SChris Mason if (!plug->info) {
17730b246afaSJeff Mahoney plug->info = fs_info;
17746ac0f488SChris Mason INIT_LIST_HEAD(&plug->rbio_list);
17756ac0f488SChris Mason }
17766ac0f488SChris Mason list_add_tail(&rbio->plug_list, &plug->rbio_list);
177793723095SQu Wenruo return;
177853b381b3SDavid Woodhouse }
1779abb49e87SChristoph Hellwig }
1780abb49e87SChristoph Hellwig
178193723095SQu Wenruo /*
178293723095SQu Wenruo * Either we don't have any existing plug, or we're doing a full stripe,
1783abb49e87SChristoph Hellwig * queue the rmw work now.
178493723095SQu Wenruo */
178593723095SQu Wenruo start_async_work(rbio, rmw_rbio_work);
17866ac0f488SChris Mason }
178753b381b3SDavid Woodhouse
verify_one_sector(struct btrfs_raid_bio * rbio,int stripe_nr,int sector_nr)17887a315072SQu Wenruo static int verify_one_sector(struct btrfs_raid_bio *rbio,
17897a315072SQu Wenruo int stripe_nr, int sector_nr)
17907a315072SQu Wenruo {
17917a315072SQu Wenruo struct btrfs_fs_info *fs_info = rbio->bioc->fs_info;
17927a315072SQu Wenruo struct sector_ptr *sector;
17937a315072SQu Wenruo u8 csum_buf[BTRFS_CSUM_SIZE];
17947a315072SQu Wenruo u8 *csum_expected;
17957a315072SQu Wenruo int ret;
17967a315072SQu Wenruo
17977a315072SQu Wenruo if (!rbio->csum_bitmap || !rbio->csum_buf)
17987a315072SQu Wenruo return 0;
17997a315072SQu Wenruo
18007a315072SQu Wenruo /* No way to verify P/Q as they are not covered by data csum. */
18017a315072SQu Wenruo if (stripe_nr >= rbio->nr_data)
18027a315072SQu Wenruo return 0;
18037a315072SQu Wenruo /*
18047a315072SQu Wenruo * If we're rebuilding a read, we have to use pages from the
18057a315072SQu Wenruo * bio list if possible.
18067a315072SQu Wenruo */
18073a3c7a7fSQu Wenruo if (rbio->operation == BTRFS_RBIO_READ_REBUILD) {
18087a315072SQu Wenruo sector = sector_in_rbio(rbio, stripe_nr, sector_nr, 0);
18097a315072SQu Wenruo } else {
18107a315072SQu Wenruo sector = rbio_stripe_sector(rbio, stripe_nr, sector_nr);
18117a315072SQu Wenruo }
18127a315072SQu Wenruo
18137a315072SQu Wenruo ASSERT(sector->page);
18147a315072SQu Wenruo
18157a315072SQu Wenruo csum_expected = rbio->csum_buf +
18167a315072SQu Wenruo (stripe_nr * rbio->stripe_nsectors + sector_nr) *
18177a315072SQu Wenruo fs_info->csum_size;
18187a315072SQu Wenruo ret = btrfs_check_sector_csum(fs_info, sector->page, sector->pgoff,
18197a315072SQu Wenruo csum_buf, csum_expected);
18207a315072SQu Wenruo return ret;
18217a315072SQu Wenruo }
18227a315072SQu Wenruo
182353b381b3SDavid Woodhouse /*
18249c5ff9b4SQu Wenruo * Recover a vertical stripe specified by @sector_nr.
18259c5ff9b4SQu Wenruo * @*pointers are the pre-allocated pointers by the caller, so we don't
18269c5ff9b4SQu Wenruo * need to allocate/free the pointers again and again.
18279c5ff9b4SQu Wenruo */
recover_vertical(struct btrfs_raid_bio * rbio,int sector_nr,void ** pointers,void ** unmap_array)182875b47033SQu Wenruo static int recover_vertical(struct btrfs_raid_bio *rbio, int sector_nr,
18299c5ff9b4SQu Wenruo void **pointers, void **unmap_array)
18309c5ff9b4SQu Wenruo {
18319c5ff9b4SQu Wenruo struct btrfs_fs_info *fs_info = rbio->bioc->fs_info;
18329c5ff9b4SQu Wenruo struct sector_ptr *sector;
18339c5ff9b4SQu Wenruo const u32 sectorsize = fs_info->sectorsize;
183475b47033SQu Wenruo int found_errors;
183575b47033SQu Wenruo int faila;
183675b47033SQu Wenruo int failb;
18379c5ff9b4SQu Wenruo int stripe_nr;
18387a315072SQu Wenruo int ret = 0;
18399c5ff9b4SQu Wenruo
18409c5ff9b4SQu Wenruo /*
18419c5ff9b4SQu Wenruo * Now we just use bitmap to mark the horizontal stripes in
18429c5ff9b4SQu Wenruo * which we have data when doing parity scrub.
18439c5ff9b4SQu Wenruo */
18449c5ff9b4SQu Wenruo if (rbio->operation == BTRFS_RBIO_PARITY_SCRUB &&
18459c5ff9b4SQu Wenruo !test_bit(sector_nr, &rbio->dbitmap))
184675b47033SQu Wenruo return 0;
184775b47033SQu Wenruo
184875b47033SQu Wenruo found_errors = get_rbio_veritical_errors(rbio, sector_nr, &faila,
184975b47033SQu Wenruo &failb);
185075b47033SQu Wenruo /*
185167da05b3SColin Ian King * No errors in the vertical stripe, skip it. Can happen for recovery
185275b47033SQu Wenruo * which only part of a stripe failed csum check.
185375b47033SQu Wenruo */
185475b47033SQu Wenruo if (!found_errors)
185575b47033SQu Wenruo return 0;
185675b47033SQu Wenruo
185775b47033SQu Wenruo if (found_errors > rbio->bioc->max_errors)
185875b47033SQu Wenruo return -EIO;
18599c5ff9b4SQu Wenruo
18609c5ff9b4SQu Wenruo /*
18619c5ff9b4SQu Wenruo * Setup our array of pointers with sectors from each stripe
18629c5ff9b4SQu Wenruo *
18639c5ff9b4SQu Wenruo * NOTE: store a duplicate array of pointers to preserve the
18649c5ff9b4SQu Wenruo * pointer order.
18659c5ff9b4SQu Wenruo */
18669c5ff9b4SQu Wenruo for (stripe_nr = 0; stripe_nr < rbio->real_stripes; stripe_nr++) {
18679c5ff9b4SQu Wenruo /*
186875b47033SQu Wenruo * If we're rebuilding a read, we have to use pages from the
186975b47033SQu Wenruo * bio list if possible.
18709c5ff9b4SQu Wenruo */
18713a3c7a7fSQu Wenruo if (rbio->operation == BTRFS_RBIO_READ_REBUILD) {
18729c5ff9b4SQu Wenruo sector = sector_in_rbio(rbio, stripe_nr, sector_nr, 0);
18739c5ff9b4SQu Wenruo } else {
18749c5ff9b4SQu Wenruo sector = rbio_stripe_sector(rbio, stripe_nr, sector_nr);
18759c5ff9b4SQu Wenruo }
18769c5ff9b4SQu Wenruo ASSERT(sector->page);
18779c5ff9b4SQu Wenruo pointers[stripe_nr] = kmap_local_page(sector->page) +
18789c5ff9b4SQu Wenruo sector->pgoff;
18799c5ff9b4SQu Wenruo unmap_array[stripe_nr] = pointers[stripe_nr];
18809c5ff9b4SQu Wenruo }
18819c5ff9b4SQu Wenruo
18829c5ff9b4SQu Wenruo /* All raid6 handling here */
18839c5ff9b4SQu Wenruo if (rbio->bioc->map_type & BTRFS_BLOCK_GROUP_RAID6) {
18849c5ff9b4SQu Wenruo /* Single failure, rebuild from parity raid5 style */
18859c5ff9b4SQu Wenruo if (failb < 0) {
18869c5ff9b4SQu Wenruo if (faila == rbio->nr_data)
18879c5ff9b4SQu Wenruo /*
18889c5ff9b4SQu Wenruo * Just the P stripe has failed, without
18899c5ff9b4SQu Wenruo * a bad data or Q stripe.
18909c5ff9b4SQu Wenruo * We have nothing to do, just skip the
18919c5ff9b4SQu Wenruo * recovery for this stripe.
18929c5ff9b4SQu Wenruo */
18939c5ff9b4SQu Wenruo goto cleanup;
18949c5ff9b4SQu Wenruo /*
18959c5ff9b4SQu Wenruo * a single failure in raid6 is rebuilt
18969c5ff9b4SQu Wenruo * in the pstripe code below
18979c5ff9b4SQu Wenruo */
18989c5ff9b4SQu Wenruo goto pstripe;
18999c5ff9b4SQu Wenruo }
19009c5ff9b4SQu Wenruo
19019c5ff9b4SQu Wenruo /*
19029c5ff9b4SQu Wenruo * If the q stripe is failed, do a pstripe reconstruction from
19039c5ff9b4SQu Wenruo * the xors.
19049c5ff9b4SQu Wenruo * If both the q stripe and the P stripe are failed, we're
19059c5ff9b4SQu Wenruo * here due to a crc mismatch and we can't give them the
19069c5ff9b4SQu Wenruo * data they want.
19079c5ff9b4SQu Wenruo */
190818d758a2SQu Wenruo if (failb == rbio->real_stripes - 1) {
190918d758a2SQu Wenruo if (faila == rbio->real_stripes - 2)
19109c5ff9b4SQu Wenruo /*
19119c5ff9b4SQu Wenruo * Only P and Q are corrupted.
19129c5ff9b4SQu Wenruo * We only care about data stripes recovery,
19139c5ff9b4SQu Wenruo * can skip this vertical stripe.
19149c5ff9b4SQu Wenruo */
19159c5ff9b4SQu Wenruo goto cleanup;
19169c5ff9b4SQu Wenruo /*
19179c5ff9b4SQu Wenruo * Otherwise we have one bad data stripe and
19189c5ff9b4SQu Wenruo * a good P stripe. raid5!
19199c5ff9b4SQu Wenruo */
19209c5ff9b4SQu Wenruo goto pstripe;
19219c5ff9b4SQu Wenruo }
19229c5ff9b4SQu Wenruo
192318d758a2SQu Wenruo if (failb == rbio->real_stripes - 2) {
19249c5ff9b4SQu Wenruo raid6_datap_recov(rbio->real_stripes, sectorsize,
19259c5ff9b4SQu Wenruo faila, pointers);
19269c5ff9b4SQu Wenruo } else {
19279c5ff9b4SQu Wenruo raid6_2data_recov(rbio->real_stripes, sectorsize,
19289c5ff9b4SQu Wenruo faila, failb, pointers);
19299c5ff9b4SQu Wenruo }
19309c5ff9b4SQu Wenruo } else {
19319c5ff9b4SQu Wenruo void *p;
19329c5ff9b4SQu Wenruo
19339c5ff9b4SQu Wenruo /* Rebuild from P stripe here (raid5 or raid6). */
19349c5ff9b4SQu Wenruo ASSERT(failb == -1);
19359c5ff9b4SQu Wenruo pstripe:
19369c5ff9b4SQu Wenruo /* Copy parity block into failed block to start with */
19379c5ff9b4SQu Wenruo memcpy(pointers[faila], pointers[rbio->nr_data], sectorsize);
19389c5ff9b4SQu Wenruo
19399c5ff9b4SQu Wenruo /* Rearrange the pointer array */
19409c5ff9b4SQu Wenruo p = pointers[faila];
19419c5ff9b4SQu Wenruo for (stripe_nr = faila; stripe_nr < rbio->nr_data - 1;
19429c5ff9b4SQu Wenruo stripe_nr++)
19439c5ff9b4SQu Wenruo pointers[stripe_nr] = pointers[stripe_nr + 1];
19449c5ff9b4SQu Wenruo pointers[rbio->nr_data - 1] = p;
19459c5ff9b4SQu Wenruo
19469c5ff9b4SQu Wenruo /* Xor in the rest */
19479c5ff9b4SQu Wenruo run_xor(pointers, rbio->nr_data - 1, sectorsize);
19489c5ff9b4SQu Wenruo
19499c5ff9b4SQu Wenruo }
19509c5ff9b4SQu Wenruo
19519c5ff9b4SQu Wenruo /*
19529c5ff9b4SQu Wenruo * No matter if this is a RMW or recovery, we should have all
19539c5ff9b4SQu Wenruo * failed sectors repaired in the vertical stripe, thus they are now
19549c5ff9b4SQu Wenruo * uptodate.
19559c5ff9b4SQu Wenruo * Especially if we determine to cache the rbio, we need to
19569c5ff9b4SQu Wenruo * have at least all data sectors uptodate.
19577a315072SQu Wenruo *
19587a315072SQu Wenruo * If possible, also check if the repaired sector matches its data
19597a315072SQu Wenruo * checksum.
19609c5ff9b4SQu Wenruo */
196175b47033SQu Wenruo if (faila >= 0) {
19627a315072SQu Wenruo ret = verify_one_sector(rbio, faila, sector_nr);
19637a315072SQu Wenruo if (ret < 0)
19647a315072SQu Wenruo goto cleanup;
19657a315072SQu Wenruo
196675b47033SQu Wenruo sector = rbio_stripe_sector(rbio, faila, sector_nr);
19679c5ff9b4SQu Wenruo sector->uptodate = 1;
19689c5ff9b4SQu Wenruo }
196975b47033SQu Wenruo if (failb >= 0) {
1970f7c11affSTanmay Bhushan ret = verify_one_sector(rbio, failb, sector_nr);
19717a315072SQu Wenruo if (ret < 0)
19727a315072SQu Wenruo goto cleanup;
19737a315072SQu Wenruo
197475b47033SQu Wenruo sector = rbio_stripe_sector(rbio, failb, sector_nr);
19759c5ff9b4SQu Wenruo sector->uptodate = 1;
19769c5ff9b4SQu Wenruo }
19779c5ff9b4SQu Wenruo
19789c5ff9b4SQu Wenruo cleanup:
19799c5ff9b4SQu Wenruo for (stripe_nr = rbio->real_stripes - 1; stripe_nr >= 0; stripe_nr--)
19809c5ff9b4SQu Wenruo kunmap_local(unmap_array[stripe_nr]);
19817a315072SQu Wenruo return ret;
19829c5ff9b4SQu Wenruo }
19839c5ff9b4SQu Wenruo
recover_sectors(struct btrfs_raid_bio * rbio)1984ec936b03SQu Wenruo static int recover_sectors(struct btrfs_raid_bio *rbio)
198553b381b3SDavid Woodhouse {
19869c5ff9b4SQu Wenruo void **pointers = NULL;
19879c5ff9b4SQu Wenruo void **unmap_array = NULL;
1988ec936b03SQu Wenruo int sectornr;
1989ec936b03SQu Wenruo int ret = 0;
199053b381b3SDavid Woodhouse
199107e4d380SQu Wenruo /*
1992ec936b03SQu Wenruo * @pointers array stores the pointer for each sector.
1993ec936b03SQu Wenruo *
1994ec936b03SQu Wenruo * @unmap_array stores copy of pointers that does not get reordered
1995ec936b03SQu Wenruo * during reconstruction so that kunmap_local works.
199607e4d380SQu Wenruo */
199731e818feSDavid Sterba pointers = kcalloc(rbio->real_stripes, sizeof(void *), GFP_NOFS);
199894a0b58dSIra Weiny unmap_array = kcalloc(rbio->real_stripes, sizeof(void *), GFP_NOFS);
1999ec936b03SQu Wenruo if (!pointers || !unmap_array) {
2000ec936b03SQu Wenruo ret = -ENOMEM;
2001ec936b03SQu Wenruo goto out;
200294a0b58dSIra Weiny }
200394a0b58dSIra Weiny
20043a3c7a7fSQu Wenruo if (rbio->operation == BTRFS_RBIO_READ_REBUILD) {
200574cc3600SChristoph Hellwig spin_lock(&rbio->bio_list_lock);
200653b381b3SDavid Woodhouse set_bit(RBIO_RMW_LOCKED_BIT, &rbio->flags);
200774cc3600SChristoph Hellwig spin_unlock(&rbio->bio_list_lock);
200853b381b3SDavid Woodhouse }
200953b381b3SDavid Woodhouse
201053b381b3SDavid Woodhouse index_rbio_pages(rbio);
201153b381b3SDavid Woodhouse
201275b47033SQu Wenruo for (sectornr = 0; sectornr < rbio->stripe_nsectors; sectornr++) {
201375b47033SQu Wenruo ret = recover_vertical(rbio, sectornr, pointers, unmap_array);
201475b47033SQu Wenruo if (ret < 0)
201575b47033SQu Wenruo break;
201675b47033SQu Wenruo }
201753b381b3SDavid Woodhouse
2018ec936b03SQu Wenruo out:
201953b381b3SDavid Woodhouse kfree(pointers);
2020ec936b03SQu Wenruo kfree(unmap_array);
2021ec936b03SQu Wenruo return ret;
2022ec936b03SQu Wenruo }
2023ec936b03SQu Wenruo
recover_rbio(struct btrfs_raid_bio * rbio)202440f87ddbSChristoph Hellwig static void recover_rbio(struct btrfs_raid_bio *rbio)
202553b381b3SDavid Woodhouse {
2026d838d05eSChristoph Hellwig struct bio_list bio_list = BIO_EMPTY_LIST;
2027d31968d9SQu Wenruo int total_sector_nr;
2028d31968d9SQu Wenruo int ret = 0;
202953b381b3SDavid Woodhouse
2030d838d05eSChristoph Hellwig /*
2031d838d05eSChristoph Hellwig * Either we're doing recover for a read failure or degraded write,
2032d838d05eSChristoph Hellwig * caller should have set error bitmap correctly.
2033d838d05eSChristoph Hellwig */
2034d838d05eSChristoph Hellwig ASSERT(bitmap_weight(rbio->error_bitmap, rbio->nr_sectors));
2035d838d05eSChristoph Hellwig
2036d838d05eSChristoph Hellwig /* For recovery, we need to read all sectors including P/Q. */
2037d838d05eSChristoph Hellwig ret = alloc_rbio_pages(rbio);
2038d838d05eSChristoph Hellwig if (ret < 0)
203940f87ddbSChristoph Hellwig goto out;
2040d838d05eSChristoph Hellwig
2041d838d05eSChristoph Hellwig index_rbio_pages(rbio);
2042d838d05eSChristoph Hellwig
204353b381b3SDavid Woodhouse /*
2044f6065f8eSQu Wenruo * Read everything that hasn't failed. However this time we will
2045f6065f8eSQu Wenruo * not trust any cached sector.
2046f6065f8eSQu Wenruo * As we may read out some stale data but higher layer is not reading
2047f6065f8eSQu Wenruo * that stale part.
2048f6065f8eSQu Wenruo *
2049f6065f8eSQu Wenruo * So here we always re-read everything in recovery path.
205053b381b3SDavid Woodhouse */
2051ef340fccSQu Wenruo for (total_sector_nr = 0; total_sector_nr < rbio->nr_sectors;
2052ef340fccSQu Wenruo total_sector_nr++) {
2053ef340fccSQu Wenruo int stripe = total_sector_nr / rbio->stripe_nsectors;
2054ef340fccSQu Wenruo int sectornr = total_sector_nr % rbio->stripe_nsectors;
20553e77605dSQu Wenruo struct sector_ptr *sector;
205653b381b3SDavid Woodhouse
205775b47033SQu Wenruo /*
205875b47033SQu Wenruo * Skip the range which has error. It can be a range which is
205975b47033SQu Wenruo * marked error (for csum mismatch), or it can be a missing
206075b47033SQu Wenruo * device.
206175b47033SQu Wenruo */
206275b47033SQu Wenruo if (!rbio->bioc->stripes[stripe].dev->bdev ||
206375b47033SQu Wenruo test_bit(total_sector_nr, rbio->error_bitmap)) {
206475b47033SQu Wenruo /*
206575b47033SQu Wenruo * Also set the error bit for missing device, which
206675b47033SQu Wenruo * may not yet have its error bit set.
206775b47033SQu Wenruo */
206875b47033SQu Wenruo set_bit(total_sector_nr, rbio->error_bitmap);
206953b381b3SDavid Woodhouse continue;
2070ef340fccSQu Wenruo }
207175b47033SQu Wenruo
207253b381b3SDavid Woodhouse sector = rbio_stripe_sector(rbio, stripe, sectornr);
2073d838d05eSChristoph Hellwig ret = rbio_add_io_sector(rbio, &bio_list, sector, stripe,
2074ff18a4afSChristoph Hellwig sectornr, REQ_OP_READ);
2075d838d05eSChristoph Hellwig if (ret < 0) {
2076801fcfc5SChristoph Hellwig bio_list_put(&bio_list);
207740f87ddbSChristoph Hellwig goto out;
2078d817ce35SQu Wenruo }
2079d838d05eSChristoph Hellwig }
2080d838d05eSChristoph Hellwig
2081d838d05eSChristoph Hellwig submit_read_wait_bio_list(rbio, &bio_list);
208240f87ddbSChristoph Hellwig ret = recover_sectors(rbio);
208340f87ddbSChristoph Hellwig out:
208440f87ddbSChristoph Hellwig rbio_orig_end_io(rbio, errno_to_blk_status(ret));
2085d838d05eSChristoph Hellwig }
2086d817ce35SQu Wenruo
recover_rbio_work(struct work_struct * work)2087d817ce35SQu Wenruo static void recover_rbio_work(struct work_struct *work)
2088d817ce35SQu Wenruo {
2089d817ce35SQu Wenruo struct btrfs_raid_bio *rbio;
2090d817ce35SQu Wenruo
2091d817ce35SQu Wenruo rbio = container_of(work, struct btrfs_raid_bio, work);
209240f87ddbSChristoph Hellwig if (!lock_stripe_add(rbio))
209340f87ddbSChristoph Hellwig recover_rbio(rbio);
2094d817ce35SQu Wenruo }
2095d817ce35SQu Wenruo
recover_rbio_work_locked(struct work_struct * work)2096d817ce35SQu Wenruo static void recover_rbio_work_locked(struct work_struct *work)
2097d817ce35SQu Wenruo {
209840f87ddbSChristoph Hellwig recover_rbio(container_of(work, struct btrfs_raid_bio, work));
2099d817ce35SQu Wenruo }
2100d817ce35SQu Wenruo
set_rbio_raid6_extra_error(struct btrfs_raid_bio * rbio,int mirror_num)210175b47033SQu Wenruo static void set_rbio_raid6_extra_error(struct btrfs_raid_bio *rbio, int mirror_num)
210275b47033SQu Wenruo {
210375b47033SQu Wenruo bool found = false;
210475b47033SQu Wenruo int sector_nr;
210575b47033SQu Wenruo
210675b47033SQu Wenruo /*
210775b47033SQu Wenruo * This is for RAID6 extra recovery tries, thus mirror number should
210875b47033SQu Wenruo * be large than 2.
210975b47033SQu Wenruo * Mirror 1 means read from data stripes. Mirror 2 means rebuild using
211075b47033SQu Wenruo * RAID5 methods.
211175b47033SQu Wenruo */
211275b47033SQu Wenruo ASSERT(mirror_num > 2);
211375b47033SQu Wenruo for (sector_nr = 0; sector_nr < rbio->stripe_nsectors; sector_nr++) {
211475b47033SQu Wenruo int found_errors;
211575b47033SQu Wenruo int faila;
211675b47033SQu Wenruo int failb;
211775b47033SQu Wenruo
211875b47033SQu Wenruo found_errors = get_rbio_veritical_errors(rbio, sector_nr,
211975b47033SQu Wenruo &faila, &failb);
212075b47033SQu Wenruo /* This vertical stripe doesn't have errors. */
212175b47033SQu Wenruo if (!found_errors)
212275b47033SQu Wenruo continue;
212375b47033SQu Wenruo
212475b47033SQu Wenruo /*
212575b47033SQu Wenruo * If we found errors, there should be only one error marked
212675b47033SQu Wenruo * by previous set_rbio_range_error().
212775b47033SQu Wenruo */
212875b47033SQu Wenruo ASSERT(found_errors == 1);
212975b47033SQu Wenruo found = true;
213075b47033SQu Wenruo
213175b47033SQu Wenruo /* Now select another stripe to mark as error. */
213275b47033SQu Wenruo failb = rbio->real_stripes - (mirror_num - 1);
213375b47033SQu Wenruo if (failb <= faila)
213475b47033SQu Wenruo failb--;
213575b47033SQu Wenruo
213675b47033SQu Wenruo /* Set the extra bit in error bitmap. */
213775b47033SQu Wenruo if (failb >= 0)
213875b47033SQu Wenruo set_bit(failb * rbio->stripe_nsectors + sector_nr,
213975b47033SQu Wenruo rbio->error_bitmap);
214075b47033SQu Wenruo }
214175b47033SQu Wenruo
214275b47033SQu Wenruo /* We should found at least one vertical stripe with error.*/
214375b47033SQu Wenruo ASSERT(found);
214475b47033SQu Wenruo }
214575b47033SQu Wenruo
2146d31968d9SQu Wenruo /*
214753b381b3SDavid Woodhouse * the main entry point for reads from the higher layers. This
214853b381b3SDavid Woodhouse * is really only called when the normal read path had a failure,
214953b381b3SDavid Woodhouse * so we assume the bio they send down corresponds to a failed part
215053b381b3SDavid Woodhouse * of the drive.
215153b381b3SDavid Woodhouse */
raid56_parity_recover(struct bio * bio,struct btrfs_io_context * bioc,int mirror_num)21526065fd95SChristoph Hellwig void raid56_parity_recover(struct bio *bio, struct btrfs_io_context *bioc,
2153f1c29379SChristoph Hellwig int mirror_num)
215453b381b3SDavid Woodhouse {
21556a258d72SQu Wenruo struct btrfs_fs_info *fs_info = bioc->fs_info;
215653b381b3SDavid Woodhouse struct btrfs_raid_bio *rbio;
215753b381b3SDavid Woodhouse
2158ff18a4afSChristoph Hellwig rbio = alloc_rbio(fs_info, bioc);
2159af8e2d1dSMiao Xie if (IS_ERR(rbio)) {
21606065fd95SChristoph Hellwig bio->bi_status = errno_to_blk_status(PTR_ERR(rbio));
2161d817ce35SQu Wenruo bio_endio(bio);
2162d817ce35SQu Wenruo return;
2163af8e2d1dSMiao Xie }
216453b381b3SDavid Woodhouse
21651b94b556SMiao Xie rbio->operation = BTRFS_RBIO_READ_REBUILD;
2166bd8f7e62SQu Wenruo rbio_add_bio(rbio, bio);
216753b381b3SDavid Woodhouse
21682942a50dSQu Wenruo set_rbio_range_error(rbio, bio);
21692942a50dSQu Wenruo
217053b381b3SDavid Woodhouse /*
21718810f751SLiu Bo * Loop retry:
21728810f751SLiu Bo * for 'mirror == 2', reconstruct from all other stripes.
21738810f751SLiu Bo * for 'mirror_num > 2', select a stripe to fail on every retry.
217453b381b3SDavid Woodhouse */
2175ad3daf1cSQu Wenruo if (mirror_num > 2)
217675b47033SQu Wenruo set_rbio_raid6_extra_error(rbio, mirror_num);
217753b381b3SDavid Woodhouse
2178d817ce35SQu Wenruo start_async_work(rbio, recover_rbio_work);
217953b381b3SDavid Woodhouse }
218053b381b3SDavid Woodhouse
fill_data_csums(struct btrfs_raid_bio * rbio)2181c5a41562SQu Wenruo static void fill_data_csums(struct btrfs_raid_bio *rbio)
2182c5a41562SQu Wenruo {
2183c5a41562SQu Wenruo struct btrfs_fs_info *fs_info = rbio->bioc->fs_info;
2184c5a41562SQu Wenruo struct btrfs_root *csum_root = btrfs_csum_root(fs_info,
218518d758a2SQu Wenruo rbio->bioc->full_stripe_logical);
218618d758a2SQu Wenruo const u64 start = rbio->bioc->full_stripe_logical;
2187c5a41562SQu Wenruo const u32 len = (rbio->nr_data * rbio->stripe_nsectors) <<
2188c5a41562SQu Wenruo fs_info->sectorsize_bits;
2189c5a41562SQu Wenruo int ret;
2190c5a41562SQu Wenruo
2191c5a41562SQu Wenruo /* The rbio should not have its csum buffer initialized. */
2192c5a41562SQu Wenruo ASSERT(!rbio->csum_buf && !rbio->csum_bitmap);
2193c5a41562SQu Wenruo
2194c5a41562SQu Wenruo /*
2195c5a41562SQu Wenruo * Skip the csum search if:
2196c5a41562SQu Wenruo *
2197c5a41562SQu Wenruo * - The rbio doesn't belong to data block groups
2198c5a41562SQu Wenruo * Then we are doing IO for tree blocks, no need to search csums.
2199c5a41562SQu Wenruo *
2200c5a41562SQu Wenruo * - The rbio belongs to mixed block groups
2201c5a41562SQu Wenruo * This is to avoid deadlock, as we're already holding the full
2202c5a41562SQu Wenruo * stripe lock, if we trigger a metadata read, and it needs to do
2203c5a41562SQu Wenruo * raid56 recovery, we will deadlock.
2204c5a41562SQu Wenruo */
2205c5a41562SQu Wenruo if (!(rbio->bioc->map_type & BTRFS_BLOCK_GROUP_DATA) ||
2206c5a41562SQu Wenruo rbio->bioc->map_type & BTRFS_BLOCK_GROUP_METADATA)
2207c5a41562SQu Wenruo return;
2208c5a41562SQu Wenruo
2209c5a41562SQu Wenruo rbio->csum_buf = kzalloc(rbio->nr_data * rbio->stripe_nsectors *
2210c5a41562SQu Wenruo fs_info->csum_size, GFP_NOFS);
2211c5a41562SQu Wenruo rbio->csum_bitmap = bitmap_zalloc(rbio->nr_data * rbio->stripe_nsectors,
2212c5a41562SQu Wenruo GFP_NOFS);
2213c5a41562SQu Wenruo if (!rbio->csum_buf || !rbio->csum_bitmap) {
2214c5a41562SQu Wenruo ret = -ENOMEM;
2215c5a41562SQu Wenruo goto error;
2216c5a41562SQu Wenruo }
2217c5a41562SQu Wenruo
22183c771c19SQu Wenruo ret = btrfs_lookup_csums_bitmap(csum_root, NULL, start, start + len - 1,
22193c771c19SQu Wenruo rbio->csum_buf, rbio->csum_bitmap);
2220c5a41562SQu Wenruo if (ret < 0)
2221c5a41562SQu Wenruo goto error;
2222c5a41562SQu Wenruo if (bitmap_empty(rbio->csum_bitmap, len >> fs_info->sectorsize_bits))
2223c5a41562SQu Wenruo goto no_csum;
2224c5a41562SQu Wenruo return;
2225c5a41562SQu Wenruo
2226c5a41562SQu Wenruo error:
2227c5a41562SQu Wenruo /*
2228c5a41562SQu Wenruo * We failed to allocate memory or grab the csum, but it's not fatal,
2229c5a41562SQu Wenruo * we can still continue. But better to warn users that RMW is no
2230c5a41562SQu Wenruo * longer safe for this particular sub-stripe write.
2231c5a41562SQu Wenruo */
2232c5a41562SQu Wenruo btrfs_warn_rl(fs_info,
2233c5a41562SQu Wenruo "sub-stripe write for full stripe %llu is not safe, failed to get csum: %d",
223418d758a2SQu Wenruo rbio->bioc->full_stripe_logical, ret);
2235c5a41562SQu Wenruo no_csum:
2236c5a41562SQu Wenruo kfree(rbio->csum_buf);
2237c5a41562SQu Wenruo bitmap_free(rbio->csum_bitmap);
2238c5a41562SQu Wenruo rbio->csum_buf = NULL;
2239c5a41562SQu Wenruo rbio->csum_bitmap = NULL;
2240c5a41562SQu Wenruo }
2241c5a41562SQu Wenruo
rmw_read_wait_recover(struct btrfs_raid_bio * rbio)22427a315072SQu Wenruo static int rmw_read_wait_recover(struct btrfs_raid_bio *rbio)
22435eb30ee2SQu Wenruo {
224402efa3a6SChristoph Hellwig struct bio_list bio_list = BIO_EMPTY_LIST;
224502efa3a6SChristoph Hellwig int total_sector_nr;
224602efa3a6SChristoph Hellwig int ret = 0;
22475eb30ee2SQu Wenruo
2248c5a41562SQu Wenruo /*
2249c5a41562SQu Wenruo * Fill the data csums we need for data verification. We need to fill
2250c5a41562SQu Wenruo * the csum_bitmap/csum_buf first, as our endio function will try to
2251c5a41562SQu Wenruo * verify the data sectors.
2252c5a41562SQu Wenruo */
2253c5a41562SQu Wenruo fill_data_csums(rbio);
2254c5a41562SQu Wenruo
225502efa3a6SChristoph Hellwig /*
225602efa3a6SChristoph Hellwig * Build a list of bios to read all sectors (including data and P/Q).
225702efa3a6SChristoph Hellwig *
225802efa3a6SChristoph Hellwig * This behavior is to compensate the later csum verification and recovery.
225902efa3a6SChristoph Hellwig */
226002efa3a6SChristoph Hellwig for (total_sector_nr = 0; total_sector_nr < rbio->nr_sectors;
226102efa3a6SChristoph Hellwig total_sector_nr++) {
226202efa3a6SChristoph Hellwig struct sector_ptr *sector;
226302efa3a6SChristoph Hellwig int stripe = total_sector_nr / rbio->stripe_nsectors;
226402efa3a6SChristoph Hellwig int sectornr = total_sector_nr % rbio->stripe_nsectors;
22655eb30ee2SQu Wenruo
226602efa3a6SChristoph Hellwig sector = rbio_stripe_sector(rbio, stripe, sectornr);
226702efa3a6SChristoph Hellwig ret = rbio_add_io_sector(rbio, &bio_list, sector,
226802efa3a6SChristoph Hellwig stripe, sectornr, REQ_OP_READ);
226902efa3a6SChristoph Hellwig if (ret) {
227002efa3a6SChristoph Hellwig bio_list_put(&bio_list);
227102efa3a6SChristoph Hellwig return ret;
227202efa3a6SChristoph Hellwig }
227302efa3a6SChristoph Hellwig }
22747a315072SQu Wenruo
22757a315072SQu Wenruo /*
22767a315072SQu Wenruo * We may or may not have any corrupted sectors (including missing dev
22777a315072SQu Wenruo * and csum mismatch), just let recover_sectors() to handle them all.
22787a315072SQu Wenruo */
227902efa3a6SChristoph Hellwig submit_read_wait_bio_list(rbio, &bio_list);
228002efa3a6SChristoph Hellwig return recover_sectors(rbio);
22815eb30ee2SQu Wenruo }
22825eb30ee2SQu Wenruo
raid_wait_write_end_io(struct bio * bio)22835eb30ee2SQu Wenruo static void raid_wait_write_end_io(struct bio *bio)
22845eb30ee2SQu Wenruo {
22855eb30ee2SQu Wenruo struct btrfs_raid_bio *rbio = bio->bi_private;
22865eb30ee2SQu Wenruo blk_status_t err = bio->bi_status;
22875eb30ee2SQu Wenruo
2288ad3daf1cSQu Wenruo if (err)
22892942a50dSQu Wenruo rbio_update_error_bitmap(rbio, bio);
22905eb30ee2SQu Wenruo bio_put(bio);
22915eb30ee2SQu Wenruo if (atomic_dec_and_test(&rbio->stripes_pending))
22925eb30ee2SQu Wenruo wake_up(&rbio->io_wait);
22935eb30ee2SQu Wenruo }
22945eb30ee2SQu Wenruo
submit_write_bios(struct btrfs_raid_bio * rbio,struct bio_list * bio_list)22955eb30ee2SQu Wenruo static void submit_write_bios(struct btrfs_raid_bio *rbio,
22965eb30ee2SQu Wenruo struct bio_list *bio_list)
22975eb30ee2SQu Wenruo {
22985eb30ee2SQu Wenruo struct bio *bio;
22995eb30ee2SQu Wenruo
23005eb30ee2SQu Wenruo atomic_set(&rbio->stripes_pending, bio_list_size(bio_list));
23015eb30ee2SQu Wenruo while ((bio = bio_list_pop(bio_list))) {
23025eb30ee2SQu Wenruo bio->bi_end_io = raid_wait_write_end_io;
23035eb30ee2SQu Wenruo
2304dbb6ecb3SQu Wenruo if (trace_raid56_write_enabled()) {
23055eb30ee2SQu Wenruo struct raid56_bio_trace_info trace_info = { 0 };
23065eb30ee2SQu Wenruo
23075eb30ee2SQu Wenruo bio_get_trace_info(rbio, bio, &trace_info);
2308dbb6ecb3SQu Wenruo trace_raid56_write(rbio, bio, &trace_info);
23095eb30ee2SQu Wenruo }
23105eb30ee2SQu Wenruo submit_bio(bio);
23115eb30ee2SQu Wenruo }
23125eb30ee2SQu Wenruo }
23135eb30ee2SQu Wenruo
23147a315072SQu Wenruo /*
23157a315072SQu Wenruo * To determine if we need to read any sector from the disk.
23167a315072SQu Wenruo * Should only be utilized in RMW path, to skip cached rbio.
23177a315072SQu Wenruo */
need_read_stripe_sectors(struct btrfs_raid_bio * rbio)23187a315072SQu Wenruo static bool need_read_stripe_sectors(struct btrfs_raid_bio *rbio)
23197a315072SQu Wenruo {
23207a315072SQu Wenruo int i;
23217a315072SQu Wenruo
23227a315072SQu Wenruo for (i = 0; i < rbio->nr_data * rbio->stripe_nsectors; i++) {
23237a315072SQu Wenruo struct sector_ptr *sector = &rbio->stripe_sectors[i];
23247a315072SQu Wenruo
23257a315072SQu Wenruo /*
23267a315072SQu Wenruo * We have a sector which doesn't have page nor uptodate,
23277a315072SQu Wenruo * thus this rbio can not be cached one, as cached one must
23287a315072SQu Wenruo * have all its data sectors present and uptodate.
23297a315072SQu Wenruo */
23307a315072SQu Wenruo if (!sector->page || !sector->uptodate)
23317a315072SQu Wenruo return true;
23327a315072SQu Wenruo }
23337a315072SQu Wenruo return false;
23347a315072SQu Wenruo }
23357a315072SQu Wenruo
rmw_rbio(struct btrfs_raid_bio * rbio)23361d0ef1caSChristoph Hellwig static void rmw_rbio(struct btrfs_raid_bio *rbio)
23375eb30ee2SQu Wenruo {
23385eb30ee2SQu Wenruo struct bio_list bio_list;
23395eb30ee2SQu Wenruo int sectornr;
23405eb30ee2SQu Wenruo int ret = 0;
23415eb30ee2SQu Wenruo
23425eb30ee2SQu Wenruo /*
23435eb30ee2SQu Wenruo * Allocate the pages for parity first, as P/Q pages will always be
23445eb30ee2SQu Wenruo * needed for both full-stripe and sub-stripe writes.
23455eb30ee2SQu Wenruo */
23465eb30ee2SQu Wenruo ret = alloc_rbio_parity_pages(rbio);
23475eb30ee2SQu Wenruo if (ret < 0)
23481d0ef1caSChristoph Hellwig goto out;
23495eb30ee2SQu Wenruo
23507a315072SQu Wenruo /*
23517a315072SQu Wenruo * Either full stripe write, or we have every data sector already
23527a315072SQu Wenruo * cached, can go to write path immediately.
23537a315072SQu Wenruo */
23544d762701SChristoph Hellwig if (!rbio_is_full(rbio) && need_read_stripe_sectors(rbio)) {
23555eb30ee2SQu Wenruo /*
23564d762701SChristoph Hellwig * Now we're doing sub-stripe write, also need all data stripes
23574d762701SChristoph Hellwig * to do the full RMW.
23585eb30ee2SQu Wenruo */
23595eb30ee2SQu Wenruo ret = alloc_rbio_data_pages(rbio);
23605eb30ee2SQu Wenruo if (ret < 0)
23611d0ef1caSChristoph Hellwig goto out;
23625eb30ee2SQu Wenruo
23635eb30ee2SQu Wenruo index_rbio_pages(rbio);
23645eb30ee2SQu Wenruo
23657a315072SQu Wenruo ret = rmw_read_wait_recover(rbio);
23665eb30ee2SQu Wenruo if (ret < 0)
23671d0ef1caSChristoph Hellwig goto out;
23684d762701SChristoph Hellwig }
23695eb30ee2SQu Wenruo
23705eb30ee2SQu Wenruo /*
23715eb30ee2SQu Wenruo * At this stage we're not allowed to add any new bios to the
23725eb30ee2SQu Wenruo * bio list any more, anyone else that wants to change this stripe
23735eb30ee2SQu Wenruo * needs to do their own rmw.
23745eb30ee2SQu Wenruo */
237574cc3600SChristoph Hellwig spin_lock(&rbio->bio_list_lock);
23765eb30ee2SQu Wenruo set_bit(RBIO_RMW_LOCKED_BIT, &rbio->flags);
237774cc3600SChristoph Hellwig spin_unlock(&rbio->bio_list_lock);
23785eb30ee2SQu Wenruo
23792942a50dSQu Wenruo bitmap_clear(rbio->error_bitmap, 0, rbio->nr_sectors);
23805eb30ee2SQu Wenruo
23815eb30ee2SQu Wenruo index_rbio_pages(rbio);
23825eb30ee2SQu Wenruo
23835eb30ee2SQu Wenruo /*
23845eb30ee2SQu Wenruo * We don't cache full rbios because we're assuming
23855eb30ee2SQu Wenruo * the higher layers are unlikely to use this area of
23865eb30ee2SQu Wenruo * the disk again soon. If they do use it again,
23875eb30ee2SQu Wenruo * hopefully they will send another full bio.
23885eb30ee2SQu Wenruo */
23895eb30ee2SQu Wenruo if (!rbio_is_full(rbio))
23905eb30ee2SQu Wenruo cache_rbio_pages(rbio);
23915eb30ee2SQu Wenruo else
23925eb30ee2SQu Wenruo clear_bit(RBIO_CACHE_READY_BIT, &rbio->flags);
23935eb30ee2SQu Wenruo
23945eb30ee2SQu Wenruo for (sectornr = 0; sectornr < rbio->stripe_nsectors; sectornr++)
23955eb30ee2SQu Wenruo generate_pq_vertical(rbio, sectornr);
23965eb30ee2SQu Wenruo
23975eb30ee2SQu Wenruo bio_list_init(&bio_list);
23985eb30ee2SQu Wenruo ret = rmw_assemble_write_bios(rbio, &bio_list);
23995eb30ee2SQu Wenruo if (ret < 0)
24001d0ef1caSChristoph Hellwig goto out;
24015eb30ee2SQu Wenruo
24025eb30ee2SQu Wenruo /* We should have at least one bio assembled. */
24035eb30ee2SQu Wenruo ASSERT(bio_list_size(&bio_list));
24045eb30ee2SQu Wenruo submit_write_bios(rbio, &bio_list);
24055eb30ee2SQu Wenruo wait_event(rbio->io_wait, atomic_read(&rbio->stripes_pending) == 0);
24065eb30ee2SQu Wenruo
2407ad3daf1cSQu Wenruo /* We may have more errors than our tolerance during the read. */
2408ad3daf1cSQu Wenruo for (sectornr = 0; sectornr < rbio->stripe_nsectors; sectornr++) {
2409ad3daf1cSQu Wenruo int found_errors;
2410ad3daf1cSQu Wenruo
2411ad3daf1cSQu Wenruo found_errors = get_rbio_veritical_errors(rbio, sectornr, NULL, NULL);
2412ad3daf1cSQu Wenruo if (found_errors > rbio->bioc->max_errors) {
24135eb30ee2SQu Wenruo ret = -EIO;
2414ad3daf1cSQu Wenruo break;
2415ad3daf1cSQu Wenruo }
2416ad3daf1cSQu Wenruo }
24171d0ef1caSChristoph Hellwig out:
24181d0ef1caSChristoph Hellwig rbio_orig_end_io(rbio, errno_to_blk_status(ret));
24195eb30ee2SQu Wenruo }
24205eb30ee2SQu Wenruo
rmw_rbio_work(struct work_struct * work)242193723095SQu Wenruo static void rmw_rbio_work(struct work_struct *work)
242253b381b3SDavid Woodhouse {
242353b381b3SDavid Woodhouse struct btrfs_raid_bio *rbio;
242453b381b3SDavid Woodhouse
242553b381b3SDavid Woodhouse rbio = container_of(work, struct btrfs_raid_bio, work);
24261d0ef1caSChristoph Hellwig if (lock_stripe_add(rbio) == 0)
24271d0ef1caSChristoph Hellwig rmw_rbio(rbio);
242893723095SQu Wenruo }
242993723095SQu Wenruo
rmw_rbio_work_locked(struct work_struct * work)243093723095SQu Wenruo static void rmw_rbio_work_locked(struct work_struct *work)
243193723095SQu Wenruo {
24321d0ef1caSChristoph Hellwig rmw_rbio(container_of(work, struct btrfs_raid_bio, work));
243353b381b3SDavid Woodhouse }
243453b381b3SDavid Woodhouse
24355a6ac9eaSMiao Xie /*
24365a6ac9eaSMiao Xie * The following code is used to scrub/replace the parity stripe
24375a6ac9eaSMiao Xie *
24384c664611SQu Wenruo * Caller must have already increased bio_counter for getting @bioc.
2439ae6529c3SQu Wenruo *
24405a6ac9eaSMiao Xie * Note: We need make sure all the pages that add into the scrub/replace
24415a6ac9eaSMiao Xie * raid bio are correct and not be changed during the scrub/replace. That
24425a6ac9eaSMiao Xie * is those pages just hold metadata or file data with checksum.
24435a6ac9eaSMiao Xie */
24445a6ac9eaSMiao Xie
raid56_parity_alloc_scrub_rbio(struct bio * bio,struct btrfs_io_context * bioc,struct btrfs_device * scrub_dev,unsigned long * dbitmap,int stripe_nsectors)24456a258d72SQu Wenruo struct btrfs_raid_bio *raid56_parity_alloc_scrub_rbio(struct bio *bio,
24466a258d72SQu Wenruo struct btrfs_io_context *bioc,
2447ff18a4afSChristoph Hellwig struct btrfs_device *scrub_dev,
24485a6ac9eaSMiao Xie unsigned long *dbitmap, int stripe_nsectors)
24495a6ac9eaSMiao Xie {
24506a258d72SQu Wenruo struct btrfs_fs_info *fs_info = bioc->fs_info;
24515a6ac9eaSMiao Xie struct btrfs_raid_bio *rbio;
24525a6ac9eaSMiao Xie int i;
24535a6ac9eaSMiao Xie
2454ff18a4afSChristoph Hellwig rbio = alloc_rbio(fs_info, bioc);
24555a6ac9eaSMiao Xie if (IS_ERR(rbio))
24565a6ac9eaSMiao Xie return NULL;
24575a6ac9eaSMiao Xie bio_list_add(&rbio->bio_list, bio);
24585a6ac9eaSMiao Xie /*
24595a6ac9eaSMiao Xie * This is a special bio which is used to hold the completion handler
24605a6ac9eaSMiao Xie * and make the scrub rbio is similar to the other types
24615a6ac9eaSMiao Xie */
24625a6ac9eaSMiao Xie ASSERT(!bio->bi_iter.bi_size);
24635a6ac9eaSMiao Xie rbio->operation = BTRFS_RBIO_PARITY_SCRUB;
24645a6ac9eaSMiao Xie
24659cd3a7ebSLiu Bo /*
24664c664611SQu Wenruo * After mapping bioc with BTRFS_MAP_WRITE, parities have been sorted
24679cd3a7ebSLiu Bo * to the end position, so this search can start from the first parity
24689cd3a7ebSLiu Bo * stripe.
24699cd3a7ebSLiu Bo */
24709cd3a7ebSLiu Bo for (i = rbio->nr_data; i < rbio->real_stripes; i++) {
24714c664611SQu Wenruo if (bioc->stripes[i].dev == scrub_dev) {
24725a6ac9eaSMiao Xie rbio->scrubp = i;
24735a6ac9eaSMiao Xie break;
24745a6ac9eaSMiao Xie }
24755a6ac9eaSMiao Xie }
2476bbbee460SQu Wenruo ASSERT_RBIO_STRIPE(i < rbio->real_stripes, rbio, i);
24775a6ac9eaSMiao Xie
2478c67c68ebSQu Wenruo bitmap_copy(&rbio->dbitmap, dbitmap, stripe_nsectors);
24795a6ac9eaSMiao Xie return rbio;
24805a6ac9eaSMiao Xie }
24815a6ac9eaSMiao Xie
24825a6ac9eaSMiao Xie /*
24835a6ac9eaSMiao Xie * We just scrub the parity that we have correct data on the same horizontal,
24845a6ac9eaSMiao Xie * so we needn't allocate all pages for all the stripes.
24855a6ac9eaSMiao Xie */
alloc_rbio_essential_pages(struct btrfs_raid_bio * rbio)24865a6ac9eaSMiao Xie static int alloc_rbio_essential_pages(struct btrfs_raid_bio *rbio)
24875a6ac9eaSMiao Xie {
24883907ce29SQu Wenruo const u32 sectorsize = rbio->bioc->fs_info->sectorsize;
2489aee35e4bSQu Wenruo int total_sector_nr;
24905a6ac9eaSMiao Xie
2491aee35e4bSQu Wenruo for (total_sector_nr = 0; total_sector_nr < rbio->nr_sectors;
2492aee35e4bSQu Wenruo total_sector_nr++) {
24933907ce29SQu Wenruo struct page *page;
2494aee35e4bSQu Wenruo int sectornr = total_sector_nr % rbio->stripe_nsectors;
2495aee35e4bSQu Wenruo int index = (total_sector_nr * sectorsize) >> PAGE_SHIFT;
24963907ce29SQu Wenruo
2497aee35e4bSQu Wenruo if (!test_bit(sectornr, &rbio->dbitmap))
2498aee35e4bSQu Wenruo continue;
24995a6ac9eaSMiao Xie if (rbio->stripe_pages[index])
25005a6ac9eaSMiao Xie continue;
2501b0ee5e1eSDavid Sterba page = alloc_page(GFP_NOFS);
25025a6ac9eaSMiao Xie if (!page)
25035a6ac9eaSMiao Xie return -ENOMEM;
25045a6ac9eaSMiao Xie rbio->stripe_pages[index] = page;
25055a6ac9eaSMiao Xie }
2506eb357060SQu Wenruo index_stripe_sectors(rbio);
25075a6ac9eaSMiao Xie return 0;
25085a6ac9eaSMiao Xie }
25095a6ac9eaSMiao Xie
finish_parity_scrub(struct btrfs_raid_bio * rbio)2510486c737fSQu Wenruo static int finish_parity_scrub(struct btrfs_raid_bio *rbio)
25115a6ac9eaSMiao Xie {
25124c664611SQu Wenruo struct btrfs_io_context *bioc = rbio->bioc;
251346900662SQu Wenruo const u32 sectorsize = bioc->fs_info->sectorsize;
25141389053eSKees Cook void **pointers = rbio->finish_pointers;
2515c67c68ebSQu Wenruo unsigned long *pbitmap = &rbio->finish_pbitmap;
25165a6ac9eaSMiao Xie int nr_data = rbio->nr_data;
25175a6ac9eaSMiao Xie int stripe;
25183e77605dSQu Wenruo int sectornr;
2519c17af965SDavid Sterba bool has_qstripe;
252046900662SQu Wenruo struct sector_ptr p_sector = { 0 };
252146900662SQu Wenruo struct sector_ptr q_sector = { 0 };
25225a6ac9eaSMiao Xie struct bio_list bio_list;
252376035976SMiao Xie int is_replace = 0;
25245a6ac9eaSMiao Xie int ret;
25255a6ac9eaSMiao Xie
25265a6ac9eaSMiao Xie bio_list_init(&bio_list);
25275a6ac9eaSMiao Xie
2528c17af965SDavid Sterba if (rbio->real_stripes - rbio->nr_data == 1)
2529c17af965SDavid Sterba has_qstripe = false;
2530c17af965SDavid Sterba else if (rbio->real_stripes - rbio->nr_data == 2)
2531c17af965SDavid Sterba has_qstripe = true;
2532c17af965SDavid Sterba else
25335a6ac9eaSMiao Xie BUG();
25345a6ac9eaSMiao Xie
25351faf3885SQu Wenruo /*
25361faf3885SQu Wenruo * Replace is running and our P/Q stripe is being replaced, then we
25371faf3885SQu Wenruo * need to duplicate the final write to replace target.
25381faf3885SQu Wenruo */
25391faf3885SQu Wenruo if (bioc->replace_nr_stripes && bioc->replace_stripe_src == rbio->scrubp) {
254076035976SMiao Xie is_replace = 1;
2541c67c68ebSQu Wenruo bitmap_copy(pbitmap, &rbio->dbitmap, rbio->stripe_nsectors);
254276035976SMiao Xie }
254376035976SMiao Xie
25445a6ac9eaSMiao Xie /*
25455a6ac9eaSMiao Xie * Because the higher layers(scrubber) are unlikely to
25465a6ac9eaSMiao Xie * use this area of the disk again soon, so don't cache
25475a6ac9eaSMiao Xie * it.
25485a6ac9eaSMiao Xie */
25495a6ac9eaSMiao Xie clear_bit(RBIO_CACHE_READY_BIT, &rbio->flags);
25505a6ac9eaSMiao Xie
255146900662SQu Wenruo p_sector.page = alloc_page(GFP_NOFS);
255246900662SQu Wenruo if (!p_sector.page)
25536bfd0133SQu Wenruo return -ENOMEM;
255446900662SQu Wenruo p_sector.pgoff = 0;
255546900662SQu Wenruo p_sector.uptodate = 1;
25565a6ac9eaSMiao Xie
2557c17af965SDavid Sterba if (has_qstripe) {
2558d70cef0dSIra Weiny /* RAID6, allocate and map temp space for the Q stripe */
255946900662SQu Wenruo q_sector.page = alloc_page(GFP_NOFS);
256046900662SQu Wenruo if (!q_sector.page) {
256146900662SQu Wenruo __free_page(p_sector.page);
256246900662SQu Wenruo p_sector.page = NULL;
25636bfd0133SQu Wenruo return -ENOMEM;
25645a6ac9eaSMiao Xie }
256546900662SQu Wenruo q_sector.pgoff = 0;
256646900662SQu Wenruo q_sector.uptodate = 1;
256746900662SQu Wenruo pointers[rbio->real_stripes - 1] = kmap_local_page(q_sector.page);
25685a6ac9eaSMiao Xie }
25695a6ac9eaSMiao Xie
25702942a50dSQu Wenruo bitmap_clear(rbio->error_bitmap, 0, rbio->nr_sectors);
25715a6ac9eaSMiao Xie
2572d70cef0dSIra Weiny /* Map the parity stripe just once */
257346900662SQu Wenruo pointers[nr_data] = kmap_local_page(p_sector.page);
2574d70cef0dSIra Weiny
2575c67c68ebSQu Wenruo for_each_set_bit(sectornr, &rbio->dbitmap, rbio->stripe_nsectors) {
257646900662SQu Wenruo struct sector_ptr *sector;
25775a6ac9eaSMiao Xie void *parity;
257846900662SQu Wenruo
25795a6ac9eaSMiao Xie /* first collect one page from each data stripe */
25805a6ac9eaSMiao Xie for (stripe = 0; stripe < nr_data; stripe++) {
258146900662SQu Wenruo sector = sector_in_rbio(rbio, stripe, sectornr, 0);
258246900662SQu Wenruo pointers[stripe] = kmap_local_page(sector->page) +
258346900662SQu Wenruo sector->pgoff;
25845a6ac9eaSMiao Xie }
25855a6ac9eaSMiao Xie
2586c17af965SDavid Sterba if (has_qstripe) {
2587b2324e08SQu Wenruo assert_rbio(rbio);
2588d70cef0dSIra Weiny /* RAID6, call the library function to fill in our P/Q */
258946900662SQu Wenruo raid6_call.gen_syndrome(rbio->real_stripes, sectorsize,
25905a6ac9eaSMiao Xie pointers);
25915a6ac9eaSMiao Xie } else {
25925a6ac9eaSMiao Xie /* raid5 */
259346900662SQu Wenruo memcpy(pointers[nr_data], pointers[0], sectorsize);
259446900662SQu Wenruo run_xor(pointers + 1, nr_data - 1, sectorsize);
25955a6ac9eaSMiao Xie }
25965a6ac9eaSMiao Xie
259701327610SNicholas D Steeves /* Check scrubbing parity and repair it */
259846900662SQu Wenruo sector = rbio_stripe_sector(rbio, rbio->scrubp, sectornr);
259946900662SQu Wenruo parity = kmap_local_page(sector->page) + sector->pgoff;
260046900662SQu Wenruo if (memcmp(parity, pointers[rbio->scrubp], sectorsize) != 0)
260146900662SQu Wenruo memcpy(parity, pointers[rbio->scrubp], sectorsize);
26025a6ac9eaSMiao Xie else
26035a6ac9eaSMiao Xie /* Parity is right, needn't writeback */
2604c67c68ebSQu Wenruo bitmap_clear(&rbio->dbitmap, sectornr, 1);
260558c1a35cSIra Weiny kunmap_local(parity);
26065a6ac9eaSMiao Xie
260794a0b58dSIra Weiny for (stripe = nr_data - 1; stripe >= 0; stripe--)
260894a0b58dSIra Weiny kunmap_local(pointers[stripe]);
26095a6ac9eaSMiao Xie }
26105a6ac9eaSMiao Xie
261194a0b58dSIra Weiny kunmap_local(pointers[nr_data]);
261246900662SQu Wenruo __free_page(p_sector.page);
261346900662SQu Wenruo p_sector.page = NULL;
261446900662SQu Wenruo if (q_sector.page) {
261594a0b58dSIra Weiny kunmap_local(pointers[rbio->real_stripes - 1]);
261646900662SQu Wenruo __free_page(q_sector.page);
261746900662SQu Wenruo q_sector.page = NULL;
2618d70cef0dSIra Weiny }
26195a6ac9eaSMiao Xie
26205a6ac9eaSMiao Xie /*
26215a6ac9eaSMiao Xie * time to start writing. Make bios for everything from the
26225a6ac9eaSMiao Xie * higher layers (the bio_list in our rbio) and our p/q. Ignore
26235a6ac9eaSMiao Xie * everything else.
26245a6ac9eaSMiao Xie */
2625c67c68ebSQu Wenruo for_each_set_bit(sectornr, &rbio->dbitmap, rbio->stripe_nsectors) {
26263e77605dSQu Wenruo struct sector_ptr *sector;
26275a6ac9eaSMiao Xie
26283e77605dSQu Wenruo sector = rbio_stripe_sector(rbio, rbio->scrubp, sectornr);
26293e77605dSQu Wenruo ret = rbio_add_io_sector(rbio, &bio_list, sector, rbio->scrubp,
2630ff18a4afSChristoph Hellwig sectornr, REQ_OP_WRITE);
26315a6ac9eaSMiao Xie if (ret)
26325a6ac9eaSMiao Xie goto cleanup;
26335a6ac9eaSMiao Xie }
26345a6ac9eaSMiao Xie
263576035976SMiao Xie if (!is_replace)
263676035976SMiao Xie goto submit_write;
263776035976SMiao Xie
26381faf3885SQu Wenruo /*
26391faf3885SQu Wenruo * Replace is running and our parity stripe needs to be duplicated to
26401faf3885SQu Wenruo * the target device. Check we have a valid source stripe number.
26411faf3885SQu Wenruo */
2642bbbee460SQu Wenruo ASSERT_RBIO(rbio->bioc->replace_stripe_src >= 0, rbio);
26433e77605dSQu Wenruo for_each_set_bit(sectornr, pbitmap, rbio->stripe_nsectors) {
26443e77605dSQu Wenruo struct sector_ptr *sector;
264576035976SMiao Xie
26463e77605dSQu Wenruo sector = rbio_stripe_sector(rbio, rbio->scrubp, sectornr);
26473e77605dSQu Wenruo ret = rbio_add_io_sector(rbio, &bio_list, sector,
26481faf3885SQu Wenruo rbio->real_stripes,
2649ff18a4afSChristoph Hellwig sectornr, REQ_OP_WRITE);
265076035976SMiao Xie if (ret)
265176035976SMiao Xie goto cleanup;
265276035976SMiao Xie }
265376035976SMiao Xie
265476035976SMiao Xie submit_write:
26556bfd0133SQu Wenruo submit_write_bios(rbio, &bio_list);
26566bfd0133SQu Wenruo return 0;
26575a6ac9eaSMiao Xie
26585a6ac9eaSMiao Xie cleanup:
2659801fcfc5SChristoph Hellwig bio_list_put(&bio_list);
26606bfd0133SQu Wenruo return ret;
26615a6ac9eaSMiao Xie }
26625a6ac9eaSMiao Xie
is_data_stripe(struct btrfs_raid_bio * rbio,int stripe)26635a6ac9eaSMiao Xie static inline int is_data_stripe(struct btrfs_raid_bio *rbio, int stripe)
26645a6ac9eaSMiao Xie {
26655a6ac9eaSMiao Xie if (stripe >= 0 && stripe < rbio->nr_data)
26665a6ac9eaSMiao Xie return 1;
26675a6ac9eaSMiao Xie return 0;
26685a6ac9eaSMiao Xie }
26695a6ac9eaSMiao Xie
recover_scrub_rbio(struct btrfs_raid_bio * rbio)26706bfd0133SQu Wenruo static int recover_scrub_rbio(struct btrfs_raid_bio *rbio)
26715a6ac9eaSMiao Xie {
267275b47033SQu Wenruo void **pointers = NULL;
267375b47033SQu Wenruo void **unmap_array = NULL;
267475b47033SQu Wenruo int sector_nr;
2675e7fc357eSJosef Bacik int ret = 0;
26766bfd0133SQu Wenruo
26775a6ac9eaSMiao Xie /*
267875b47033SQu Wenruo * @pointers array stores the pointer for each sector.
267975b47033SQu Wenruo *
268075b47033SQu Wenruo * @unmap_array stores copy of pointers that does not get reordered
268175b47033SQu Wenruo * during reconstruction so that kunmap_local works.
26825a6ac9eaSMiao Xie */
268375b47033SQu Wenruo pointers = kcalloc(rbio->real_stripes, sizeof(void *), GFP_NOFS);
268475b47033SQu Wenruo unmap_array = kcalloc(rbio->real_stripes, sizeof(void *), GFP_NOFS);
268575b47033SQu Wenruo if (!pointers || !unmap_array) {
268675b47033SQu Wenruo ret = -ENOMEM;
268775b47033SQu Wenruo goto out;
268875b47033SQu Wenruo }
26895a6ac9eaSMiao Xie
269075b47033SQu Wenruo for (sector_nr = 0; sector_nr < rbio->stripe_nsectors; sector_nr++) {
269175b47033SQu Wenruo int dfail = 0, failp = -1;
269275b47033SQu Wenruo int faila;
269375b47033SQu Wenruo int failb;
269475b47033SQu Wenruo int found_errors;
269575b47033SQu Wenruo
269675b47033SQu Wenruo found_errors = get_rbio_veritical_errors(rbio, sector_nr,
269775b47033SQu Wenruo &faila, &failb);
269875b47033SQu Wenruo if (found_errors > rbio->bioc->max_errors) {
269975b47033SQu Wenruo ret = -EIO;
270075b47033SQu Wenruo goto out;
270175b47033SQu Wenruo }
270275b47033SQu Wenruo if (found_errors == 0)
270375b47033SQu Wenruo continue;
270475b47033SQu Wenruo
270575b47033SQu Wenruo /* We should have at least one error here. */
270675b47033SQu Wenruo ASSERT(faila >= 0 || failb >= 0);
270775b47033SQu Wenruo
270875b47033SQu Wenruo if (is_data_stripe(rbio, faila))
270975b47033SQu Wenruo dfail++;
271075b47033SQu Wenruo else if (is_parity_stripe(faila))
271175b47033SQu Wenruo failp = faila;
271275b47033SQu Wenruo
271375b47033SQu Wenruo if (is_data_stripe(rbio, failb))
271475b47033SQu Wenruo dfail++;
271575b47033SQu Wenruo else if (is_parity_stripe(failb))
271675b47033SQu Wenruo failp = failb;
27175a6ac9eaSMiao Xie /*
271875b47033SQu Wenruo * Because we can not use a scrubbing parity to repair the
271975b47033SQu Wenruo * data, so the capability of the repair is declined. (In the
272075b47033SQu Wenruo * case of RAID5, we can not repair anything.)
272175b47033SQu Wenruo */
272275b47033SQu Wenruo if (dfail > rbio->bioc->max_errors - 1) {
272375b47033SQu Wenruo ret = -EIO;
272475b47033SQu Wenruo goto out;
272575b47033SQu Wenruo }
272675b47033SQu Wenruo /*
272775b47033SQu Wenruo * If all data is good, only parity is correctly, just repair
272875b47033SQu Wenruo * the parity, no need to recover data stripes.
27295a6ac9eaSMiao Xie */
27306bfd0133SQu Wenruo if (dfail == 0)
273175b47033SQu Wenruo continue;
27325a6ac9eaSMiao Xie
27335a6ac9eaSMiao Xie /*
27345a6ac9eaSMiao Xie * Here means we got one corrupted data stripe and one
273575b47033SQu Wenruo * corrupted parity on RAID6, if the corrupted parity is
273675b47033SQu Wenruo * scrubbing parity, luckily, use the other one to repair the
273775b47033SQu Wenruo * data, or we can not repair the data stripe.
27385a6ac9eaSMiao Xie */
273975b47033SQu Wenruo if (failp != rbio->scrubp) {
274075b47033SQu Wenruo ret = -EIO;
274175b47033SQu Wenruo goto out;
274275b47033SQu Wenruo }
27435a6ac9eaSMiao Xie
274475b47033SQu Wenruo ret = recover_vertical(rbio, sector_nr, pointers, unmap_array);
274575b47033SQu Wenruo if (ret < 0)
274675b47033SQu Wenruo goto out;
274775b47033SQu Wenruo }
274875b47033SQu Wenruo out:
274975b47033SQu Wenruo kfree(pointers);
275075b47033SQu Wenruo kfree(unmap_array);
27516bfd0133SQu Wenruo return ret;
27525a6ac9eaSMiao Xie }
27535a6ac9eaSMiao Xie
scrub_assemble_read_bios(struct btrfs_raid_bio * rbio)275452f0c198SChristoph Hellwig static int scrub_assemble_read_bios(struct btrfs_raid_bio *rbio)
27555a6ac9eaSMiao Xie {
275652f0c198SChristoph Hellwig struct bio_list bio_list = BIO_EMPTY_LIST;
2757cb3450b7SQu Wenruo int total_sector_nr;
2758cb3450b7SQu Wenruo int ret = 0;
27595a6ac9eaSMiao Xie
27601c10702eSQu Wenruo /* Build a list of bios to read all the missing parts. */
27611c10702eSQu Wenruo for (total_sector_nr = 0; total_sector_nr < rbio->nr_sectors;
27621c10702eSQu Wenruo total_sector_nr++) {
27631c10702eSQu Wenruo int sectornr = total_sector_nr % rbio->stripe_nsectors;
27641c10702eSQu Wenruo int stripe = total_sector_nr / rbio->stripe_nsectors;
27653e77605dSQu Wenruo struct sector_ptr *sector;
27661c10702eSQu Wenruo
27671c10702eSQu Wenruo /* No data in the vertical stripe, no need to read. */
27681c10702eSQu Wenruo if (!test_bit(sectornr, &rbio->dbitmap))
27691c10702eSQu Wenruo continue;
27701c10702eSQu Wenruo
27715a6ac9eaSMiao Xie /*
27721c10702eSQu Wenruo * We want to find all the sectors missing from the rbio and
27731c10702eSQu Wenruo * read them from the disk. If sector_in_rbio() finds a sector
27741c10702eSQu Wenruo * in the bio list we don't need to read it off the stripe.
27755a6ac9eaSMiao Xie */
27763e77605dSQu Wenruo sector = sector_in_rbio(rbio, stripe, sectornr, 1);
27773e77605dSQu Wenruo if (sector)
27785a6ac9eaSMiao Xie continue;
27795a6ac9eaSMiao Xie
27803e77605dSQu Wenruo sector = rbio_stripe_sector(rbio, stripe, sectornr);
27815a6ac9eaSMiao Xie /*
27821c10702eSQu Wenruo * The bio cache may have handed us an uptodate sector. If so,
27831c10702eSQu Wenruo * use it.
27845a6ac9eaSMiao Xie */
27853e77605dSQu Wenruo if (sector->uptodate)
27865a6ac9eaSMiao Xie continue;
27875a6ac9eaSMiao Xie
278852f0c198SChristoph Hellwig ret = rbio_add_io_sector(rbio, &bio_list, sector, stripe,
2789ff18a4afSChristoph Hellwig sectornr, REQ_OP_READ);
279052f0c198SChristoph Hellwig if (ret) {
279152f0c198SChristoph Hellwig bio_list_put(&bio_list);
2792cb3450b7SQu Wenruo return ret;
2793cb3450b7SQu Wenruo }
279452f0c198SChristoph Hellwig }
279552f0c198SChristoph Hellwig
279652f0c198SChristoph Hellwig submit_read_wait_bio_list(rbio, &bio_list);
279752f0c198SChristoph Hellwig return 0;
279852f0c198SChristoph Hellwig }
2799cb3450b7SQu Wenruo
scrub_rbio(struct btrfs_raid_bio * rbio)280008241d3cSChristoph Hellwig static void scrub_rbio(struct btrfs_raid_bio *rbio)
2801cb3450b7SQu Wenruo {
2802ad3daf1cSQu Wenruo int sector_nr;
2803cb3450b7SQu Wenruo int ret;
2804cb3450b7SQu Wenruo
2805cb3450b7SQu Wenruo ret = alloc_rbio_essential_pages(rbio);
2806cb3450b7SQu Wenruo if (ret)
280708241d3cSChristoph Hellwig goto out;
2808cb3450b7SQu Wenruo
28092942a50dSQu Wenruo bitmap_clear(rbio->error_bitmap, 0, rbio->nr_sectors);
28102942a50dSQu Wenruo
281152f0c198SChristoph Hellwig ret = scrub_assemble_read_bios(rbio);
2812cb3450b7SQu Wenruo if (ret < 0)
281308241d3cSChristoph Hellwig goto out;
28146bfd0133SQu Wenruo
281575b47033SQu Wenruo /* We may have some failures, recover the failed sectors first. */
28166bfd0133SQu Wenruo ret = recover_scrub_rbio(rbio);
28176bfd0133SQu Wenruo if (ret < 0)
281808241d3cSChristoph Hellwig goto out;
28196bfd0133SQu Wenruo
28205a6ac9eaSMiao Xie /*
28216bfd0133SQu Wenruo * We have every sector properly prepared. Can finish the scrub
28226bfd0133SQu Wenruo * and writeback the good content.
28235a6ac9eaSMiao Xie */
2824486c737fSQu Wenruo ret = finish_parity_scrub(rbio);
28256bfd0133SQu Wenruo wait_event(rbio->io_wait, atomic_read(&rbio->stripes_pending) == 0);
2826ad3daf1cSQu Wenruo for (sector_nr = 0; sector_nr < rbio->stripe_nsectors; sector_nr++) {
2827ad3daf1cSQu Wenruo int found_errors;
2828ad3daf1cSQu Wenruo
2829ad3daf1cSQu Wenruo found_errors = get_rbio_veritical_errors(rbio, sector_nr, NULL, NULL);
2830ad3daf1cSQu Wenruo if (found_errors > rbio->bioc->max_errors) {
28316bfd0133SQu Wenruo ret = -EIO;
2832ad3daf1cSQu Wenruo break;
2833ad3daf1cSQu Wenruo }
2834ad3daf1cSQu Wenruo }
283508241d3cSChristoph Hellwig out:
283608241d3cSChristoph Hellwig rbio_orig_end_io(rbio, errno_to_blk_status(ret));
28375a6ac9eaSMiao Xie }
28385a6ac9eaSMiao Xie
scrub_rbio_work_locked(struct work_struct * work)28396bfd0133SQu Wenruo static void scrub_rbio_work_locked(struct work_struct *work)
28405a6ac9eaSMiao Xie {
284108241d3cSChristoph Hellwig scrub_rbio(container_of(work, struct btrfs_raid_bio, work));
28425a6ac9eaSMiao Xie }
28435a6ac9eaSMiao Xie
raid56_parity_submit_scrub_rbio(struct btrfs_raid_bio * rbio)28445a6ac9eaSMiao Xie void raid56_parity_submit_scrub_rbio(struct btrfs_raid_bio *rbio)
28455a6ac9eaSMiao Xie {
28465a6ac9eaSMiao Xie if (!lock_stripe_add(rbio))
28476bfd0133SQu Wenruo start_async_work(rbio, scrub_rbio_work_locked);
28485a6ac9eaSMiao Xie }
284994ead93eSQu Wenruo
285094ead93eSQu Wenruo /*
285194ead93eSQu Wenruo * This is for scrub call sites where we already have correct data contents.
285294ead93eSQu Wenruo * This allows us to avoid reading data stripes again.
285394ead93eSQu Wenruo *
285494ead93eSQu Wenruo * Unfortunately here we have to do page copy, other than reusing the pages.
285594ead93eSQu Wenruo * This is due to the fact rbio has its own page management for its cache.
285694ead93eSQu Wenruo */
raid56_parity_cache_data_pages(struct btrfs_raid_bio * rbio,struct page ** data_pages,u64 data_logical)285794ead93eSQu Wenruo void raid56_parity_cache_data_pages(struct btrfs_raid_bio *rbio,
285894ead93eSQu Wenruo struct page **data_pages, u64 data_logical)
285994ead93eSQu Wenruo {
286094ead93eSQu Wenruo const u64 offset_in_full_stripe = data_logical -
286194ead93eSQu Wenruo rbio->bioc->full_stripe_logical;
286294ead93eSQu Wenruo const int page_index = offset_in_full_stripe >> PAGE_SHIFT;
286394ead93eSQu Wenruo const u32 sectorsize = rbio->bioc->fs_info->sectorsize;
286494ead93eSQu Wenruo const u32 sectors_per_page = PAGE_SIZE / sectorsize;
286594ead93eSQu Wenruo int ret;
286694ead93eSQu Wenruo
286794ead93eSQu Wenruo /*
286894ead93eSQu Wenruo * If we hit ENOMEM temporarily, but later at
286994ead93eSQu Wenruo * raid56_parity_submit_scrub_rbio() time it succeeded, we just do
287094ead93eSQu Wenruo * the extra read, not a big deal.
287194ead93eSQu Wenruo *
287294ead93eSQu Wenruo * If we hit ENOMEM later at raid56_parity_submit_scrub_rbio() time,
287394ead93eSQu Wenruo * the bio would got proper error number set.
287494ead93eSQu Wenruo */
287594ead93eSQu Wenruo ret = alloc_rbio_data_pages(rbio);
287694ead93eSQu Wenruo if (ret < 0)
287794ead93eSQu Wenruo return;
287894ead93eSQu Wenruo
287994ead93eSQu Wenruo /* data_logical must be at stripe boundary and inside the full stripe. */
288094ead93eSQu Wenruo ASSERT(IS_ALIGNED(offset_in_full_stripe, BTRFS_STRIPE_LEN));
288194ead93eSQu Wenruo ASSERT(offset_in_full_stripe < (rbio->nr_data << BTRFS_STRIPE_LEN_SHIFT));
288294ead93eSQu Wenruo
288394ead93eSQu Wenruo for (int page_nr = 0; page_nr < (BTRFS_STRIPE_LEN >> PAGE_SHIFT); page_nr++) {
288494ead93eSQu Wenruo struct page *dst = rbio->stripe_pages[page_nr + page_index];
288594ead93eSQu Wenruo struct page *src = data_pages[page_nr];
288694ead93eSQu Wenruo
288794ead93eSQu Wenruo memcpy_page(dst, 0, src, 0, PAGE_SIZE);
288894ead93eSQu Wenruo for (int sector_nr = sectors_per_page * page_index;
288994ead93eSQu Wenruo sector_nr < sectors_per_page * (page_index + 1);
289094ead93eSQu Wenruo sector_nr++)
289194ead93eSQu Wenruo rbio->stripe_sectors[sector_nr].uptodate = true;
289294ead93eSQu Wenruo }
289394ead93eSQu Wenruo }
2894