Lines Matching refs:sh
26 * (in add_stripe_bio) we update the in-memory bitmap and record in sh->bm_seq
120 static inline int raid6_d0(struct stripe_head *sh)
122 if (sh->ddf_layout)
126 if (sh->qd_idx == sh->disks - 1)
129 return sh->qd_idx + 1;
142 static int raid6_idx_to_slot(int idx, struct stripe_head *sh,
147 if (sh->ddf_layout)
149 if (idx == sh->pd_idx)
151 if (idx == sh->qd_idx)
153 if (!sh->ddf_layout)
160 static int stripe_operations_active(struct stripe_head *sh)
162 return sh->check_state || sh->reconstruct_state ||
163 test_bit(STRIPE_BIOFILL_RUN, &sh->state) ||
164 test_bit(STRIPE_COMPUTE_RUN, &sh->state);
167 static bool stripe_is_lowprio(struct stripe_head *sh)
169 return (test_bit(STRIPE_R5C_FULL_STRIPE, &sh->state) ||
170 test_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state)) &&
171 !test_bit(STRIPE_R5C_CACHING, &sh->state);
174 static void raid5_wakeup_stripe_thread(struct stripe_head *sh)
175 __must_hold(&sh->raid_conf->device_lock)
177 struct r5conf *conf = sh->raid_conf;
180 int i, cpu = sh->cpu;
184 sh->cpu = cpu;
187 if (list_empty(&sh->lru)) {
190 if (stripe_is_lowprio(sh))
191 list_add_tail(&sh->lru, &group->loprio_list);
193 list_add_tail(&sh->lru, &group->handle_list);
195 sh->group = group;
203 group = conf->worker_groups + cpu_to_group(sh->cpu);
207 queue_work_on(sh->cpu, raid5_wq, &group->workers[0].work);
214 queue_work_on(sh->cpu, raid5_wq,
221 static void do_release_stripe(struct r5conf *conf, struct stripe_head *sh,
228 BUG_ON(!list_empty(&sh->lru));
232 for (i = sh->disks; i--; )
233 if (test_bit(R5_InJournal, &sh->dev[i].flags))
242 if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state) ||
244 !test_bit(STRIPE_HANDLE, &sh->state) && injournal != 0)) {
245 if (test_bit(STRIPE_R5C_CACHING, &sh->state))
246 r5c_make_stripe_write_out(sh);
247 set_bit(STRIPE_HANDLE, &sh->state);
250 if (test_bit(STRIPE_HANDLE, &sh->state)) {
251 if (test_bit(STRIPE_DELAYED, &sh->state) &&
252 !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
253 list_add_tail(&sh->lru, &conf->delayed_list);
254 else if (test_bit(STRIPE_BIT_DELAY, &sh->state) &&
255 sh->bm_seq - conf->seq_write > 0)
256 list_add_tail(&sh->lru, &conf->bitmap_list);
258 clear_bit(STRIPE_DELAYED, &sh->state);
259 clear_bit(STRIPE_BIT_DELAY, &sh->state);
261 if (stripe_is_lowprio(sh))
262 list_add_tail(&sh->lru,
265 list_add_tail(&sh->lru,
268 raid5_wakeup_stripe_thread(sh);
274 BUG_ON(stripe_operations_active(sh));
275 if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
280 if (!test_bit(STRIPE_EXPANDING, &sh->state)) {
282 list_add_tail(&sh->lru, temp_inactive_list);
284 WARN_ON(test_bit(R5_InJournal, &sh->dev[sh->pd_idx].flags));
286 list_add_tail(&sh->lru, temp_inactive_list);
289 if (!test_and_set_bit(STRIPE_R5C_FULL_STRIPE, &sh->state))
291 if (test_and_clear_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state))
293 list_add_tail(&sh->lru, &conf->r5c_full_stripe_list);
301 list_add_tail(&sh->lru, &conf->r5c_partial_stripe_list);
307 static void __release_stripe(struct r5conf *conf, struct stripe_head *sh,
311 if (atomic_dec_and_test(&sh->count))
312 do_release_stripe(conf, sh, temp_inactive_list);
368 struct stripe_head *sh, *t;
374 llist_for_each_entry_safe(sh, t, head, release_list) {
377 /* sh could be readded after STRIPE_ON_RELEASE_LIST is cleard */
379 clear_bit(STRIPE_ON_RELEASE_LIST, &sh->state);
385 hash = sh->hash_lock_index;
386 __release_stripe(conf, sh, &temp_inactive_list[hash]);
393 void raid5_release_stripe(struct stripe_head *sh)
395 struct r5conf *conf = sh->raid_conf;
403 if (atomic_add_unless(&sh->count, -1, 1))
407 test_and_set_bit(STRIPE_ON_RELEASE_LIST, &sh->state))
409 wakeup = llist_add(&sh->release_list, &conf->released_stripes);
415 if (atomic_dec_and_lock_irqsave(&sh->count, &conf->device_lock, flags)) {
417 hash = sh->hash_lock_index;
418 do_release_stripe(conf, sh, &list);
424 static inline void remove_hash(struct stripe_head *sh)
427 (unsigned long long)sh->sector);
429 hlist_del_init(&sh->hash);
432 static inline void insert_hash(struct r5conf *conf, struct stripe_head *sh)
434 struct hlist_head *hp = stripe_hash(conf, sh->sector);
437 (unsigned long long)sh->sector);
439 hlist_add_head(&sh->hash, hp);
445 struct stripe_head *sh = NULL;
451 sh = list_entry(first, struct stripe_head, lru);
453 remove_hash(sh);
455 BUG_ON(hash != sh->hash_lock_index);
459 return sh;
463 static void free_stripe_pages(struct stripe_head *sh)
469 if (!sh->pages)
472 for (i = 0; i < sh->nr_pages; i++) {
473 p = sh->pages[i];
476 sh->pages[i] = NULL;
480 static int alloc_stripe_pages(struct stripe_head *sh, gfp_t gfp)
485 for (i = 0; i < sh->nr_pages; i++) {
487 if (sh->pages[i])
492 free_stripe_pages(sh);
495 sh->pages[i] = p;
501 init_stripe_shared_pages(struct stripe_head *sh, struct r5conf *conf, int disks)
505 if (sh->pages)
508 /* Each of the sh->dev[i] need one conf->stripe_size */
512 sh->pages = kcalloc(nr_pages, sizeof(struct page *), GFP_KERNEL);
513 if (!sh->pages)
515 sh->nr_pages = nr_pages;
516 sh->stripes_per_page = cnt;
521 static void shrink_buffers(struct stripe_head *sh)
524 int num = sh->raid_conf->pool_size;
530 WARN_ON(sh->dev[i].page != sh->dev[i].orig_page);
531 p = sh->dev[i].page;
534 sh->dev[i].page = NULL;
539 sh->dev[i].page = NULL;
540 free_stripe_pages(sh); /* Free pages */
544 static int grow_buffers(struct stripe_head *sh, gfp_t gfp)
547 int num = sh->raid_conf->pool_size;
556 sh->dev[i].page = page;
557 sh->dev[i].orig_page = page;
558 sh->dev[i].offset = 0;
561 if (alloc_stripe_pages(sh, gfp))
565 sh->dev[i].page = raid5_get_dev_page(sh, i);
566 sh->dev[i].orig_page = sh->dev[i].page;
567 sh->dev[i].offset = raid5_get_page_offset(sh, i);
574 struct stripe_head *sh);
576 static void init_stripe(struct stripe_head *sh, sector_t sector, int previous)
578 struct r5conf *conf = sh->raid_conf;
581 BUG_ON(atomic_read(&sh->count) != 0);
582 BUG_ON(test_bit(STRIPE_HANDLE, &sh->state));
583 BUG_ON(stripe_operations_active(sh));
584 BUG_ON(sh->batch_head);
590 sh->generation = conf->generation - previous;
591 sh->disks = previous ? conf->previous_raid_disks : conf->raid_disks;
592 sh->sector = sector;
593 stripe_set_idx(sector, conf, previous, sh);
594 sh->state = 0;
596 for (i = sh->disks; i--; ) {
597 struct r5dev *dev = &sh->dev[i];
602 (unsigned long long)sh->sector, i, dev->toread,
608 dev->sector = raid5_compute_blocknr(sh, i, previous);
612 sh->overwrite_disks = 0;
613 insert_hash(conf, sh);
614 sh->cpu = smp_processor_id();
615 set_bit(STRIPE_BATCH_READY, &sh->state);
621 struct stripe_head *sh;
624 hlist_for_each_entry(sh, stripe_hash(conf, sector), hash)
625 if (sh->sector == sector && sh->generation == generation)
626 return sh;
635 struct stripe_head *sh;
637 sh = __find_stripe(conf, sector, generation);
638 if (!sh)
641 if (atomic_inc_not_zero(&sh->count))
642 return sh;
646 * be on a list (sh->lru). Must remove the stripe from the list that
651 if (!atomic_read(&sh->count)) {
652 if (!test_bit(STRIPE_HANDLE, &sh->state))
654 BUG_ON(list_empty(&sh->lru) &&
655 !test_bit(STRIPE_EXPANDING, &sh->state));
659 list_del_init(&sh->lru);
663 if (sh->group) {
664 sh->group->stripes_cnt--;
665 sh->group = NULL;
668 atomic_inc(&sh->count);
671 return sh;
807 struct stripe_head *sh;
834 sh = find_get_stripe(conf, sector, conf->generation - previous,
836 if (sh)
840 sh = get_free_stripe(conf, hash);
841 if (sh) {
843 init_stripe(sh, sector, previous);
844 atomic_inc(&sh->count);
871 return sh;
874 static bool is_full_stripe_write(struct stripe_head *sh)
876 BUG_ON(sh->overwrite_disks > (sh->disks - sh->raid_conf->max_degraded));
877 return sh->overwrite_disks == (sh->disks - sh->raid_conf->max_degraded);
902 static bool stripe_can_batch(struct stripe_head *sh)
904 struct r5conf *conf = sh->raid_conf;
908 return test_bit(STRIPE_BATCH_READY, &sh->state) &&
909 is_full_stripe_write(sh);
914 struct stripe_head *sh, struct stripe_head *last_sh)
922 tmp_sec = sh->sector;
925 head_sector = sh->sector - RAID5_STRIPE_SECTORS(conf);
942 lock_two_stripes(head, sh);
944 if (!stripe_can_batch(head) || !stripe_can_batch(sh))
947 if (sh->batch_head)
951 while (dd_idx == sh->pd_idx || dd_idx == sh->qd_idx)
953 if (head->dev[dd_idx].towrite->bi_opf != sh->dev[dd_idx].towrite->bi_opf ||
954 bio_op(head->dev[dd_idx].towrite) != bio_op(sh->dev[dd_idx].towrite))
971 sh->batch_head = head->batch_head;
977 list_add(&sh->batch_list, &head->batch_list);
981 sh->batch_head = head->batch_head;
983 list_add_tail(&sh->batch_list, &head->batch_list);
987 if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
992 if (test_and_clear_bit(STRIPE_BIT_DELAY, &sh->state)) {
993 int seq = sh->bm_seq;
994 if (test_bit(STRIPE_BIT_DELAY, &sh->batch_head->state) &&
995 sh->batch_head->bm_seq > seq)
996 seq = sh->batch_head->bm_seq;
997 set_bit(STRIPE_BIT_DELAY, &sh->batch_head->state);
998 sh->batch_head->bm_seq = seq;
1001 atomic_inc(&sh->count);
1003 unlock_two_stripes(head, sh);
1011 static int use_new_offset(struct r5conf *conf, struct stripe_head *sh)
1021 if (sh->generation == conf->generation - 1)
1138 static void ops_run_io(struct stripe_head *sh, struct stripe_head_state *s)
1140 struct r5conf *conf = sh->raid_conf;
1141 int i, disks = sh->disks;
1142 struct stripe_head *head_sh = sh;
1149 if (log_stripe(sh, s) == 0)
1161 sh = head_sh;
1162 if (test_and_clear_bit(R5_Wantwrite, &sh->dev[i].flags)) {
1164 if (test_and_clear_bit(R5_WantFUA, &sh->dev[i].flags))
1166 if (test_bit(R5_Discard, &sh->dev[i].flags))
1168 } else if (test_and_clear_bit(R5_Wantread, &sh->dev[i].flags))
1171 &sh->dev[i].flags)) {
1176 if (test_and_clear_bit(R5_SyncIO, &sh->dev[i].flags))
1180 dev = &sh->dev[i];
1213 int bad = rdev_has_badblock(rdev, sh->sector,
1247 set_bit(STRIPE_IO_STARTED, &sh->state);
1253 bi->bi_private = sh;
1256 __func__, (unsigned long long)sh->sector,
1258 atomic_inc(&sh->count);
1259 if (sh != head_sh)
1261 if (use_new_offset(conf, sh))
1262 bi->bi_iter.bi_sector = (sh->sector
1265 bi->bi_iter.bi_sector = (sh->sector
1270 if (test_bit(R5_SkipCopy, &sh->dev[i].flags))
1271 WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags));
1274 test_bit(R5_InJournal, &sh->dev[i].flags))
1280 sh->dev[i].vec.bv_page = sh->dev[i].orig_page;
1282 sh->dev[i].vec.bv_page = sh->dev[i].page;
1285 bi->bi_io_vec[0].bv_offset = sh->dev[i].offset;
1294 set_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags);
1296 mddev_trace_remap(conf->mddev, bi, sh->dev[i].sector);
1307 set_bit(STRIPE_IO_STARTED, &sh->state);
1312 rbi->bi_private = sh;
1316 __func__, (unsigned long long)sh->sector,
1318 atomic_inc(&sh->count);
1319 if (sh != head_sh)
1321 if (use_new_offset(conf, sh))
1322 rbi->bi_iter.bi_sector = (sh->sector
1325 rbi->bi_iter.bi_sector = (sh->sector
1327 if (test_bit(R5_SkipCopy, &sh->dev[i].flags))
1328 WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags));
1329 sh->dev[i].rvec.bv_page = sh->dev[i].page;
1332 rbi->bi_io_vec[0].bv_offset = sh->dev[i].offset;
1340 mddev_trace_remap(conf->mddev, rbi, sh->dev[i].sector);
1348 bi->bi_opf, i, (unsigned long long)sh->sector);
1349 clear_bit(R5_LOCKED, &sh->dev[i].flags);
1350 set_bit(STRIPE_HANDLE, &sh->state);
1355 sh = list_first_entry(&sh->batch_list, struct stripe_head,
1357 if (sh != head_sh)
1368 struct stripe_head *sh, int no_skipcopy)
1376 struct r5conf *conf = sh->raid_conf;
1432 struct stripe_head *sh = stripe_head_ref;
1434 struct r5conf *conf = sh->raid_conf;
1437 (unsigned long long)sh->sector);
1440 for (i = sh->disks; i--; ) {
1441 struct r5dev *dev = &sh->dev[i];
1462 clear_bit(STRIPE_BIOFILL_RUN, &sh->state);
1464 set_bit(STRIPE_HANDLE, &sh->state);
1465 raid5_release_stripe(sh);
1468 static void ops_run_biofill(struct stripe_head *sh)
1473 struct r5conf *conf = sh->raid_conf;
1475 BUG_ON(sh->batch_head);
1477 (unsigned long long)sh->sector);
1479 for (i = sh->disks; i--; ) {
1480 struct r5dev *dev = &sh->dev[i];
1483 spin_lock_irq(&sh->stripe_lock);
1486 spin_unlock_irq(&sh->stripe_lock);
1491 dev->sector, tx, sh, 0);
1497 atomic_inc(&sh->count);
1498 init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_biofill, sh, NULL);
1502 static void mark_target_uptodate(struct stripe_head *sh, int target)
1509 tgt = &sh->dev[target];
1517 struct stripe_head *sh = stripe_head_ref;
1520 (unsigned long long)sh->sector);
1523 mark_target_uptodate(sh, sh->ops.target);
1524 mark_target_uptodate(sh, sh->ops.target2);
1526 clear_bit(STRIPE_COMPUTE_RUN, &sh->state);
1527 if (sh->check_state == check_state_compute_run)
1528 sh->check_state = check_state_compute_result;
1529 set_bit(STRIPE_HANDLE, &sh->state);
1530 raid5_release_stripe(sh);
1540 static addr_conv_t *to_addr_conv(struct stripe_head *sh,
1543 return (void *) (to_addr_page(percpu, i) + sh->disks + 2);
1550 to_addr_offs(struct stripe_head *sh, struct raid5_percpu *percpu)
1552 return (unsigned int *) (to_addr_conv(sh, percpu, 0) + sh->disks + 2);
1556 ops_run_compute5(struct stripe_head *sh, struct raid5_percpu *percpu)
1558 int disks = sh->disks;
1560 unsigned int *off_srcs = to_addr_offs(sh, percpu);
1561 int target = sh->ops.target;
1562 struct r5dev *tgt = &sh->dev[target];
1570 BUG_ON(sh->batch_head);
1573 __func__, (unsigned long long)sh->sector, target);
1578 off_srcs[count] = sh->dev[i].offset;
1579 xor_srcs[count++] = sh->dev[i].page;
1583 atomic_inc(&sh->count);
1586 ops_complete_compute, sh, to_addr_conv(sh, percpu, 0));
1589 RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
1592 RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
1598 * @srcs - (struct page *) array of size sh->disks
1600 * @sh - stripe_head to parse
1609 struct stripe_head *sh,
1612 int disks = sh->disks;
1613 int syndrome_disks = sh->ddf_layout ? disks : (disks - 2);
1614 int d0_idx = raid6_d0(sh);
1624 int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks);
1625 struct r5dev *dev = &sh->dev[i];
1627 if (i == sh->qd_idx || i == sh->pd_idx ||
1636 srcs[slot] = sh->dev[i].orig_page;
1638 srcs[slot] = sh->dev[i].page;
1644 offs[slot] = sh->dev[i].offset;
1653 ops_run_compute6_1(struct stripe_head *sh, struct raid5_percpu *percpu)
1655 int disks = sh->disks;
1657 unsigned int *offs = to_addr_offs(sh, percpu);
1659 int qd_idx = sh->qd_idx;
1668 BUG_ON(sh->batch_head);
1669 if (sh->ops.target < 0)
1670 target = sh->ops.target2;
1671 else if (sh->ops.target2 < 0)
1672 target = sh->ops.target;
1678 __func__, (unsigned long long)sh->sector, target);
1680 tgt = &sh->dev[target];
1685 atomic_inc(&sh->count);
1688 count = set_syndrome_sources(blocks, offs, sh, SYNDROME_SRC_ALL);
1692 ops_complete_compute, sh,
1693 to_addr_conv(sh, percpu, 0));
1695 RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
1702 offs[count] = sh->dev[i].offset;
1703 blocks[count++] = sh->dev[i].page;
1707 NULL, ops_complete_compute, sh,
1708 to_addr_conv(sh, percpu, 0));
1710 RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
1717 ops_run_compute6_2(struct stripe_head *sh, struct raid5_percpu *percpu)
1719 int i, count, disks = sh->disks;
1720 int syndrome_disks = sh->ddf_layout ? disks : disks-2;
1721 int d0_idx = raid6_d0(sh);
1723 int target = sh->ops.target;
1724 int target2 = sh->ops.target2;
1725 struct r5dev *tgt = &sh->dev[target];
1726 struct r5dev *tgt2 = &sh->dev[target2];
1729 unsigned int *offs = to_addr_offs(sh, percpu);
1732 BUG_ON(sh->batch_head);
1734 __func__, (unsigned long long)sh->sector, target, target2);
1749 int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks);
1751 offs[slot] = sh->dev[i].offset;
1752 blocks[slot] = sh->dev[i].page;
1765 __func__, (unsigned long long)sh->sector, faila, failb);
1767 atomic_inc(&sh->count);
1774 ops_complete_compute, sh,
1775 to_addr_conv(sh, percpu, 0));
1777 RAID5_STRIPE_SIZE(sh->raid_conf),
1783 int qd_idx = sh->qd_idx;
1795 offs[count] = sh->dev[i].offset;
1796 blocks[count++] = sh->dev[i].page;
1798 dest = sh->dev[data_target].page;
1799 dest_off = sh->dev[data_target].offset;
1803 to_addr_conv(sh, percpu, 0));
1805 RAID5_STRIPE_SIZE(sh->raid_conf),
1808 count = set_syndrome_sources(blocks, offs, sh, SYNDROME_SRC_ALL);
1810 ops_complete_compute, sh,
1811 to_addr_conv(sh, percpu, 0));
1813 RAID5_STRIPE_SIZE(sh->raid_conf),
1818 ops_complete_compute, sh,
1819 to_addr_conv(sh, percpu, 0));
1823 RAID5_STRIPE_SIZE(sh->raid_conf),
1829 RAID5_STRIPE_SIZE(sh->raid_conf),
1838 struct stripe_head *sh = stripe_head_ref;
1841 (unsigned long long)sh->sector);
1843 if (r5c_is_writeback(sh->raid_conf->log))
1848 r5c_release_extra_page(sh);
1852 ops_run_prexor5(struct stripe_head *sh, struct raid5_percpu *percpu,
1855 int disks = sh->disks;
1857 unsigned int *off_srcs = to_addr_offs(sh, percpu);
1858 int count = 0, pd_idx = sh->pd_idx, i;
1862 unsigned int off_dest = off_srcs[count] = sh->dev[pd_idx].offset;
1863 struct page *xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;
1865 BUG_ON(sh->batch_head);
1867 (unsigned long long)sh->sector);
1870 struct r5dev *dev = &sh->dev[i];
1886 ops_complete_prexor, sh, to_addr_conv(sh, percpu, 0));
1888 RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
1894 ops_run_prexor6(struct stripe_head *sh, struct raid5_percpu *percpu,
1898 unsigned int *offs = to_addr_offs(sh, percpu);
1903 (unsigned long long)sh->sector);
1905 count = set_syndrome_sources(blocks, offs, sh, SYNDROME_SRC_WANT_DRAIN);
1908 ops_complete_prexor, sh, to_addr_conv(sh, percpu, 0));
1910 RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
1916 ops_run_biodrain(struct stripe_head *sh, struct dma_async_tx_descriptor *tx)
1918 struct r5conf *conf = sh->raid_conf;
1919 int disks = sh->disks;
1921 struct stripe_head *head_sh = sh;
1924 (unsigned long long)sh->sector);
1930 sh = head_sh;
1935 dev = &sh->dev[i];
1941 spin_lock_irq(&sh->stripe_lock);
1944 sh->overwrite_disks = 0;
1947 spin_unlock_irq(&sh->stripe_lock);
1961 dev->sector, tx, sh,
1974 sh = list_first_entry(&sh->batch_list,
1977 if (sh == head_sh)
1989 struct stripe_head *sh = stripe_head_ref;
1990 int disks = sh->disks;
1991 int pd_idx = sh->pd_idx;
1992 int qd_idx = sh->qd_idx;
1997 (unsigned long long)sh->sector);
2000 fua |= test_bit(R5_WantFUA, &sh->dev[i].flags);
2001 sync |= test_bit(R5_SyncIO, &sh->dev[i].flags);
2002 discard |= test_bit(R5_Discard, &sh->dev[i].flags);
2006 struct r5dev *dev = &sh->dev[i];
2011 if (test_bit(STRIPE_EXPAND_READY, &sh->state))
2021 if (sh->reconstruct_state == reconstruct_state_drain_run)
2022 sh->reconstruct_state = reconstruct_state_drain_result;
2023 else if (sh->reconstruct_state == reconstruct_state_prexor_drain_run)
2024 sh->reconstruct_state = reconstruct_state_prexor_drain_result;
2026 BUG_ON(sh->reconstruct_state != reconstruct_state_run);
2027 sh->reconstruct_state = reconstruct_state_result;
2030 set_bit(STRIPE_HANDLE, &sh->state);
2031 raid5_release_stripe(sh);
2035 ops_run_reconstruct5(struct stripe_head *sh, struct raid5_percpu *percpu,
2038 int disks = sh->disks;
2042 int count, pd_idx = sh->pd_idx, i;
2048 struct stripe_head *head_sh = sh;
2052 (unsigned long long)sh->sector);
2054 for (i = 0; i < sh->disks; i++) {
2057 if (!test_bit(R5_Discard, &sh->dev[i].flags))
2060 if (i >= sh->disks) {
2061 atomic_inc(&sh->count);
2062 set_bit(R5_Discard, &sh->dev[pd_idx].flags);
2063 ops_complete_reconstruct(sh);
2069 off_srcs = to_addr_offs(sh, percpu);
2075 off_dest = off_srcs[count] = sh->dev[pd_idx].offset;
2076 xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;
2078 struct r5dev *dev = &sh->dev[i];
2086 xor_dest = sh->dev[pd_idx].page;
2087 off_dest = sh->dev[pd_idx].offset;
2089 struct r5dev *dev = &sh->dev[i];
2103 list_first_entry(&sh->batch_list,
2111 to_addr_conv(sh, percpu, j));
2115 to_addr_conv(sh, percpu, j));
2120 RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
2123 RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
2126 sh = list_first_entry(&sh->batch_list, struct stripe_head,
2133 ops_run_reconstruct6(struct stripe_head *sh, struct raid5_percpu *percpu,
2140 struct stripe_head *head_sh = sh;
2145 pr_debug("%s: stripe %llu\n", __func__, (unsigned long long)sh->sector);
2147 for (i = 0; i < sh->disks; i++) {
2148 if (sh->pd_idx == i || sh->qd_idx == i)
2150 if (!test_bit(R5_Discard, &sh->dev[i].flags))
2153 if (i >= sh->disks) {
2154 atomic_inc(&sh->count);
2155 set_bit(R5_Discard, &sh->dev[sh->pd_idx].flags);
2156 set_bit(R5_Discard, &sh->dev[sh->qd_idx].flags);
2157 ops_complete_reconstruct(sh);
2163 offs = to_addr_offs(sh, percpu);
2165 if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) {
2173 count = set_syndrome_sources(blocks, offs, sh, synflags);
2175 list_first_entry(&sh->batch_list,
2181 head_sh, to_addr_conv(sh, percpu, j));
2184 to_addr_conv(sh, percpu, j));
2186 RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
2189 sh = list_first_entry(&sh->batch_list, struct stripe_head,
2197 struct stripe_head *sh = stripe_head_ref;
2200 (unsigned long long)sh->sector);
2202 sh->check_state = check_state_check_result;
2203 set_bit(STRIPE_HANDLE, &sh->state);
2204 raid5_release_stripe(sh);
2207 static void ops_run_check_p(struct stripe_head *sh, struct raid5_percpu *percpu)
2209 int disks = sh->disks;
2210 int pd_idx = sh->pd_idx;
2211 int qd_idx = sh->qd_idx;
2215 unsigned int *off_srcs = to_addr_offs(sh, percpu);
2222 (unsigned long long)sh->sector);
2224 BUG_ON(sh->batch_head);
2226 xor_dest = sh->dev[pd_idx].page;
2227 off_dest = sh->dev[pd_idx].offset;
2233 off_srcs[count] = sh->dev[i].offset;
2234 xor_srcs[count++] = sh->dev[i].page;
2238 to_addr_conv(sh, percpu, 0));
2240 RAID5_STRIPE_SIZE(sh->raid_conf),
2241 &sh->ops.zero_sum_result, &submit);
2243 atomic_inc(&sh->count);
2244 init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_check, sh, NULL);
2248 static void ops_run_check_pq(struct stripe_head *sh, struct raid5_percpu *percpu, int checkp)
2251 unsigned int *offs = to_addr_offs(sh, percpu);
2256 (unsigned long long)sh->sector, checkp);
2258 BUG_ON(sh->batch_head);
2259 count = set_syndrome_sources(srcs, offs, sh, SYNDROME_SRC_ALL);
2263 atomic_inc(&sh->count);
2265 sh, to_addr_conv(sh, percpu, 0));
2267 RAID5_STRIPE_SIZE(sh->raid_conf),
2268 &sh->ops.zero_sum_result, percpu->spare_page, 0, &submit);
2271 static void raid_run_ops(struct stripe_head *sh, unsigned long ops_request)
2273 int overlap_clear = 0, i, disks = sh->disks;
2275 struct r5conf *conf = sh->raid_conf;
2282 ops_run_biofill(sh);
2288 tx = ops_run_compute5(sh, percpu);
2290 if (sh->ops.target2 < 0 || sh->ops.target < 0)
2291 tx = ops_run_compute6_1(sh, percpu);
2293 tx = ops_run_compute6_2(sh, percpu);
2302 tx = ops_run_prexor5(sh, percpu, tx);
2304 tx = ops_run_prexor6(sh, percpu, tx);
2308 tx = ops_run_partial_parity(sh, percpu, tx);
2311 tx = ops_run_biodrain(sh, tx);
2317 ops_run_reconstruct5(sh, percpu, tx);
2319 ops_run_reconstruct6(sh, percpu, tx);
2323 if (sh->check_state == check_state_run)
2324 ops_run_check_p(sh, percpu);
2325 else if (sh->check_state == check_state_run_q)
2326 ops_run_check_pq(sh, percpu, 0);
2327 else if (sh->check_state == check_state_run_pq)
2328 ops_run_check_pq(sh, percpu, 1);
2333 if (overlap_clear && !sh->batch_head) {
2335 struct r5dev *dev = &sh->dev[i];
2343 static void free_stripe(struct kmem_cache *sc, struct stripe_head *sh)
2346 kfree(sh->pages);
2348 if (sh->ppl_page)
2349 __free_page(sh->ppl_page);
2350 kmem_cache_free(sc, sh);
2356 struct stripe_head *sh;
2358 sh = kmem_cache_zalloc(sc, gfp);
2359 if (sh) {
2360 spin_lock_init(&sh->stripe_lock);
2361 spin_lock_init(&sh->batch_lock);
2362 INIT_LIST_HEAD(&sh->batch_list);
2363 INIT_LIST_HEAD(&sh->lru);
2364 INIT_LIST_HEAD(&sh->r5c);
2365 INIT_LIST_HEAD(&sh->log_list);
2366 atomic_set(&sh->count, 1);
2367 sh->raid_conf = conf;
2368 sh->log_start = MaxSector;
2371 sh->ppl_page = alloc_page(gfp);
2372 if (!sh->ppl_page) {
2373 free_stripe(sc, sh);
2378 if (init_stripe_shared_pages(sh, conf, disks)) {
2379 free_stripe(sc, sh);
2384 return sh;
2388 struct stripe_head *sh;
2390 sh = alloc_stripe(conf->slab_cache, gfp, conf->pool_size, conf);
2391 if (!sh)
2394 if (grow_buffers(sh, gfp)) {
2395 shrink_buffers(sh);
2396 free_stripe(conf->slab_cache, sh);
2399 sh->hash_lock_index =
2404 raid5_release_stripe(sh);
2688 struct stripe_head *sh;
2692 sh = get_free_stripe(conf, hash);
2694 if (!sh)
2696 BUG_ON(atomic_read(&sh->count));
2697 shrink_buffers(sh);
2698 free_stripe(conf->slab_cache, sh);
2716 struct stripe_head *sh = bi->bi_private;
2717 struct r5conf *conf = sh->raid_conf;
2718 int disks = sh->disks, i;
2723 if (bi == &sh->dev[i].req)
2727 (unsigned long long)sh->sector, i, atomic_read(&sh->count),
2733 if (test_bit(R5_ReadRepl, &sh->dev[i].flags))
2743 if (use_new_offset(conf, sh))
2744 s = sh->sector + rdev->new_data_offset;
2746 s = sh->sector + rdev->data_offset;
2748 set_bit(R5_UPTODATE, &sh->dev[i].flags);
2749 if (test_bit(R5_ReadError, &sh->dev[i].flags)) {
2760 clear_bit(R5_ReadError, &sh->dev[i].flags);
2761 clear_bit(R5_ReWrite, &sh->dev[i].flags);
2762 } else if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags))
2763 clear_bit(R5_ReadNoMerge, &sh->dev[i].flags);
2765 if (test_bit(R5_InJournal, &sh->dev[i].flags))
2770 set_bit(R5_OrigPageUPTDODATE, &sh->dev[i].flags);
2778 clear_bit(R5_UPTODATE, &sh->dev[i].flags);
2781 if (test_bit(R5_ReadRepl, &sh->dev[i].flags))
2794 } else if (test_bit(R5_ReWrite, &sh->dev[i].flags)) {
2815 && !test_bit(R5_ReadNoMerge, &sh->dev[i].flags))
2818 if (sh->qd_idx >= 0 && sh->pd_idx == i)
2819 set_bit(R5_ReadError, &sh->dev[i].flags);
2820 else if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags)) {
2821 set_bit(R5_ReadError, &sh->dev[i].flags);
2822 clear_bit(R5_ReadNoMerge, &sh->dev[i].flags);
2824 set_bit(R5_ReadNoMerge, &sh->dev[i].flags);
2826 clear_bit(R5_ReadError, &sh->dev[i].flags);
2827 clear_bit(R5_ReWrite, &sh->dev[i].flags);
2831 rdev, sh->sector, RAID5_STRIPE_SECTORS(conf), 0)))
2837 clear_bit(R5_LOCKED, &sh->dev[i].flags);
2838 set_bit(STRIPE_HANDLE, &sh->state);
2839 raid5_release_stripe(sh);
2844 struct stripe_head *sh = bi->bi_private;
2845 struct r5conf *conf = sh->raid_conf;
2846 int disks = sh->disks, i;
2851 if (bi == &sh->dev[i].req) {
2855 if (bi == &sh->dev[i].rreq) {
2869 (unsigned long long)sh->sector, i, atomic_read(&sh->count),
2879 else if (rdev_has_badblock(rdev, sh->sector,
2881 set_bit(R5_MadeGoodRepl, &sh->dev[i].flags);
2885 set_bit(R5_WriteError, &sh->dev[i].flags);
2889 } else if (rdev_has_badblock(rdev, sh->sector,
2891 set_bit(R5_MadeGood, &sh->dev[i].flags);
2892 if (test_bit(R5_ReadError, &sh->dev[i].flags))
2897 set_bit(R5_ReWrite, &sh->dev[i].flags);
2902 if (sh->batch_head && bi->bi_status && !replacement)
2903 set_bit(STRIPE_BATCH_ERR, &sh->batch_head->state);
2906 if (!test_and_clear_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags))
2907 clear_bit(R5_LOCKED, &sh->dev[i].flags);
2908 set_bit(STRIPE_HANDLE, &sh->state);
2910 if (sh->batch_head && sh != sh->batch_head)
2911 raid5_release_stripe(sh->batch_head);
2912 raid5_release_stripe(sh);
2955 struct stripe_head *sh)
3143 if (sh) {
3144 sh->pd_idx = pd_idx;
3145 sh->qd_idx = qd_idx;
3146 sh->ddf_layout = ddf_layout;
3155 sector_t raid5_compute_blocknr(struct stripe_head *sh, int i, int previous)
3157 struct r5conf *conf = sh->raid_conf;
3158 int raid_disks = sh->disks;
3160 sector_t new_sector = sh->sector, check;
3175 if (i == sh->pd_idx)
3183 if (i > sh->pd_idx)
3188 if (i < sh->pd_idx)
3190 i -= (sh->pd_idx + 1);
3202 if (i == sh->qd_idx)
3209 if (sh->pd_idx == raid_disks-1)
3211 else if (i > sh->pd_idx)
3216 if (sh->pd_idx == raid_disks-1)
3220 if (i < sh->pd_idx)
3222 i -= (sh->pd_idx + 2);
3232 if (sh->pd_idx == 0)
3236 if (i < sh->pd_idx)
3238 i -= (sh->pd_idx + 1);
3243 if (i > sh->pd_idx)
3248 if (i < sh->pd_idx)
3250 i -= (sh->pd_idx + 1);
3266 if (check != sh->sector || dummy1 != dd_idx || sh2.pd_idx != sh->pd_idx
3267 || sh2.qd_idx != sh->qd_idx) {
3332 schedule_reconstruction(struct stripe_head *sh, struct stripe_head_state *s,
3335 int i, pd_idx = sh->pd_idx, qd_idx = sh->qd_idx, disks = sh->disks;
3336 struct r5conf *conf = sh->raid_conf;
3346 r5c_release_extra_page(sh);
3349 struct r5dev *dev = &sh->dev[i];
3370 sh->reconstruct_state = reconstruct_state_drain_run;
3373 sh->reconstruct_state = reconstruct_state_run;
3378 if (!test_and_set_bit(STRIPE_FULL_WRITE, &sh->state))
3381 BUG_ON(!(test_bit(R5_UPTODATE, &sh->dev[pd_idx].flags) ||
3382 test_bit(R5_Wantcompute, &sh->dev[pd_idx].flags)));
3384 (!(test_bit(R5_UPTODATE, &sh->dev[qd_idx].flags) ||
3385 test_bit(R5_Wantcompute, &sh->dev[qd_idx].flags))));
3388 struct r5dev *dev = &sh->dev[i];
3407 sh->reconstruct_state = reconstruct_state_prexor_drain_run;
3416 set_bit(R5_LOCKED, &sh->dev[pd_idx].flags);
3417 clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags);
3421 int qd_idx = sh->qd_idx;
3422 struct r5dev *dev = &sh->dev[qd_idx];
3429 if (raid5_has_ppl(sh->raid_conf) && sh->ppl_page &&
3431 !test_bit(STRIPE_FULL_WRITE, &sh->state) &&
3432 test_bit(R5_Insync, &sh->dev[pd_idx].flags))
3436 __func__, (unsigned long long)sh->sector,
3440 static bool stripe_bio_overlaps(struct stripe_head *sh, struct bio *bi,
3443 struct r5conf *conf = sh->raid_conf;
3447 bi->bi_iter.bi_sector, sh->sector);
3450 if (sh->batch_head)
3454 bip = &sh->dev[dd_idx].towrite;
3456 bip = &sh->dev[dd_idx].toread;
3481 for (i = 0; i < sh->disks; i++) {
3482 if (i != sh->pd_idx &&
3483 (i == dd_idx || sh->dev[i].towrite)) {
3484 sector = sh->dev[i].sector;
3500 static void __add_stripe_bio(struct stripe_head *sh, struct bio *bi,
3503 struct r5conf *conf = sh->raid_conf;
3508 bip = &sh->dev[dd_idx].towrite;
3512 bip = &sh->dev[dd_idx].toread;
3519 clear_bit(STRIPE_BATCH_READY, &sh->state);
3530 sector_t sector = sh->dev[dd_idx].sector;
3531 for (bi=sh->dev[dd_idx].towrite;
3532 sector < sh->dev[dd_idx].sector + RAID5_STRIPE_SECTORS(conf) &&
3534 bi = r5_next_bio(conf, bi, sh->dev[dd_idx].sector)) {
3538 if (sector >= sh->dev[dd_idx].sector + RAID5_STRIPE_SECTORS(conf))
3539 if (!test_and_set_bit(R5_OVERWRITE, &sh->dev[dd_idx].flags))
3540 sh->overwrite_disks++;
3544 (*bip)->bi_iter.bi_sector, sh->sector, dd_idx,
3545 sh->dev[dd_idx].sector);
3547 if (conf->mddev->bitmap && firstwrite && !sh->batch_head) {
3548 sh->bm_seq = conf->seq_flush+1;
3549 set_bit(STRIPE_BIT_DELAY, &sh->state);
3558 static bool add_stripe_bio(struct stripe_head *sh, struct bio *bi,
3561 spin_lock_irq(&sh->stripe_lock);
3563 if (stripe_bio_overlaps(sh, bi, dd_idx, forwrite)) {
3564 set_bit(R5_Overlap, &sh->dev[dd_idx].flags);
3565 spin_unlock_irq(&sh->stripe_lock);
3569 __add_stripe_bio(sh, bi, dd_idx, forwrite, previous);
3570 spin_unlock_irq(&sh->stripe_lock);
3577 struct stripe_head *sh)
3589 &dd_idx, sh);
3593 handle_failed_stripe(struct r5conf *conf, struct stripe_head *sh,
3597 BUG_ON(sh->batch_head);
3601 if (test_bit(R5_ReadError, &sh->dev[i].flags)) {
3612 sh->sector,
3618 spin_lock_irq(&sh->stripe_lock);
3620 bi = sh->dev[i].towrite;
3621 sh->dev[i].towrite = NULL;
3622 sh->overwrite_disks = 0;
3623 spin_unlock_irq(&sh->stripe_lock);
3625 log_stripe_write_finished(sh);
3627 if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
3628 wake_up_bit(&sh->dev[i].flags, R5_Overlap);
3631 sh->dev[i].sector + RAID5_STRIPE_SECTORS(conf)) {
3632 struct bio *nextbi = r5_next_bio(conf, bi, sh->dev[i].sector);
3639 bi = sh->dev[i].written;
3640 sh->dev[i].written = NULL;
3641 if (test_and_clear_bit(R5_SkipCopy, &sh->dev[i].flags)) {
3642 WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags));
3643 sh->dev[i].page = sh->dev[i].orig_page;
3647 sh->dev[i].sector + RAID5_STRIPE_SECTORS(conf)) {
3648 struct bio *bi2 = r5_next_bio(conf, bi, sh->dev[i].sector);
3658 if (!test_bit(R5_Wantfill, &sh->dev[i].flags) &&
3660 (!test_bit(R5_Insync, &sh->dev[i].flags) ||
3661 test_bit(R5_ReadError, &sh->dev[i].flags))) {
3662 spin_lock_irq(&sh->stripe_lock);
3663 bi = sh->dev[i].toread;
3664 sh->dev[i].toread = NULL;
3665 spin_unlock_irq(&sh->stripe_lock);
3666 if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
3667 wake_up_bit(&sh->dev[i].flags, R5_Overlap);
3671 sh->dev[i].sector + RAID5_STRIPE_SECTORS(conf)) {
3673 r5_next_bio(conf, bi, sh->dev[i].sector);
3682 clear_bit(R5_LOCKED, &sh->dev[i].flags);
3687 if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state))
3693 handle_failed_sync(struct r5conf *conf, struct stripe_head *sh,
3699 BUG_ON(sh->batch_head);
3700 clear_bit(STRIPE_SYNCING, &sh->state);
3701 if (test_and_clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags))
3702 wake_up_bit(&sh->dev[sh->pd_idx].flags, R5_Overlap);
3722 && !rdev_set_badblocks(rdev, sh->sector,
3730 && !rdev_set_badblocks(rdev, sh->sector,
3741 static int want_replace(struct stripe_head *sh, int disk_idx)
3746 rdev = sh->raid_conf->disks[disk_idx].replacement;
3750 && (rdev->recovery_offset <= sh->sector
3751 || rdev->mddev->recovery_cp <= sh->sector))
3756 static int need_this_block(struct stripe_head *sh, struct stripe_head_state *s,
3759 struct r5dev *dev = &sh->dev[disk_idx];
3760 struct r5dev *fdev[2] = { &sh->dev[s->failed_num[0]],
3761 &sh->dev[s->failed_num[1]] };
3763 bool force_rcw = (sh->raid_conf->rmw_level == PARITY_DISABLE_RMW);
3779 (s->replacing && want_replace(sh, disk_idx)))
3804 !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
3825 s->failed_num[i] == sh->pd_idx ||
3826 s->failed_num[i] == sh->qd_idx) &&
3843 sh->sector < sh->raid_conf->mddev->recovery_cp)
3847 if (s->failed_num[i] != sh->pd_idx &&
3848 s->failed_num[i] != sh->qd_idx &&
3863 static int fetch_block(struct stripe_head *sh, struct stripe_head_state *s,
3866 struct r5dev *dev = &sh->dev[disk_idx];
3869 if (need_this_block(sh, s, disk_idx, disks)) {
3875 BUG_ON(sh->batch_head);
3887 ((sh->qd_idx >= 0 && sh->pd_idx == disk_idx) ||
3894 (unsigned long long)sh->sector, disk_idx);
3895 set_bit(STRIPE_COMPUTE_RUN, &sh->state);
3898 sh->ops.target = disk_idx;
3899 sh->ops.target2 = -1; /* no 2nd target */
3918 &sh->dev[other].flags))
3923 (unsigned long long)sh->sector,
3925 set_bit(STRIPE_COMPUTE_RUN, &sh->state);
3927 set_bit(R5_Wantcompute, &sh->dev[disk_idx].flags);
3928 set_bit(R5_Wantcompute, &sh->dev[other].flags);
3929 sh->ops.target = disk_idx;
3930 sh->ops.target2 = other;
3949 static void handle_stripe_fill(struct stripe_head *sh,
3959 if (!test_bit(STRIPE_COMPUTE_RUN, &sh->state) && !sh->check_state &&
3960 !sh->reconstruct_state) {
3970 if (test_bit(STRIPE_R5C_CACHING, &sh->state))
3971 r5c_make_stripe_write_out(sh);
3976 if (fetch_block(sh, s, i, disks))
3980 set_bit(STRIPE_HANDLE, &sh->state);
3991 struct stripe_head *sh, int disks)
3996 struct stripe_head *head_sh = sh;
4000 if (sh->dev[i].written) {
4001 dev = &sh->dev[i];
4029 sh = list_first_entry(&sh->batch_list,
4032 if (sh != head_sh) {
4033 dev = &sh->dev[i];
4037 sh = head_sh;
4038 dev = &sh->dev[i];
4043 log_stripe_write_finished(sh);
4046 test_bit(R5_Discard, &sh->dev[sh->pd_idx].flags)) {
4048 clear_bit(R5_Discard, &sh->dev[sh->pd_idx].flags);
4049 clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags);
4050 if (sh->qd_idx >= 0) {
4051 clear_bit(R5_Discard, &sh->dev[sh->qd_idx].flags);
4052 clear_bit(R5_UPTODATE, &sh->dev[sh->qd_idx].flags);
4055 clear_bit(STRIPE_DISCARD, &sh->state);
4062 hash = sh->hash_lock_index;
4064 remove_hash(sh);
4067 sh = list_first_entry(&sh->batch_list,
4069 if (sh != head_sh)
4072 sh = head_sh;
4074 if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state))
4075 set_bit(STRIPE_HANDLE, &sh->state);
4079 if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state))
4103 struct stripe_head *sh,
4118 (recovery_cp < MaxSector && sh->sector >= recovery_cp &&
4124 pr_debug("force RCW rmw_level=%u, recovery_cp=%llu sh->sector=%llu\n",
4126 (unsigned long long)sh->sector);
4129 struct r5dev *dev = &sh->dev[i];
4131 i == sh->pd_idx || i == sh->qd_idx ||
4143 i != sh->pd_idx && i != sh->qd_idx &&
4155 (unsigned long long)sh->sector, sh->state, rmw, rcw);
4156 set_bit(STRIPE_HANDLE, &sh->state);
4160 sh->sector, rmw);
4163 struct r5dev *dev = &sh->dev[i];
4166 !test_bit(R5_LOCKED, &sh->dev[sh->pd_idx].flags)) {
4181 r5c_use_extra_page(sh);
4186 set_bit(STRIPE_DELAYED, &sh->state);
4193 struct r5dev *dev = &sh->dev[i];
4195 i == sh->pd_idx || i == sh->qd_idx ||
4202 &sh->state)) {
4209 set_bit(STRIPE_DELAYED, &sh->state);
4218 struct r5dev *dev = &sh->dev[i];
4220 i != sh->pd_idx && i != sh->qd_idx &&
4227 &sh->state)) {
4235 set_bit(STRIPE_DELAYED, &sh->state);
4241 (unsigned long long)sh->sector, rcw, qread,
4242 test_bit(STRIPE_DELAYED, &sh->state));
4246 !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
4247 set_bit(STRIPE_DELAYED, &sh->state);
4259 if ((s->req_compute || !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) &&
4261 !test_bit(STRIPE_BIT_DELAY, &sh->state)))
4262 schedule_reconstruction(sh, s, rcw == 0, 0);
4266 static void handle_parity_checks5(struct r5conf *conf, struct stripe_head *sh,
4271 BUG_ON(sh->batch_head);
4272 set_bit(STRIPE_HANDLE, &sh->state);
4274 switch (sh->check_state) {
4279 sh->check_state = check_state_run;
4281 clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags);
4285 dev = &sh->dev[s->failed_num[0]];
4288 sh->check_state = check_state_idle;
4290 dev = &sh->dev[sh->pd_idx];
4293 if (test_bit(STRIPE_INSYNC, &sh->state))
4304 set_bit(STRIPE_INSYNC, &sh->state);
4309 sh->check_state = check_state_idle;
4321 if ((sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) == 0)
4325 set_bit(STRIPE_INSYNC, &sh->state);
4330 set_bit(STRIPE_INSYNC, &sh->state);
4333 (unsigned long long) sh->sector,
4334 (unsigned long long) sh->sector +
4337 sh->check_state = check_state_compute_run;
4338 set_bit(STRIPE_COMPUTE_RUN, &sh->state);
4341 &sh->dev[sh->pd_idx].flags);
4342 sh->ops.target = sh->pd_idx;
4343 sh->ops.target2 = -1;
4352 __func__, sh->check_state,
4353 (unsigned long long) sh->sector);
4358 static void handle_parity_checks6(struct r5conf *conf, struct stripe_head *sh,
4362 int pd_idx = sh->pd_idx;
4363 int qd_idx = sh->qd_idx;
4366 BUG_ON(sh->batch_head);
4367 set_bit(STRIPE_HANDLE, &sh->state);
4377 switch (sh->check_state) {
4385 sh->check_state = check_state_run;
4391 if (sh->check_state == check_state_run)
4392 sh->check_state = check_state_run_pq;
4394 sh->check_state = check_state_run_q;
4398 sh->ops.zero_sum_result = 0;
4400 if (sh->check_state == check_state_run) {
4402 clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags);
4405 if (sh->check_state >= check_state_run &&
4406 sh->check_state <= check_state_run_pq) {
4418 sh->check_state = check_state_idle;
4421 if (test_bit(STRIPE_INSYNC, &sh->state))
4429 dev = &sh->dev[s->failed_num[1]];
4435 dev = &sh->dev[s->failed_num[0]];
4440 if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) {
4441 dev = &sh->dev[pd_idx];
4446 if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) {
4447 dev = &sh->dev[qd_idx];
4455 dev - (struct r5dev *) &sh->dev)) {
4461 set_bit(STRIPE_INSYNC, &sh->state);
4468 sh->check_state = check_state_idle;
4474 if (sh->ops.zero_sum_result == 0) {
4477 set_bit(STRIPE_INSYNC, &sh->state);
4483 sh->check_state = check_state_compute_result;
4494 set_bit(STRIPE_INSYNC, &sh->state);
4497 (unsigned long long) sh->sector,
4498 (unsigned long long) sh->sector +
4501 int *target = &sh->ops.target;
4503 sh->ops.target = -1;
4504 sh->ops.target2 = -1;
4505 sh->check_state = check_state_compute_run;
4506 set_bit(STRIPE_COMPUTE_RUN, &sh->state);
4508 if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) {
4510 &sh->dev[pd_idx].flags);
4512 target = &sh->ops.target2;
4515 if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) {
4517 &sh->dev[qd_idx].flags);
4528 __func__, sh->check_state,
4529 (unsigned long long) sh->sector);
4534 static void handle_stripe_expansion(struct r5conf *conf, struct stripe_head *sh)
4542 BUG_ON(sh->batch_head);
4543 clear_bit(STRIPE_EXPAND_SOURCE, &sh->state);
4544 for (i = 0; i < sh->disks; i++)
4545 if (i != sh->pd_idx && i != sh->qd_idx) {
4550 sector_t bn = raid5_compute_blocknr(sh, i, 1);
4571 sh->dev[i].page, sh2->dev[dd_idx].offset,
4572 sh->dev[i].offset, RAID5_STRIPE_SIZE(conf),
4607 static void analyse_stripe(struct stripe_head *sh, struct stripe_head_state *s)
4609 struct r5conf *conf = sh->raid_conf;
4610 int disks = sh->disks;
4617 s->expanding = test_bit(STRIPE_EXPAND_SOURCE, &sh->state) && !sh->batch_head;
4618 s->expanded = test_bit(STRIPE_EXPAND_READY, &sh->state) && !sh->batch_head;
4628 dev = &sh->dev[i];
4639 !test_bit(STRIPE_BIOFILL_RUN, &sh->state))
4668 rdev->recovery_offset >= sh->sector + RAID5_STRIPE_SECTORS(conf) &&
4669 !rdev_has_badblock(rdev, sh->sector,
4683 is_bad = rdev_has_badblock(rdev, sh->sector,
4709 else if (sh->sector + RAID5_STRIPE_SECTORS(conf) <= rdev->recovery_offset)
4778 if (test_bit(STRIPE_SYNCING, &sh->state)) {
4788 sh->sector >= conf->mddev->recovery_cp ||
4800 static int clear_batch_ready(struct stripe_head *sh)
4803 if (!test_and_clear_bit(STRIPE_BATCH_READY, &sh->state))
4804 return (sh->batch_head && sh->batch_head != sh);
4805 spin_lock(&sh->stripe_lock);
4806 if (!sh->batch_head) {
4807 spin_unlock(&sh->stripe_lock);
4815 if (sh->batch_head != sh) {
4816 spin_unlock(&sh->stripe_lock);
4819 spin_lock(&sh->batch_lock);
4820 list_for_each_entry(tmp, &sh->batch_list, batch_list)
4822 spin_unlock(&sh->batch_lock);
4823 spin_unlock(&sh->stripe_lock);
4835 struct stripe_head *sh, *next;
4838 list_for_each_entry_safe(sh, next, &head_sh->batch_list, batch_list) {
4840 list_del_init(&sh->batch_list);
4842 WARN_ONCE(sh->state & ((1 << STRIPE_ACTIVE) |
4853 "stripe state: %lx\n", sh->state);
4858 set_mask_bits(&sh->state, ~(STRIPE_EXPAND_SYNC_FLAGS |
4863 sh->check_state = head_sh->check_state;
4864 sh->reconstruct_state = head_sh->reconstruct_state;
4865 spin_lock_irq(&sh->stripe_lock);
4866 sh->batch_head = NULL;
4867 spin_unlock_irq(&sh->stripe_lock);
4868 for (i = 0; i < sh->disks; i++) {
4869 if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
4870 wake_up_bit(&sh->dev[i].flags, R5_Overlap);
4871 sh->dev[i].flags = head_sh->dev[i].flags &
4875 sh->state & handle_flags)
4876 set_bit(STRIPE_HANDLE, &sh->state);
4877 raid5_release_stripe(sh);
4889 static void handle_stripe(struct stripe_head *sh)
4892 struct r5conf *conf = sh->raid_conf;
4895 int disks = sh->disks;
4898 clear_bit(STRIPE_HANDLE, &sh->state);
4906 if (clear_batch_ready(sh))
4909 if (test_and_set_bit_lock(STRIPE_ACTIVE, &sh->state)) {
4912 set_bit(STRIPE_HANDLE, &sh->state);
4916 if (test_and_clear_bit(STRIPE_BATCH_ERR, &sh->state))
4917 break_stripe_batch_list(sh, 0);
4919 if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state) && !sh->batch_head) {
4920 spin_lock(&sh->stripe_lock);
4925 if (!test_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state) &&
4926 !test_bit(STRIPE_R5C_FULL_STRIPE, &sh->state) &&
4927 !test_bit(STRIPE_DISCARD, &sh->state) &&
4928 test_and_clear_bit(STRIPE_SYNC_REQUESTED, &sh->state)) {
4929 set_bit(STRIPE_SYNCING, &sh->state);
4930 clear_bit(STRIPE_INSYNC, &sh->state);
4931 clear_bit(STRIPE_REPLACED, &sh->state);
4933 spin_unlock(&sh->stripe_lock);
4935 clear_bit(STRIPE_DELAYED, &sh->state);
4939 (unsigned long long)sh->sector, sh->state,
4940 atomic_read(&sh->count), sh->pd_idx, sh->qd_idx,
4941 sh->check_state, sh->reconstruct_state);
4943 analyse_stripe(sh, &s);
4945 if (test_bit(STRIPE_LOG_TRAPPED, &sh->state))
4950 set_bit(STRIPE_HANDLE, &sh->state);
4957 set_bit(STRIPE_HANDLE, &sh->state);
4965 if (s.to_fill && !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) {
4967 set_bit(STRIPE_BIOFILL_RUN, &sh->state);
4983 sh->check_state = 0;
4984 sh->reconstruct_state = 0;
4985 break_stripe_batch_list(sh, 0);
4987 handle_failed_stripe(conf, sh, &s, disks);
4989 handle_failed_sync(conf, sh, &s);
4996 if (sh->reconstruct_state == reconstruct_state_prexor_drain_result)
4998 if (sh->reconstruct_state == reconstruct_state_drain_result ||
4999 sh->reconstruct_state == reconstruct_state_prexor_drain_result) {
5000 sh->reconstruct_state = reconstruct_state_idle;
5005 BUG_ON(!test_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags) &&
5006 !test_bit(R5_Discard, &sh->dev[sh->pd_idx].flags));
5007 BUG_ON(sh->qd_idx >= 0 &&
5008 !test_bit(R5_UPTODATE, &sh->dev[sh->qd_idx].flags) &&
5009 !test_bit(R5_Discard, &sh->dev[sh->qd_idx].flags));
5011 struct r5dev *dev = &sh->dev[i];
5013 (i == sh->pd_idx || i == sh->qd_idx ||
5023 ((i == sh->pd_idx || i == sh->qd_idx) &&
5025 set_bit(STRIPE_INSYNC, &sh->state);
5028 if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
5036 pdev = &sh->dev[sh->pd_idx];
5037 s.p_failed = (s.failed >= 1 && s.failed_num[0] == sh->pd_idx)
5038 || (s.failed >= 2 && s.failed_num[1] == sh->pd_idx);
5039 qdev = &sh->dev[sh->qd_idx];
5040 s.q_failed = (s.failed >= 1 && s.failed_num[0] == sh->qd_idx)
5041 || (s.failed >= 2 && s.failed_num[1] == sh->qd_idx)
5053 handle_stripe_clean_event(conf, sh, disks);
5056 r5c_handle_cached_data_endio(conf, sh, disks);
5057 log_stripe_write_finished(sh);
5068 handle_stripe_fill(sh, &s, disks);
5075 r5c_finish_stripe_write_out(conf, sh, &s);
5086 if (!sh->reconstruct_state && !sh->check_state && !sh->log_io) {
5089 handle_stripe_dirtying(conf, sh, &s, disks);
5095 ret = r5c_try_caching_write(conf, sh, &s,
5106 (!test_bit(STRIPE_R5C_CACHING, &sh->state) &&
5108 ret = handle_stripe_dirtying(conf, sh, &s,
5121 if (sh->check_state ||
5123 !test_bit(STRIPE_COMPUTE_RUN, &sh->state) &&
5124 !test_bit(STRIPE_INSYNC, &sh->state))) {
5126 handle_parity_checks6(conf, sh, &s, disks);
5128 handle_parity_checks5(conf, sh, &s, disks);
5132 && !test_bit(STRIPE_COMPUTE_RUN, &sh->state)
5133 && !test_bit(STRIPE_REPLACED, &sh->state)) {
5136 if (test_bit(R5_NeedReplace, &sh->dev[i].flags)) {
5137 WARN_ON(!test_bit(R5_UPTODATE, &sh->dev[i].flags));
5138 set_bit(R5_WantReplace, &sh->dev[i].flags);
5139 set_bit(R5_LOCKED, &sh->dev[i].flags);
5143 set_bit(STRIPE_INSYNC, &sh->state);
5144 set_bit(STRIPE_REPLACED, &sh->state);
5147 !test_bit(STRIPE_COMPUTE_RUN, &sh->state) &&
5148 test_bit(STRIPE_INSYNC, &sh->state)) {
5150 clear_bit(STRIPE_SYNCING, &sh->state);
5151 if (test_and_clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags))
5152 wake_up_bit(&sh->dev[sh->pd_idx].flags, R5_Overlap);
5160 struct r5dev *dev = &sh->dev[s.failed_num[i]];
5177 if (sh->reconstruct_state == reconstruct_state_result) {
5179 = raid5_get_active_stripe(conf, NULL, sh->sector,
5183 /* sh cannot be written until sh_src has been read.
5184 * so arrange for sh to be delayed a little
5186 set_bit(STRIPE_DELAYED, &sh->state);
5187 set_bit(STRIPE_HANDLE, &sh->state);
5197 sh->reconstruct_state = reconstruct_state_idle;
5198 clear_bit(STRIPE_EXPANDING, &sh->state);
5200 set_bit(R5_Wantwrite, &sh->dev[i].flags);
5201 set_bit(R5_LOCKED, &sh->dev[i].flags);
5206 if (s.expanded && test_bit(STRIPE_EXPANDING, &sh->state) &&
5207 !sh->reconstruct_state) {
5209 sh->disks = conf->raid_disks;
5210 stripe_set_idx(sh->sector, conf, 0, sh);
5211 schedule_reconstruction(sh, &s, 1, 1);
5212 } else if (s.expanded && !sh->reconstruct_state && s.locked == 0) {
5213 clear_bit(STRIPE_EXPAND_READY, &sh->state);
5220 !test_bit(STRIPE_COMPUTE_RUN, &sh->state))
5221 handle_stripe_expansion(conf, sh);
5241 struct r5dev *dev = &sh->dev[i];
5245 if (!rdev_set_badblocks(rdev, sh->sector,
5252 rdev_clear_badblocks(rdev, sh->sector,
5261 rdev_clear_badblocks(rdev, sh->sector,
5268 raid_run_ops(sh, s.ops_request);
5270 ops_run_io(sh, &s);
5283 clear_bit_unlock(STRIPE_ACTIVE, &sh->state);
5292 struct stripe_head *sh;
5293 sh = list_entry(l, struct stripe_head, lru);
5295 clear_bit(STRIPE_DELAYED, &sh->state);
5296 if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
5298 list_add_tail(&sh->lru, &conf->hold_list);
5299 raid5_wakeup_stripe_thread(sh);
5312 struct stripe_head *sh = list_entry(head.next, struct stripe_head, lru);
5314 list_del_init(&sh->lru);
5315 atomic_inc(&sh->count);
5316 hash = sh->hash_lock_index;
5317 __release_stripe(conf, sh, &temp_inactive_list[hash]);
5511 struct stripe_head *sh, *tmp;
5521 sh = NULL;
5547 sh = list_entry(handle_list->next, typeof(*sh), lru);
5551 else if (!test_bit(STRIPE_IO_STARTED, &sh->state)) {
5571 sh = tmp;
5576 if (sh) {
5584 if (!sh) {
5594 sh->group = NULL;
5596 list_del_init(&sh->lru);
5597 BUG_ON(atomic_inc_return(&sh->count) != 1);
5598 return sh;
5611 struct stripe_head *sh;
5620 sh = list_first_entry(&cb->list, struct stripe_head, lru);
5621 list_del_init(&sh->lru);
5628 clear_bit(STRIPE_ON_UNPLUG_LIST, &sh->state);
5633 hash = sh->hash_lock_index;
5634 __release_stripe(conf, sh, &cb->temp_inactive_list[hash]);
5647 struct stripe_head *sh)
5655 raid5_release_stripe(sh);
5668 if (!test_and_set_bit(STRIPE_ON_UNPLUG_LIST, &sh->state))
5669 list_add_tail(&sh->lru, &cb->list);
5671 raid5_release_stripe(sh);
5678 struct stripe_head *sh;
5708 sh = raid5_get_active_stripe(conf, NULL, logical_sector, 0);
5709 set_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags);
5710 if (test_bit(STRIPE_SYNCING, &sh->state)) {
5711 raid5_release_stripe(sh);
5712 wait_on_bit(&sh->dev[sh->pd_idx].flags, R5_Overlap,
5716 clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags);
5717 spin_lock_irq(&sh->stripe_lock);
5719 if (d == sh->pd_idx || d == sh->qd_idx)
5721 if (sh->dev[d].towrite || sh->dev[d].toread) {
5722 set_bit(R5_Overlap, &sh->dev[d].flags);
5723 spin_unlock_irq(&sh->stripe_lock);
5724 raid5_release_stripe(sh);
5725 wait_on_bit(&sh->dev[d].flags, R5_Overlap,
5730 set_bit(STRIPE_DISCARD, &sh->state);
5731 sh->overwrite_disks = 0;
5733 if (d == sh->pd_idx || d == sh->qd_idx)
5735 sh->dev[d].towrite = bi;
5736 set_bit(R5_OVERWRITE, &sh->dev[d].flags);
5739 sh->overwrite_disks++;
5741 spin_unlock_irq(&sh->stripe_lock);
5743 sh->bm_seq = conf->seq_flush + 1;
5744 set_bit(STRIPE_BIT_DELAY, &sh->state);
5747 set_bit(STRIPE_HANDLE, &sh->state);
5748 clear_bit(STRIPE_DELAYED, &sh->state);
5749 if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
5751 release_stripe_plug(mddev, sh);
5772 struct stripe_head *sh)
5778 for (dd_idx = 0; dd_idx < sh->disks; dd_idx++) {
5779 if (dd_idx == sh->pd_idx || dd_idx == sh->qd_idx)
5782 min_sector = min(min_sector, sh->dev[dd_idx].sector);
5783 max_sector = max(max_sector, sh->dev[dd_idx].sector);
5799 struct stripe_request_ctx *ctx, struct stripe_head *sh,
5804 spin_lock_irq(&sh->stripe_lock);
5806 for (dd_idx = 0; dd_idx < sh->disks; dd_idx++) {
5807 struct r5dev *dev = &sh->dev[dd_idx];
5809 if (dd_idx == sh->pd_idx || dd_idx == sh->qd_idx)
5816 if (stripe_bio_overlaps(sh, bi, dd_idx, forwrite)) {
5818 spin_unlock_irq(&sh->stripe_lock);
5819 raid5_release_stripe(sh);
5831 for (dd_idx = 0; dd_idx < sh->disks; dd_idx++) {
5832 struct r5dev *dev = &sh->dev[dd_idx];
5834 if (dd_idx == sh->pd_idx || dd_idx == sh->qd_idx)
5841 __add_stripe_bio(sh, bi, dd_idx, forwrite, previous);
5846 spin_unlock_irq(&sh->stripe_lock);
5937 struct stripe_head *sh;
5964 sh = raid5_get_active_stripe(conf, ctx, new_sector, flags);
5965 if (unlikely(!sh)) {
5972 stripe_ahead_of_reshape(mddev, conf, sh)) {
5977 * 'sh', we know that if that happens,
5991 if (test_bit(STRIPE_EXPANDING, &sh->state)) {
5997 if (!add_all_stripe_bios(conf, ctx, sh, bi, rw, previous)) {
6002 if (stripe_can_batch(sh)) {
6003 stripe_add_to_batch_list(conf, sh, ctx->batch_last);
6006 atomic_inc(&sh->count);
6007 ctx->batch_last = sh;
6011 set_bit(STRIPE_R5C_PREFLUSH, &sh->state);
6016 set_bit(STRIPE_HANDLE, &sh->state);
6017 clear_bit(STRIPE_DELAYED, &sh->state);
6018 if ((!sh->batch_head || sh == sh->batch_head) &&
6020 !test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
6023 release_stripe_plug(mddev, sh);
6027 raid5_release_stripe(sh);
6047 struct stripe_head sh;
6053 sector = raid5_compute_sector(conf, r_sector, 0, &dd_idx, &sh);
6062 while (dd_idx == sh.pd_idx || dd_idx == sh.qd_idx)
6221 struct stripe_head *sh;
6378 sh = raid5_get_active_stripe(conf, NULL, stripe_addr+i,
6380 set_bit(STRIPE_EXPANDING, &sh->state);
6385 for (j=sh->disks; j--;) {
6387 if (j == sh->pd_idx)
6390 j == sh->qd_idx)
6392 s = raid5_compute_blocknr(sh, j, 0);
6397 memset(page_address(sh->dev[j].page), 0, RAID5_STRIPE_SIZE(conf));
6398 set_bit(R5_Expanded, &sh->dev[j].flags);
6399 set_bit(R5_UPTODATE, &sh->dev[j].flags);
6402 set_bit(STRIPE_EXPAND_READY, &sh->state);
6403 set_bit(STRIPE_HANDLE, &sh->state);
6405 list_add(&sh->lru, &stripes);
6428 sh = raid5_get_active_stripe(conf, NULL, first_sector,
6430 set_bit(STRIPE_EXPAND_SOURCE, &sh->state);
6431 set_bit(STRIPE_HANDLE, &sh->state);
6432 raid5_release_stripe(sh);
6439 sh = list_entry(stripes.next, struct stripe_head, lru);
6440 list_del_init(&sh->lru);
6441 raid5_release_stripe(sh);
6490 struct stripe_head *sh;
6549 sh = raid5_get_active_stripe(conf, NULL, sector_nr,
6551 if (sh == NULL) {
6552 sh = raid5_get_active_stripe(conf, NULL, sector_nr, 0);
6572 set_bit(STRIPE_SYNC_REQUESTED, &sh->state);
6573 set_bit(STRIPE_HANDLE, &sh->state);
6575 raid5_release_stripe(sh);
6593 struct stripe_head *sh;
6614 sh = raid5_get_active_stripe(conf, NULL, sector,
6616 if (!sh) {
6623 if (!add_stripe_bio(sh, raid_bio, dd_idx, 0, 0)) {
6624 raid5_release_stripe(sh);
6630 set_bit(R5_ReadNoMerge, &sh->dev[dd_idx].flags);
6631 handle_stripe(sh);
6632 raid5_release_stripe(sh);
6648 struct stripe_head *batch[MAX_STRIPE_BATCH], *sh;
6653 (sh = __get_priority_stripe(conf, group)) != NULL)
6654 batch[batch_size++] = sh;