1 /* 2 * raid5.c : Multiple Devices driver for Linux 3 * Copyright (C) 1996, 1997 Ingo Molnar, Miguel de Icaza, Gadi Oxman 4 * Copyright (C) 1999, 2000 Ingo Molnar 5 * Copyright (C) 2002, 2003 H. Peter Anvin 6 * 7 * RAID-4/5/6 management functions. 8 * Thanks to Penguin Computing for making the RAID-6 development possible 9 * by donating a test server! 10 * 11 * This program is free software; you can redistribute it and/or modify 12 * it under the terms of the GNU General Public License as published by 13 * the Free Software Foundation; either version 2, or (at your option) 14 * any later version. 15 * 16 * You should have received a copy of the GNU General Public License 17 * (for example /usr/src/linux/COPYING); if not, write to the Free 18 * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. 19 */ 20 21 /* 22 * BITMAP UNPLUGGING: 23 * 24 * The sequencing for updating the bitmap reliably is a little 25 * subtle (and I got it wrong the first time) so it deserves some 26 * explanation. 27 * 28 * We group bitmap updates into batches. Each batch has a number. 29 * We may write out several batches at once, but that isn't very important. 30 * conf->seq_write is the number of the last batch successfully written. 31 * conf->seq_flush is the number of the last batch that was closed to 32 * new additions. 33 * When we discover that we will need to write to any block in a stripe 34 * (in add_stripe_bio) we update the in-memory bitmap and record in sh->bm_seq 35 * the number of the batch it will be in. This is seq_flush+1. 36 * When we are ready to do a write, if that batch hasn't been written yet, 37 * we plug the array and queue the stripe for later. 38 * When an unplug happens, we increment bm_flush, thus closing the current 39 * batch. 40 * When we notice that bm_flush > bm_write, we write out all pending updates 41 * to the bitmap, and advance bm_write to where bm_flush was. 42 * This may occasionally write a bit out twice, but is sure never to 43 * miss any bits. 44 */ 45 46 #include <linux/blkdev.h> 47 #include <linux/kthread.h> 48 #include <linux/raid/pq.h> 49 #include <linux/async_tx.h> 50 #include <linux/module.h> 51 #include <linux/async.h> 52 #include <linux/seq_file.h> 53 #include <linux/cpu.h> 54 #include <linux/slab.h> 55 #include <linux/ratelimit.h> 56 #include "md.h" 57 #include "raid5.h" 58 #include "raid0.h" 59 #include "bitmap.h" 60 61 /* 62 * Stripe cache 63 */ 64 65 #define NR_STRIPES 256 66 #define STRIPE_SIZE PAGE_SIZE 67 #define STRIPE_SHIFT (PAGE_SHIFT - 9) 68 #define STRIPE_SECTORS (STRIPE_SIZE>>9) 69 #define IO_THRESHOLD 1 70 #define BYPASS_THRESHOLD 1 71 #define NR_HASH (PAGE_SIZE / sizeof(struct hlist_head)) 72 #define HASH_MASK (NR_HASH - 1) 73 74 static inline struct hlist_head *stripe_hash(struct r5conf *conf, sector_t sect) 75 { 76 int hash = (sect >> STRIPE_SHIFT) & HASH_MASK; 77 return &conf->stripe_hashtbl[hash]; 78 } 79 80 /* bio's attached to a stripe+device for I/O are linked together in bi_sector 81 * order without overlap. There may be several bio's per stripe+device, and 82 * a bio could span several devices. 83 * When walking this list for a particular stripe+device, we must never proceed 84 * beyond a bio that extends past this device, as the next bio might no longer 85 * be valid. 86 * This function is used to determine the 'next' bio in the list, given the sector 87 * of the current stripe+device 88 */ 89 static inline struct bio *r5_next_bio(struct bio *bio, sector_t sector) 90 { 91 int sectors = bio->bi_size >> 9; 92 if (bio->bi_sector + sectors < sector + STRIPE_SECTORS) 93 return bio->bi_next; 94 else 95 return NULL; 96 } 97 98 /* 99 * We maintain a biased count of active stripes in the bottom 16 bits of 100 * bi_phys_segments, and a count of processed stripes in the upper 16 bits 101 */ 102 static inline int raid5_bi_phys_segments(struct bio *bio) 103 { 104 return bio->bi_phys_segments & 0xffff; 105 } 106 107 static inline int raid5_bi_hw_segments(struct bio *bio) 108 { 109 return (bio->bi_phys_segments >> 16) & 0xffff; 110 } 111 112 static inline int raid5_dec_bi_phys_segments(struct bio *bio) 113 { 114 --bio->bi_phys_segments; 115 return raid5_bi_phys_segments(bio); 116 } 117 118 static inline int raid5_dec_bi_hw_segments(struct bio *bio) 119 { 120 unsigned short val = raid5_bi_hw_segments(bio); 121 122 --val; 123 bio->bi_phys_segments = (val << 16) | raid5_bi_phys_segments(bio); 124 return val; 125 } 126 127 static inline void raid5_set_bi_hw_segments(struct bio *bio, unsigned int cnt) 128 { 129 bio->bi_phys_segments = raid5_bi_phys_segments(bio) | (cnt << 16); 130 } 131 132 /* Find first data disk in a raid6 stripe */ 133 static inline int raid6_d0(struct stripe_head *sh) 134 { 135 if (sh->ddf_layout) 136 /* ddf always start from first device */ 137 return 0; 138 /* md starts just after Q block */ 139 if (sh->qd_idx == sh->disks - 1) 140 return 0; 141 else 142 return sh->qd_idx + 1; 143 } 144 static inline int raid6_next_disk(int disk, int raid_disks) 145 { 146 disk++; 147 return (disk < raid_disks) ? disk : 0; 148 } 149 150 /* When walking through the disks in a raid5, starting at raid6_d0, 151 * We need to map each disk to a 'slot', where the data disks are slot 152 * 0 .. raid_disks-3, the parity disk is raid_disks-2 and the Q disk 153 * is raid_disks-1. This help does that mapping. 154 */ 155 static int raid6_idx_to_slot(int idx, struct stripe_head *sh, 156 int *count, int syndrome_disks) 157 { 158 int slot = *count; 159 160 if (sh->ddf_layout) 161 (*count)++; 162 if (idx == sh->pd_idx) 163 return syndrome_disks; 164 if (idx == sh->qd_idx) 165 return syndrome_disks + 1; 166 if (!sh->ddf_layout) 167 (*count)++; 168 return slot; 169 } 170 171 static void return_io(struct bio *return_bi) 172 { 173 struct bio *bi = return_bi; 174 while (bi) { 175 176 return_bi = bi->bi_next; 177 bi->bi_next = NULL; 178 bi->bi_size = 0; 179 bio_endio(bi, 0); 180 bi = return_bi; 181 } 182 } 183 184 static void print_raid5_conf (struct r5conf *conf); 185 186 static int stripe_operations_active(struct stripe_head *sh) 187 { 188 return sh->check_state || sh->reconstruct_state || 189 test_bit(STRIPE_BIOFILL_RUN, &sh->state) || 190 test_bit(STRIPE_COMPUTE_RUN, &sh->state); 191 } 192 193 static void __release_stripe(struct r5conf *conf, struct stripe_head *sh) 194 { 195 if (atomic_dec_and_test(&sh->count)) { 196 BUG_ON(!list_empty(&sh->lru)); 197 BUG_ON(atomic_read(&conf->active_stripes)==0); 198 if (test_bit(STRIPE_HANDLE, &sh->state)) { 199 if (test_bit(STRIPE_DELAYED, &sh->state)) 200 list_add_tail(&sh->lru, &conf->delayed_list); 201 else if (test_bit(STRIPE_BIT_DELAY, &sh->state) && 202 sh->bm_seq - conf->seq_write > 0) 203 list_add_tail(&sh->lru, &conf->bitmap_list); 204 else { 205 clear_bit(STRIPE_BIT_DELAY, &sh->state); 206 list_add_tail(&sh->lru, &conf->handle_list); 207 } 208 md_wakeup_thread(conf->mddev->thread); 209 } else { 210 BUG_ON(stripe_operations_active(sh)); 211 if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) 212 if (atomic_dec_return(&conf->preread_active_stripes) 213 < IO_THRESHOLD) 214 md_wakeup_thread(conf->mddev->thread); 215 atomic_dec(&conf->active_stripes); 216 if (!test_bit(STRIPE_EXPANDING, &sh->state)) { 217 list_add_tail(&sh->lru, &conf->inactive_list); 218 wake_up(&conf->wait_for_stripe); 219 if (conf->retry_read_aligned) 220 md_wakeup_thread(conf->mddev->thread); 221 } 222 } 223 } 224 } 225 226 static void release_stripe(struct stripe_head *sh) 227 { 228 struct r5conf *conf = sh->raid_conf; 229 unsigned long flags; 230 231 spin_lock_irqsave(&conf->device_lock, flags); 232 __release_stripe(conf, sh); 233 spin_unlock_irqrestore(&conf->device_lock, flags); 234 } 235 236 static inline void remove_hash(struct stripe_head *sh) 237 { 238 pr_debug("remove_hash(), stripe %llu\n", 239 (unsigned long long)sh->sector); 240 241 hlist_del_init(&sh->hash); 242 } 243 244 static inline void insert_hash(struct r5conf *conf, struct stripe_head *sh) 245 { 246 struct hlist_head *hp = stripe_hash(conf, sh->sector); 247 248 pr_debug("insert_hash(), stripe %llu\n", 249 (unsigned long long)sh->sector); 250 251 hlist_add_head(&sh->hash, hp); 252 } 253 254 255 /* find an idle stripe, make sure it is unhashed, and return it. */ 256 static struct stripe_head *get_free_stripe(struct r5conf *conf) 257 { 258 struct stripe_head *sh = NULL; 259 struct list_head *first; 260 261 if (list_empty(&conf->inactive_list)) 262 goto out; 263 first = conf->inactive_list.next; 264 sh = list_entry(first, struct stripe_head, lru); 265 list_del_init(first); 266 remove_hash(sh); 267 atomic_inc(&conf->active_stripes); 268 out: 269 return sh; 270 } 271 272 static void shrink_buffers(struct stripe_head *sh) 273 { 274 struct page *p; 275 int i; 276 int num = sh->raid_conf->pool_size; 277 278 for (i = 0; i < num ; i++) { 279 p = sh->dev[i].page; 280 if (!p) 281 continue; 282 sh->dev[i].page = NULL; 283 put_page(p); 284 } 285 } 286 287 static int grow_buffers(struct stripe_head *sh) 288 { 289 int i; 290 int num = sh->raid_conf->pool_size; 291 292 for (i = 0; i < num; i++) { 293 struct page *page; 294 295 if (!(page = alloc_page(GFP_KERNEL))) { 296 return 1; 297 } 298 sh->dev[i].page = page; 299 } 300 return 0; 301 } 302 303 static void raid5_build_block(struct stripe_head *sh, int i, int previous); 304 static void stripe_set_idx(sector_t stripe, struct r5conf *conf, int previous, 305 struct stripe_head *sh); 306 307 static void init_stripe(struct stripe_head *sh, sector_t sector, int previous) 308 { 309 struct r5conf *conf = sh->raid_conf; 310 int i; 311 312 BUG_ON(atomic_read(&sh->count) != 0); 313 BUG_ON(test_bit(STRIPE_HANDLE, &sh->state)); 314 BUG_ON(stripe_operations_active(sh)); 315 316 pr_debug("init_stripe called, stripe %llu\n", 317 (unsigned long long)sh->sector); 318 319 remove_hash(sh); 320 321 sh->generation = conf->generation - previous; 322 sh->disks = previous ? conf->previous_raid_disks : conf->raid_disks; 323 sh->sector = sector; 324 stripe_set_idx(sector, conf, previous, sh); 325 sh->state = 0; 326 327 328 for (i = sh->disks; i--; ) { 329 struct r5dev *dev = &sh->dev[i]; 330 331 if (dev->toread || dev->read || dev->towrite || dev->written || 332 test_bit(R5_LOCKED, &dev->flags)) { 333 printk(KERN_ERR "sector=%llx i=%d %p %p %p %p %d\n", 334 (unsigned long long)sh->sector, i, dev->toread, 335 dev->read, dev->towrite, dev->written, 336 test_bit(R5_LOCKED, &dev->flags)); 337 WARN_ON(1); 338 } 339 dev->flags = 0; 340 raid5_build_block(sh, i, previous); 341 } 342 insert_hash(conf, sh); 343 } 344 345 static struct stripe_head *__find_stripe(struct r5conf *conf, sector_t sector, 346 short generation) 347 { 348 struct stripe_head *sh; 349 struct hlist_node *hn; 350 351 pr_debug("__find_stripe, sector %llu\n", (unsigned long long)sector); 352 hlist_for_each_entry(sh, hn, stripe_hash(conf, sector), hash) 353 if (sh->sector == sector && sh->generation == generation) 354 return sh; 355 pr_debug("__stripe %llu not in cache\n", (unsigned long long)sector); 356 return NULL; 357 } 358 359 /* 360 * Need to check if array has failed when deciding whether to: 361 * - start an array 362 * - remove non-faulty devices 363 * - add a spare 364 * - allow a reshape 365 * This determination is simple when no reshape is happening. 366 * However if there is a reshape, we need to carefully check 367 * both the before and after sections. 368 * This is because some failed devices may only affect one 369 * of the two sections, and some non-in_sync devices may 370 * be insync in the section most affected by failed devices. 371 */ 372 static int calc_degraded(struct r5conf *conf) 373 { 374 int degraded, degraded2; 375 int i; 376 377 rcu_read_lock(); 378 degraded = 0; 379 for (i = 0; i < conf->previous_raid_disks; i++) { 380 struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev); 381 if (!rdev || test_bit(Faulty, &rdev->flags)) 382 degraded++; 383 else if (test_bit(In_sync, &rdev->flags)) 384 ; 385 else 386 /* not in-sync or faulty. 387 * If the reshape increases the number of devices, 388 * this is being recovered by the reshape, so 389 * this 'previous' section is not in_sync. 390 * If the number of devices is being reduced however, 391 * the device can only be part of the array if 392 * we are reverting a reshape, so this section will 393 * be in-sync. 394 */ 395 if (conf->raid_disks >= conf->previous_raid_disks) 396 degraded++; 397 } 398 rcu_read_unlock(); 399 if (conf->raid_disks == conf->previous_raid_disks) 400 return degraded; 401 rcu_read_lock(); 402 degraded2 = 0; 403 for (i = 0; i < conf->raid_disks; i++) { 404 struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev); 405 if (!rdev || test_bit(Faulty, &rdev->flags)) 406 degraded2++; 407 else if (test_bit(In_sync, &rdev->flags)) 408 ; 409 else 410 /* not in-sync or faulty. 411 * If reshape increases the number of devices, this 412 * section has already been recovered, else it 413 * almost certainly hasn't. 414 */ 415 if (conf->raid_disks <= conf->previous_raid_disks) 416 degraded2++; 417 } 418 rcu_read_unlock(); 419 if (degraded2 > degraded) 420 return degraded2; 421 return degraded; 422 } 423 424 static int has_failed(struct r5conf *conf) 425 { 426 int degraded; 427 428 if (conf->mddev->reshape_position == MaxSector) 429 return conf->mddev->degraded > conf->max_degraded; 430 431 degraded = calc_degraded(conf); 432 if (degraded > conf->max_degraded) 433 return 1; 434 return 0; 435 } 436 437 static struct stripe_head * 438 get_active_stripe(struct r5conf *conf, sector_t sector, 439 int previous, int noblock, int noquiesce) 440 { 441 struct stripe_head *sh; 442 443 pr_debug("get_stripe, sector %llu\n", (unsigned long long)sector); 444 445 spin_lock_irq(&conf->device_lock); 446 447 do { 448 wait_event_lock_irq(conf->wait_for_stripe, 449 conf->quiesce == 0 || noquiesce, 450 conf->device_lock, /* nothing */); 451 sh = __find_stripe(conf, sector, conf->generation - previous); 452 if (!sh) { 453 if (!conf->inactive_blocked) 454 sh = get_free_stripe(conf); 455 if (noblock && sh == NULL) 456 break; 457 if (!sh) { 458 conf->inactive_blocked = 1; 459 wait_event_lock_irq(conf->wait_for_stripe, 460 !list_empty(&conf->inactive_list) && 461 (atomic_read(&conf->active_stripes) 462 < (conf->max_nr_stripes *3/4) 463 || !conf->inactive_blocked), 464 conf->device_lock, 465 ); 466 conf->inactive_blocked = 0; 467 } else 468 init_stripe(sh, sector, previous); 469 } else { 470 if (atomic_read(&sh->count)) { 471 BUG_ON(!list_empty(&sh->lru) 472 && !test_bit(STRIPE_EXPANDING, &sh->state)); 473 } else { 474 if (!test_bit(STRIPE_HANDLE, &sh->state)) 475 atomic_inc(&conf->active_stripes); 476 if (list_empty(&sh->lru) && 477 !test_bit(STRIPE_EXPANDING, &sh->state)) 478 BUG(); 479 list_del_init(&sh->lru); 480 } 481 } 482 } while (sh == NULL); 483 484 if (sh) 485 atomic_inc(&sh->count); 486 487 spin_unlock_irq(&conf->device_lock); 488 return sh; 489 } 490 491 /* Determine if 'data_offset' or 'new_data_offset' should be used 492 * in this stripe_head. 493 */ 494 static int use_new_offset(struct r5conf *conf, struct stripe_head *sh) 495 { 496 sector_t progress = conf->reshape_progress; 497 /* Need a memory barrier to make sure we see the value 498 * of conf->generation, or ->data_offset that was set before 499 * reshape_progress was updated. 500 */ 501 smp_rmb(); 502 if (progress == MaxSector) 503 return 0; 504 if (sh->generation == conf->generation - 1) 505 return 0; 506 /* We are in a reshape, and this is a new-generation stripe, 507 * so use new_data_offset. 508 */ 509 return 1; 510 } 511 512 static void 513 raid5_end_read_request(struct bio *bi, int error); 514 static void 515 raid5_end_write_request(struct bio *bi, int error); 516 517 static void ops_run_io(struct stripe_head *sh, struct stripe_head_state *s) 518 { 519 struct r5conf *conf = sh->raid_conf; 520 int i, disks = sh->disks; 521 522 might_sleep(); 523 524 for (i = disks; i--; ) { 525 int rw; 526 int replace_only = 0; 527 struct bio *bi, *rbi; 528 struct md_rdev *rdev, *rrdev = NULL; 529 if (test_and_clear_bit(R5_Wantwrite, &sh->dev[i].flags)) { 530 if (test_and_clear_bit(R5_WantFUA, &sh->dev[i].flags)) 531 rw = WRITE_FUA; 532 else 533 rw = WRITE; 534 } else if (test_and_clear_bit(R5_Wantread, &sh->dev[i].flags)) 535 rw = READ; 536 else if (test_and_clear_bit(R5_WantReplace, 537 &sh->dev[i].flags)) { 538 rw = WRITE; 539 replace_only = 1; 540 } else 541 continue; 542 543 bi = &sh->dev[i].req; 544 rbi = &sh->dev[i].rreq; /* For writing to replacement */ 545 546 bi->bi_rw = rw; 547 rbi->bi_rw = rw; 548 if (rw & WRITE) { 549 bi->bi_end_io = raid5_end_write_request; 550 rbi->bi_end_io = raid5_end_write_request; 551 } else 552 bi->bi_end_io = raid5_end_read_request; 553 554 rcu_read_lock(); 555 rrdev = rcu_dereference(conf->disks[i].replacement); 556 smp_mb(); /* Ensure that if rrdev is NULL, rdev won't be */ 557 rdev = rcu_dereference(conf->disks[i].rdev); 558 if (!rdev) { 559 rdev = rrdev; 560 rrdev = NULL; 561 } 562 if (rw & WRITE) { 563 if (replace_only) 564 rdev = NULL; 565 if (rdev == rrdev) 566 /* We raced and saw duplicates */ 567 rrdev = NULL; 568 } else { 569 if (test_bit(R5_ReadRepl, &sh->dev[i].flags) && rrdev) 570 rdev = rrdev; 571 rrdev = NULL; 572 } 573 574 if (rdev && test_bit(Faulty, &rdev->flags)) 575 rdev = NULL; 576 if (rdev) 577 atomic_inc(&rdev->nr_pending); 578 if (rrdev && test_bit(Faulty, &rrdev->flags)) 579 rrdev = NULL; 580 if (rrdev) 581 atomic_inc(&rrdev->nr_pending); 582 rcu_read_unlock(); 583 584 /* We have already checked bad blocks for reads. Now 585 * need to check for writes. We never accept write errors 586 * on the replacement, so we don't to check rrdev. 587 */ 588 while ((rw & WRITE) && rdev && 589 test_bit(WriteErrorSeen, &rdev->flags)) { 590 sector_t first_bad; 591 int bad_sectors; 592 int bad = is_badblock(rdev, sh->sector, STRIPE_SECTORS, 593 &first_bad, &bad_sectors); 594 if (!bad) 595 break; 596 597 if (bad < 0) { 598 set_bit(BlockedBadBlocks, &rdev->flags); 599 if (!conf->mddev->external && 600 conf->mddev->flags) { 601 /* It is very unlikely, but we might 602 * still need to write out the 603 * bad block log - better give it 604 * a chance*/ 605 md_check_recovery(conf->mddev); 606 } 607 md_wait_for_blocked_rdev(rdev, conf->mddev); 608 } else { 609 /* Acknowledged bad block - skip the write */ 610 rdev_dec_pending(rdev, conf->mddev); 611 rdev = NULL; 612 } 613 } 614 615 if (rdev) { 616 if (s->syncing || s->expanding || s->expanded 617 || s->replacing) 618 md_sync_acct(rdev->bdev, STRIPE_SECTORS); 619 620 set_bit(STRIPE_IO_STARTED, &sh->state); 621 622 bi->bi_bdev = rdev->bdev; 623 pr_debug("%s: for %llu schedule op %ld on disc %d\n", 624 __func__, (unsigned long long)sh->sector, 625 bi->bi_rw, i); 626 atomic_inc(&sh->count); 627 if (use_new_offset(conf, sh)) 628 bi->bi_sector = (sh->sector 629 + rdev->new_data_offset); 630 else 631 bi->bi_sector = (sh->sector 632 + rdev->data_offset); 633 bi->bi_flags = 1 << BIO_UPTODATE; 634 bi->bi_idx = 0; 635 bi->bi_io_vec[0].bv_len = STRIPE_SIZE; 636 bi->bi_io_vec[0].bv_offset = 0; 637 bi->bi_size = STRIPE_SIZE; 638 bi->bi_next = NULL; 639 if (rrdev) 640 set_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags); 641 generic_make_request(bi); 642 } 643 if (rrdev) { 644 if (s->syncing || s->expanding || s->expanded 645 || s->replacing) 646 md_sync_acct(rrdev->bdev, STRIPE_SECTORS); 647 648 set_bit(STRIPE_IO_STARTED, &sh->state); 649 650 rbi->bi_bdev = rrdev->bdev; 651 pr_debug("%s: for %llu schedule op %ld on " 652 "replacement disc %d\n", 653 __func__, (unsigned long long)sh->sector, 654 rbi->bi_rw, i); 655 atomic_inc(&sh->count); 656 if (use_new_offset(conf, sh)) 657 rbi->bi_sector = (sh->sector 658 + rrdev->new_data_offset); 659 else 660 rbi->bi_sector = (sh->sector 661 + rrdev->data_offset); 662 rbi->bi_flags = 1 << BIO_UPTODATE; 663 rbi->bi_idx = 0; 664 rbi->bi_io_vec[0].bv_len = STRIPE_SIZE; 665 rbi->bi_io_vec[0].bv_offset = 0; 666 rbi->bi_size = STRIPE_SIZE; 667 rbi->bi_next = NULL; 668 generic_make_request(rbi); 669 } 670 if (!rdev && !rrdev) { 671 if (rw & WRITE) 672 set_bit(STRIPE_DEGRADED, &sh->state); 673 pr_debug("skip op %ld on disc %d for sector %llu\n", 674 bi->bi_rw, i, (unsigned long long)sh->sector); 675 clear_bit(R5_LOCKED, &sh->dev[i].flags); 676 set_bit(STRIPE_HANDLE, &sh->state); 677 } 678 } 679 } 680 681 static struct dma_async_tx_descriptor * 682 async_copy_data(int frombio, struct bio *bio, struct page *page, 683 sector_t sector, struct dma_async_tx_descriptor *tx) 684 { 685 struct bio_vec *bvl; 686 struct page *bio_page; 687 int i; 688 int page_offset; 689 struct async_submit_ctl submit; 690 enum async_tx_flags flags = 0; 691 692 if (bio->bi_sector >= sector) 693 page_offset = (signed)(bio->bi_sector - sector) * 512; 694 else 695 page_offset = (signed)(sector - bio->bi_sector) * -512; 696 697 if (frombio) 698 flags |= ASYNC_TX_FENCE; 699 init_async_submit(&submit, flags, tx, NULL, NULL, NULL); 700 701 bio_for_each_segment(bvl, bio, i) { 702 int len = bvl->bv_len; 703 int clen; 704 int b_offset = 0; 705 706 if (page_offset < 0) { 707 b_offset = -page_offset; 708 page_offset += b_offset; 709 len -= b_offset; 710 } 711 712 if (len > 0 && page_offset + len > STRIPE_SIZE) 713 clen = STRIPE_SIZE - page_offset; 714 else 715 clen = len; 716 717 if (clen > 0) { 718 b_offset += bvl->bv_offset; 719 bio_page = bvl->bv_page; 720 if (frombio) 721 tx = async_memcpy(page, bio_page, page_offset, 722 b_offset, clen, &submit); 723 else 724 tx = async_memcpy(bio_page, page, b_offset, 725 page_offset, clen, &submit); 726 } 727 /* chain the operations */ 728 submit.depend_tx = tx; 729 730 if (clen < len) /* hit end of page */ 731 break; 732 page_offset += len; 733 } 734 735 return tx; 736 } 737 738 static void ops_complete_biofill(void *stripe_head_ref) 739 { 740 struct stripe_head *sh = stripe_head_ref; 741 struct bio *return_bi = NULL; 742 struct r5conf *conf = sh->raid_conf; 743 int i; 744 745 pr_debug("%s: stripe %llu\n", __func__, 746 (unsigned long long)sh->sector); 747 748 /* clear completed biofills */ 749 spin_lock_irq(&conf->device_lock); 750 for (i = sh->disks; i--; ) { 751 struct r5dev *dev = &sh->dev[i]; 752 753 /* acknowledge completion of a biofill operation */ 754 /* and check if we need to reply to a read request, 755 * new R5_Wantfill requests are held off until 756 * !STRIPE_BIOFILL_RUN 757 */ 758 if (test_and_clear_bit(R5_Wantfill, &dev->flags)) { 759 struct bio *rbi, *rbi2; 760 761 BUG_ON(!dev->read); 762 rbi = dev->read; 763 dev->read = NULL; 764 while (rbi && rbi->bi_sector < 765 dev->sector + STRIPE_SECTORS) { 766 rbi2 = r5_next_bio(rbi, dev->sector); 767 if (!raid5_dec_bi_phys_segments(rbi)) { 768 rbi->bi_next = return_bi; 769 return_bi = rbi; 770 } 771 rbi = rbi2; 772 } 773 } 774 } 775 spin_unlock_irq(&conf->device_lock); 776 clear_bit(STRIPE_BIOFILL_RUN, &sh->state); 777 778 return_io(return_bi); 779 780 set_bit(STRIPE_HANDLE, &sh->state); 781 release_stripe(sh); 782 } 783 784 static void ops_run_biofill(struct stripe_head *sh) 785 { 786 struct dma_async_tx_descriptor *tx = NULL; 787 struct r5conf *conf = sh->raid_conf; 788 struct async_submit_ctl submit; 789 int i; 790 791 pr_debug("%s: stripe %llu\n", __func__, 792 (unsigned long long)sh->sector); 793 794 for (i = sh->disks; i--; ) { 795 struct r5dev *dev = &sh->dev[i]; 796 if (test_bit(R5_Wantfill, &dev->flags)) { 797 struct bio *rbi; 798 spin_lock_irq(&conf->device_lock); 799 dev->read = rbi = dev->toread; 800 dev->toread = NULL; 801 spin_unlock_irq(&conf->device_lock); 802 while (rbi && rbi->bi_sector < 803 dev->sector + STRIPE_SECTORS) { 804 tx = async_copy_data(0, rbi, dev->page, 805 dev->sector, tx); 806 rbi = r5_next_bio(rbi, dev->sector); 807 } 808 } 809 } 810 811 atomic_inc(&sh->count); 812 init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_biofill, sh, NULL); 813 async_trigger_callback(&submit); 814 } 815 816 static void mark_target_uptodate(struct stripe_head *sh, int target) 817 { 818 struct r5dev *tgt; 819 820 if (target < 0) 821 return; 822 823 tgt = &sh->dev[target]; 824 set_bit(R5_UPTODATE, &tgt->flags); 825 BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags)); 826 clear_bit(R5_Wantcompute, &tgt->flags); 827 } 828 829 static void ops_complete_compute(void *stripe_head_ref) 830 { 831 struct stripe_head *sh = stripe_head_ref; 832 833 pr_debug("%s: stripe %llu\n", __func__, 834 (unsigned long long)sh->sector); 835 836 /* mark the computed target(s) as uptodate */ 837 mark_target_uptodate(sh, sh->ops.target); 838 mark_target_uptodate(sh, sh->ops.target2); 839 840 clear_bit(STRIPE_COMPUTE_RUN, &sh->state); 841 if (sh->check_state == check_state_compute_run) 842 sh->check_state = check_state_compute_result; 843 set_bit(STRIPE_HANDLE, &sh->state); 844 release_stripe(sh); 845 } 846 847 /* return a pointer to the address conversion region of the scribble buffer */ 848 static addr_conv_t *to_addr_conv(struct stripe_head *sh, 849 struct raid5_percpu *percpu) 850 { 851 return percpu->scribble + sizeof(struct page *) * (sh->disks + 2); 852 } 853 854 static struct dma_async_tx_descriptor * 855 ops_run_compute5(struct stripe_head *sh, struct raid5_percpu *percpu) 856 { 857 int disks = sh->disks; 858 struct page **xor_srcs = percpu->scribble; 859 int target = sh->ops.target; 860 struct r5dev *tgt = &sh->dev[target]; 861 struct page *xor_dest = tgt->page; 862 int count = 0; 863 struct dma_async_tx_descriptor *tx; 864 struct async_submit_ctl submit; 865 int i; 866 867 pr_debug("%s: stripe %llu block: %d\n", 868 __func__, (unsigned long long)sh->sector, target); 869 BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags)); 870 871 for (i = disks; i--; ) 872 if (i != target) 873 xor_srcs[count++] = sh->dev[i].page; 874 875 atomic_inc(&sh->count); 876 877 init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, NULL, 878 ops_complete_compute, sh, to_addr_conv(sh, percpu)); 879 if (unlikely(count == 1)) 880 tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit); 881 else 882 tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit); 883 884 return tx; 885 } 886 887 /* set_syndrome_sources - populate source buffers for gen_syndrome 888 * @srcs - (struct page *) array of size sh->disks 889 * @sh - stripe_head to parse 890 * 891 * Populates srcs in proper layout order for the stripe and returns the 892 * 'count' of sources to be used in a call to async_gen_syndrome. The P 893 * destination buffer is recorded in srcs[count] and the Q destination 894 * is recorded in srcs[count+1]]. 895 */ 896 static int set_syndrome_sources(struct page **srcs, struct stripe_head *sh) 897 { 898 int disks = sh->disks; 899 int syndrome_disks = sh->ddf_layout ? disks : (disks - 2); 900 int d0_idx = raid6_d0(sh); 901 int count; 902 int i; 903 904 for (i = 0; i < disks; i++) 905 srcs[i] = NULL; 906 907 count = 0; 908 i = d0_idx; 909 do { 910 int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks); 911 912 srcs[slot] = sh->dev[i].page; 913 i = raid6_next_disk(i, disks); 914 } while (i != d0_idx); 915 916 return syndrome_disks; 917 } 918 919 static struct dma_async_tx_descriptor * 920 ops_run_compute6_1(struct stripe_head *sh, struct raid5_percpu *percpu) 921 { 922 int disks = sh->disks; 923 struct page **blocks = percpu->scribble; 924 int target; 925 int qd_idx = sh->qd_idx; 926 struct dma_async_tx_descriptor *tx; 927 struct async_submit_ctl submit; 928 struct r5dev *tgt; 929 struct page *dest; 930 int i; 931 int count; 932 933 if (sh->ops.target < 0) 934 target = sh->ops.target2; 935 else if (sh->ops.target2 < 0) 936 target = sh->ops.target; 937 else 938 /* we should only have one valid target */ 939 BUG(); 940 BUG_ON(target < 0); 941 pr_debug("%s: stripe %llu block: %d\n", 942 __func__, (unsigned long long)sh->sector, target); 943 944 tgt = &sh->dev[target]; 945 BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags)); 946 dest = tgt->page; 947 948 atomic_inc(&sh->count); 949 950 if (target == qd_idx) { 951 count = set_syndrome_sources(blocks, sh); 952 blocks[count] = NULL; /* regenerating p is not necessary */ 953 BUG_ON(blocks[count+1] != dest); /* q should already be set */ 954 init_async_submit(&submit, ASYNC_TX_FENCE, NULL, 955 ops_complete_compute, sh, 956 to_addr_conv(sh, percpu)); 957 tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit); 958 } else { 959 /* Compute any data- or p-drive using XOR */ 960 count = 0; 961 for (i = disks; i-- ; ) { 962 if (i == target || i == qd_idx) 963 continue; 964 blocks[count++] = sh->dev[i].page; 965 } 966 967 init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, 968 NULL, ops_complete_compute, sh, 969 to_addr_conv(sh, percpu)); 970 tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE, &submit); 971 } 972 973 return tx; 974 } 975 976 static struct dma_async_tx_descriptor * 977 ops_run_compute6_2(struct stripe_head *sh, struct raid5_percpu *percpu) 978 { 979 int i, count, disks = sh->disks; 980 int syndrome_disks = sh->ddf_layout ? disks : disks-2; 981 int d0_idx = raid6_d0(sh); 982 int faila = -1, failb = -1; 983 int target = sh->ops.target; 984 int target2 = sh->ops.target2; 985 struct r5dev *tgt = &sh->dev[target]; 986 struct r5dev *tgt2 = &sh->dev[target2]; 987 struct dma_async_tx_descriptor *tx; 988 struct page **blocks = percpu->scribble; 989 struct async_submit_ctl submit; 990 991 pr_debug("%s: stripe %llu block1: %d block2: %d\n", 992 __func__, (unsigned long long)sh->sector, target, target2); 993 BUG_ON(target < 0 || target2 < 0); 994 BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags)); 995 BUG_ON(!test_bit(R5_Wantcompute, &tgt2->flags)); 996 997 /* we need to open-code set_syndrome_sources to handle the 998 * slot number conversion for 'faila' and 'failb' 999 */ 1000 for (i = 0; i < disks ; i++) 1001 blocks[i] = NULL; 1002 count = 0; 1003 i = d0_idx; 1004 do { 1005 int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks); 1006 1007 blocks[slot] = sh->dev[i].page; 1008 1009 if (i == target) 1010 faila = slot; 1011 if (i == target2) 1012 failb = slot; 1013 i = raid6_next_disk(i, disks); 1014 } while (i != d0_idx); 1015 1016 BUG_ON(faila == failb); 1017 if (failb < faila) 1018 swap(faila, failb); 1019 pr_debug("%s: stripe: %llu faila: %d failb: %d\n", 1020 __func__, (unsigned long long)sh->sector, faila, failb); 1021 1022 atomic_inc(&sh->count); 1023 1024 if (failb == syndrome_disks+1) { 1025 /* Q disk is one of the missing disks */ 1026 if (faila == syndrome_disks) { 1027 /* Missing P+Q, just recompute */ 1028 init_async_submit(&submit, ASYNC_TX_FENCE, NULL, 1029 ops_complete_compute, sh, 1030 to_addr_conv(sh, percpu)); 1031 return async_gen_syndrome(blocks, 0, syndrome_disks+2, 1032 STRIPE_SIZE, &submit); 1033 } else { 1034 struct page *dest; 1035 int data_target; 1036 int qd_idx = sh->qd_idx; 1037 1038 /* Missing D+Q: recompute D from P, then recompute Q */ 1039 if (target == qd_idx) 1040 data_target = target2; 1041 else 1042 data_target = target; 1043 1044 count = 0; 1045 for (i = disks; i-- ; ) { 1046 if (i == data_target || i == qd_idx) 1047 continue; 1048 blocks[count++] = sh->dev[i].page; 1049 } 1050 dest = sh->dev[data_target].page; 1051 init_async_submit(&submit, 1052 ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, 1053 NULL, NULL, NULL, 1054 to_addr_conv(sh, percpu)); 1055 tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE, 1056 &submit); 1057 1058 count = set_syndrome_sources(blocks, sh); 1059 init_async_submit(&submit, ASYNC_TX_FENCE, tx, 1060 ops_complete_compute, sh, 1061 to_addr_conv(sh, percpu)); 1062 return async_gen_syndrome(blocks, 0, count+2, 1063 STRIPE_SIZE, &submit); 1064 } 1065 } else { 1066 init_async_submit(&submit, ASYNC_TX_FENCE, NULL, 1067 ops_complete_compute, sh, 1068 to_addr_conv(sh, percpu)); 1069 if (failb == syndrome_disks) { 1070 /* We're missing D+P. */ 1071 return async_raid6_datap_recov(syndrome_disks+2, 1072 STRIPE_SIZE, faila, 1073 blocks, &submit); 1074 } else { 1075 /* We're missing D+D. */ 1076 return async_raid6_2data_recov(syndrome_disks+2, 1077 STRIPE_SIZE, faila, failb, 1078 blocks, &submit); 1079 } 1080 } 1081 } 1082 1083 1084 static void ops_complete_prexor(void *stripe_head_ref) 1085 { 1086 struct stripe_head *sh = stripe_head_ref; 1087 1088 pr_debug("%s: stripe %llu\n", __func__, 1089 (unsigned long long)sh->sector); 1090 } 1091 1092 static struct dma_async_tx_descriptor * 1093 ops_run_prexor(struct stripe_head *sh, struct raid5_percpu *percpu, 1094 struct dma_async_tx_descriptor *tx) 1095 { 1096 int disks = sh->disks; 1097 struct page **xor_srcs = percpu->scribble; 1098 int count = 0, pd_idx = sh->pd_idx, i; 1099 struct async_submit_ctl submit; 1100 1101 /* existing parity data subtracted */ 1102 struct page *xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page; 1103 1104 pr_debug("%s: stripe %llu\n", __func__, 1105 (unsigned long long)sh->sector); 1106 1107 for (i = disks; i--; ) { 1108 struct r5dev *dev = &sh->dev[i]; 1109 /* Only process blocks that are known to be uptodate */ 1110 if (test_bit(R5_Wantdrain, &dev->flags)) 1111 xor_srcs[count++] = dev->page; 1112 } 1113 1114 init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx, 1115 ops_complete_prexor, sh, to_addr_conv(sh, percpu)); 1116 tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit); 1117 1118 return tx; 1119 } 1120 1121 static struct dma_async_tx_descriptor * 1122 ops_run_biodrain(struct stripe_head *sh, struct dma_async_tx_descriptor *tx) 1123 { 1124 int disks = sh->disks; 1125 int i; 1126 1127 pr_debug("%s: stripe %llu\n", __func__, 1128 (unsigned long long)sh->sector); 1129 1130 for (i = disks; i--; ) { 1131 struct r5dev *dev = &sh->dev[i]; 1132 struct bio *chosen; 1133 1134 if (test_and_clear_bit(R5_Wantdrain, &dev->flags)) { 1135 struct bio *wbi; 1136 1137 spin_lock_irq(&sh->raid_conf->device_lock); 1138 chosen = dev->towrite; 1139 dev->towrite = NULL; 1140 BUG_ON(dev->written); 1141 wbi = dev->written = chosen; 1142 spin_unlock_irq(&sh->raid_conf->device_lock); 1143 1144 while (wbi && wbi->bi_sector < 1145 dev->sector + STRIPE_SECTORS) { 1146 if (wbi->bi_rw & REQ_FUA) 1147 set_bit(R5_WantFUA, &dev->flags); 1148 tx = async_copy_data(1, wbi, dev->page, 1149 dev->sector, tx); 1150 wbi = r5_next_bio(wbi, dev->sector); 1151 } 1152 } 1153 } 1154 1155 return tx; 1156 } 1157 1158 static void ops_complete_reconstruct(void *stripe_head_ref) 1159 { 1160 struct stripe_head *sh = stripe_head_ref; 1161 int disks = sh->disks; 1162 int pd_idx = sh->pd_idx; 1163 int qd_idx = sh->qd_idx; 1164 int i; 1165 bool fua = false; 1166 1167 pr_debug("%s: stripe %llu\n", __func__, 1168 (unsigned long long)sh->sector); 1169 1170 for (i = disks; i--; ) 1171 fua |= test_bit(R5_WantFUA, &sh->dev[i].flags); 1172 1173 for (i = disks; i--; ) { 1174 struct r5dev *dev = &sh->dev[i]; 1175 1176 if (dev->written || i == pd_idx || i == qd_idx) { 1177 set_bit(R5_UPTODATE, &dev->flags); 1178 if (fua) 1179 set_bit(R5_WantFUA, &dev->flags); 1180 } 1181 } 1182 1183 if (sh->reconstruct_state == reconstruct_state_drain_run) 1184 sh->reconstruct_state = reconstruct_state_drain_result; 1185 else if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) 1186 sh->reconstruct_state = reconstruct_state_prexor_drain_result; 1187 else { 1188 BUG_ON(sh->reconstruct_state != reconstruct_state_run); 1189 sh->reconstruct_state = reconstruct_state_result; 1190 } 1191 1192 set_bit(STRIPE_HANDLE, &sh->state); 1193 release_stripe(sh); 1194 } 1195 1196 static void 1197 ops_run_reconstruct5(struct stripe_head *sh, struct raid5_percpu *percpu, 1198 struct dma_async_tx_descriptor *tx) 1199 { 1200 int disks = sh->disks; 1201 struct page **xor_srcs = percpu->scribble; 1202 struct async_submit_ctl submit; 1203 int count = 0, pd_idx = sh->pd_idx, i; 1204 struct page *xor_dest; 1205 int prexor = 0; 1206 unsigned long flags; 1207 1208 pr_debug("%s: stripe %llu\n", __func__, 1209 (unsigned long long)sh->sector); 1210 1211 /* check if prexor is active which means only process blocks 1212 * that are part of a read-modify-write (written) 1213 */ 1214 if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) { 1215 prexor = 1; 1216 xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page; 1217 for (i = disks; i--; ) { 1218 struct r5dev *dev = &sh->dev[i]; 1219 if (dev->written) 1220 xor_srcs[count++] = dev->page; 1221 } 1222 } else { 1223 xor_dest = sh->dev[pd_idx].page; 1224 for (i = disks; i--; ) { 1225 struct r5dev *dev = &sh->dev[i]; 1226 if (i != pd_idx) 1227 xor_srcs[count++] = dev->page; 1228 } 1229 } 1230 1231 /* 1/ if we prexor'd then the dest is reused as a source 1232 * 2/ if we did not prexor then we are redoing the parity 1233 * set ASYNC_TX_XOR_DROP_DST and ASYNC_TX_XOR_ZERO_DST 1234 * for the synchronous xor case 1235 */ 1236 flags = ASYNC_TX_ACK | 1237 (prexor ? ASYNC_TX_XOR_DROP_DST : ASYNC_TX_XOR_ZERO_DST); 1238 1239 atomic_inc(&sh->count); 1240 1241 init_async_submit(&submit, flags, tx, ops_complete_reconstruct, sh, 1242 to_addr_conv(sh, percpu)); 1243 if (unlikely(count == 1)) 1244 tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit); 1245 else 1246 tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit); 1247 } 1248 1249 static void 1250 ops_run_reconstruct6(struct stripe_head *sh, struct raid5_percpu *percpu, 1251 struct dma_async_tx_descriptor *tx) 1252 { 1253 struct async_submit_ctl submit; 1254 struct page **blocks = percpu->scribble; 1255 int count; 1256 1257 pr_debug("%s: stripe %llu\n", __func__, (unsigned long long)sh->sector); 1258 1259 count = set_syndrome_sources(blocks, sh); 1260 1261 atomic_inc(&sh->count); 1262 1263 init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_reconstruct, 1264 sh, to_addr_conv(sh, percpu)); 1265 async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit); 1266 } 1267 1268 static void ops_complete_check(void *stripe_head_ref) 1269 { 1270 struct stripe_head *sh = stripe_head_ref; 1271 1272 pr_debug("%s: stripe %llu\n", __func__, 1273 (unsigned long long)sh->sector); 1274 1275 sh->check_state = check_state_check_result; 1276 set_bit(STRIPE_HANDLE, &sh->state); 1277 release_stripe(sh); 1278 } 1279 1280 static void ops_run_check_p(struct stripe_head *sh, struct raid5_percpu *percpu) 1281 { 1282 int disks = sh->disks; 1283 int pd_idx = sh->pd_idx; 1284 int qd_idx = sh->qd_idx; 1285 struct page *xor_dest; 1286 struct page **xor_srcs = percpu->scribble; 1287 struct dma_async_tx_descriptor *tx; 1288 struct async_submit_ctl submit; 1289 int count; 1290 int i; 1291 1292 pr_debug("%s: stripe %llu\n", __func__, 1293 (unsigned long long)sh->sector); 1294 1295 count = 0; 1296 xor_dest = sh->dev[pd_idx].page; 1297 xor_srcs[count++] = xor_dest; 1298 for (i = disks; i--; ) { 1299 if (i == pd_idx || i == qd_idx) 1300 continue; 1301 xor_srcs[count++] = sh->dev[i].page; 1302 } 1303 1304 init_async_submit(&submit, 0, NULL, NULL, NULL, 1305 to_addr_conv(sh, percpu)); 1306 tx = async_xor_val(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, 1307 &sh->ops.zero_sum_result, &submit); 1308 1309 atomic_inc(&sh->count); 1310 init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_check, sh, NULL); 1311 tx = async_trigger_callback(&submit); 1312 } 1313 1314 static void ops_run_check_pq(struct stripe_head *sh, struct raid5_percpu *percpu, int checkp) 1315 { 1316 struct page **srcs = percpu->scribble; 1317 struct async_submit_ctl submit; 1318 int count; 1319 1320 pr_debug("%s: stripe %llu checkp: %d\n", __func__, 1321 (unsigned long long)sh->sector, checkp); 1322 1323 count = set_syndrome_sources(srcs, sh); 1324 if (!checkp) 1325 srcs[count] = NULL; 1326 1327 atomic_inc(&sh->count); 1328 init_async_submit(&submit, ASYNC_TX_ACK, NULL, ops_complete_check, 1329 sh, to_addr_conv(sh, percpu)); 1330 async_syndrome_val(srcs, 0, count+2, STRIPE_SIZE, 1331 &sh->ops.zero_sum_result, percpu->spare_page, &submit); 1332 } 1333 1334 static void __raid_run_ops(struct stripe_head *sh, unsigned long ops_request) 1335 { 1336 int overlap_clear = 0, i, disks = sh->disks; 1337 struct dma_async_tx_descriptor *tx = NULL; 1338 struct r5conf *conf = sh->raid_conf; 1339 int level = conf->level; 1340 struct raid5_percpu *percpu; 1341 unsigned long cpu; 1342 1343 cpu = get_cpu(); 1344 percpu = per_cpu_ptr(conf->percpu, cpu); 1345 if (test_bit(STRIPE_OP_BIOFILL, &ops_request)) { 1346 ops_run_biofill(sh); 1347 overlap_clear++; 1348 } 1349 1350 if (test_bit(STRIPE_OP_COMPUTE_BLK, &ops_request)) { 1351 if (level < 6) 1352 tx = ops_run_compute5(sh, percpu); 1353 else { 1354 if (sh->ops.target2 < 0 || sh->ops.target < 0) 1355 tx = ops_run_compute6_1(sh, percpu); 1356 else 1357 tx = ops_run_compute6_2(sh, percpu); 1358 } 1359 /* terminate the chain if reconstruct is not set to be run */ 1360 if (tx && !test_bit(STRIPE_OP_RECONSTRUCT, &ops_request)) 1361 async_tx_ack(tx); 1362 } 1363 1364 if (test_bit(STRIPE_OP_PREXOR, &ops_request)) 1365 tx = ops_run_prexor(sh, percpu, tx); 1366 1367 if (test_bit(STRIPE_OP_BIODRAIN, &ops_request)) { 1368 tx = ops_run_biodrain(sh, tx); 1369 overlap_clear++; 1370 } 1371 1372 if (test_bit(STRIPE_OP_RECONSTRUCT, &ops_request)) { 1373 if (level < 6) 1374 ops_run_reconstruct5(sh, percpu, tx); 1375 else 1376 ops_run_reconstruct6(sh, percpu, tx); 1377 } 1378 1379 if (test_bit(STRIPE_OP_CHECK, &ops_request)) { 1380 if (sh->check_state == check_state_run) 1381 ops_run_check_p(sh, percpu); 1382 else if (sh->check_state == check_state_run_q) 1383 ops_run_check_pq(sh, percpu, 0); 1384 else if (sh->check_state == check_state_run_pq) 1385 ops_run_check_pq(sh, percpu, 1); 1386 else 1387 BUG(); 1388 } 1389 1390 if (overlap_clear) 1391 for (i = disks; i--; ) { 1392 struct r5dev *dev = &sh->dev[i]; 1393 if (test_and_clear_bit(R5_Overlap, &dev->flags)) 1394 wake_up(&sh->raid_conf->wait_for_overlap); 1395 } 1396 put_cpu(); 1397 } 1398 1399 #ifdef CONFIG_MULTICORE_RAID456 1400 static void async_run_ops(void *param, async_cookie_t cookie) 1401 { 1402 struct stripe_head *sh = param; 1403 unsigned long ops_request = sh->ops.request; 1404 1405 clear_bit_unlock(STRIPE_OPS_REQ_PENDING, &sh->state); 1406 wake_up(&sh->ops.wait_for_ops); 1407 1408 __raid_run_ops(sh, ops_request); 1409 release_stripe(sh); 1410 } 1411 1412 static void raid_run_ops(struct stripe_head *sh, unsigned long ops_request) 1413 { 1414 /* since handle_stripe can be called outside of raid5d context 1415 * we need to ensure sh->ops.request is de-staged before another 1416 * request arrives 1417 */ 1418 wait_event(sh->ops.wait_for_ops, 1419 !test_and_set_bit_lock(STRIPE_OPS_REQ_PENDING, &sh->state)); 1420 sh->ops.request = ops_request; 1421 1422 atomic_inc(&sh->count); 1423 async_schedule(async_run_ops, sh); 1424 } 1425 #else 1426 #define raid_run_ops __raid_run_ops 1427 #endif 1428 1429 static int grow_one_stripe(struct r5conf *conf) 1430 { 1431 struct stripe_head *sh; 1432 sh = kmem_cache_zalloc(conf->slab_cache, GFP_KERNEL); 1433 if (!sh) 1434 return 0; 1435 1436 sh->raid_conf = conf; 1437 #ifdef CONFIG_MULTICORE_RAID456 1438 init_waitqueue_head(&sh->ops.wait_for_ops); 1439 #endif 1440 1441 if (grow_buffers(sh)) { 1442 shrink_buffers(sh); 1443 kmem_cache_free(conf->slab_cache, sh); 1444 return 0; 1445 } 1446 /* we just created an active stripe so... */ 1447 atomic_set(&sh->count, 1); 1448 atomic_inc(&conf->active_stripes); 1449 INIT_LIST_HEAD(&sh->lru); 1450 release_stripe(sh); 1451 return 1; 1452 } 1453 1454 static int grow_stripes(struct r5conf *conf, int num) 1455 { 1456 struct kmem_cache *sc; 1457 int devs = max(conf->raid_disks, conf->previous_raid_disks); 1458 1459 if (conf->mddev->gendisk) 1460 sprintf(conf->cache_name[0], 1461 "raid%d-%s", conf->level, mdname(conf->mddev)); 1462 else 1463 sprintf(conf->cache_name[0], 1464 "raid%d-%p", conf->level, conf->mddev); 1465 sprintf(conf->cache_name[1], "%s-alt", conf->cache_name[0]); 1466 1467 conf->active_name = 0; 1468 sc = kmem_cache_create(conf->cache_name[conf->active_name], 1469 sizeof(struct stripe_head)+(devs-1)*sizeof(struct r5dev), 1470 0, 0, NULL); 1471 if (!sc) 1472 return 1; 1473 conf->slab_cache = sc; 1474 conf->pool_size = devs; 1475 while (num--) 1476 if (!grow_one_stripe(conf)) 1477 return 1; 1478 return 0; 1479 } 1480 1481 /** 1482 * scribble_len - return the required size of the scribble region 1483 * @num - total number of disks in the array 1484 * 1485 * The size must be enough to contain: 1486 * 1/ a struct page pointer for each device in the array +2 1487 * 2/ room to convert each entry in (1) to its corresponding dma 1488 * (dma_map_page()) or page (page_address()) address. 1489 * 1490 * Note: the +2 is for the destination buffers of the ddf/raid6 case where we 1491 * calculate over all devices (not just the data blocks), using zeros in place 1492 * of the P and Q blocks. 1493 */ 1494 static size_t scribble_len(int num) 1495 { 1496 size_t len; 1497 1498 len = sizeof(struct page *) * (num+2) + sizeof(addr_conv_t) * (num+2); 1499 1500 return len; 1501 } 1502 1503 static int resize_stripes(struct r5conf *conf, int newsize) 1504 { 1505 /* Make all the stripes able to hold 'newsize' devices. 1506 * New slots in each stripe get 'page' set to a new page. 1507 * 1508 * This happens in stages: 1509 * 1/ create a new kmem_cache and allocate the required number of 1510 * stripe_heads. 1511 * 2/ gather all the old stripe_heads and tranfer the pages across 1512 * to the new stripe_heads. This will have the side effect of 1513 * freezing the array as once all stripe_heads have been collected, 1514 * no IO will be possible. Old stripe heads are freed once their 1515 * pages have been transferred over, and the old kmem_cache is 1516 * freed when all stripes are done. 1517 * 3/ reallocate conf->disks to be suitable bigger. If this fails, 1518 * we simple return a failre status - no need to clean anything up. 1519 * 4/ allocate new pages for the new slots in the new stripe_heads. 1520 * If this fails, we don't bother trying the shrink the 1521 * stripe_heads down again, we just leave them as they are. 1522 * As each stripe_head is processed the new one is released into 1523 * active service. 1524 * 1525 * Once step2 is started, we cannot afford to wait for a write, 1526 * so we use GFP_NOIO allocations. 1527 */ 1528 struct stripe_head *osh, *nsh; 1529 LIST_HEAD(newstripes); 1530 struct disk_info *ndisks; 1531 unsigned long cpu; 1532 int err; 1533 struct kmem_cache *sc; 1534 int i; 1535 1536 if (newsize <= conf->pool_size) 1537 return 0; /* never bother to shrink */ 1538 1539 err = md_allow_write(conf->mddev); 1540 if (err) 1541 return err; 1542 1543 /* Step 1 */ 1544 sc = kmem_cache_create(conf->cache_name[1-conf->active_name], 1545 sizeof(struct stripe_head)+(newsize-1)*sizeof(struct r5dev), 1546 0, 0, NULL); 1547 if (!sc) 1548 return -ENOMEM; 1549 1550 for (i = conf->max_nr_stripes; i; i--) { 1551 nsh = kmem_cache_zalloc(sc, GFP_KERNEL); 1552 if (!nsh) 1553 break; 1554 1555 nsh->raid_conf = conf; 1556 #ifdef CONFIG_MULTICORE_RAID456 1557 init_waitqueue_head(&nsh->ops.wait_for_ops); 1558 #endif 1559 1560 list_add(&nsh->lru, &newstripes); 1561 } 1562 if (i) { 1563 /* didn't get enough, give up */ 1564 while (!list_empty(&newstripes)) { 1565 nsh = list_entry(newstripes.next, struct stripe_head, lru); 1566 list_del(&nsh->lru); 1567 kmem_cache_free(sc, nsh); 1568 } 1569 kmem_cache_destroy(sc); 1570 return -ENOMEM; 1571 } 1572 /* Step 2 - Must use GFP_NOIO now. 1573 * OK, we have enough stripes, start collecting inactive 1574 * stripes and copying them over 1575 */ 1576 list_for_each_entry(nsh, &newstripes, lru) { 1577 spin_lock_irq(&conf->device_lock); 1578 wait_event_lock_irq(conf->wait_for_stripe, 1579 !list_empty(&conf->inactive_list), 1580 conf->device_lock, 1581 ); 1582 osh = get_free_stripe(conf); 1583 spin_unlock_irq(&conf->device_lock); 1584 atomic_set(&nsh->count, 1); 1585 for(i=0; i<conf->pool_size; i++) 1586 nsh->dev[i].page = osh->dev[i].page; 1587 for( ; i<newsize; i++) 1588 nsh->dev[i].page = NULL; 1589 kmem_cache_free(conf->slab_cache, osh); 1590 } 1591 kmem_cache_destroy(conf->slab_cache); 1592 1593 /* Step 3. 1594 * At this point, we are holding all the stripes so the array 1595 * is completely stalled, so now is a good time to resize 1596 * conf->disks and the scribble region 1597 */ 1598 ndisks = kzalloc(newsize * sizeof(struct disk_info), GFP_NOIO); 1599 if (ndisks) { 1600 for (i=0; i<conf->raid_disks; i++) 1601 ndisks[i] = conf->disks[i]; 1602 kfree(conf->disks); 1603 conf->disks = ndisks; 1604 } else 1605 err = -ENOMEM; 1606 1607 get_online_cpus(); 1608 conf->scribble_len = scribble_len(newsize); 1609 for_each_present_cpu(cpu) { 1610 struct raid5_percpu *percpu; 1611 void *scribble; 1612 1613 percpu = per_cpu_ptr(conf->percpu, cpu); 1614 scribble = kmalloc(conf->scribble_len, GFP_NOIO); 1615 1616 if (scribble) { 1617 kfree(percpu->scribble); 1618 percpu->scribble = scribble; 1619 } else { 1620 err = -ENOMEM; 1621 break; 1622 } 1623 } 1624 put_online_cpus(); 1625 1626 /* Step 4, return new stripes to service */ 1627 while(!list_empty(&newstripes)) { 1628 nsh = list_entry(newstripes.next, struct stripe_head, lru); 1629 list_del_init(&nsh->lru); 1630 1631 for (i=conf->raid_disks; i < newsize; i++) 1632 if (nsh->dev[i].page == NULL) { 1633 struct page *p = alloc_page(GFP_NOIO); 1634 nsh->dev[i].page = p; 1635 if (!p) 1636 err = -ENOMEM; 1637 } 1638 release_stripe(nsh); 1639 } 1640 /* critical section pass, GFP_NOIO no longer needed */ 1641 1642 conf->slab_cache = sc; 1643 conf->active_name = 1-conf->active_name; 1644 conf->pool_size = newsize; 1645 return err; 1646 } 1647 1648 static int drop_one_stripe(struct r5conf *conf) 1649 { 1650 struct stripe_head *sh; 1651 1652 spin_lock_irq(&conf->device_lock); 1653 sh = get_free_stripe(conf); 1654 spin_unlock_irq(&conf->device_lock); 1655 if (!sh) 1656 return 0; 1657 BUG_ON(atomic_read(&sh->count)); 1658 shrink_buffers(sh); 1659 kmem_cache_free(conf->slab_cache, sh); 1660 atomic_dec(&conf->active_stripes); 1661 return 1; 1662 } 1663 1664 static void shrink_stripes(struct r5conf *conf) 1665 { 1666 while (drop_one_stripe(conf)) 1667 ; 1668 1669 if (conf->slab_cache) 1670 kmem_cache_destroy(conf->slab_cache); 1671 conf->slab_cache = NULL; 1672 } 1673 1674 static void raid5_end_read_request(struct bio * bi, int error) 1675 { 1676 struct stripe_head *sh = bi->bi_private; 1677 struct r5conf *conf = sh->raid_conf; 1678 int disks = sh->disks, i; 1679 int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags); 1680 char b[BDEVNAME_SIZE]; 1681 struct md_rdev *rdev = NULL; 1682 sector_t s; 1683 1684 for (i=0 ; i<disks; i++) 1685 if (bi == &sh->dev[i].req) 1686 break; 1687 1688 pr_debug("end_read_request %llu/%d, count: %d, uptodate %d.\n", 1689 (unsigned long long)sh->sector, i, atomic_read(&sh->count), 1690 uptodate); 1691 if (i == disks) { 1692 BUG(); 1693 return; 1694 } 1695 if (test_bit(R5_ReadRepl, &sh->dev[i].flags)) 1696 /* If replacement finished while this request was outstanding, 1697 * 'replacement' might be NULL already. 1698 * In that case it moved down to 'rdev'. 1699 * rdev is not removed until all requests are finished. 1700 */ 1701 rdev = conf->disks[i].replacement; 1702 if (!rdev) 1703 rdev = conf->disks[i].rdev; 1704 1705 if (use_new_offset(conf, sh)) 1706 s = sh->sector + rdev->new_data_offset; 1707 else 1708 s = sh->sector + rdev->data_offset; 1709 if (uptodate) { 1710 set_bit(R5_UPTODATE, &sh->dev[i].flags); 1711 if (test_bit(R5_ReadError, &sh->dev[i].flags)) { 1712 /* Note that this cannot happen on a 1713 * replacement device. We just fail those on 1714 * any error 1715 */ 1716 printk_ratelimited( 1717 KERN_INFO 1718 "md/raid:%s: read error corrected" 1719 " (%lu sectors at %llu on %s)\n", 1720 mdname(conf->mddev), STRIPE_SECTORS, 1721 (unsigned long long)s, 1722 bdevname(rdev->bdev, b)); 1723 atomic_add(STRIPE_SECTORS, &rdev->corrected_errors); 1724 clear_bit(R5_ReadError, &sh->dev[i].flags); 1725 clear_bit(R5_ReWrite, &sh->dev[i].flags); 1726 } 1727 if (atomic_read(&rdev->read_errors)) 1728 atomic_set(&rdev->read_errors, 0); 1729 } else { 1730 const char *bdn = bdevname(rdev->bdev, b); 1731 int retry = 0; 1732 1733 clear_bit(R5_UPTODATE, &sh->dev[i].flags); 1734 atomic_inc(&rdev->read_errors); 1735 if (test_bit(R5_ReadRepl, &sh->dev[i].flags)) 1736 printk_ratelimited( 1737 KERN_WARNING 1738 "md/raid:%s: read error on replacement device " 1739 "(sector %llu on %s).\n", 1740 mdname(conf->mddev), 1741 (unsigned long long)s, 1742 bdn); 1743 else if (conf->mddev->degraded >= conf->max_degraded) 1744 printk_ratelimited( 1745 KERN_WARNING 1746 "md/raid:%s: read error not correctable " 1747 "(sector %llu on %s).\n", 1748 mdname(conf->mddev), 1749 (unsigned long long)s, 1750 bdn); 1751 else if (test_bit(R5_ReWrite, &sh->dev[i].flags)) 1752 /* Oh, no!!! */ 1753 printk_ratelimited( 1754 KERN_WARNING 1755 "md/raid:%s: read error NOT corrected!! " 1756 "(sector %llu on %s).\n", 1757 mdname(conf->mddev), 1758 (unsigned long long)s, 1759 bdn); 1760 else if (atomic_read(&rdev->read_errors) 1761 > conf->max_nr_stripes) 1762 printk(KERN_WARNING 1763 "md/raid:%s: Too many read errors, failing device %s.\n", 1764 mdname(conf->mddev), bdn); 1765 else 1766 retry = 1; 1767 if (retry) 1768 set_bit(R5_ReadError, &sh->dev[i].flags); 1769 else { 1770 clear_bit(R5_ReadError, &sh->dev[i].flags); 1771 clear_bit(R5_ReWrite, &sh->dev[i].flags); 1772 md_error(conf->mddev, rdev); 1773 } 1774 } 1775 rdev_dec_pending(rdev, conf->mddev); 1776 clear_bit(R5_LOCKED, &sh->dev[i].flags); 1777 set_bit(STRIPE_HANDLE, &sh->state); 1778 release_stripe(sh); 1779 } 1780 1781 static void raid5_end_write_request(struct bio *bi, int error) 1782 { 1783 struct stripe_head *sh = bi->bi_private; 1784 struct r5conf *conf = sh->raid_conf; 1785 int disks = sh->disks, i; 1786 struct md_rdev *uninitialized_var(rdev); 1787 int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags); 1788 sector_t first_bad; 1789 int bad_sectors; 1790 int replacement = 0; 1791 1792 for (i = 0 ; i < disks; i++) { 1793 if (bi == &sh->dev[i].req) { 1794 rdev = conf->disks[i].rdev; 1795 break; 1796 } 1797 if (bi == &sh->dev[i].rreq) { 1798 rdev = conf->disks[i].replacement; 1799 if (rdev) 1800 replacement = 1; 1801 else 1802 /* rdev was removed and 'replacement' 1803 * replaced it. rdev is not removed 1804 * until all requests are finished. 1805 */ 1806 rdev = conf->disks[i].rdev; 1807 break; 1808 } 1809 } 1810 pr_debug("end_write_request %llu/%d, count %d, uptodate: %d.\n", 1811 (unsigned long long)sh->sector, i, atomic_read(&sh->count), 1812 uptodate); 1813 if (i == disks) { 1814 BUG(); 1815 return; 1816 } 1817 1818 if (replacement) { 1819 if (!uptodate) 1820 md_error(conf->mddev, rdev); 1821 else if (is_badblock(rdev, sh->sector, 1822 STRIPE_SECTORS, 1823 &first_bad, &bad_sectors)) 1824 set_bit(R5_MadeGoodRepl, &sh->dev[i].flags); 1825 } else { 1826 if (!uptodate) { 1827 set_bit(WriteErrorSeen, &rdev->flags); 1828 set_bit(R5_WriteError, &sh->dev[i].flags); 1829 if (!test_and_set_bit(WantReplacement, &rdev->flags)) 1830 set_bit(MD_RECOVERY_NEEDED, 1831 &rdev->mddev->recovery); 1832 } else if (is_badblock(rdev, sh->sector, 1833 STRIPE_SECTORS, 1834 &first_bad, &bad_sectors)) 1835 set_bit(R5_MadeGood, &sh->dev[i].flags); 1836 } 1837 rdev_dec_pending(rdev, conf->mddev); 1838 1839 if (!test_and_clear_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags)) 1840 clear_bit(R5_LOCKED, &sh->dev[i].flags); 1841 set_bit(STRIPE_HANDLE, &sh->state); 1842 release_stripe(sh); 1843 } 1844 1845 static sector_t compute_blocknr(struct stripe_head *sh, int i, int previous); 1846 1847 static void raid5_build_block(struct stripe_head *sh, int i, int previous) 1848 { 1849 struct r5dev *dev = &sh->dev[i]; 1850 1851 bio_init(&dev->req); 1852 dev->req.bi_io_vec = &dev->vec; 1853 dev->req.bi_vcnt++; 1854 dev->req.bi_max_vecs++; 1855 dev->req.bi_private = sh; 1856 dev->vec.bv_page = dev->page; 1857 1858 bio_init(&dev->rreq); 1859 dev->rreq.bi_io_vec = &dev->rvec; 1860 dev->rreq.bi_vcnt++; 1861 dev->rreq.bi_max_vecs++; 1862 dev->rreq.bi_private = sh; 1863 dev->rvec.bv_page = dev->page; 1864 1865 dev->flags = 0; 1866 dev->sector = compute_blocknr(sh, i, previous); 1867 } 1868 1869 static void error(struct mddev *mddev, struct md_rdev *rdev) 1870 { 1871 char b[BDEVNAME_SIZE]; 1872 struct r5conf *conf = mddev->private; 1873 unsigned long flags; 1874 pr_debug("raid456: error called\n"); 1875 1876 spin_lock_irqsave(&conf->device_lock, flags); 1877 clear_bit(In_sync, &rdev->flags); 1878 mddev->degraded = calc_degraded(conf); 1879 spin_unlock_irqrestore(&conf->device_lock, flags); 1880 set_bit(MD_RECOVERY_INTR, &mddev->recovery); 1881 1882 set_bit(Blocked, &rdev->flags); 1883 set_bit(Faulty, &rdev->flags); 1884 set_bit(MD_CHANGE_DEVS, &mddev->flags); 1885 printk(KERN_ALERT 1886 "md/raid:%s: Disk failure on %s, disabling device.\n" 1887 "md/raid:%s: Operation continuing on %d devices.\n", 1888 mdname(mddev), 1889 bdevname(rdev->bdev, b), 1890 mdname(mddev), 1891 conf->raid_disks - mddev->degraded); 1892 } 1893 1894 /* 1895 * Input: a 'big' sector number, 1896 * Output: index of the data and parity disk, and the sector # in them. 1897 */ 1898 static sector_t raid5_compute_sector(struct r5conf *conf, sector_t r_sector, 1899 int previous, int *dd_idx, 1900 struct stripe_head *sh) 1901 { 1902 sector_t stripe, stripe2; 1903 sector_t chunk_number; 1904 unsigned int chunk_offset; 1905 int pd_idx, qd_idx; 1906 int ddf_layout = 0; 1907 sector_t new_sector; 1908 int algorithm = previous ? conf->prev_algo 1909 : conf->algorithm; 1910 int sectors_per_chunk = previous ? conf->prev_chunk_sectors 1911 : conf->chunk_sectors; 1912 int raid_disks = previous ? conf->previous_raid_disks 1913 : conf->raid_disks; 1914 int data_disks = raid_disks - conf->max_degraded; 1915 1916 /* First compute the information on this sector */ 1917 1918 /* 1919 * Compute the chunk number and the sector offset inside the chunk 1920 */ 1921 chunk_offset = sector_div(r_sector, sectors_per_chunk); 1922 chunk_number = r_sector; 1923 1924 /* 1925 * Compute the stripe number 1926 */ 1927 stripe = chunk_number; 1928 *dd_idx = sector_div(stripe, data_disks); 1929 stripe2 = stripe; 1930 /* 1931 * Select the parity disk based on the user selected algorithm. 1932 */ 1933 pd_idx = qd_idx = -1; 1934 switch(conf->level) { 1935 case 4: 1936 pd_idx = data_disks; 1937 break; 1938 case 5: 1939 switch (algorithm) { 1940 case ALGORITHM_LEFT_ASYMMETRIC: 1941 pd_idx = data_disks - sector_div(stripe2, raid_disks); 1942 if (*dd_idx >= pd_idx) 1943 (*dd_idx)++; 1944 break; 1945 case ALGORITHM_RIGHT_ASYMMETRIC: 1946 pd_idx = sector_div(stripe2, raid_disks); 1947 if (*dd_idx >= pd_idx) 1948 (*dd_idx)++; 1949 break; 1950 case ALGORITHM_LEFT_SYMMETRIC: 1951 pd_idx = data_disks - sector_div(stripe2, raid_disks); 1952 *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks; 1953 break; 1954 case ALGORITHM_RIGHT_SYMMETRIC: 1955 pd_idx = sector_div(stripe2, raid_disks); 1956 *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks; 1957 break; 1958 case ALGORITHM_PARITY_0: 1959 pd_idx = 0; 1960 (*dd_idx)++; 1961 break; 1962 case ALGORITHM_PARITY_N: 1963 pd_idx = data_disks; 1964 break; 1965 default: 1966 BUG(); 1967 } 1968 break; 1969 case 6: 1970 1971 switch (algorithm) { 1972 case ALGORITHM_LEFT_ASYMMETRIC: 1973 pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks); 1974 qd_idx = pd_idx + 1; 1975 if (pd_idx == raid_disks-1) { 1976 (*dd_idx)++; /* Q D D D P */ 1977 qd_idx = 0; 1978 } else if (*dd_idx >= pd_idx) 1979 (*dd_idx) += 2; /* D D P Q D */ 1980 break; 1981 case ALGORITHM_RIGHT_ASYMMETRIC: 1982 pd_idx = sector_div(stripe2, raid_disks); 1983 qd_idx = pd_idx + 1; 1984 if (pd_idx == raid_disks-1) { 1985 (*dd_idx)++; /* Q D D D P */ 1986 qd_idx = 0; 1987 } else if (*dd_idx >= pd_idx) 1988 (*dd_idx) += 2; /* D D P Q D */ 1989 break; 1990 case ALGORITHM_LEFT_SYMMETRIC: 1991 pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks); 1992 qd_idx = (pd_idx + 1) % raid_disks; 1993 *dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks; 1994 break; 1995 case ALGORITHM_RIGHT_SYMMETRIC: 1996 pd_idx = sector_div(stripe2, raid_disks); 1997 qd_idx = (pd_idx + 1) % raid_disks; 1998 *dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks; 1999 break; 2000 2001 case ALGORITHM_PARITY_0: 2002 pd_idx = 0; 2003 qd_idx = 1; 2004 (*dd_idx) += 2; 2005 break; 2006 case ALGORITHM_PARITY_N: 2007 pd_idx = data_disks; 2008 qd_idx = data_disks + 1; 2009 break; 2010 2011 case ALGORITHM_ROTATING_ZERO_RESTART: 2012 /* Exactly the same as RIGHT_ASYMMETRIC, but or 2013 * of blocks for computing Q is different. 2014 */ 2015 pd_idx = sector_div(stripe2, raid_disks); 2016 qd_idx = pd_idx + 1; 2017 if (pd_idx == raid_disks-1) { 2018 (*dd_idx)++; /* Q D D D P */ 2019 qd_idx = 0; 2020 } else if (*dd_idx >= pd_idx) 2021 (*dd_idx) += 2; /* D D P Q D */ 2022 ddf_layout = 1; 2023 break; 2024 2025 case ALGORITHM_ROTATING_N_RESTART: 2026 /* Same a left_asymmetric, by first stripe is 2027 * D D D P Q rather than 2028 * Q D D D P 2029 */ 2030 stripe2 += 1; 2031 pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks); 2032 qd_idx = pd_idx + 1; 2033 if (pd_idx == raid_disks-1) { 2034 (*dd_idx)++; /* Q D D D P */ 2035 qd_idx = 0; 2036 } else if (*dd_idx >= pd_idx) 2037 (*dd_idx) += 2; /* D D P Q D */ 2038 ddf_layout = 1; 2039 break; 2040 2041 case ALGORITHM_ROTATING_N_CONTINUE: 2042 /* Same as left_symmetric but Q is before P */ 2043 pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks); 2044 qd_idx = (pd_idx + raid_disks - 1) % raid_disks; 2045 *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks; 2046 ddf_layout = 1; 2047 break; 2048 2049 case ALGORITHM_LEFT_ASYMMETRIC_6: 2050 /* RAID5 left_asymmetric, with Q on last device */ 2051 pd_idx = data_disks - sector_div(stripe2, raid_disks-1); 2052 if (*dd_idx >= pd_idx) 2053 (*dd_idx)++; 2054 qd_idx = raid_disks - 1; 2055 break; 2056 2057 case ALGORITHM_RIGHT_ASYMMETRIC_6: 2058 pd_idx = sector_div(stripe2, raid_disks-1); 2059 if (*dd_idx >= pd_idx) 2060 (*dd_idx)++; 2061 qd_idx = raid_disks - 1; 2062 break; 2063 2064 case ALGORITHM_LEFT_SYMMETRIC_6: 2065 pd_idx = data_disks - sector_div(stripe2, raid_disks-1); 2066 *dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1); 2067 qd_idx = raid_disks - 1; 2068 break; 2069 2070 case ALGORITHM_RIGHT_SYMMETRIC_6: 2071 pd_idx = sector_div(stripe2, raid_disks-1); 2072 *dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1); 2073 qd_idx = raid_disks - 1; 2074 break; 2075 2076 case ALGORITHM_PARITY_0_6: 2077 pd_idx = 0; 2078 (*dd_idx)++; 2079 qd_idx = raid_disks - 1; 2080 break; 2081 2082 default: 2083 BUG(); 2084 } 2085 break; 2086 } 2087 2088 if (sh) { 2089 sh->pd_idx = pd_idx; 2090 sh->qd_idx = qd_idx; 2091 sh->ddf_layout = ddf_layout; 2092 } 2093 /* 2094 * Finally, compute the new sector number 2095 */ 2096 new_sector = (sector_t)stripe * sectors_per_chunk + chunk_offset; 2097 return new_sector; 2098 } 2099 2100 2101 static sector_t compute_blocknr(struct stripe_head *sh, int i, int previous) 2102 { 2103 struct r5conf *conf = sh->raid_conf; 2104 int raid_disks = sh->disks; 2105 int data_disks = raid_disks - conf->max_degraded; 2106 sector_t new_sector = sh->sector, check; 2107 int sectors_per_chunk = previous ? conf->prev_chunk_sectors 2108 : conf->chunk_sectors; 2109 int algorithm = previous ? conf->prev_algo 2110 : conf->algorithm; 2111 sector_t stripe; 2112 int chunk_offset; 2113 sector_t chunk_number; 2114 int dummy1, dd_idx = i; 2115 sector_t r_sector; 2116 struct stripe_head sh2; 2117 2118 2119 chunk_offset = sector_div(new_sector, sectors_per_chunk); 2120 stripe = new_sector; 2121 2122 if (i == sh->pd_idx) 2123 return 0; 2124 switch(conf->level) { 2125 case 4: break; 2126 case 5: 2127 switch (algorithm) { 2128 case ALGORITHM_LEFT_ASYMMETRIC: 2129 case ALGORITHM_RIGHT_ASYMMETRIC: 2130 if (i > sh->pd_idx) 2131 i--; 2132 break; 2133 case ALGORITHM_LEFT_SYMMETRIC: 2134 case ALGORITHM_RIGHT_SYMMETRIC: 2135 if (i < sh->pd_idx) 2136 i += raid_disks; 2137 i -= (sh->pd_idx + 1); 2138 break; 2139 case ALGORITHM_PARITY_0: 2140 i -= 1; 2141 break; 2142 case ALGORITHM_PARITY_N: 2143 break; 2144 default: 2145 BUG(); 2146 } 2147 break; 2148 case 6: 2149 if (i == sh->qd_idx) 2150 return 0; /* It is the Q disk */ 2151 switch (algorithm) { 2152 case ALGORITHM_LEFT_ASYMMETRIC: 2153 case ALGORITHM_RIGHT_ASYMMETRIC: 2154 case ALGORITHM_ROTATING_ZERO_RESTART: 2155 case ALGORITHM_ROTATING_N_RESTART: 2156 if (sh->pd_idx == raid_disks-1) 2157 i--; /* Q D D D P */ 2158 else if (i > sh->pd_idx) 2159 i -= 2; /* D D P Q D */ 2160 break; 2161 case ALGORITHM_LEFT_SYMMETRIC: 2162 case ALGORITHM_RIGHT_SYMMETRIC: 2163 if (sh->pd_idx == raid_disks-1) 2164 i--; /* Q D D D P */ 2165 else { 2166 /* D D P Q D */ 2167 if (i < sh->pd_idx) 2168 i += raid_disks; 2169 i -= (sh->pd_idx + 2); 2170 } 2171 break; 2172 case ALGORITHM_PARITY_0: 2173 i -= 2; 2174 break; 2175 case ALGORITHM_PARITY_N: 2176 break; 2177 case ALGORITHM_ROTATING_N_CONTINUE: 2178 /* Like left_symmetric, but P is before Q */ 2179 if (sh->pd_idx == 0) 2180 i--; /* P D D D Q */ 2181 else { 2182 /* D D Q P D */ 2183 if (i < sh->pd_idx) 2184 i += raid_disks; 2185 i -= (sh->pd_idx + 1); 2186 } 2187 break; 2188 case ALGORITHM_LEFT_ASYMMETRIC_6: 2189 case ALGORITHM_RIGHT_ASYMMETRIC_6: 2190 if (i > sh->pd_idx) 2191 i--; 2192 break; 2193 case ALGORITHM_LEFT_SYMMETRIC_6: 2194 case ALGORITHM_RIGHT_SYMMETRIC_6: 2195 if (i < sh->pd_idx) 2196 i += data_disks + 1; 2197 i -= (sh->pd_idx + 1); 2198 break; 2199 case ALGORITHM_PARITY_0_6: 2200 i -= 1; 2201 break; 2202 default: 2203 BUG(); 2204 } 2205 break; 2206 } 2207 2208 chunk_number = stripe * data_disks + i; 2209 r_sector = chunk_number * sectors_per_chunk + chunk_offset; 2210 2211 check = raid5_compute_sector(conf, r_sector, 2212 previous, &dummy1, &sh2); 2213 if (check != sh->sector || dummy1 != dd_idx || sh2.pd_idx != sh->pd_idx 2214 || sh2.qd_idx != sh->qd_idx) { 2215 printk(KERN_ERR "md/raid:%s: compute_blocknr: map not correct\n", 2216 mdname(conf->mddev)); 2217 return 0; 2218 } 2219 return r_sector; 2220 } 2221 2222 2223 static void 2224 schedule_reconstruction(struct stripe_head *sh, struct stripe_head_state *s, 2225 int rcw, int expand) 2226 { 2227 int i, pd_idx = sh->pd_idx, disks = sh->disks; 2228 struct r5conf *conf = sh->raid_conf; 2229 int level = conf->level; 2230 2231 if (rcw) { 2232 /* if we are not expanding this is a proper write request, and 2233 * there will be bios with new data to be drained into the 2234 * stripe cache 2235 */ 2236 if (!expand) { 2237 sh->reconstruct_state = reconstruct_state_drain_run; 2238 set_bit(STRIPE_OP_BIODRAIN, &s->ops_request); 2239 } else 2240 sh->reconstruct_state = reconstruct_state_run; 2241 2242 set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request); 2243 2244 for (i = disks; i--; ) { 2245 struct r5dev *dev = &sh->dev[i]; 2246 2247 if (dev->towrite) { 2248 set_bit(R5_LOCKED, &dev->flags); 2249 set_bit(R5_Wantdrain, &dev->flags); 2250 if (!expand) 2251 clear_bit(R5_UPTODATE, &dev->flags); 2252 s->locked++; 2253 } 2254 } 2255 if (s->locked + conf->max_degraded == disks) 2256 if (!test_and_set_bit(STRIPE_FULL_WRITE, &sh->state)) 2257 atomic_inc(&conf->pending_full_writes); 2258 } else { 2259 BUG_ON(level == 6); 2260 BUG_ON(!(test_bit(R5_UPTODATE, &sh->dev[pd_idx].flags) || 2261 test_bit(R5_Wantcompute, &sh->dev[pd_idx].flags))); 2262 2263 sh->reconstruct_state = reconstruct_state_prexor_drain_run; 2264 set_bit(STRIPE_OP_PREXOR, &s->ops_request); 2265 set_bit(STRIPE_OP_BIODRAIN, &s->ops_request); 2266 set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request); 2267 2268 for (i = disks; i--; ) { 2269 struct r5dev *dev = &sh->dev[i]; 2270 if (i == pd_idx) 2271 continue; 2272 2273 if (dev->towrite && 2274 (test_bit(R5_UPTODATE, &dev->flags) || 2275 test_bit(R5_Wantcompute, &dev->flags))) { 2276 set_bit(R5_Wantdrain, &dev->flags); 2277 set_bit(R5_LOCKED, &dev->flags); 2278 clear_bit(R5_UPTODATE, &dev->flags); 2279 s->locked++; 2280 } 2281 } 2282 } 2283 2284 /* keep the parity disk(s) locked while asynchronous operations 2285 * are in flight 2286 */ 2287 set_bit(R5_LOCKED, &sh->dev[pd_idx].flags); 2288 clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags); 2289 s->locked++; 2290 2291 if (level == 6) { 2292 int qd_idx = sh->qd_idx; 2293 struct r5dev *dev = &sh->dev[qd_idx]; 2294 2295 set_bit(R5_LOCKED, &dev->flags); 2296 clear_bit(R5_UPTODATE, &dev->flags); 2297 s->locked++; 2298 } 2299 2300 pr_debug("%s: stripe %llu locked: %d ops_request: %lx\n", 2301 __func__, (unsigned long long)sh->sector, 2302 s->locked, s->ops_request); 2303 } 2304 2305 /* 2306 * Each stripe/dev can have one or more bion attached. 2307 * toread/towrite point to the first in a chain. 2308 * The bi_next chain must be in order. 2309 */ 2310 static int add_stripe_bio(struct stripe_head *sh, struct bio *bi, int dd_idx, int forwrite) 2311 { 2312 struct bio **bip; 2313 struct r5conf *conf = sh->raid_conf; 2314 int firstwrite=0; 2315 2316 pr_debug("adding bi b#%llu to stripe s#%llu\n", 2317 (unsigned long long)bi->bi_sector, 2318 (unsigned long long)sh->sector); 2319 2320 2321 spin_lock_irq(&conf->device_lock); 2322 if (forwrite) { 2323 bip = &sh->dev[dd_idx].towrite; 2324 if (*bip == NULL && sh->dev[dd_idx].written == NULL) 2325 firstwrite = 1; 2326 } else 2327 bip = &sh->dev[dd_idx].toread; 2328 while (*bip && (*bip)->bi_sector < bi->bi_sector) { 2329 if ((*bip)->bi_sector + ((*bip)->bi_size >> 9) > bi->bi_sector) 2330 goto overlap; 2331 bip = & (*bip)->bi_next; 2332 } 2333 if (*bip && (*bip)->bi_sector < bi->bi_sector + ((bi->bi_size)>>9)) 2334 goto overlap; 2335 2336 BUG_ON(*bip && bi->bi_next && (*bip) != bi->bi_next); 2337 if (*bip) 2338 bi->bi_next = *bip; 2339 *bip = bi; 2340 bi->bi_phys_segments++; 2341 2342 if (forwrite) { 2343 /* check if page is covered */ 2344 sector_t sector = sh->dev[dd_idx].sector; 2345 for (bi=sh->dev[dd_idx].towrite; 2346 sector < sh->dev[dd_idx].sector + STRIPE_SECTORS && 2347 bi && bi->bi_sector <= sector; 2348 bi = r5_next_bio(bi, sh->dev[dd_idx].sector)) { 2349 if (bi->bi_sector + (bi->bi_size>>9) >= sector) 2350 sector = bi->bi_sector + (bi->bi_size>>9); 2351 } 2352 if (sector >= sh->dev[dd_idx].sector + STRIPE_SECTORS) 2353 set_bit(R5_OVERWRITE, &sh->dev[dd_idx].flags); 2354 } 2355 spin_unlock_irq(&conf->device_lock); 2356 2357 pr_debug("added bi b#%llu to stripe s#%llu, disk %d.\n", 2358 (unsigned long long)(*bip)->bi_sector, 2359 (unsigned long long)sh->sector, dd_idx); 2360 2361 if (conf->mddev->bitmap && firstwrite) { 2362 bitmap_startwrite(conf->mddev->bitmap, sh->sector, 2363 STRIPE_SECTORS, 0); 2364 sh->bm_seq = conf->seq_flush+1; 2365 set_bit(STRIPE_BIT_DELAY, &sh->state); 2366 } 2367 return 1; 2368 2369 overlap: 2370 set_bit(R5_Overlap, &sh->dev[dd_idx].flags); 2371 spin_unlock_irq(&conf->device_lock); 2372 return 0; 2373 } 2374 2375 static void end_reshape(struct r5conf *conf); 2376 2377 static void stripe_set_idx(sector_t stripe, struct r5conf *conf, int previous, 2378 struct stripe_head *sh) 2379 { 2380 int sectors_per_chunk = 2381 previous ? conf->prev_chunk_sectors : conf->chunk_sectors; 2382 int dd_idx; 2383 int chunk_offset = sector_div(stripe, sectors_per_chunk); 2384 int disks = previous ? conf->previous_raid_disks : conf->raid_disks; 2385 2386 raid5_compute_sector(conf, 2387 stripe * (disks - conf->max_degraded) 2388 *sectors_per_chunk + chunk_offset, 2389 previous, 2390 &dd_idx, sh); 2391 } 2392 2393 static void 2394 handle_failed_stripe(struct r5conf *conf, struct stripe_head *sh, 2395 struct stripe_head_state *s, int disks, 2396 struct bio **return_bi) 2397 { 2398 int i; 2399 for (i = disks; i--; ) { 2400 struct bio *bi; 2401 int bitmap_end = 0; 2402 2403 if (test_bit(R5_ReadError, &sh->dev[i].flags)) { 2404 struct md_rdev *rdev; 2405 rcu_read_lock(); 2406 rdev = rcu_dereference(conf->disks[i].rdev); 2407 if (rdev && test_bit(In_sync, &rdev->flags)) 2408 atomic_inc(&rdev->nr_pending); 2409 else 2410 rdev = NULL; 2411 rcu_read_unlock(); 2412 if (rdev) { 2413 if (!rdev_set_badblocks( 2414 rdev, 2415 sh->sector, 2416 STRIPE_SECTORS, 0)) 2417 md_error(conf->mddev, rdev); 2418 rdev_dec_pending(rdev, conf->mddev); 2419 } 2420 } 2421 spin_lock_irq(&conf->device_lock); 2422 /* fail all writes first */ 2423 bi = sh->dev[i].towrite; 2424 sh->dev[i].towrite = NULL; 2425 if (bi) { 2426 s->to_write--; 2427 bitmap_end = 1; 2428 } 2429 2430 if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags)) 2431 wake_up(&conf->wait_for_overlap); 2432 2433 while (bi && bi->bi_sector < 2434 sh->dev[i].sector + STRIPE_SECTORS) { 2435 struct bio *nextbi = r5_next_bio(bi, sh->dev[i].sector); 2436 clear_bit(BIO_UPTODATE, &bi->bi_flags); 2437 if (!raid5_dec_bi_phys_segments(bi)) { 2438 md_write_end(conf->mddev); 2439 bi->bi_next = *return_bi; 2440 *return_bi = bi; 2441 } 2442 bi = nextbi; 2443 } 2444 /* and fail all 'written' */ 2445 bi = sh->dev[i].written; 2446 sh->dev[i].written = NULL; 2447 if (bi) bitmap_end = 1; 2448 while (bi && bi->bi_sector < 2449 sh->dev[i].sector + STRIPE_SECTORS) { 2450 struct bio *bi2 = r5_next_bio(bi, sh->dev[i].sector); 2451 clear_bit(BIO_UPTODATE, &bi->bi_flags); 2452 if (!raid5_dec_bi_phys_segments(bi)) { 2453 md_write_end(conf->mddev); 2454 bi->bi_next = *return_bi; 2455 *return_bi = bi; 2456 } 2457 bi = bi2; 2458 } 2459 2460 /* fail any reads if this device is non-operational and 2461 * the data has not reached the cache yet. 2462 */ 2463 if (!test_bit(R5_Wantfill, &sh->dev[i].flags) && 2464 (!test_bit(R5_Insync, &sh->dev[i].flags) || 2465 test_bit(R5_ReadError, &sh->dev[i].flags))) { 2466 bi = sh->dev[i].toread; 2467 sh->dev[i].toread = NULL; 2468 if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags)) 2469 wake_up(&conf->wait_for_overlap); 2470 if (bi) s->to_read--; 2471 while (bi && bi->bi_sector < 2472 sh->dev[i].sector + STRIPE_SECTORS) { 2473 struct bio *nextbi = 2474 r5_next_bio(bi, sh->dev[i].sector); 2475 clear_bit(BIO_UPTODATE, &bi->bi_flags); 2476 if (!raid5_dec_bi_phys_segments(bi)) { 2477 bi->bi_next = *return_bi; 2478 *return_bi = bi; 2479 } 2480 bi = nextbi; 2481 } 2482 } 2483 spin_unlock_irq(&conf->device_lock); 2484 if (bitmap_end) 2485 bitmap_endwrite(conf->mddev->bitmap, sh->sector, 2486 STRIPE_SECTORS, 0, 0); 2487 /* If we were in the middle of a write the parity block might 2488 * still be locked - so just clear all R5_LOCKED flags 2489 */ 2490 clear_bit(R5_LOCKED, &sh->dev[i].flags); 2491 } 2492 2493 if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state)) 2494 if (atomic_dec_and_test(&conf->pending_full_writes)) 2495 md_wakeup_thread(conf->mddev->thread); 2496 } 2497 2498 static void 2499 handle_failed_sync(struct r5conf *conf, struct stripe_head *sh, 2500 struct stripe_head_state *s) 2501 { 2502 int abort = 0; 2503 int i; 2504 2505 clear_bit(STRIPE_SYNCING, &sh->state); 2506 s->syncing = 0; 2507 s->replacing = 0; 2508 /* There is nothing more to do for sync/check/repair. 2509 * Don't even need to abort as that is handled elsewhere 2510 * if needed, and not always wanted e.g. if there is a known 2511 * bad block here. 2512 * For recover/replace we need to record a bad block on all 2513 * non-sync devices, or abort the recovery 2514 */ 2515 if (test_bit(MD_RECOVERY_RECOVER, &conf->mddev->recovery)) { 2516 /* During recovery devices cannot be removed, so 2517 * locking and refcounting of rdevs is not needed 2518 */ 2519 for (i = 0; i < conf->raid_disks; i++) { 2520 struct md_rdev *rdev = conf->disks[i].rdev; 2521 if (rdev 2522 && !test_bit(Faulty, &rdev->flags) 2523 && !test_bit(In_sync, &rdev->flags) 2524 && !rdev_set_badblocks(rdev, sh->sector, 2525 STRIPE_SECTORS, 0)) 2526 abort = 1; 2527 rdev = conf->disks[i].replacement; 2528 if (rdev 2529 && !test_bit(Faulty, &rdev->flags) 2530 && !test_bit(In_sync, &rdev->flags) 2531 && !rdev_set_badblocks(rdev, sh->sector, 2532 STRIPE_SECTORS, 0)) 2533 abort = 1; 2534 } 2535 if (abort) 2536 conf->recovery_disabled = 2537 conf->mddev->recovery_disabled; 2538 } 2539 md_done_sync(conf->mddev, STRIPE_SECTORS, !abort); 2540 } 2541 2542 static int want_replace(struct stripe_head *sh, int disk_idx) 2543 { 2544 struct md_rdev *rdev; 2545 int rv = 0; 2546 /* Doing recovery so rcu locking not required */ 2547 rdev = sh->raid_conf->disks[disk_idx].replacement; 2548 if (rdev 2549 && !test_bit(Faulty, &rdev->flags) 2550 && !test_bit(In_sync, &rdev->flags) 2551 && (rdev->recovery_offset <= sh->sector 2552 || rdev->mddev->recovery_cp <= sh->sector)) 2553 rv = 1; 2554 2555 return rv; 2556 } 2557 2558 /* fetch_block - checks the given member device to see if its data needs 2559 * to be read or computed to satisfy a request. 2560 * 2561 * Returns 1 when no more member devices need to be checked, otherwise returns 2562 * 0 to tell the loop in handle_stripe_fill to continue 2563 */ 2564 static int fetch_block(struct stripe_head *sh, struct stripe_head_state *s, 2565 int disk_idx, int disks) 2566 { 2567 struct r5dev *dev = &sh->dev[disk_idx]; 2568 struct r5dev *fdev[2] = { &sh->dev[s->failed_num[0]], 2569 &sh->dev[s->failed_num[1]] }; 2570 2571 /* is the data in this block needed, and can we get it? */ 2572 if (!test_bit(R5_LOCKED, &dev->flags) && 2573 !test_bit(R5_UPTODATE, &dev->flags) && 2574 (dev->toread || 2575 (dev->towrite && !test_bit(R5_OVERWRITE, &dev->flags)) || 2576 s->syncing || s->expanding || 2577 (s->replacing && want_replace(sh, disk_idx)) || 2578 (s->failed >= 1 && fdev[0]->toread) || 2579 (s->failed >= 2 && fdev[1]->toread) || 2580 (sh->raid_conf->level <= 5 && s->failed && fdev[0]->towrite && 2581 !test_bit(R5_OVERWRITE, &fdev[0]->flags)) || 2582 (sh->raid_conf->level == 6 && s->failed && s->to_write))) { 2583 /* we would like to get this block, possibly by computing it, 2584 * otherwise read it if the backing disk is insync 2585 */ 2586 BUG_ON(test_bit(R5_Wantcompute, &dev->flags)); 2587 BUG_ON(test_bit(R5_Wantread, &dev->flags)); 2588 if ((s->uptodate == disks - 1) && 2589 (s->failed && (disk_idx == s->failed_num[0] || 2590 disk_idx == s->failed_num[1]))) { 2591 /* have disk failed, and we're requested to fetch it; 2592 * do compute it 2593 */ 2594 pr_debug("Computing stripe %llu block %d\n", 2595 (unsigned long long)sh->sector, disk_idx); 2596 set_bit(STRIPE_COMPUTE_RUN, &sh->state); 2597 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request); 2598 set_bit(R5_Wantcompute, &dev->flags); 2599 sh->ops.target = disk_idx; 2600 sh->ops.target2 = -1; /* no 2nd target */ 2601 s->req_compute = 1; 2602 /* Careful: from this point on 'uptodate' is in the eye 2603 * of raid_run_ops which services 'compute' operations 2604 * before writes. R5_Wantcompute flags a block that will 2605 * be R5_UPTODATE by the time it is needed for a 2606 * subsequent operation. 2607 */ 2608 s->uptodate++; 2609 return 1; 2610 } else if (s->uptodate == disks-2 && s->failed >= 2) { 2611 /* Computing 2-failure is *very* expensive; only 2612 * do it if failed >= 2 2613 */ 2614 int other; 2615 for (other = disks; other--; ) { 2616 if (other == disk_idx) 2617 continue; 2618 if (!test_bit(R5_UPTODATE, 2619 &sh->dev[other].flags)) 2620 break; 2621 } 2622 BUG_ON(other < 0); 2623 pr_debug("Computing stripe %llu blocks %d,%d\n", 2624 (unsigned long long)sh->sector, 2625 disk_idx, other); 2626 set_bit(STRIPE_COMPUTE_RUN, &sh->state); 2627 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request); 2628 set_bit(R5_Wantcompute, &sh->dev[disk_idx].flags); 2629 set_bit(R5_Wantcompute, &sh->dev[other].flags); 2630 sh->ops.target = disk_idx; 2631 sh->ops.target2 = other; 2632 s->uptodate += 2; 2633 s->req_compute = 1; 2634 return 1; 2635 } else if (test_bit(R5_Insync, &dev->flags)) { 2636 set_bit(R5_LOCKED, &dev->flags); 2637 set_bit(R5_Wantread, &dev->flags); 2638 s->locked++; 2639 pr_debug("Reading block %d (sync=%d)\n", 2640 disk_idx, s->syncing); 2641 } 2642 } 2643 2644 return 0; 2645 } 2646 2647 /** 2648 * handle_stripe_fill - read or compute data to satisfy pending requests. 2649 */ 2650 static void handle_stripe_fill(struct stripe_head *sh, 2651 struct stripe_head_state *s, 2652 int disks) 2653 { 2654 int i; 2655 2656 /* look for blocks to read/compute, skip this if a compute 2657 * is already in flight, or if the stripe contents are in the 2658 * midst of changing due to a write 2659 */ 2660 if (!test_bit(STRIPE_COMPUTE_RUN, &sh->state) && !sh->check_state && 2661 !sh->reconstruct_state) 2662 for (i = disks; i--; ) 2663 if (fetch_block(sh, s, i, disks)) 2664 break; 2665 set_bit(STRIPE_HANDLE, &sh->state); 2666 } 2667 2668 2669 /* handle_stripe_clean_event 2670 * any written block on an uptodate or failed drive can be returned. 2671 * Note that if we 'wrote' to a failed drive, it will be UPTODATE, but 2672 * never LOCKED, so we don't need to test 'failed' directly. 2673 */ 2674 static void handle_stripe_clean_event(struct r5conf *conf, 2675 struct stripe_head *sh, int disks, struct bio **return_bi) 2676 { 2677 int i; 2678 struct r5dev *dev; 2679 2680 for (i = disks; i--; ) 2681 if (sh->dev[i].written) { 2682 dev = &sh->dev[i]; 2683 if (!test_bit(R5_LOCKED, &dev->flags) && 2684 test_bit(R5_UPTODATE, &dev->flags)) { 2685 /* We can return any write requests */ 2686 struct bio *wbi, *wbi2; 2687 int bitmap_end = 0; 2688 pr_debug("Return write for disc %d\n", i); 2689 spin_lock_irq(&conf->device_lock); 2690 wbi = dev->written; 2691 dev->written = NULL; 2692 while (wbi && wbi->bi_sector < 2693 dev->sector + STRIPE_SECTORS) { 2694 wbi2 = r5_next_bio(wbi, dev->sector); 2695 if (!raid5_dec_bi_phys_segments(wbi)) { 2696 md_write_end(conf->mddev); 2697 wbi->bi_next = *return_bi; 2698 *return_bi = wbi; 2699 } 2700 wbi = wbi2; 2701 } 2702 if (dev->towrite == NULL) 2703 bitmap_end = 1; 2704 spin_unlock_irq(&conf->device_lock); 2705 if (bitmap_end) 2706 bitmap_endwrite(conf->mddev->bitmap, 2707 sh->sector, 2708 STRIPE_SECTORS, 2709 !test_bit(STRIPE_DEGRADED, &sh->state), 2710 0); 2711 } 2712 } 2713 2714 if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state)) 2715 if (atomic_dec_and_test(&conf->pending_full_writes)) 2716 md_wakeup_thread(conf->mddev->thread); 2717 } 2718 2719 static void handle_stripe_dirtying(struct r5conf *conf, 2720 struct stripe_head *sh, 2721 struct stripe_head_state *s, 2722 int disks) 2723 { 2724 int rmw = 0, rcw = 0, i; 2725 if (conf->max_degraded == 2) { 2726 /* RAID6 requires 'rcw' in current implementation 2727 * Calculate the real rcw later - for now fake it 2728 * look like rcw is cheaper 2729 */ 2730 rcw = 1; rmw = 2; 2731 } else for (i = disks; i--; ) { 2732 /* would I have to read this buffer for read_modify_write */ 2733 struct r5dev *dev = &sh->dev[i]; 2734 if ((dev->towrite || i == sh->pd_idx) && 2735 !test_bit(R5_LOCKED, &dev->flags) && 2736 !(test_bit(R5_UPTODATE, &dev->flags) || 2737 test_bit(R5_Wantcompute, &dev->flags))) { 2738 if (test_bit(R5_Insync, &dev->flags)) 2739 rmw++; 2740 else 2741 rmw += 2*disks; /* cannot read it */ 2742 } 2743 /* Would I have to read this buffer for reconstruct_write */ 2744 if (!test_bit(R5_OVERWRITE, &dev->flags) && i != sh->pd_idx && 2745 !test_bit(R5_LOCKED, &dev->flags) && 2746 !(test_bit(R5_UPTODATE, &dev->flags) || 2747 test_bit(R5_Wantcompute, &dev->flags))) { 2748 if (test_bit(R5_Insync, &dev->flags)) rcw++; 2749 else 2750 rcw += 2*disks; 2751 } 2752 } 2753 pr_debug("for sector %llu, rmw=%d rcw=%d\n", 2754 (unsigned long long)sh->sector, rmw, rcw); 2755 set_bit(STRIPE_HANDLE, &sh->state); 2756 if (rmw < rcw && rmw > 0) 2757 /* prefer read-modify-write, but need to get some data */ 2758 for (i = disks; i--; ) { 2759 struct r5dev *dev = &sh->dev[i]; 2760 if ((dev->towrite || i == sh->pd_idx) && 2761 !test_bit(R5_LOCKED, &dev->flags) && 2762 !(test_bit(R5_UPTODATE, &dev->flags) || 2763 test_bit(R5_Wantcompute, &dev->flags)) && 2764 test_bit(R5_Insync, &dev->flags)) { 2765 if ( 2766 test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) { 2767 pr_debug("Read_old block " 2768 "%d for r-m-w\n", i); 2769 set_bit(R5_LOCKED, &dev->flags); 2770 set_bit(R5_Wantread, &dev->flags); 2771 s->locked++; 2772 } else { 2773 set_bit(STRIPE_DELAYED, &sh->state); 2774 set_bit(STRIPE_HANDLE, &sh->state); 2775 } 2776 } 2777 } 2778 if (rcw <= rmw && rcw > 0) { 2779 /* want reconstruct write, but need to get some data */ 2780 rcw = 0; 2781 for (i = disks; i--; ) { 2782 struct r5dev *dev = &sh->dev[i]; 2783 if (!test_bit(R5_OVERWRITE, &dev->flags) && 2784 i != sh->pd_idx && i != sh->qd_idx && 2785 !test_bit(R5_LOCKED, &dev->flags) && 2786 !(test_bit(R5_UPTODATE, &dev->flags) || 2787 test_bit(R5_Wantcompute, &dev->flags))) { 2788 rcw++; 2789 if (!test_bit(R5_Insync, &dev->flags)) 2790 continue; /* it's a failed drive */ 2791 if ( 2792 test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) { 2793 pr_debug("Read_old block " 2794 "%d for Reconstruct\n", i); 2795 set_bit(R5_LOCKED, &dev->flags); 2796 set_bit(R5_Wantread, &dev->flags); 2797 s->locked++; 2798 } else { 2799 set_bit(STRIPE_DELAYED, &sh->state); 2800 set_bit(STRIPE_HANDLE, &sh->state); 2801 } 2802 } 2803 } 2804 } 2805 /* now if nothing is locked, and if we have enough data, 2806 * we can start a write request 2807 */ 2808 /* since handle_stripe can be called at any time we need to handle the 2809 * case where a compute block operation has been submitted and then a 2810 * subsequent call wants to start a write request. raid_run_ops only 2811 * handles the case where compute block and reconstruct are requested 2812 * simultaneously. If this is not the case then new writes need to be 2813 * held off until the compute completes. 2814 */ 2815 if ((s->req_compute || !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) && 2816 (s->locked == 0 && (rcw == 0 || rmw == 0) && 2817 !test_bit(STRIPE_BIT_DELAY, &sh->state))) 2818 schedule_reconstruction(sh, s, rcw == 0, 0); 2819 } 2820 2821 static void handle_parity_checks5(struct r5conf *conf, struct stripe_head *sh, 2822 struct stripe_head_state *s, int disks) 2823 { 2824 struct r5dev *dev = NULL; 2825 2826 set_bit(STRIPE_HANDLE, &sh->state); 2827 2828 switch (sh->check_state) { 2829 case check_state_idle: 2830 /* start a new check operation if there are no failures */ 2831 if (s->failed == 0) { 2832 BUG_ON(s->uptodate != disks); 2833 sh->check_state = check_state_run; 2834 set_bit(STRIPE_OP_CHECK, &s->ops_request); 2835 clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags); 2836 s->uptodate--; 2837 break; 2838 } 2839 dev = &sh->dev[s->failed_num[0]]; 2840 /* fall through */ 2841 case check_state_compute_result: 2842 sh->check_state = check_state_idle; 2843 if (!dev) 2844 dev = &sh->dev[sh->pd_idx]; 2845 2846 /* check that a write has not made the stripe insync */ 2847 if (test_bit(STRIPE_INSYNC, &sh->state)) 2848 break; 2849 2850 /* either failed parity check, or recovery is happening */ 2851 BUG_ON(!test_bit(R5_UPTODATE, &dev->flags)); 2852 BUG_ON(s->uptodate != disks); 2853 2854 set_bit(R5_LOCKED, &dev->flags); 2855 s->locked++; 2856 set_bit(R5_Wantwrite, &dev->flags); 2857 2858 clear_bit(STRIPE_DEGRADED, &sh->state); 2859 set_bit(STRIPE_INSYNC, &sh->state); 2860 break; 2861 case check_state_run: 2862 break; /* we will be called again upon completion */ 2863 case check_state_check_result: 2864 sh->check_state = check_state_idle; 2865 2866 /* if a failure occurred during the check operation, leave 2867 * STRIPE_INSYNC not set and let the stripe be handled again 2868 */ 2869 if (s->failed) 2870 break; 2871 2872 /* handle a successful check operation, if parity is correct 2873 * we are done. Otherwise update the mismatch count and repair 2874 * parity if !MD_RECOVERY_CHECK 2875 */ 2876 if ((sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) == 0) 2877 /* parity is correct (on disc, 2878 * not in buffer any more) 2879 */ 2880 set_bit(STRIPE_INSYNC, &sh->state); 2881 else { 2882 conf->mddev->resync_mismatches += STRIPE_SECTORS; 2883 if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery)) 2884 /* don't try to repair!! */ 2885 set_bit(STRIPE_INSYNC, &sh->state); 2886 else { 2887 sh->check_state = check_state_compute_run; 2888 set_bit(STRIPE_COMPUTE_RUN, &sh->state); 2889 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request); 2890 set_bit(R5_Wantcompute, 2891 &sh->dev[sh->pd_idx].flags); 2892 sh->ops.target = sh->pd_idx; 2893 sh->ops.target2 = -1; 2894 s->uptodate++; 2895 } 2896 } 2897 break; 2898 case check_state_compute_run: 2899 break; 2900 default: 2901 printk(KERN_ERR "%s: unknown check_state: %d sector: %llu\n", 2902 __func__, sh->check_state, 2903 (unsigned long long) sh->sector); 2904 BUG(); 2905 } 2906 } 2907 2908 2909 static void handle_parity_checks6(struct r5conf *conf, struct stripe_head *sh, 2910 struct stripe_head_state *s, 2911 int disks) 2912 { 2913 int pd_idx = sh->pd_idx; 2914 int qd_idx = sh->qd_idx; 2915 struct r5dev *dev; 2916 2917 set_bit(STRIPE_HANDLE, &sh->state); 2918 2919 BUG_ON(s->failed > 2); 2920 2921 /* Want to check and possibly repair P and Q. 2922 * However there could be one 'failed' device, in which 2923 * case we can only check one of them, possibly using the 2924 * other to generate missing data 2925 */ 2926 2927 switch (sh->check_state) { 2928 case check_state_idle: 2929 /* start a new check operation if there are < 2 failures */ 2930 if (s->failed == s->q_failed) { 2931 /* The only possible failed device holds Q, so it 2932 * makes sense to check P (If anything else were failed, 2933 * we would have used P to recreate it). 2934 */ 2935 sh->check_state = check_state_run; 2936 } 2937 if (!s->q_failed && s->failed < 2) { 2938 /* Q is not failed, and we didn't use it to generate 2939 * anything, so it makes sense to check it 2940 */ 2941 if (sh->check_state == check_state_run) 2942 sh->check_state = check_state_run_pq; 2943 else 2944 sh->check_state = check_state_run_q; 2945 } 2946 2947 /* discard potentially stale zero_sum_result */ 2948 sh->ops.zero_sum_result = 0; 2949 2950 if (sh->check_state == check_state_run) { 2951 /* async_xor_zero_sum destroys the contents of P */ 2952 clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags); 2953 s->uptodate--; 2954 } 2955 if (sh->check_state >= check_state_run && 2956 sh->check_state <= check_state_run_pq) { 2957 /* async_syndrome_zero_sum preserves P and Q, so 2958 * no need to mark them !uptodate here 2959 */ 2960 set_bit(STRIPE_OP_CHECK, &s->ops_request); 2961 break; 2962 } 2963 2964 /* we have 2-disk failure */ 2965 BUG_ON(s->failed != 2); 2966 /* fall through */ 2967 case check_state_compute_result: 2968 sh->check_state = check_state_idle; 2969 2970 /* check that a write has not made the stripe insync */ 2971 if (test_bit(STRIPE_INSYNC, &sh->state)) 2972 break; 2973 2974 /* now write out any block on a failed drive, 2975 * or P or Q if they were recomputed 2976 */ 2977 BUG_ON(s->uptodate < disks - 1); /* We don't need Q to recover */ 2978 if (s->failed == 2) { 2979 dev = &sh->dev[s->failed_num[1]]; 2980 s->locked++; 2981 set_bit(R5_LOCKED, &dev->flags); 2982 set_bit(R5_Wantwrite, &dev->flags); 2983 } 2984 if (s->failed >= 1) { 2985 dev = &sh->dev[s->failed_num[0]]; 2986 s->locked++; 2987 set_bit(R5_LOCKED, &dev->flags); 2988 set_bit(R5_Wantwrite, &dev->flags); 2989 } 2990 if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) { 2991 dev = &sh->dev[pd_idx]; 2992 s->locked++; 2993 set_bit(R5_LOCKED, &dev->flags); 2994 set_bit(R5_Wantwrite, &dev->flags); 2995 } 2996 if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) { 2997 dev = &sh->dev[qd_idx]; 2998 s->locked++; 2999 set_bit(R5_LOCKED, &dev->flags); 3000 set_bit(R5_Wantwrite, &dev->flags); 3001 } 3002 clear_bit(STRIPE_DEGRADED, &sh->state); 3003 3004 set_bit(STRIPE_INSYNC, &sh->state); 3005 break; 3006 case check_state_run: 3007 case check_state_run_q: 3008 case check_state_run_pq: 3009 break; /* we will be called again upon completion */ 3010 case check_state_check_result: 3011 sh->check_state = check_state_idle; 3012 3013 /* handle a successful check operation, if parity is correct 3014 * we are done. Otherwise update the mismatch count and repair 3015 * parity if !MD_RECOVERY_CHECK 3016 */ 3017 if (sh->ops.zero_sum_result == 0) { 3018 /* both parities are correct */ 3019 if (!s->failed) 3020 set_bit(STRIPE_INSYNC, &sh->state); 3021 else { 3022 /* in contrast to the raid5 case we can validate 3023 * parity, but still have a failure to write 3024 * back 3025 */ 3026 sh->check_state = check_state_compute_result; 3027 /* Returning at this point means that we may go 3028 * off and bring p and/or q uptodate again so 3029 * we make sure to check zero_sum_result again 3030 * to verify if p or q need writeback 3031 */ 3032 } 3033 } else { 3034 conf->mddev->resync_mismatches += STRIPE_SECTORS; 3035 if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery)) 3036 /* don't try to repair!! */ 3037 set_bit(STRIPE_INSYNC, &sh->state); 3038 else { 3039 int *target = &sh->ops.target; 3040 3041 sh->ops.target = -1; 3042 sh->ops.target2 = -1; 3043 sh->check_state = check_state_compute_run; 3044 set_bit(STRIPE_COMPUTE_RUN, &sh->state); 3045 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request); 3046 if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) { 3047 set_bit(R5_Wantcompute, 3048 &sh->dev[pd_idx].flags); 3049 *target = pd_idx; 3050 target = &sh->ops.target2; 3051 s->uptodate++; 3052 } 3053 if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) { 3054 set_bit(R5_Wantcompute, 3055 &sh->dev[qd_idx].flags); 3056 *target = qd_idx; 3057 s->uptodate++; 3058 } 3059 } 3060 } 3061 break; 3062 case check_state_compute_run: 3063 break; 3064 default: 3065 printk(KERN_ERR "%s: unknown check_state: %d sector: %llu\n", 3066 __func__, sh->check_state, 3067 (unsigned long long) sh->sector); 3068 BUG(); 3069 } 3070 } 3071 3072 static void handle_stripe_expansion(struct r5conf *conf, struct stripe_head *sh) 3073 { 3074 int i; 3075 3076 /* We have read all the blocks in this stripe and now we need to 3077 * copy some of them into a target stripe for expand. 3078 */ 3079 struct dma_async_tx_descriptor *tx = NULL; 3080 clear_bit(STRIPE_EXPAND_SOURCE, &sh->state); 3081 for (i = 0; i < sh->disks; i++) 3082 if (i != sh->pd_idx && i != sh->qd_idx) { 3083 int dd_idx, j; 3084 struct stripe_head *sh2; 3085 struct async_submit_ctl submit; 3086 3087 sector_t bn = compute_blocknr(sh, i, 1); 3088 sector_t s = raid5_compute_sector(conf, bn, 0, 3089 &dd_idx, NULL); 3090 sh2 = get_active_stripe(conf, s, 0, 1, 1); 3091 if (sh2 == NULL) 3092 /* so far only the early blocks of this stripe 3093 * have been requested. When later blocks 3094 * get requested, we will try again 3095 */ 3096 continue; 3097 if (!test_bit(STRIPE_EXPANDING, &sh2->state) || 3098 test_bit(R5_Expanded, &sh2->dev[dd_idx].flags)) { 3099 /* must have already done this block */ 3100 release_stripe(sh2); 3101 continue; 3102 } 3103 3104 /* place all the copies on one channel */ 3105 init_async_submit(&submit, 0, tx, NULL, NULL, NULL); 3106 tx = async_memcpy(sh2->dev[dd_idx].page, 3107 sh->dev[i].page, 0, 0, STRIPE_SIZE, 3108 &submit); 3109 3110 set_bit(R5_Expanded, &sh2->dev[dd_idx].flags); 3111 set_bit(R5_UPTODATE, &sh2->dev[dd_idx].flags); 3112 for (j = 0; j < conf->raid_disks; j++) 3113 if (j != sh2->pd_idx && 3114 j != sh2->qd_idx && 3115 !test_bit(R5_Expanded, &sh2->dev[j].flags)) 3116 break; 3117 if (j == conf->raid_disks) { 3118 set_bit(STRIPE_EXPAND_READY, &sh2->state); 3119 set_bit(STRIPE_HANDLE, &sh2->state); 3120 } 3121 release_stripe(sh2); 3122 3123 } 3124 /* done submitting copies, wait for them to complete */ 3125 if (tx) { 3126 async_tx_ack(tx); 3127 dma_wait_for_async_tx(tx); 3128 } 3129 } 3130 3131 /* 3132 * handle_stripe - do things to a stripe. 3133 * 3134 * We lock the stripe by setting STRIPE_ACTIVE and then examine the 3135 * state of various bits to see what needs to be done. 3136 * Possible results: 3137 * return some read requests which now have data 3138 * return some write requests which are safely on storage 3139 * schedule a read on some buffers 3140 * schedule a write of some buffers 3141 * return confirmation of parity correctness 3142 * 3143 */ 3144 3145 static void analyse_stripe(struct stripe_head *sh, struct stripe_head_state *s) 3146 { 3147 struct r5conf *conf = sh->raid_conf; 3148 int disks = sh->disks; 3149 struct r5dev *dev; 3150 int i; 3151 int do_recovery = 0; 3152 3153 memset(s, 0, sizeof(*s)); 3154 3155 s->expanding = test_bit(STRIPE_EXPAND_SOURCE, &sh->state); 3156 s->expanded = test_bit(STRIPE_EXPAND_READY, &sh->state); 3157 s->failed_num[0] = -1; 3158 s->failed_num[1] = -1; 3159 3160 /* Now to look around and see what can be done */ 3161 rcu_read_lock(); 3162 spin_lock_irq(&conf->device_lock); 3163 for (i=disks; i--; ) { 3164 struct md_rdev *rdev; 3165 sector_t first_bad; 3166 int bad_sectors; 3167 int is_bad = 0; 3168 3169 dev = &sh->dev[i]; 3170 3171 pr_debug("check %d: state 0x%lx read %p write %p written %p\n", 3172 i, dev->flags, 3173 dev->toread, dev->towrite, dev->written); 3174 /* maybe we can reply to a read 3175 * 3176 * new wantfill requests are only permitted while 3177 * ops_complete_biofill is guaranteed to be inactive 3178 */ 3179 if (test_bit(R5_UPTODATE, &dev->flags) && dev->toread && 3180 !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) 3181 set_bit(R5_Wantfill, &dev->flags); 3182 3183 /* now count some things */ 3184 if (test_bit(R5_LOCKED, &dev->flags)) 3185 s->locked++; 3186 if (test_bit(R5_UPTODATE, &dev->flags)) 3187 s->uptodate++; 3188 if (test_bit(R5_Wantcompute, &dev->flags)) { 3189 s->compute++; 3190 BUG_ON(s->compute > 2); 3191 } 3192 3193 if (test_bit(R5_Wantfill, &dev->flags)) 3194 s->to_fill++; 3195 else if (dev->toread) 3196 s->to_read++; 3197 if (dev->towrite) { 3198 s->to_write++; 3199 if (!test_bit(R5_OVERWRITE, &dev->flags)) 3200 s->non_overwrite++; 3201 } 3202 if (dev->written) 3203 s->written++; 3204 /* Prefer to use the replacement for reads, but only 3205 * if it is recovered enough and has no bad blocks. 3206 */ 3207 rdev = rcu_dereference(conf->disks[i].replacement); 3208 if (rdev && !test_bit(Faulty, &rdev->flags) && 3209 rdev->recovery_offset >= sh->sector + STRIPE_SECTORS && 3210 !is_badblock(rdev, sh->sector, STRIPE_SECTORS, 3211 &first_bad, &bad_sectors)) 3212 set_bit(R5_ReadRepl, &dev->flags); 3213 else { 3214 if (rdev) 3215 set_bit(R5_NeedReplace, &dev->flags); 3216 rdev = rcu_dereference(conf->disks[i].rdev); 3217 clear_bit(R5_ReadRepl, &dev->flags); 3218 } 3219 if (rdev && test_bit(Faulty, &rdev->flags)) 3220 rdev = NULL; 3221 if (rdev) { 3222 is_bad = is_badblock(rdev, sh->sector, STRIPE_SECTORS, 3223 &first_bad, &bad_sectors); 3224 if (s->blocked_rdev == NULL 3225 && (test_bit(Blocked, &rdev->flags) 3226 || is_bad < 0)) { 3227 if (is_bad < 0) 3228 set_bit(BlockedBadBlocks, 3229 &rdev->flags); 3230 s->blocked_rdev = rdev; 3231 atomic_inc(&rdev->nr_pending); 3232 } 3233 } 3234 clear_bit(R5_Insync, &dev->flags); 3235 if (!rdev) 3236 /* Not in-sync */; 3237 else if (is_bad) { 3238 /* also not in-sync */ 3239 if (!test_bit(WriteErrorSeen, &rdev->flags) && 3240 test_bit(R5_UPTODATE, &dev->flags)) { 3241 /* treat as in-sync, but with a read error 3242 * which we can now try to correct 3243 */ 3244 set_bit(R5_Insync, &dev->flags); 3245 set_bit(R5_ReadError, &dev->flags); 3246 } 3247 } else if (test_bit(In_sync, &rdev->flags)) 3248 set_bit(R5_Insync, &dev->flags); 3249 else if (sh->sector + STRIPE_SECTORS <= rdev->recovery_offset) 3250 /* in sync if before recovery_offset */ 3251 set_bit(R5_Insync, &dev->flags); 3252 else if (test_bit(R5_UPTODATE, &dev->flags) && 3253 test_bit(R5_Expanded, &dev->flags)) 3254 /* If we've reshaped into here, we assume it is Insync. 3255 * We will shortly update recovery_offset to make 3256 * it official. 3257 */ 3258 set_bit(R5_Insync, &dev->flags); 3259 3260 if (rdev && test_bit(R5_WriteError, &dev->flags)) { 3261 /* This flag does not apply to '.replacement' 3262 * only to .rdev, so make sure to check that*/ 3263 struct md_rdev *rdev2 = rcu_dereference( 3264 conf->disks[i].rdev); 3265 if (rdev2 == rdev) 3266 clear_bit(R5_Insync, &dev->flags); 3267 if (rdev2 && !test_bit(Faulty, &rdev2->flags)) { 3268 s->handle_bad_blocks = 1; 3269 atomic_inc(&rdev2->nr_pending); 3270 } else 3271 clear_bit(R5_WriteError, &dev->flags); 3272 } 3273 if (rdev && test_bit(R5_MadeGood, &dev->flags)) { 3274 /* This flag does not apply to '.replacement' 3275 * only to .rdev, so make sure to check that*/ 3276 struct md_rdev *rdev2 = rcu_dereference( 3277 conf->disks[i].rdev); 3278 if (rdev2 && !test_bit(Faulty, &rdev2->flags)) { 3279 s->handle_bad_blocks = 1; 3280 atomic_inc(&rdev2->nr_pending); 3281 } else 3282 clear_bit(R5_MadeGood, &dev->flags); 3283 } 3284 if (test_bit(R5_MadeGoodRepl, &dev->flags)) { 3285 struct md_rdev *rdev2 = rcu_dereference( 3286 conf->disks[i].replacement); 3287 if (rdev2 && !test_bit(Faulty, &rdev2->flags)) { 3288 s->handle_bad_blocks = 1; 3289 atomic_inc(&rdev2->nr_pending); 3290 } else 3291 clear_bit(R5_MadeGoodRepl, &dev->flags); 3292 } 3293 if (!test_bit(R5_Insync, &dev->flags)) { 3294 /* The ReadError flag will just be confusing now */ 3295 clear_bit(R5_ReadError, &dev->flags); 3296 clear_bit(R5_ReWrite, &dev->flags); 3297 } 3298 if (test_bit(R5_ReadError, &dev->flags)) 3299 clear_bit(R5_Insync, &dev->flags); 3300 if (!test_bit(R5_Insync, &dev->flags)) { 3301 if (s->failed < 2) 3302 s->failed_num[s->failed] = i; 3303 s->failed++; 3304 if (rdev && !test_bit(Faulty, &rdev->flags)) 3305 do_recovery = 1; 3306 } 3307 } 3308 spin_unlock_irq(&conf->device_lock); 3309 if (test_bit(STRIPE_SYNCING, &sh->state)) { 3310 /* If there is a failed device being replaced, 3311 * we must be recovering. 3312 * else if we are after recovery_cp, we must be syncing 3313 * else if MD_RECOVERY_REQUESTED is set, we also are syncing. 3314 * else we can only be replacing 3315 * sync and recovery both need to read all devices, and so 3316 * use the same flag. 3317 */ 3318 if (do_recovery || 3319 sh->sector >= conf->mddev->recovery_cp || 3320 test_bit(MD_RECOVERY_REQUESTED, &(conf->mddev->recovery))) 3321 s->syncing = 1; 3322 else 3323 s->replacing = 1; 3324 } 3325 rcu_read_unlock(); 3326 } 3327 3328 static void handle_stripe(struct stripe_head *sh) 3329 { 3330 struct stripe_head_state s; 3331 struct r5conf *conf = sh->raid_conf; 3332 int i; 3333 int prexor; 3334 int disks = sh->disks; 3335 struct r5dev *pdev, *qdev; 3336 3337 clear_bit(STRIPE_HANDLE, &sh->state); 3338 if (test_and_set_bit_lock(STRIPE_ACTIVE, &sh->state)) { 3339 /* already being handled, ensure it gets handled 3340 * again when current action finishes */ 3341 set_bit(STRIPE_HANDLE, &sh->state); 3342 return; 3343 } 3344 3345 if (test_and_clear_bit(STRIPE_SYNC_REQUESTED, &sh->state)) { 3346 set_bit(STRIPE_SYNCING, &sh->state); 3347 clear_bit(STRIPE_INSYNC, &sh->state); 3348 } 3349 clear_bit(STRIPE_DELAYED, &sh->state); 3350 3351 pr_debug("handling stripe %llu, state=%#lx cnt=%d, " 3352 "pd_idx=%d, qd_idx=%d\n, check:%d, reconstruct:%d\n", 3353 (unsigned long long)sh->sector, sh->state, 3354 atomic_read(&sh->count), sh->pd_idx, sh->qd_idx, 3355 sh->check_state, sh->reconstruct_state); 3356 3357 analyse_stripe(sh, &s); 3358 3359 if (s.handle_bad_blocks) { 3360 set_bit(STRIPE_HANDLE, &sh->state); 3361 goto finish; 3362 } 3363 3364 if (unlikely(s.blocked_rdev)) { 3365 if (s.syncing || s.expanding || s.expanded || 3366 s.replacing || s.to_write || s.written) { 3367 set_bit(STRIPE_HANDLE, &sh->state); 3368 goto finish; 3369 } 3370 /* There is nothing for the blocked_rdev to block */ 3371 rdev_dec_pending(s.blocked_rdev, conf->mddev); 3372 s.blocked_rdev = NULL; 3373 } 3374 3375 if (s.to_fill && !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) { 3376 set_bit(STRIPE_OP_BIOFILL, &s.ops_request); 3377 set_bit(STRIPE_BIOFILL_RUN, &sh->state); 3378 } 3379 3380 pr_debug("locked=%d uptodate=%d to_read=%d" 3381 " to_write=%d failed=%d failed_num=%d,%d\n", 3382 s.locked, s.uptodate, s.to_read, s.to_write, s.failed, 3383 s.failed_num[0], s.failed_num[1]); 3384 /* check if the array has lost more than max_degraded devices and, 3385 * if so, some requests might need to be failed. 3386 */ 3387 if (s.failed > conf->max_degraded) { 3388 sh->check_state = 0; 3389 sh->reconstruct_state = 0; 3390 if (s.to_read+s.to_write+s.written) 3391 handle_failed_stripe(conf, sh, &s, disks, &s.return_bi); 3392 if (s.syncing + s.replacing) 3393 handle_failed_sync(conf, sh, &s); 3394 } 3395 3396 /* 3397 * might be able to return some write requests if the parity blocks 3398 * are safe, or on a failed drive 3399 */ 3400 pdev = &sh->dev[sh->pd_idx]; 3401 s.p_failed = (s.failed >= 1 && s.failed_num[0] == sh->pd_idx) 3402 || (s.failed >= 2 && s.failed_num[1] == sh->pd_idx); 3403 qdev = &sh->dev[sh->qd_idx]; 3404 s.q_failed = (s.failed >= 1 && s.failed_num[0] == sh->qd_idx) 3405 || (s.failed >= 2 && s.failed_num[1] == sh->qd_idx) 3406 || conf->level < 6; 3407 3408 if (s.written && 3409 (s.p_failed || ((test_bit(R5_Insync, &pdev->flags) 3410 && !test_bit(R5_LOCKED, &pdev->flags) 3411 && test_bit(R5_UPTODATE, &pdev->flags)))) && 3412 (s.q_failed || ((test_bit(R5_Insync, &qdev->flags) 3413 && !test_bit(R5_LOCKED, &qdev->flags) 3414 && test_bit(R5_UPTODATE, &qdev->flags))))) 3415 handle_stripe_clean_event(conf, sh, disks, &s.return_bi); 3416 3417 /* Now we might consider reading some blocks, either to check/generate 3418 * parity, or to satisfy requests 3419 * or to load a block that is being partially written. 3420 */ 3421 if (s.to_read || s.non_overwrite 3422 || (conf->level == 6 && s.to_write && s.failed) 3423 || (s.syncing && (s.uptodate + s.compute < disks)) 3424 || s.replacing 3425 || s.expanding) 3426 handle_stripe_fill(sh, &s, disks); 3427 3428 /* Now we check to see if any write operations have recently 3429 * completed 3430 */ 3431 prexor = 0; 3432 if (sh->reconstruct_state == reconstruct_state_prexor_drain_result) 3433 prexor = 1; 3434 if (sh->reconstruct_state == reconstruct_state_drain_result || 3435 sh->reconstruct_state == reconstruct_state_prexor_drain_result) { 3436 sh->reconstruct_state = reconstruct_state_idle; 3437 3438 /* All the 'written' buffers and the parity block are ready to 3439 * be written back to disk 3440 */ 3441 BUG_ON(!test_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags)); 3442 BUG_ON(sh->qd_idx >= 0 && 3443 !test_bit(R5_UPTODATE, &sh->dev[sh->qd_idx].flags)); 3444 for (i = disks; i--; ) { 3445 struct r5dev *dev = &sh->dev[i]; 3446 if (test_bit(R5_LOCKED, &dev->flags) && 3447 (i == sh->pd_idx || i == sh->qd_idx || 3448 dev->written)) { 3449 pr_debug("Writing block %d\n", i); 3450 set_bit(R5_Wantwrite, &dev->flags); 3451 if (prexor) 3452 continue; 3453 if (!test_bit(R5_Insync, &dev->flags) || 3454 ((i == sh->pd_idx || i == sh->qd_idx) && 3455 s.failed == 0)) 3456 set_bit(STRIPE_INSYNC, &sh->state); 3457 } 3458 } 3459 if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) 3460 s.dec_preread_active = 1; 3461 } 3462 3463 /* Now to consider new write requests and what else, if anything 3464 * should be read. We do not handle new writes when: 3465 * 1/ A 'write' operation (copy+xor) is already in flight. 3466 * 2/ A 'check' operation is in flight, as it may clobber the parity 3467 * block. 3468 */ 3469 if (s.to_write && !sh->reconstruct_state && !sh->check_state) 3470 handle_stripe_dirtying(conf, sh, &s, disks); 3471 3472 /* maybe we need to check and possibly fix the parity for this stripe 3473 * Any reads will already have been scheduled, so we just see if enough 3474 * data is available. The parity check is held off while parity 3475 * dependent operations are in flight. 3476 */ 3477 if (sh->check_state || 3478 (s.syncing && s.locked == 0 && 3479 !test_bit(STRIPE_COMPUTE_RUN, &sh->state) && 3480 !test_bit(STRIPE_INSYNC, &sh->state))) { 3481 if (conf->level == 6) 3482 handle_parity_checks6(conf, sh, &s, disks); 3483 else 3484 handle_parity_checks5(conf, sh, &s, disks); 3485 } 3486 3487 if (s.replacing && s.locked == 0 3488 && !test_bit(STRIPE_INSYNC, &sh->state)) { 3489 /* Write out to replacement devices where possible */ 3490 for (i = 0; i < conf->raid_disks; i++) 3491 if (test_bit(R5_UPTODATE, &sh->dev[i].flags) && 3492 test_bit(R5_NeedReplace, &sh->dev[i].flags)) { 3493 set_bit(R5_WantReplace, &sh->dev[i].flags); 3494 set_bit(R5_LOCKED, &sh->dev[i].flags); 3495 s.locked++; 3496 } 3497 set_bit(STRIPE_INSYNC, &sh->state); 3498 } 3499 if ((s.syncing || s.replacing) && s.locked == 0 && 3500 test_bit(STRIPE_INSYNC, &sh->state)) { 3501 md_done_sync(conf->mddev, STRIPE_SECTORS, 1); 3502 clear_bit(STRIPE_SYNCING, &sh->state); 3503 } 3504 3505 /* If the failed drives are just a ReadError, then we might need 3506 * to progress the repair/check process 3507 */ 3508 if (s.failed <= conf->max_degraded && !conf->mddev->ro) 3509 for (i = 0; i < s.failed; i++) { 3510 struct r5dev *dev = &sh->dev[s.failed_num[i]]; 3511 if (test_bit(R5_ReadError, &dev->flags) 3512 && !test_bit(R5_LOCKED, &dev->flags) 3513 && test_bit(R5_UPTODATE, &dev->flags) 3514 ) { 3515 if (!test_bit(R5_ReWrite, &dev->flags)) { 3516 set_bit(R5_Wantwrite, &dev->flags); 3517 set_bit(R5_ReWrite, &dev->flags); 3518 set_bit(R5_LOCKED, &dev->flags); 3519 s.locked++; 3520 } else { 3521 /* let's read it back */ 3522 set_bit(R5_Wantread, &dev->flags); 3523 set_bit(R5_LOCKED, &dev->flags); 3524 s.locked++; 3525 } 3526 } 3527 } 3528 3529 3530 /* Finish reconstruct operations initiated by the expansion process */ 3531 if (sh->reconstruct_state == reconstruct_state_result) { 3532 struct stripe_head *sh_src 3533 = get_active_stripe(conf, sh->sector, 1, 1, 1); 3534 if (sh_src && test_bit(STRIPE_EXPAND_SOURCE, &sh_src->state)) { 3535 /* sh cannot be written until sh_src has been read. 3536 * so arrange for sh to be delayed a little 3537 */ 3538 set_bit(STRIPE_DELAYED, &sh->state); 3539 set_bit(STRIPE_HANDLE, &sh->state); 3540 if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, 3541 &sh_src->state)) 3542 atomic_inc(&conf->preread_active_stripes); 3543 release_stripe(sh_src); 3544 goto finish; 3545 } 3546 if (sh_src) 3547 release_stripe(sh_src); 3548 3549 sh->reconstruct_state = reconstruct_state_idle; 3550 clear_bit(STRIPE_EXPANDING, &sh->state); 3551 for (i = conf->raid_disks; i--; ) { 3552 set_bit(R5_Wantwrite, &sh->dev[i].flags); 3553 set_bit(R5_LOCKED, &sh->dev[i].flags); 3554 s.locked++; 3555 } 3556 } 3557 3558 if (s.expanded && test_bit(STRIPE_EXPANDING, &sh->state) && 3559 !sh->reconstruct_state) { 3560 /* Need to write out all blocks after computing parity */ 3561 sh->disks = conf->raid_disks; 3562 stripe_set_idx(sh->sector, conf, 0, sh); 3563 schedule_reconstruction(sh, &s, 1, 1); 3564 } else if (s.expanded && !sh->reconstruct_state && s.locked == 0) { 3565 clear_bit(STRIPE_EXPAND_READY, &sh->state); 3566 atomic_dec(&conf->reshape_stripes); 3567 wake_up(&conf->wait_for_overlap); 3568 md_done_sync(conf->mddev, STRIPE_SECTORS, 1); 3569 } 3570 3571 if (s.expanding && s.locked == 0 && 3572 !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) 3573 handle_stripe_expansion(conf, sh); 3574 3575 finish: 3576 /* wait for this device to become unblocked */ 3577 if (conf->mddev->external && unlikely(s.blocked_rdev)) 3578 md_wait_for_blocked_rdev(s.blocked_rdev, conf->mddev); 3579 3580 if (s.handle_bad_blocks) 3581 for (i = disks; i--; ) { 3582 struct md_rdev *rdev; 3583 struct r5dev *dev = &sh->dev[i]; 3584 if (test_and_clear_bit(R5_WriteError, &dev->flags)) { 3585 /* We own a safe reference to the rdev */ 3586 rdev = conf->disks[i].rdev; 3587 if (!rdev_set_badblocks(rdev, sh->sector, 3588 STRIPE_SECTORS, 0)) 3589 md_error(conf->mddev, rdev); 3590 rdev_dec_pending(rdev, conf->mddev); 3591 } 3592 if (test_and_clear_bit(R5_MadeGood, &dev->flags)) { 3593 rdev = conf->disks[i].rdev; 3594 rdev_clear_badblocks(rdev, sh->sector, 3595 STRIPE_SECTORS, 0); 3596 rdev_dec_pending(rdev, conf->mddev); 3597 } 3598 if (test_and_clear_bit(R5_MadeGoodRepl, &dev->flags)) { 3599 rdev = conf->disks[i].replacement; 3600 if (!rdev) 3601 /* rdev have been moved down */ 3602 rdev = conf->disks[i].rdev; 3603 rdev_clear_badblocks(rdev, sh->sector, 3604 STRIPE_SECTORS, 0); 3605 rdev_dec_pending(rdev, conf->mddev); 3606 } 3607 } 3608 3609 if (s.ops_request) 3610 raid_run_ops(sh, s.ops_request); 3611 3612 ops_run_io(sh, &s); 3613 3614 if (s.dec_preread_active) { 3615 /* We delay this until after ops_run_io so that if make_request 3616 * is waiting on a flush, it won't continue until the writes 3617 * have actually been submitted. 3618 */ 3619 atomic_dec(&conf->preread_active_stripes); 3620 if (atomic_read(&conf->preread_active_stripes) < 3621 IO_THRESHOLD) 3622 md_wakeup_thread(conf->mddev->thread); 3623 } 3624 3625 return_io(s.return_bi); 3626 3627 clear_bit_unlock(STRIPE_ACTIVE, &sh->state); 3628 } 3629 3630 static void raid5_activate_delayed(struct r5conf *conf) 3631 { 3632 if (atomic_read(&conf->preread_active_stripes) < IO_THRESHOLD) { 3633 while (!list_empty(&conf->delayed_list)) { 3634 struct list_head *l = conf->delayed_list.next; 3635 struct stripe_head *sh; 3636 sh = list_entry(l, struct stripe_head, lru); 3637 list_del_init(l); 3638 clear_bit(STRIPE_DELAYED, &sh->state); 3639 if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) 3640 atomic_inc(&conf->preread_active_stripes); 3641 list_add_tail(&sh->lru, &conf->hold_list); 3642 } 3643 } 3644 } 3645 3646 static void activate_bit_delay(struct r5conf *conf) 3647 { 3648 /* device_lock is held */ 3649 struct list_head head; 3650 list_add(&head, &conf->bitmap_list); 3651 list_del_init(&conf->bitmap_list); 3652 while (!list_empty(&head)) { 3653 struct stripe_head *sh = list_entry(head.next, struct stripe_head, lru); 3654 list_del_init(&sh->lru); 3655 atomic_inc(&sh->count); 3656 __release_stripe(conf, sh); 3657 } 3658 } 3659 3660 int md_raid5_congested(struct mddev *mddev, int bits) 3661 { 3662 struct r5conf *conf = mddev->private; 3663 3664 /* No difference between reads and writes. Just check 3665 * how busy the stripe_cache is 3666 */ 3667 3668 if (conf->inactive_blocked) 3669 return 1; 3670 if (conf->quiesce) 3671 return 1; 3672 if (list_empty_careful(&conf->inactive_list)) 3673 return 1; 3674 3675 return 0; 3676 } 3677 EXPORT_SYMBOL_GPL(md_raid5_congested); 3678 3679 static int raid5_congested(void *data, int bits) 3680 { 3681 struct mddev *mddev = data; 3682 3683 return mddev_congested(mddev, bits) || 3684 md_raid5_congested(mddev, bits); 3685 } 3686 3687 /* We want read requests to align with chunks where possible, 3688 * but write requests don't need to. 3689 */ 3690 static int raid5_mergeable_bvec(struct request_queue *q, 3691 struct bvec_merge_data *bvm, 3692 struct bio_vec *biovec) 3693 { 3694 struct mddev *mddev = q->queuedata; 3695 sector_t sector = bvm->bi_sector + get_start_sect(bvm->bi_bdev); 3696 int max; 3697 unsigned int chunk_sectors = mddev->chunk_sectors; 3698 unsigned int bio_sectors = bvm->bi_size >> 9; 3699 3700 if ((bvm->bi_rw & 1) == WRITE) 3701 return biovec->bv_len; /* always allow writes to be mergeable */ 3702 3703 if (mddev->new_chunk_sectors < mddev->chunk_sectors) 3704 chunk_sectors = mddev->new_chunk_sectors; 3705 max = (chunk_sectors - ((sector & (chunk_sectors - 1)) + bio_sectors)) << 9; 3706 if (max < 0) max = 0; 3707 if (max <= biovec->bv_len && bio_sectors == 0) 3708 return biovec->bv_len; 3709 else 3710 return max; 3711 } 3712 3713 3714 static int in_chunk_boundary(struct mddev *mddev, struct bio *bio) 3715 { 3716 sector_t sector = bio->bi_sector + get_start_sect(bio->bi_bdev); 3717 unsigned int chunk_sectors = mddev->chunk_sectors; 3718 unsigned int bio_sectors = bio->bi_size >> 9; 3719 3720 if (mddev->new_chunk_sectors < mddev->chunk_sectors) 3721 chunk_sectors = mddev->new_chunk_sectors; 3722 return chunk_sectors >= 3723 ((sector & (chunk_sectors - 1)) + bio_sectors); 3724 } 3725 3726 /* 3727 * add bio to the retry LIFO ( in O(1) ... we are in interrupt ) 3728 * later sampled by raid5d. 3729 */ 3730 static void add_bio_to_retry(struct bio *bi,struct r5conf *conf) 3731 { 3732 unsigned long flags; 3733 3734 spin_lock_irqsave(&conf->device_lock, flags); 3735 3736 bi->bi_next = conf->retry_read_aligned_list; 3737 conf->retry_read_aligned_list = bi; 3738 3739 spin_unlock_irqrestore(&conf->device_lock, flags); 3740 md_wakeup_thread(conf->mddev->thread); 3741 } 3742 3743 3744 static struct bio *remove_bio_from_retry(struct r5conf *conf) 3745 { 3746 struct bio *bi; 3747 3748 bi = conf->retry_read_aligned; 3749 if (bi) { 3750 conf->retry_read_aligned = NULL; 3751 return bi; 3752 } 3753 bi = conf->retry_read_aligned_list; 3754 if(bi) { 3755 conf->retry_read_aligned_list = bi->bi_next; 3756 bi->bi_next = NULL; 3757 /* 3758 * this sets the active strip count to 1 and the processed 3759 * strip count to zero (upper 8 bits) 3760 */ 3761 bi->bi_phys_segments = 1; /* biased count of active stripes */ 3762 } 3763 3764 return bi; 3765 } 3766 3767 3768 /* 3769 * The "raid5_align_endio" should check if the read succeeded and if it 3770 * did, call bio_endio on the original bio (having bio_put the new bio 3771 * first). 3772 * If the read failed.. 3773 */ 3774 static void raid5_align_endio(struct bio *bi, int error) 3775 { 3776 struct bio* raid_bi = bi->bi_private; 3777 struct mddev *mddev; 3778 struct r5conf *conf; 3779 int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags); 3780 struct md_rdev *rdev; 3781 3782 bio_put(bi); 3783 3784 rdev = (void*)raid_bi->bi_next; 3785 raid_bi->bi_next = NULL; 3786 mddev = rdev->mddev; 3787 conf = mddev->private; 3788 3789 rdev_dec_pending(rdev, conf->mddev); 3790 3791 if (!error && uptodate) { 3792 bio_endio(raid_bi, 0); 3793 if (atomic_dec_and_test(&conf->active_aligned_reads)) 3794 wake_up(&conf->wait_for_stripe); 3795 return; 3796 } 3797 3798 3799 pr_debug("raid5_align_endio : io error...handing IO for a retry\n"); 3800 3801 add_bio_to_retry(raid_bi, conf); 3802 } 3803 3804 static int bio_fits_rdev(struct bio *bi) 3805 { 3806 struct request_queue *q = bdev_get_queue(bi->bi_bdev); 3807 3808 if ((bi->bi_size>>9) > queue_max_sectors(q)) 3809 return 0; 3810 blk_recount_segments(q, bi); 3811 if (bi->bi_phys_segments > queue_max_segments(q)) 3812 return 0; 3813 3814 if (q->merge_bvec_fn) 3815 /* it's too hard to apply the merge_bvec_fn at this stage, 3816 * just just give up 3817 */ 3818 return 0; 3819 3820 return 1; 3821 } 3822 3823 3824 static int chunk_aligned_read(struct mddev *mddev, struct bio * raid_bio) 3825 { 3826 struct r5conf *conf = mddev->private; 3827 int dd_idx; 3828 struct bio* align_bi; 3829 struct md_rdev *rdev; 3830 sector_t end_sector; 3831 3832 if (!in_chunk_boundary(mddev, raid_bio)) { 3833 pr_debug("chunk_aligned_read : non aligned\n"); 3834 return 0; 3835 } 3836 /* 3837 * use bio_clone_mddev to make a copy of the bio 3838 */ 3839 align_bi = bio_clone_mddev(raid_bio, GFP_NOIO, mddev); 3840 if (!align_bi) 3841 return 0; 3842 /* 3843 * set bi_end_io to a new function, and set bi_private to the 3844 * original bio. 3845 */ 3846 align_bi->bi_end_io = raid5_align_endio; 3847 align_bi->bi_private = raid_bio; 3848 /* 3849 * compute position 3850 */ 3851 align_bi->bi_sector = raid5_compute_sector(conf, raid_bio->bi_sector, 3852 0, 3853 &dd_idx, NULL); 3854 3855 end_sector = align_bi->bi_sector + (align_bi->bi_size >> 9); 3856 rcu_read_lock(); 3857 rdev = rcu_dereference(conf->disks[dd_idx].replacement); 3858 if (!rdev || test_bit(Faulty, &rdev->flags) || 3859 rdev->recovery_offset < end_sector) { 3860 rdev = rcu_dereference(conf->disks[dd_idx].rdev); 3861 if (rdev && 3862 (test_bit(Faulty, &rdev->flags) || 3863 !(test_bit(In_sync, &rdev->flags) || 3864 rdev->recovery_offset >= end_sector))) 3865 rdev = NULL; 3866 } 3867 if (rdev) { 3868 sector_t first_bad; 3869 int bad_sectors; 3870 3871 atomic_inc(&rdev->nr_pending); 3872 rcu_read_unlock(); 3873 raid_bio->bi_next = (void*)rdev; 3874 align_bi->bi_bdev = rdev->bdev; 3875 align_bi->bi_flags &= ~(1 << BIO_SEG_VALID); 3876 /* No reshape active, so we can trust rdev->data_offset */ 3877 align_bi->bi_sector += rdev->data_offset; 3878 3879 if (!bio_fits_rdev(align_bi) || 3880 is_badblock(rdev, align_bi->bi_sector, align_bi->bi_size>>9, 3881 &first_bad, &bad_sectors)) { 3882 /* too big in some way, or has a known bad block */ 3883 bio_put(align_bi); 3884 rdev_dec_pending(rdev, mddev); 3885 return 0; 3886 } 3887 3888 spin_lock_irq(&conf->device_lock); 3889 wait_event_lock_irq(conf->wait_for_stripe, 3890 conf->quiesce == 0, 3891 conf->device_lock, /* nothing */); 3892 atomic_inc(&conf->active_aligned_reads); 3893 spin_unlock_irq(&conf->device_lock); 3894 3895 generic_make_request(align_bi); 3896 return 1; 3897 } else { 3898 rcu_read_unlock(); 3899 bio_put(align_bi); 3900 return 0; 3901 } 3902 } 3903 3904 /* __get_priority_stripe - get the next stripe to process 3905 * 3906 * Full stripe writes are allowed to pass preread active stripes up until 3907 * the bypass_threshold is exceeded. In general the bypass_count 3908 * increments when the handle_list is handled before the hold_list; however, it 3909 * will not be incremented when STRIPE_IO_STARTED is sampled set signifying a 3910 * stripe with in flight i/o. The bypass_count will be reset when the 3911 * head of the hold_list has changed, i.e. the head was promoted to the 3912 * handle_list. 3913 */ 3914 static struct stripe_head *__get_priority_stripe(struct r5conf *conf) 3915 { 3916 struct stripe_head *sh; 3917 3918 pr_debug("%s: handle: %s hold: %s full_writes: %d bypass_count: %d\n", 3919 __func__, 3920 list_empty(&conf->handle_list) ? "empty" : "busy", 3921 list_empty(&conf->hold_list) ? "empty" : "busy", 3922 atomic_read(&conf->pending_full_writes), conf->bypass_count); 3923 3924 if (!list_empty(&conf->handle_list)) { 3925 sh = list_entry(conf->handle_list.next, typeof(*sh), lru); 3926 3927 if (list_empty(&conf->hold_list)) 3928 conf->bypass_count = 0; 3929 else if (!test_bit(STRIPE_IO_STARTED, &sh->state)) { 3930 if (conf->hold_list.next == conf->last_hold) 3931 conf->bypass_count++; 3932 else { 3933 conf->last_hold = conf->hold_list.next; 3934 conf->bypass_count -= conf->bypass_threshold; 3935 if (conf->bypass_count < 0) 3936 conf->bypass_count = 0; 3937 } 3938 } 3939 } else if (!list_empty(&conf->hold_list) && 3940 ((conf->bypass_threshold && 3941 conf->bypass_count > conf->bypass_threshold) || 3942 atomic_read(&conf->pending_full_writes) == 0)) { 3943 sh = list_entry(conf->hold_list.next, 3944 typeof(*sh), lru); 3945 conf->bypass_count -= conf->bypass_threshold; 3946 if (conf->bypass_count < 0) 3947 conf->bypass_count = 0; 3948 } else 3949 return NULL; 3950 3951 list_del_init(&sh->lru); 3952 atomic_inc(&sh->count); 3953 BUG_ON(atomic_read(&sh->count) != 1); 3954 return sh; 3955 } 3956 3957 static void make_request(struct mddev *mddev, struct bio * bi) 3958 { 3959 struct r5conf *conf = mddev->private; 3960 int dd_idx; 3961 sector_t new_sector; 3962 sector_t logical_sector, last_sector; 3963 struct stripe_head *sh; 3964 const int rw = bio_data_dir(bi); 3965 int remaining; 3966 int plugged; 3967 3968 if (unlikely(bi->bi_rw & REQ_FLUSH)) { 3969 md_flush_request(mddev, bi); 3970 return; 3971 } 3972 3973 md_write_start(mddev, bi); 3974 3975 if (rw == READ && 3976 mddev->reshape_position == MaxSector && 3977 chunk_aligned_read(mddev,bi)) 3978 return; 3979 3980 logical_sector = bi->bi_sector & ~((sector_t)STRIPE_SECTORS-1); 3981 last_sector = bi->bi_sector + (bi->bi_size>>9); 3982 bi->bi_next = NULL; 3983 bi->bi_phys_segments = 1; /* over-loaded to count active stripes */ 3984 3985 plugged = mddev_check_plugged(mddev); 3986 for (;logical_sector < last_sector; logical_sector += STRIPE_SECTORS) { 3987 DEFINE_WAIT(w); 3988 int disks, data_disks; 3989 int previous; 3990 3991 retry: 3992 previous = 0; 3993 disks = conf->raid_disks; 3994 prepare_to_wait(&conf->wait_for_overlap, &w, TASK_UNINTERRUPTIBLE); 3995 if (unlikely(conf->reshape_progress != MaxSector)) { 3996 /* spinlock is needed as reshape_progress may be 3997 * 64bit on a 32bit platform, and so it might be 3998 * possible to see a half-updated value 3999 * Of course reshape_progress could change after 4000 * the lock is dropped, so once we get a reference 4001 * to the stripe that we think it is, we will have 4002 * to check again. 4003 */ 4004 spin_lock_irq(&conf->device_lock); 4005 if (mddev->reshape_backwards 4006 ? logical_sector < conf->reshape_progress 4007 : logical_sector >= conf->reshape_progress) { 4008 disks = conf->previous_raid_disks; 4009 previous = 1; 4010 } else { 4011 if (mddev->reshape_backwards 4012 ? logical_sector < conf->reshape_safe 4013 : logical_sector >= conf->reshape_safe) { 4014 spin_unlock_irq(&conf->device_lock); 4015 schedule(); 4016 goto retry; 4017 } 4018 } 4019 spin_unlock_irq(&conf->device_lock); 4020 } 4021 data_disks = disks - conf->max_degraded; 4022 4023 new_sector = raid5_compute_sector(conf, logical_sector, 4024 previous, 4025 &dd_idx, NULL); 4026 pr_debug("raid456: make_request, sector %llu logical %llu\n", 4027 (unsigned long long)new_sector, 4028 (unsigned long long)logical_sector); 4029 4030 sh = get_active_stripe(conf, new_sector, previous, 4031 (bi->bi_rw&RWA_MASK), 0); 4032 if (sh) { 4033 if (unlikely(previous)) { 4034 /* expansion might have moved on while waiting for a 4035 * stripe, so we must do the range check again. 4036 * Expansion could still move past after this 4037 * test, but as we are holding a reference to 4038 * 'sh', we know that if that happens, 4039 * STRIPE_EXPANDING will get set and the expansion 4040 * won't proceed until we finish with the stripe. 4041 */ 4042 int must_retry = 0; 4043 spin_lock_irq(&conf->device_lock); 4044 if (mddev->reshape_backwards 4045 ? logical_sector >= conf->reshape_progress 4046 : logical_sector < conf->reshape_progress) 4047 /* mismatch, need to try again */ 4048 must_retry = 1; 4049 spin_unlock_irq(&conf->device_lock); 4050 if (must_retry) { 4051 release_stripe(sh); 4052 schedule(); 4053 goto retry; 4054 } 4055 } 4056 4057 if (rw == WRITE && 4058 logical_sector >= mddev->suspend_lo && 4059 logical_sector < mddev->suspend_hi) { 4060 release_stripe(sh); 4061 /* As the suspend_* range is controlled by 4062 * userspace, we want an interruptible 4063 * wait. 4064 */ 4065 flush_signals(current); 4066 prepare_to_wait(&conf->wait_for_overlap, 4067 &w, TASK_INTERRUPTIBLE); 4068 if (logical_sector >= mddev->suspend_lo && 4069 logical_sector < mddev->suspend_hi) 4070 schedule(); 4071 goto retry; 4072 } 4073 4074 if (test_bit(STRIPE_EXPANDING, &sh->state) || 4075 !add_stripe_bio(sh, bi, dd_idx, rw)) { 4076 /* Stripe is busy expanding or 4077 * add failed due to overlap. Flush everything 4078 * and wait a while 4079 */ 4080 md_wakeup_thread(mddev->thread); 4081 release_stripe(sh); 4082 schedule(); 4083 goto retry; 4084 } 4085 finish_wait(&conf->wait_for_overlap, &w); 4086 set_bit(STRIPE_HANDLE, &sh->state); 4087 clear_bit(STRIPE_DELAYED, &sh->state); 4088 if ((bi->bi_rw & REQ_SYNC) && 4089 !test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) 4090 atomic_inc(&conf->preread_active_stripes); 4091 release_stripe(sh); 4092 } else { 4093 /* cannot get stripe for read-ahead, just give-up */ 4094 clear_bit(BIO_UPTODATE, &bi->bi_flags); 4095 finish_wait(&conf->wait_for_overlap, &w); 4096 break; 4097 } 4098 4099 } 4100 if (!plugged) 4101 md_wakeup_thread(mddev->thread); 4102 4103 spin_lock_irq(&conf->device_lock); 4104 remaining = raid5_dec_bi_phys_segments(bi); 4105 spin_unlock_irq(&conf->device_lock); 4106 if (remaining == 0) { 4107 4108 if ( rw == WRITE ) 4109 md_write_end(mddev); 4110 4111 bio_endio(bi, 0); 4112 } 4113 } 4114 4115 static sector_t raid5_size(struct mddev *mddev, sector_t sectors, int raid_disks); 4116 4117 static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, int *skipped) 4118 { 4119 /* reshaping is quite different to recovery/resync so it is 4120 * handled quite separately ... here. 4121 * 4122 * On each call to sync_request, we gather one chunk worth of 4123 * destination stripes and flag them as expanding. 4124 * Then we find all the source stripes and request reads. 4125 * As the reads complete, handle_stripe will copy the data 4126 * into the destination stripe and release that stripe. 4127 */ 4128 struct r5conf *conf = mddev->private; 4129 struct stripe_head *sh; 4130 sector_t first_sector, last_sector; 4131 int raid_disks = conf->previous_raid_disks; 4132 int data_disks = raid_disks - conf->max_degraded; 4133 int new_data_disks = conf->raid_disks - conf->max_degraded; 4134 int i; 4135 int dd_idx; 4136 sector_t writepos, readpos, safepos; 4137 sector_t stripe_addr; 4138 int reshape_sectors; 4139 struct list_head stripes; 4140 4141 if (sector_nr == 0) { 4142 /* If restarting in the middle, skip the initial sectors */ 4143 if (mddev->reshape_backwards && 4144 conf->reshape_progress < raid5_size(mddev, 0, 0)) { 4145 sector_nr = raid5_size(mddev, 0, 0) 4146 - conf->reshape_progress; 4147 } else if (!mddev->reshape_backwards && 4148 conf->reshape_progress > 0) 4149 sector_nr = conf->reshape_progress; 4150 sector_div(sector_nr, new_data_disks); 4151 if (sector_nr) { 4152 mddev->curr_resync_completed = sector_nr; 4153 sysfs_notify(&mddev->kobj, NULL, "sync_completed"); 4154 *skipped = 1; 4155 return sector_nr; 4156 } 4157 } 4158 4159 /* We need to process a full chunk at a time. 4160 * If old and new chunk sizes differ, we need to process the 4161 * largest of these 4162 */ 4163 if (mddev->new_chunk_sectors > mddev->chunk_sectors) 4164 reshape_sectors = mddev->new_chunk_sectors; 4165 else 4166 reshape_sectors = mddev->chunk_sectors; 4167 4168 /* We update the metadata at least every 10 seconds, or when 4169 * the data about to be copied would over-write the source of 4170 * the data at the front of the range. i.e. one new_stripe 4171 * along from reshape_progress new_maps to after where 4172 * reshape_safe old_maps to 4173 */ 4174 writepos = conf->reshape_progress; 4175 sector_div(writepos, new_data_disks); 4176 readpos = conf->reshape_progress; 4177 sector_div(readpos, data_disks); 4178 safepos = conf->reshape_safe; 4179 sector_div(safepos, data_disks); 4180 if (mddev->reshape_backwards) { 4181 writepos -= min_t(sector_t, reshape_sectors, writepos); 4182 readpos += reshape_sectors; 4183 safepos += reshape_sectors; 4184 } else { 4185 writepos += reshape_sectors; 4186 readpos -= min_t(sector_t, reshape_sectors, readpos); 4187 safepos -= min_t(sector_t, reshape_sectors, safepos); 4188 } 4189 4190 /* Having calculated the 'writepos' possibly use it 4191 * to set 'stripe_addr' which is where we will write to. 4192 */ 4193 if (mddev->reshape_backwards) { 4194 BUG_ON(conf->reshape_progress == 0); 4195 stripe_addr = writepos; 4196 BUG_ON((mddev->dev_sectors & 4197 ~((sector_t)reshape_sectors - 1)) 4198 - reshape_sectors - stripe_addr 4199 != sector_nr); 4200 } else { 4201 BUG_ON(writepos != sector_nr + reshape_sectors); 4202 stripe_addr = sector_nr; 4203 } 4204 4205 /* 'writepos' is the most advanced device address we might write. 4206 * 'readpos' is the least advanced device address we might read. 4207 * 'safepos' is the least address recorded in the metadata as having 4208 * been reshaped. 4209 * If there is a min_offset_diff, these are adjusted either by 4210 * increasing the safepos/readpos if diff is negative, or 4211 * increasing writepos if diff is positive. 4212 * If 'readpos' is then behind 'writepos', there is no way that we can 4213 * ensure safety in the face of a crash - that must be done by userspace 4214 * making a backup of the data. So in that case there is no particular 4215 * rush to update metadata. 4216 * Otherwise if 'safepos' is behind 'writepos', then we really need to 4217 * update the metadata to advance 'safepos' to match 'readpos' so that 4218 * we can be safe in the event of a crash. 4219 * So we insist on updating metadata if safepos is behind writepos and 4220 * readpos is beyond writepos. 4221 * In any case, update the metadata every 10 seconds. 4222 * Maybe that number should be configurable, but I'm not sure it is 4223 * worth it.... maybe it could be a multiple of safemode_delay??? 4224 */ 4225 if (conf->min_offset_diff < 0) { 4226 safepos += -conf->min_offset_diff; 4227 readpos += -conf->min_offset_diff; 4228 } else 4229 writepos += conf->min_offset_diff; 4230 4231 if ((mddev->reshape_backwards 4232 ? (safepos > writepos && readpos < writepos) 4233 : (safepos < writepos && readpos > writepos)) || 4234 time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) { 4235 /* Cannot proceed until we've updated the superblock... */ 4236 wait_event(conf->wait_for_overlap, 4237 atomic_read(&conf->reshape_stripes)==0); 4238 mddev->reshape_position = conf->reshape_progress; 4239 mddev->curr_resync_completed = sector_nr; 4240 conf->reshape_checkpoint = jiffies; 4241 set_bit(MD_CHANGE_DEVS, &mddev->flags); 4242 md_wakeup_thread(mddev->thread); 4243 wait_event(mddev->sb_wait, mddev->flags == 0 || 4244 kthread_should_stop()); 4245 spin_lock_irq(&conf->device_lock); 4246 conf->reshape_safe = mddev->reshape_position; 4247 spin_unlock_irq(&conf->device_lock); 4248 wake_up(&conf->wait_for_overlap); 4249 sysfs_notify(&mddev->kobj, NULL, "sync_completed"); 4250 } 4251 4252 INIT_LIST_HEAD(&stripes); 4253 for (i = 0; i < reshape_sectors; i += STRIPE_SECTORS) { 4254 int j; 4255 int skipped_disk = 0; 4256 sh = get_active_stripe(conf, stripe_addr+i, 0, 0, 1); 4257 set_bit(STRIPE_EXPANDING, &sh->state); 4258 atomic_inc(&conf->reshape_stripes); 4259 /* If any of this stripe is beyond the end of the old 4260 * array, then we need to zero those blocks 4261 */ 4262 for (j=sh->disks; j--;) { 4263 sector_t s; 4264 if (j == sh->pd_idx) 4265 continue; 4266 if (conf->level == 6 && 4267 j == sh->qd_idx) 4268 continue; 4269 s = compute_blocknr(sh, j, 0); 4270 if (s < raid5_size(mddev, 0, 0)) { 4271 skipped_disk = 1; 4272 continue; 4273 } 4274 memset(page_address(sh->dev[j].page), 0, STRIPE_SIZE); 4275 set_bit(R5_Expanded, &sh->dev[j].flags); 4276 set_bit(R5_UPTODATE, &sh->dev[j].flags); 4277 } 4278 if (!skipped_disk) { 4279 set_bit(STRIPE_EXPAND_READY, &sh->state); 4280 set_bit(STRIPE_HANDLE, &sh->state); 4281 } 4282 list_add(&sh->lru, &stripes); 4283 } 4284 spin_lock_irq(&conf->device_lock); 4285 if (mddev->reshape_backwards) 4286 conf->reshape_progress -= reshape_sectors * new_data_disks; 4287 else 4288 conf->reshape_progress += reshape_sectors * new_data_disks; 4289 spin_unlock_irq(&conf->device_lock); 4290 /* Ok, those stripe are ready. We can start scheduling 4291 * reads on the source stripes. 4292 * The source stripes are determined by mapping the first and last 4293 * block on the destination stripes. 4294 */ 4295 first_sector = 4296 raid5_compute_sector(conf, stripe_addr*(new_data_disks), 4297 1, &dd_idx, NULL); 4298 last_sector = 4299 raid5_compute_sector(conf, ((stripe_addr+reshape_sectors) 4300 * new_data_disks - 1), 4301 1, &dd_idx, NULL); 4302 if (last_sector >= mddev->dev_sectors) 4303 last_sector = mddev->dev_sectors - 1; 4304 while (first_sector <= last_sector) { 4305 sh = get_active_stripe(conf, first_sector, 1, 0, 1); 4306 set_bit(STRIPE_EXPAND_SOURCE, &sh->state); 4307 set_bit(STRIPE_HANDLE, &sh->state); 4308 release_stripe(sh); 4309 first_sector += STRIPE_SECTORS; 4310 } 4311 /* Now that the sources are clearly marked, we can release 4312 * the destination stripes 4313 */ 4314 while (!list_empty(&stripes)) { 4315 sh = list_entry(stripes.next, struct stripe_head, lru); 4316 list_del_init(&sh->lru); 4317 release_stripe(sh); 4318 } 4319 /* If this takes us to the resync_max point where we have to pause, 4320 * then we need to write out the superblock. 4321 */ 4322 sector_nr += reshape_sectors; 4323 if ((sector_nr - mddev->curr_resync_completed) * 2 4324 >= mddev->resync_max - mddev->curr_resync_completed) { 4325 /* Cannot proceed until we've updated the superblock... */ 4326 wait_event(conf->wait_for_overlap, 4327 atomic_read(&conf->reshape_stripes) == 0); 4328 mddev->reshape_position = conf->reshape_progress; 4329 mddev->curr_resync_completed = sector_nr; 4330 conf->reshape_checkpoint = jiffies; 4331 set_bit(MD_CHANGE_DEVS, &mddev->flags); 4332 md_wakeup_thread(mddev->thread); 4333 wait_event(mddev->sb_wait, 4334 !test_bit(MD_CHANGE_DEVS, &mddev->flags) 4335 || kthread_should_stop()); 4336 spin_lock_irq(&conf->device_lock); 4337 conf->reshape_safe = mddev->reshape_position; 4338 spin_unlock_irq(&conf->device_lock); 4339 wake_up(&conf->wait_for_overlap); 4340 sysfs_notify(&mddev->kobj, NULL, "sync_completed"); 4341 } 4342 return reshape_sectors; 4343 } 4344 4345 /* FIXME go_faster isn't used */ 4346 static inline sector_t sync_request(struct mddev *mddev, sector_t sector_nr, int *skipped, int go_faster) 4347 { 4348 struct r5conf *conf = mddev->private; 4349 struct stripe_head *sh; 4350 sector_t max_sector = mddev->dev_sectors; 4351 sector_t sync_blocks; 4352 int still_degraded = 0; 4353 int i; 4354 4355 if (sector_nr >= max_sector) { 4356 /* just being told to finish up .. nothing much to do */ 4357 4358 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) { 4359 end_reshape(conf); 4360 return 0; 4361 } 4362 4363 if (mddev->curr_resync < max_sector) /* aborted */ 4364 bitmap_end_sync(mddev->bitmap, mddev->curr_resync, 4365 &sync_blocks, 1); 4366 else /* completed sync */ 4367 conf->fullsync = 0; 4368 bitmap_close_sync(mddev->bitmap); 4369 4370 return 0; 4371 } 4372 4373 /* Allow raid5_quiesce to complete */ 4374 wait_event(conf->wait_for_overlap, conf->quiesce != 2); 4375 4376 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) 4377 return reshape_request(mddev, sector_nr, skipped); 4378 4379 /* No need to check resync_max as we never do more than one 4380 * stripe, and as resync_max will always be on a chunk boundary, 4381 * if the check in md_do_sync didn't fire, there is no chance 4382 * of overstepping resync_max here 4383 */ 4384 4385 /* if there is too many failed drives and we are trying 4386 * to resync, then assert that we are finished, because there is 4387 * nothing we can do. 4388 */ 4389 if (mddev->degraded >= conf->max_degraded && 4390 test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) { 4391 sector_t rv = mddev->dev_sectors - sector_nr; 4392 *skipped = 1; 4393 return rv; 4394 } 4395 if (!bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, 1) && 4396 !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) && 4397 !conf->fullsync && sync_blocks >= STRIPE_SECTORS) { 4398 /* we can skip this block, and probably more */ 4399 sync_blocks /= STRIPE_SECTORS; 4400 *skipped = 1; 4401 return sync_blocks * STRIPE_SECTORS; /* keep things rounded to whole stripes */ 4402 } 4403 4404 bitmap_cond_end_sync(mddev->bitmap, sector_nr); 4405 4406 sh = get_active_stripe(conf, sector_nr, 0, 1, 0); 4407 if (sh == NULL) { 4408 sh = get_active_stripe(conf, sector_nr, 0, 0, 0); 4409 /* make sure we don't swamp the stripe cache if someone else 4410 * is trying to get access 4411 */ 4412 schedule_timeout_uninterruptible(1); 4413 } 4414 /* Need to check if array will still be degraded after recovery/resync 4415 * We don't need to check the 'failed' flag as when that gets set, 4416 * recovery aborts. 4417 */ 4418 for (i = 0; i < conf->raid_disks; i++) 4419 if (conf->disks[i].rdev == NULL) 4420 still_degraded = 1; 4421 4422 bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, still_degraded); 4423 4424 set_bit(STRIPE_SYNC_REQUESTED, &sh->state); 4425 4426 handle_stripe(sh); 4427 release_stripe(sh); 4428 4429 return STRIPE_SECTORS; 4430 } 4431 4432 static int retry_aligned_read(struct r5conf *conf, struct bio *raid_bio) 4433 { 4434 /* We may not be able to submit a whole bio at once as there 4435 * may not be enough stripe_heads available. 4436 * We cannot pre-allocate enough stripe_heads as we may need 4437 * more than exist in the cache (if we allow ever large chunks). 4438 * So we do one stripe head at a time and record in 4439 * ->bi_hw_segments how many have been done. 4440 * 4441 * We *know* that this entire raid_bio is in one chunk, so 4442 * it will be only one 'dd_idx' and only need one call to raid5_compute_sector. 4443 */ 4444 struct stripe_head *sh; 4445 int dd_idx; 4446 sector_t sector, logical_sector, last_sector; 4447 int scnt = 0; 4448 int remaining; 4449 int handled = 0; 4450 4451 logical_sector = raid_bio->bi_sector & ~((sector_t)STRIPE_SECTORS-1); 4452 sector = raid5_compute_sector(conf, logical_sector, 4453 0, &dd_idx, NULL); 4454 last_sector = raid_bio->bi_sector + (raid_bio->bi_size>>9); 4455 4456 for (; logical_sector < last_sector; 4457 logical_sector += STRIPE_SECTORS, 4458 sector += STRIPE_SECTORS, 4459 scnt++) { 4460 4461 if (scnt < raid5_bi_hw_segments(raid_bio)) 4462 /* already done this stripe */ 4463 continue; 4464 4465 sh = get_active_stripe(conf, sector, 0, 1, 0); 4466 4467 if (!sh) { 4468 /* failed to get a stripe - must wait */ 4469 raid5_set_bi_hw_segments(raid_bio, scnt); 4470 conf->retry_read_aligned = raid_bio; 4471 return handled; 4472 } 4473 4474 if (!add_stripe_bio(sh, raid_bio, dd_idx, 0)) { 4475 release_stripe(sh); 4476 raid5_set_bi_hw_segments(raid_bio, scnt); 4477 conf->retry_read_aligned = raid_bio; 4478 return handled; 4479 } 4480 4481 handle_stripe(sh); 4482 release_stripe(sh); 4483 handled++; 4484 } 4485 spin_lock_irq(&conf->device_lock); 4486 remaining = raid5_dec_bi_phys_segments(raid_bio); 4487 spin_unlock_irq(&conf->device_lock); 4488 if (remaining == 0) 4489 bio_endio(raid_bio, 0); 4490 if (atomic_dec_and_test(&conf->active_aligned_reads)) 4491 wake_up(&conf->wait_for_stripe); 4492 return handled; 4493 } 4494 4495 4496 /* 4497 * This is our raid5 kernel thread. 4498 * 4499 * We scan the hash table for stripes which can be handled now. 4500 * During the scan, completed stripes are saved for us by the interrupt 4501 * handler, so that they will not have to wait for our next wakeup. 4502 */ 4503 static void raid5d(struct mddev *mddev) 4504 { 4505 struct stripe_head *sh; 4506 struct r5conf *conf = mddev->private; 4507 int handled; 4508 struct blk_plug plug; 4509 4510 pr_debug("+++ raid5d active\n"); 4511 4512 md_check_recovery(mddev); 4513 4514 blk_start_plug(&plug); 4515 handled = 0; 4516 spin_lock_irq(&conf->device_lock); 4517 while (1) { 4518 struct bio *bio; 4519 4520 if (atomic_read(&mddev->plug_cnt) == 0 && 4521 !list_empty(&conf->bitmap_list)) { 4522 /* Now is a good time to flush some bitmap updates */ 4523 conf->seq_flush++; 4524 spin_unlock_irq(&conf->device_lock); 4525 bitmap_unplug(mddev->bitmap); 4526 spin_lock_irq(&conf->device_lock); 4527 conf->seq_write = conf->seq_flush; 4528 activate_bit_delay(conf); 4529 } 4530 if (atomic_read(&mddev->plug_cnt) == 0) 4531 raid5_activate_delayed(conf); 4532 4533 while ((bio = remove_bio_from_retry(conf))) { 4534 int ok; 4535 spin_unlock_irq(&conf->device_lock); 4536 ok = retry_aligned_read(conf, bio); 4537 spin_lock_irq(&conf->device_lock); 4538 if (!ok) 4539 break; 4540 handled++; 4541 } 4542 4543 sh = __get_priority_stripe(conf); 4544 4545 if (!sh) 4546 break; 4547 spin_unlock_irq(&conf->device_lock); 4548 4549 handled++; 4550 handle_stripe(sh); 4551 release_stripe(sh); 4552 cond_resched(); 4553 4554 if (mddev->flags & ~(1<<MD_CHANGE_PENDING)) 4555 md_check_recovery(mddev); 4556 4557 spin_lock_irq(&conf->device_lock); 4558 } 4559 pr_debug("%d stripes handled\n", handled); 4560 4561 spin_unlock_irq(&conf->device_lock); 4562 4563 async_tx_issue_pending_all(); 4564 blk_finish_plug(&plug); 4565 4566 pr_debug("--- raid5d inactive\n"); 4567 } 4568 4569 static ssize_t 4570 raid5_show_stripe_cache_size(struct mddev *mddev, char *page) 4571 { 4572 struct r5conf *conf = mddev->private; 4573 if (conf) 4574 return sprintf(page, "%d\n", conf->max_nr_stripes); 4575 else 4576 return 0; 4577 } 4578 4579 int 4580 raid5_set_cache_size(struct mddev *mddev, int size) 4581 { 4582 struct r5conf *conf = mddev->private; 4583 int err; 4584 4585 if (size <= 16 || size > 32768) 4586 return -EINVAL; 4587 while (size < conf->max_nr_stripes) { 4588 if (drop_one_stripe(conf)) 4589 conf->max_nr_stripes--; 4590 else 4591 break; 4592 } 4593 err = md_allow_write(mddev); 4594 if (err) 4595 return err; 4596 while (size > conf->max_nr_stripes) { 4597 if (grow_one_stripe(conf)) 4598 conf->max_nr_stripes++; 4599 else break; 4600 } 4601 return 0; 4602 } 4603 EXPORT_SYMBOL(raid5_set_cache_size); 4604 4605 static ssize_t 4606 raid5_store_stripe_cache_size(struct mddev *mddev, const char *page, size_t len) 4607 { 4608 struct r5conf *conf = mddev->private; 4609 unsigned long new; 4610 int err; 4611 4612 if (len >= PAGE_SIZE) 4613 return -EINVAL; 4614 if (!conf) 4615 return -ENODEV; 4616 4617 if (strict_strtoul(page, 10, &new)) 4618 return -EINVAL; 4619 err = raid5_set_cache_size(mddev, new); 4620 if (err) 4621 return err; 4622 return len; 4623 } 4624 4625 static struct md_sysfs_entry 4626 raid5_stripecache_size = __ATTR(stripe_cache_size, S_IRUGO | S_IWUSR, 4627 raid5_show_stripe_cache_size, 4628 raid5_store_stripe_cache_size); 4629 4630 static ssize_t 4631 raid5_show_preread_threshold(struct mddev *mddev, char *page) 4632 { 4633 struct r5conf *conf = mddev->private; 4634 if (conf) 4635 return sprintf(page, "%d\n", conf->bypass_threshold); 4636 else 4637 return 0; 4638 } 4639 4640 static ssize_t 4641 raid5_store_preread_threshold(struct mddev *mddev, const char *page, size_t len) 4642 { 4643 struct r5conf *conf = mddev->private; 4644 unsigned long new; 4645 if (len >= PAGE_SIZE) 4646 return -EINVAL; 4647 if (!conf) 4648 return -ENODEV; 4649 4650 if (strict_strtoul(page, 10, &new)) 4651 return -EINVAL; 4652 if (new > conf->max_nr_stripes) 4653 return -EINVAL; 4654 conf->bypass_threshold = new; 4655 return len; 4656 } 4657 4658 static struct md_sysfs_entry 4659 raid5_preread_bypass_threshold = __ATTR(preread_bypass_threshold, 4660 S_IRUGO | S_IWUSR, 4661 raid5_show_preread_threshold, 4662 raid5_store_preread_threshold); 4663 4664 static ssize_t 4665 stripe_cache_active_show(struct mddev *mddev, char *page) 4666 { 4667 struct r5conf *conf = mddev->private; 4668 if (conf) 4669 return sprintf(page, "%d\n", atomic_read(&conf->active_stripes)); 4670 else 4671 return 0; 4672 } 4673 4674 static struct md_sysfs_entry 4675 raid5_stripecache_active = __ATTR_RO(stripe_cache_active); 4676 4677 static struct attribute *raid5_attrs[] = { 4678 &raid5_stripecache_size.attr, 4679 &raid5_stripecache_active.attr, 4680 &raid5_preread_bypass_threshold.attr, 4681 NULL, 4682 }; 4683 static struct attribute_group raid5_attrs_group = { 4684 .name = NULL, 4685 .attrs = raid5_attrs, 4686 }; 4687 4688 static sector_t 4689 raid5_size(struct mddev *mddev, sector_t sectors, int raid_disks) 4690 { 4691 struct r5conf *conf = mddev->private; 4692 4693 if (!sectors) 4694 sectors = mddev->dev_sectors; 4695 if (!raid_disks) 4696 /* size is defined by the smallest of previous and new size */ 4697 raid_disks = min(conf->raid_disks, conf->previous_raid_disks); 4698 4699 sectors &= ~((sector_t)mddev->chunk_sectors - 1); 4700 sectors &= ~((sector_t)mddev->new_chunk_sectors - 1); 4701 return sectors * (raid_disks - conf->max_degraded); 4702 } 4703 4704 static void raid5_free_percpu(struct r5conf *conf) 4705 { 4706 struct raid5_percpu *percpu; 4707 unsigned long cpu; 4708 4709 if (!conf->percpu) 4710 return; 4711 4712 get_online_cpus(); 4713 for_each_possible_cpu(cpu) { 4714 percpu = per_cpu_ptr(conf->percpu, cpu); 4715 safe_put_page(percpu->spare_page); 4716 kfree(percpu->scribble); 4717 } 4718 #ifdef CONFIG_HOTPLUG_CPU 4719 unregister_cpu_notifier(&conf->cpu_notify); 4720 #endif 4721 put_online_cpus(); 4722 4723 free_percpu(conf->percpu); 4724 } 4725 4726 static void free_conf(struct r5conf *conf) 4727 { 4728 shrink_stripes(conf); 4729 raid5_free_percpu(conf); 4730 kfree(conf->disks); 4731 kfree(conf->stripe_hashtbl); 4732 kfree(conf); 4733 } 4734 4735 #ifdef CONFIG_HOTPLUG_CPU 4736 static int raid456_cpu_notify(struct notifier_block *nfb, unsigned long action, 4737 void *hcpu) 4738 { 4739 struct r5conf *conf = container_of(nfb, struct r5conf, cpu_notify); 4740 long cpu = (long)hcpu; 4741 struct raid5_percpu *percpu = per_cpu_ptr(conf->percpu, cpu); 4742 4743 switch (action) { 4744 case CPU_UP_PREPARE: 4745 case CPU_UP_PREPARE_FROZEN: 4746 if (conf->level == 6 && !percpu->spare_page) 4747 percpu->spare_page = alloc_page(GFP_KERNEL); 4748 if (!percpu->scribble) 4749 percpu->scribble = kmalloc(conf->scribble_len, GFP_KERNEL); 4750 4751 if (!percpu->scribble || 4752 (conf->level == 6 && !percpu->spare_page)) { 4753 safe_put_page(percpu->spare_page); 4754 kfree(percpu->scribble); 4755 pr_err("%s: failed memory allocation for cpu%ld\n", 4756 __func__, cpu); 4757 return notifier_from_errno(-ENOMEM); 4758 } 4759 break; 4760 case CPU_DEAD: 4761 case CPU_DEAD_FROZEN: 4762 safe_put_page(percpu->spare_page); 4763 kfree(percpu->scribble); 4764 percpu->spare_page = NULL; 4765 percpu->scribble = NULL; 4766 break; 4767 default: 4768 break; 4769 } 4770 return NOTIFY_OK; 4771 } 4772 #endif 4773 4774 static int raid5_alloc_percpu(struct r5conf *conf) 4775 { 4776 unsigned long cpu; 4777 struct page *spare_page; 4778 struct raid5_percpu __percpu *allcpus; 4779 void *scribble; 4780 int err; 4781 4782 allcpus = alloc_percpu(struct raid5_percpu); 4783 if (!allcpus) 4784 return -ENOMEM; 4785 conf->percpu = allcpus; 4786 4787 get_online_cpus(); 4788 err = 0; 4789 for_each_present_cpu(cpu) { 4790 if (conf->level == 6) { 4791 spare_page = alloc_page(GFP_KERNEL); 4792 if (!spare_page) { 4793 err = -ENOMEM; 4794 break; 4795 } 4796 per_cpu_ptr(conf->percpu, cpu)->spare_page = spare_page; 4797 } 4798 scribble = kmalloc(conf->scribble_len, GFP_KERNEL); 4799 if (!scribble) { 4800 err = -ENOMEM; 4801 break; 4802 } 4803 per_cpu_ptr(conf->percpu, cpu)->scribble = scribble; 4804 } 4805 #ifdef CONFIG_HOTPLUG_CPU 4806 conf->cpu_notify.notifier_call = raid456_cpu_notify; 4807 conf->cpu_notify.priority = 0; 4808 if (err == 0) 4809 err = register_cpu_notifier(&conf->cpu_notify); 4810 #endif 4811 put_online_cpus(); 4812 4813 return err; 4814 } 4815 4816 static struct r5conf *setup_conf(struct mddev *mddev) 4817 { 4818 struct r5conf *conf; 4819 int raid_disk, memory, max_disks; 4820 struct md_rdev *rdev; 4821 struct disk_info *disk; 4822 4823 if (mddev->new_level != 5 4824 && mddev->new_level != 4 4825 && mddev->new_level != 6) { 4826 printk(KERN_ERR "md/raid:%s: raid level not set to 4/5/6 (%d)\n", 4827 mdname(mddev), mddev->new_level); 4828 return ERR_PTR(-EIO); 4829 } 4830 if ((mddev->new_level == 5 4831 && !algorithm_valid_raid5(mddev->new_layout)) || 4832 (mddev->new_level == 6 4833 && !algorithm_valid_raid6(mddev->new_layout))) { 4834 printk(KERN_ERR "md/raid:%s: layout %d not supported\n", 4835 mdname(mddev), mddev->new_layout); 4836 return ERR_PTR(-EIO); 4837 } 4838 if (mddev->new_level == 6 && mddev->raid_disks < 4) { 4839 printk(KERN_ERR "md/raid:%s: not enough configured devices (%d, minimum 4)\n", 4840 mdname(mddev), mddev->raid_disks); 4841 return ERR_PTR(-EINVAL); 4842 } 4843 4844 if (!mddev->new_chunk_sectors || 4845 (mddev->new_chunk_sectors << 9) % PAGE_SIZE || 4846 !is_power_of_2(mddev->new_chunk_sectors)) { 4847 printk(KERN_ERR "md/raid:%s: invalid chunk size %d\n", 4848 mdname(mddev), mddev->new_chunk_sectors << 9); 4849 return ERR_PTR(-EINVAL); 4850 } 4851 4852 conf = kzalloc(sizeof(struct r5conf), GFP_KERNEL); 4853 if (conf == NULL) 4854 goto abort; 4855 spin_lock_init(&conf->device_lock); 4856 init_waitqueue_head(&conf->wait_for_stripe); 4857 init_waitqueue_head(&conf->wait_for_overlap); 4858 INIT_LIST_HEAD(&conf->handle_list); 4859 INIT_LIST_HEAD(&conf->hold_list); 4860 INIT_LIST_HEAD(&conf->delayed_list); 4861 INIT_LIST_HEAD(&conf->bitmap_list); 4862 INIT_LIST_HEAD(&conf->inactive_list); 4863 atomic_set(&conf->active_stripes, 0); 4864 atomic_set(&conf->preread_active_stripes, 0); 4865 atomic_set(&conf->active_aligned_reads, 0); 4866 conf->bypass_threshold = BYPASS_THRESHOLD; 4867 conf->recovery_disabled = mddev->recovery_disabled - 1; 4868 4869 conf->raid_disks = mddev->raid_disks; 4870 if (mddev->reshape_position == MaxSector) 4871 conf->previous_raid_disks = mddev->raid_disks; 4872 else 4873 conf->previous_raid_disks = mddev->raid_disks - mddev->delta_disks; 4874 max_disks = max(conf->raid_disks, conf->previous_raid_disks); 4875 conf->scribble_len = scribble_len(max_disks); 4876 4877 conf->disks = kzalloc(max_disks * sizeof(struct disk_info), 4878 GFP_KERNEL); 4879 if (!conf->disks) 4880 goto abort; 4881 4882 conf->mddev = mddev; 4883 4884 if ((conf->stripe_hashtbl = kzalloc(PAGE_SIZE, GFP_KERNEL)) == NULL) 4885 goto abort; 4886 4887 conf->level = mddev->new_level; 4888 if (raid5_alloc_percpu(conf) != 0) 4889 goto abort; 4890 4891 pr_debug("raid456: run(%s) called.\n", mdname(mddev)); 4892 4893 rdev_for_each(rdev, mddev) { 4894 raid_disk = rdev->raid_disk; 4895 if (raid_disk >= max_disks 4896 || raid_disk < 0) 4897 continue; 4898 disk = conf->disks + raid_disk; 4899 4900 if (test_bit(Replacement, &rdev->flags)) { 4901 if (disk->replacement) 4902 goto abort; 4903 disk->replacement = rdev; 4904 } else { 4905 if (disk->rdev) 4906 goto abort; 4907 disk->rdev = rdev; 4908 } 4909 4910 if (test_bit(In_sync, &rdev->flags)) { 4911 char b[BDEVNAME_SIZE]; 4912 printk(KERN_INFO "md/raid:%s: device %s operational as raid" 4913 " disk %d\n", 4914 mdname(mddev), bdevname(rdev->bdev, b), raid_disk); 4915 } else if (rdev->saved_raid_disk != raid_disk) 4916 /* Cannot rely on bitmap to complete recovery */ 4917 conf->fullsync = 1; 4918 } 4919 4920 conf->chunk_sectors = mddev->new_chunk_sectors; 4921 conf->level = mddev->new_level; 4922 if (conf->level == 6) 4923 conf->max_degraded = 2; 4924 else 4925 conf->max_degraded = 1; 4926 conf->algorithm = mddev->new_layout; 4927 conf->max_nr_stripes = NR_STRIPES; 4928 conf->reshape_progress = mddev->reshape_position; 4929 if (conf->reshape_progress != MaxSector) { 4930 conf->prev_chunk_sectors = mddev->chunk_sectors; 4931 conf->prev_algo = mddev->layout; 4932 } 4933 4934 memory = conf->max_nr_stripes * (sizeof(struct stripe_head) + 4935 max_disks * ((sizeof(struct bio) + PAGE_SIZE))) / 1024; 4936 if (grow_stripes(conf, conf->max_nr_stripes)) { 4937 printk(KERN_ERR 4938 "md/raid:%s: couldn't allocate %dkB for buffers\n", 4939 mdname(mddev), memory); 4940 goto abort; 4941 } else 4942 printk(KERN_INFO "md/raid:%s: allocated %dkB\n", 4943 mdname(mddev), memory); 4944 4945 conf->thread = md_register_thread(raid5d, mddev, NULL); 4946 if (!conf->thread) { 4947 printk(KERN_ERR 4948 "md/raid:%s: couldn't allocate thread.\n", 4949 mdname(mddev)); 4950 goto abort; 4951 } 4952 4953 return conf; 4954 4955 abort: 4956 if (conf) { 4957 free_conf(conf); 4958 return ERR_PTR(-EIO); 4959 } else 4960 return ERR_PTR(-ENOMEM); 4961 } 4962 4963 4964 static int only_parity(int raid_disk, int algo, int raid_disks, int max_degraded) 4965 { 4966 switch (algo) { 4967 case ALGORITHM_PARITY_0: 4968 if (raid_disk < max_degraded) 4969 return 1; 4970 break; 4971 case ALGORITHM_PARITY_N: 4972 if (raid_disk >= raid_disks - max_degraded) 4973 return 1; 4974 break; 4975 case ALGORITHM_PARITY_0_6: 4976 if (raid_disk == 0 || 4977 raid_disk == raid_disks - 1) 4978 return 1; 4979 break; 4980 case ALGORITHM_LEFT_ASYMMETRIC_6: 4981 case ALGORITHM_RIGHT_ASYMMETRIC_6: 4982 case ALGORITHM_LEFT_SYMMETRIC_6: 4983 case ALGORITHM_RIGHT_SYMMETRIC_6: 4984 if (raid_disk == raid_disks - 1) 4985 return 1; 4986 } 4987 return 0; 4988 } 4989 4990 static int run(struct mddev *mddev) 4991 { 4992 struct r5conf *conf; 4993 int working_disks = 0; 4994 int dirty_parity_disks = 0; 4995 struct md_rdev *rdev; 4996 sector_t reshape_offset = 0; 4997 int i; 4998 long long min_offset_diff = 0; 4999 int first = 1; 5000 5001 if (mddev->recovery_cp != MaxSector) 5002 printk(KERN_NOTICE "md/raid:%s: not clean" 5003 " -- starting background reconstruction\n", 5004 mdname(mddev)); 5005 5006 rdev_for_each(rdev, mddev) { 5007 long long diff; 5008 if (rdev->raid_disk < 0) 5009 continue; 5010 diff = (rdev->new_data_offset - rdev->data_offset); 5011 if (first) { 5012 min_offset_diff = diff; 5013 first = 0; 5014 } else if (mddev->reshape_backwards && 5015 diff < min_offset_diff) 5016 min_offset_diff = diff; 5017 else if (!mddev->reshape_backwards && 5018 diff > min_offset_diff) 5019 min_offset_diff = diff; 5020 } 5021 5022 if (mddev->reshape_position != MaxSector) { 5023 /* Check that we can continue the reshape. 5024 * Difficulties arise if the stripe we would write to 5025 * next is at or after the stripe we would read from next. 5026 * For a reshape that changes the number of devices, this 5027 * is only possible for a very short time, and mdadm makes 5028 * sure that time appears to have past before assembling 5029 * the array. So we fail if that time hasn't passed. 5030 * For a reshape that keeps the number of devices the same 5031 * mdadm must be monitoring the reshape can keeping the 5032 * critical areas read-only and backed up. It will start 5033 * the array in read-only mode, so we check for that. 5034 */ 5035 sector_t here_new, here_old; 5036 int old_disks; 5037 int max_degraded = (mddev->level == 6 ? 2 : 1); 5038 5039 if (mddev->new_level != mddev->level) { 5040 printk(KERN_ERR "md/raid:%s: unsupported reshape " 5041 "required - aborting.\n", 5042 mdname(mddev)); 5043 return -EINVAL; 5044 } 5045 old_disks = mddev->raid_disks - mddev->delta_disks; 5046 /* reshape_position must be on a new-stripe boundary, and one 5047 * further up in new geometry must map after here in old 5048 * geometry. 5049 */ 5050 here_new = mddev->reshape_position; 5051 if (sector_div(here_new, mddev->new_chunk_sectors * 5052 (mddev->raid_disks - max_degraded))) { 5053 printk(KERN_ERR "md/raid:%s: reshape_position not " 5054 "on a stripe boundary\n", mdname(mddev)); 5055 return -EINVAL; 5056 } 5057 reshape_offset = here_new * mddev->new_chunk_sectors; 5058 /* here_new is the stripe we will write to */ 5059 here_old = mddev->reshape_position; 5060 sector_div(here_old, mddev->chunk_sectors * 5061 (old_disks-max_degraded)); 5062 /* here_old is the first stripe that we might need to read 5063 * from */ 5064 if (mddev->delta_disks == 0) { 5065 if ((here_new * mddev->new_chunk_sectors != 5066 here_old * mddev->chunk_sectors)) { 5067 printk(KERN_ERR "md/raid:%s: reshape position is" 5068 " confused - aborting\n", mdname(mddev)); 5069 return -EINVAL; 5070 } 5071 /* We cannot be sure it is safe to start an in-place 5072 * reshape. It is only safe if user-space is monitoring 5073 * and taking constant backups. 5074 * mdadm always starts a situation like this in 5075 * readonly mode so it can take control before 5076 * allowing any writes. So just check for that. 5077 */ 5078 if (abs(min_offset_diff) >= mddev->chunk_sectors && 5079 abs(min_offset_diff) >= mddev->new_chunk_sectors) 5080 /* not really in-place - so OK */; 5081 else if (mddev->ro == 0) { 5082 printk(KERN_ERR "md/raid:%s: in-place reshape " 5083 "must be started in read-only mode " 5084 "- aborting\n", 5085 mdname(mddev)); 5086 return -EINVAL; 5087 } 5088 } else if (mddev->reshape_backwards 5089 ? (here_new * mddev->new_chunk_sectors + min_offset_diff <= 5090 here_old * mddev->chunk_sectors) 5091 : (here_new * mddev->new_chunk_sectors >= 5092 here_old * mddev->chunk_sectors + (-min_offset_diff))) { 5093 /* Reading from the same stripe as writing to - bad */ 5094 printk(KERN_ERR "md/raid:%s: reshape_position too early for " 5095 "auto-recovery - aborting.\n", 5096 mdname(mddev)); 5097 return -EINVAL; 5098 } 5099 printk(KERN_INFO "md/raid:%s: reshape will continue\n", 5100 mdname(mddev)); 5101 /* OK, we should be able to continue; */ 5102 } else { 5103 BUG_ON(mddev->level != mddev->new_level); 5104 BUG_ON(mddev->layout != mddev->new_layout); 5105 BUG_ON(mddev->chunk_sectors != mddev->new_chunk_sectors); 5106 BUG_ON(mddev->delta_disks != 0); 5107 } 5108 5109 if (mddev->private == NULL) 5110 conf = setup_conf(mddev); 5111 else 5112 conf = mddev->private; 5113 5114 if (IS_ERR(conf)) 5115 return PTR_ERR(conf); 5116 5117 conf->min_offset_diff = min_offset_diff; 5118 mddev->thread = conf->thread; 5119 conf->thread = NULL; 5120 mddev->private = conf; 5121 5122 for (i = 0; i < conf->raid_disks && conf->previous_raid_disks; 5123 i++) { 5124 rdev = conf->disks[i].rdev; 5125 if (!rdev && conf->disks[i].replacement) { 5126 /* The replacement is all we have yet */ 5127 rdev = conf->disks[i].replacement; 5128 conf->disks[i].replacement = NULL; 5129 clear_bit(Replacement, &rdev->flags); 5130 conf->disks[i].rdev = rdev; 5131 } 5132 if (!rdev) 5133 continue; 5134 if (conf->disks[i].replacement && 5135 conf->reshape_progress != MaxSector) { 5136 /* replacements and reshape simply do not mix. */ 5137 printk(KERN_ERR "md: cannot handle concurrent " 5138 "replacement and reshape.\n"); 5139 goto abort; 5140 } 5141 if (test_bit(In_sync, &rdev->flags)) { 5142 working_disks++; 5143 continue; 5144 } 5145 /* This disc is not fully in-sync. However if it 5146 * just stored parity (beyond the recovery_offset), 5147 * when we don't need to be concerned about the 5148 * array being dirty. 5149 * When reshape goes 'backwards', we never have 5150 * partially completed devices, so we only need 5151 * to worry about reshape going forwards. 5152 */ 5153 /* Hack because v0.91 doesn't store recovery_offset properly. */ 5154 if (mddev->major_version == 0 && 5155 mddev->minor_version > 90) 5156 rdev->recovery_offset = reshape_offset; 5157 5158 if (rdev->recovery_offset < reshape_offset) { 5159 /* We need to check old and new layout */ 5160 if (!only_parity(rdev->raid_disk, 5161 conf->algorithm, 5162 conf->raid_disks, 5163 conf->max_degraded)) 5164 continue; 5165 } 5166 if (!only_parity(rdev->raid_disk, 5167 conf->prev_algo, 5168 conf->previous_raid_disks, 5169 conf->max_degraded)) 5170 continue; 5171 dirty_parity_disks++; 5172 } 5173 5174 /* 5175 * 0 for a fully functional array, 1 or 2 for a degraded array. 5176 */ 5177 mddev->degraded = calc_degraded(conf); 5178 5179 if (has_failed(conf)) { 5180 printk(KERN_ERR "md/raid:%s: not enough operational devices" 5181 " (%d/%d failed)\n", 5182 mdname(mddev), mddev->degraded, conf->raid_disks); 5183 goto abort; 5184 } 5185 5186 /* device size must be a multiple of chunk size */ 5187 mddev->dev_sectors &= ~(mddev->chunk_sectors - 1); 5188 mddev->resync_max_sectors = mddev->dev_sectors; 5189 5190 if (mddev->degraded > dirty_parity_disks && 5191 mddev->recovery_cp != MaxSector) { 5192 if (mddev->ok_start_degraded) 5193 printk(KERN_WARNING 5194 "md/raid:%s: starting dirty degraded array" 5195 " - data corruption possible.\n", 5196 mdname(mddev)); 5197 else { 5198 printk(KERN_ERR 5199 "md/raid:%s: cannot start dirty degraded array.\n", 5200 mdname(mddev)); 5201 goto abort; 5202 } 5203 } 5204 5205 if (mddev->degraded == 0) 5206 printk(KERN_INFO "md/raid:%s: raid level %d active with %d out of %d" 5207 " devices, algorithm %d\n", mdname(mddev), conf->level, 5208 mddev->raid_disks-mddev->degraded, mddev->raid_disks, 5209 mddev->new_layout); 5210 else 5211 printk(KERN_ALERT "md/raid:%s: raid level %d active with %d" 5212 " out of %d devices, algorithm %d\n", 5213 mdname(mddev), conf->level, 5214 mddev->raid_disks - mddev->degraded, 5215 mddev->raid_disks, mddev->new_layout); 5216 5217 print_raid5_conf(conf); 5218 5219 if (conf->reshape_progress != MaxSector) { 5220 conf->reshape_safe = conf->reshape_progress; 5221 atomic_set(&conf->reshape_stripes, 0); 5222 clear_bit(MD_RECOVERY_SYNC, &mddev->recovery); 5223 clear_bit(MD_RECOVERY_CHECK, &mddev->recovery); 5224 set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery); 5225 set_bit(MD_RECOVERY_RUNNING, &mddev->recovery); 5226 mddev->sync_thread = md_register_thread(md_do_sync, mddev, 5227 "reshape"); 5228 } 5229 5230 5231 /* Ok, everything is just fine now */ 5232 if (mddev->to_remove == &raid5_attrs_group) 5233 mddev->to_remove = NULL; 5234 else if (mddev->kobj.sd && 5235 sysfs_create_group(&mddev->kobj, &raid5_attrs_group)) 5236 printk(KERN_WARNING 5237 "raid5: failed to create sysfs attributes for %s\n", 5238 mdname(mddev)); 5239 md_set_array_sectors(mddev, raid5_size(mddev, 0, 0)); 5240 5241 if (mddev->queue) { 5242 int chunk_size; 5243 /* read-ahead size must cover two whole stripes, which 5244 * is 2 * (datadisks) * chunksize where 'n' is the 5245 * number of raid devices 5246 */ 5247 int data_disks = conf->previous_raid_disks - conf->max_degraded; 5248 int stripe = data_disks * 5249 ((mddev->chunk_sectors << 9) / PAGE_SIZE); 5250 if (mddev->queue->backing_dev_info.ra_pages < 2 * stripe) 5251 mddev->queue->backing_dev_info.ra_pages = 2 * stripe; 5252 5253 blk_queue_merge_bvec(mddev->queue, raid5_mergeable_bvec); 5254 5255 mddev->queue->backing_dev_info.congested_data = mddev; 5256 mddev->queue->backing_dev_info.congested_fn = raid5_congested; 5257 5258 chunk_size = mddev->chunk_sectors << 9; 5259 blk_queue_io_min(mddev->queue, chunk_size); 5260 blk_queue_io_opt(mddev->queue, chunk_size * 5261 (conf->raid_disks - conf->max_degraded)); 5262 5263 rdev_for_each(rdev, mddev) { 5264 disk_stack_limits(mddev->gendisk, rdev->bdev, 5265 rdev->data_offset << 9); 5266 disk_stack_limits(mddev->gendisk, rdev->bdev, 5267 rdev->new_data_offset << 9); 5268 } 5269 } 5270 5271 return 0; 5272 abort: 5273 md_unregister_thread(&mddev->thread); 5274 print_raid5_conf(conf); 5275 free_conf(conf); 5276 mddev->private = NULL; 5277 printk(KERN_ALERT "md/raid:%s: failed to run raid set.\n", mdname(mddev)); 5278 return -EIO; 5279 } 5280 5281 static int stop(struct mddev *mddev) 5282 { 5283 struct r5conf *conf = mddev->private; 5284 5285 md_unregister_thread(&mddev->thread); 5286 if (mddev->queue) 5287 mddev->queue->backing_dev_info.congested_fn = NULL; 5288 free_conf(conf); 5289 mddev->private = NULL; 5290 mddev->to_remove = &raid5_attrs_group; 5291 return 0; 5292 } 5293 5294 static void status(struct seq_file *seq, struct mddev *mddev) 5295 { 5296 struct r5conf *conf = mddev->private; 5297 int i; 5298 5299 seq_printf(seq, " level %d, %dk chunk, algorithm %d", mddev->level, 5300 mddev->chunk_sectors / 2, mddev->layout); 5301 seq_printf (seq, " [%d/%d] [", conf->raid_disks, conf->raid_disks - mddev->degraded); 5302 for (i = 0; i < conf->raid_disks; i++) 5303 seq_printf (seq, "%s", 5304 conf->disks[i].rdev && 5305 test_bit(In_sync, &conf->disks[i].rdev->flags) ? "U" : "_"); 5306 seq_printf (seq, "]"); 5307 } 5308 5309 static void print_raid5_conf (struct r5conf *conf) 5310 { 5311 int i; 5312 struct disk_info *tmp; 5313 5314 printk(KERN_DEBUG "RAID conf printout:\n"); 5315 if (!conf) { 5316 printk("(conf==NULL)\n"); 5317 return; 5318 } 5319 printk(KERN_DEBUG " --- level:%d rd:%d wd:%d\n", conf->level, 5320 conf->raid_disks, 5321 conf->raid_disks - conf->mddev->degraded); 5322 5323 for (i = 0; i < conf->raid_disks; i++) { 5324 char b[BDEVNAME_SIZE]; 5325 tmp = conf->disks + i; 5326 if (tmp->rdev) 5327 printk(KERN_DEBUG " disk %d, o:%d, dev:%s\n", 5328 i, !test_bit(Faulty, &tmp->rdev->flags), 5329 bdevname(tmp->rdev->bdev, b)); 5330 } 5331 } 5332 5333 static int raid5_spare_active(struct mddev *mddev) 5334 { 5335 int i; 5336 struct r5conf *conf = mddev->private; 5337 struct disk_info *tmp; 5338 int count = 0; 5339 unsigned long flags; 5340 5341 for (i = 0; i < conf->raid_disks; i++) { 5342 tmp = conf->disks + i; 5343 if (tmp->replacement 5344 && tmp->replacement->recovery_offset == MaxSector 5345 && !test_bit(Faulty, &tmp->replacement->flags) 5346 && !test_and_set_bit(In_sync, &tmp->replacement->flags)) { 5347 /* Replacement has just become active. */ 5348 if (!tmp->rdev 5349 || !test_and_clear_bit(In_sync, &tmp->rdev->flags)) 5350 count++; 5351 if (tmp->rdev) { 5352 /* Replaced device not technically faulty, 5353 * but we need to be sure it gets removed 5354 * and never re-added. 5355 */ 5356 set_bit(Faulty, &tmp->rdev->flags); 5357 sysfs_notify_dirent_safe( 5358 tmp->rdev->sysfs_state); 5359 } 5360 sysfs_notify_dirent_safe(tmp->replacement->sysfs_state); 5361 } else if (tmp->rdev 5362 && tmp->rdev->recovery_offset == MaxSector 5363 && !test_bit(Faulty, &tmp->rdev->flags) 5364 && !test_and_set_bit(In_sync, &tmp->rdev->flags)) { 5365 count++; 5366 sysfs_notify_dirent_safe(tmp->rdev->sysfs_state); 5367 } 5368 } 5369 spin_lock_irqsave(&conf->device_lock, flags); 5370 mddev->degraded = calc_degraded(conf); 5371 spin_unlock_irqrestore(&conf->device_lock, flags); 5372 print_raid5_conf(conf); 5373 return count; 5374 } 5375 5376 static int raid5_remove_disk(struct mddev *mddev, struct md_rdev *rdev) 5377 { 5378 struct r5conf *conf = mddev->private; 5379 int err = 0; 5380 int number = rdev->raid_disk; 5381 struct md_rdev **rdevp; 5382 struct disk_info *p = conf->disks + number; 5383 5384 print_raid5_conf(conf); 5385 if (rdev == p->rdev) 5386 rdevp = &p->rdev; 5387 else if (rdev == p->replacement) 5388 rdevp = &p->replacement; 5389 else 5390 return 0; 5391 5392 if (number >= conf->raid_disks && 5393 conf->reshape_progress == MaxSector) 5394 clear_bit(In_sync, &rdev->flags); 5395 5396 if (test_bit(In_sync, &rdev->flags) || 5397 atomic_read(&rdev->nr_pending)) { 5398 err = -EBUSY; 5399 goto abort; 5400 } 5401 /* Only remove non-faulty devices if recovery 5402 * isn't possible. 5403 */ 5404 if (!test_bit(Faulty, &rdev->flags) && 5405 mddev->recovery_disabled != conf->recovery_disabled && 5406 !has_failed(conf) && 5407 (!p->replacement || p->replacement == rdev) && 5408 number < conf->raid_disks) { 5409 err = -EBUSY; 5410 goto abort; 5411 } 5412 *rdevp = NULL; 5413 synchronize_rcu(); 5414 if (atomic_read(&rdev->nr_pending)) { 5415 /* lost the race, try later */ 5416 err = -EBUSY; 5417 *rdevp = rdev; 5418 } else if (p->replacement) { 5419 /* We must have just cleared 'rdev' */ 5420 p->rdev = p->replacement; 5421 clear_bit(Replacement, &p->replacement->flags); 5422 smp_mb(); /* Make sure other CPUs may see both as identical 5423 * but will never see neither - if they are careful 5424 */ 5425 p->replacement = NULL; 5426 clear_bit(WantReplacement, &rdev->flags); 5427 } else 5428 /* We might have just removed the Replacement as faulty- 5429 * clear the bit just in case 5430 */ 5431 clear_bit(WantReplacement, &rdev->flags); 5432 abort: 5433 5434 print_raid5_conf(conf); 5435 return err; 5436 } 5437 5438 static int raid5_add_disk(struct mddev *mddev, struct md_rdev *rdev) 5439 { 5440 struct r5conf *conf = mddev->private; 5441 int err = -EEXIST; 5442 int disk; 5443 struct disk_info *p; 5444 int first = 0; 5445 int last = conf->raid_disks - 1; 5446 5447 if (mddev->recovery_disabled == conf->recovery_disabled) 5448 return -EBUSY; 5449 5450 if (rdev->saved_raid_disk < 0 && has_failed(conf)) 5451 /* no point adding a device */ 5452 return -EINVAL; 5453 5454 if (rdev->raid_disk >= 0) 5455 first = last = rdev->raid_disk; 5456 5457 /* 5458 * find the disk ... but prefer rdev->saved_raid_disk 5459 * if possible. 5460 */ 5461 if (rdev->saved_raid_disk >= 0 && 5462 rdev->saved_raid_disk >= first && 5463 conf->disks[rdev->saved_raid_disk].rdev == NULL) 5464 disk = rdev->saved_raid_disk; 5465 else 5466 disk = first; 5467 for ( ; disk <= last ; disk++) { 5468 p = conf->disks + disk; 5469 if (p->rdev == NULL) { 5470 clear_bit(In_sync, &rdev->flags); 5471 rdev->raid_disk = disk; 5472 err = 0; 5473 if (rdev->saved_raid_disk != disk) 5474 conf->fullsync = 1; 5475 rcu_assign_pointer(p->rdev, rdev); 5476 break; 5477 } 5478 if (test_bit(WantReplacement, &p->rdev->flags) && 5479 p->replacement == NULL) { 5480 clear_bit(In_sync, &rdev->flags); 5481 set_bit(Replacement, &rdev->flags); 5482 rdev->raid_disk = disk; 5483 err = 0; 5484 conf->fullsync = 1; 5485 rcu_assign_pointer(p->replacement, rdev); 5486 break; 5487 } 5488 } 5489 print_raid5_conf(conf); 5490 return err; 5491 } 5492 5493 static int raid5_resize(struct mddev *mddev, sector_t sectors) 5494 { 5495 /* no resync is happening, and there is enough space 5496 * on all devices, so we can resize. 5497 * We need to make sure resync covers any new space. 5498 * If the array is shrinking we should possibly wait until 5499 * any io in the removed space completes, but it hardly seems 5500 * worth it. 5501 */ 5502 sectors &= ~((sector_t)mddev->chunk_sectors - 1); 5503 md_set_array_sectors(mddev, raid5_size(mddev, sectors, 5504 mddev->raid_disks)); 5505 if (mddev->array_sectors > 5506 raid5_size(mddev, sectors, mddev->raid_disks)) 5507 return -EINVAL; 5508 set_capacity(mddev->gendisk, mddev->array_sectors); 5509 revalidate_disk(mddev->gendisk); 5510 if (sectors > mddev->dev_sectors && 5511 mddev->recovery_cp > mddev->dev_sectors) { 5512 mddev->recovery_cp = mddev->dev_sectors; 5513 set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); 5514 } 5515 mddev->dev_sectors = sectors; 5516 mddev->resync_max_sectors = sectors; 5517 return 0; 5518 } 5519 5520 static int check_stripe_cache(struct mddev *mddev) 5521 { 5522 /* Can only proceed if there are plenty of stripe_heads. 5523 * We need a minimum of one full stripe,, and for sensible progress 5524 * it is best to have about 4 times that. 5525 * If we require 4 times, then the default 256 4K stripe_heads will 5526 * allow for chunk sizes up to 256K, which is probably OK. 5527 * If the chunk size is greater, user-space should request more 5528 * stripe_heads first. 5529 */ 5530 struct r5conf *conf = mddev->private; 5531 if (((mddev->chunk_sectors << 9) / STRIPE_SIZE) * 4 5532 > conf->max_nr_stripes || 5533 ((mddev->new_chunk_sectors << 9) / STRIPE_SIZE) * 4 5534 > conf->max_nr_stripes) { 5535 printk(KERN_WARNING "md/raid:%s: reshape: not enough stripes. Needed %lu\n", 5536 mdname(mddev), 5537 ((max(mddev->chunk_sectors, mddev->new_chunk_sectors) << 9) 5538 / STRIPE_SIZE)*4); 5539 return 0; 5540 } 5541 return 1; 5542 } 5543 5544 static int check_reshape(struct mddev *mddev) 5545 { 5546 struct r5conf *conf = mddev->private; 5547 5548 if (mddev->delta_disks == 0 && 5549 mddev->new_layout == mddev->layout && 5550 mddev->new_chunk_sectors == mddev->chunk_sectors) 5551 return 0; /* nothing to do */ 5552 if (mddev->bitmap) 5553 /* Cannot grow a bitmap yet */ 5554 return -EBUSY; 5555 if (has_failed(conf)) 5556 return -EINVAL; 5557 if (mddev->delta_disks < 0) { 5558 /* We might be able to shrink, but the devices must 5559 * be made bigger first. 5560 * For raid6, 4 is the minimum size. 5561 * Otherwise 2 is the minimum 5562 */ 5563 int min = 2; 5564 if (mddev->level == 6) 5565 min = 4; 5566 if (mddev->raid_disks + mddev->delta_disks < min) 5567 return -EINVAL; 5568 } 5569 5570 if (!check_stripe_cache(mddev)) 5571 return -ENOSPC; 5572 5573 return resize_stripes(conf, conf->raid_disks + mddev->delta_disks); 5574 } 5575 5576 static int raid5_start_reshape(struct mddev *mddev) 5577 { 5578 struct r5conf *conf = mddev->private; 5579 struct md_rdev *rdev; 5580 int spares = 0; 5581 unsigned long flags; 5582 5583 if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery)) 5584 return -EBUSY; 5585 5586 if (!check_stripe_cache(mddev)) 5587 return -ENOSPC; 5588 5589 rdev_for_each(rdev, mddev) { 5590 if (!test_bit(In_sync, &rdev->flags) 5591 && !test_bit(Faulty, &rdev->flags)) 5592 spares++; 5593 } 5594 5595 if (spares - mddev->degraded < mddev->delta_disks - conf->max_degraded) 5596 /* Not enough devices even to make a degraded array 5597 * of that size 5598 */ 5599 return -EINVAL; 5600 5601 /* Refuse to reduce size of the array. Any reductions in 5602 * array size must be through explicit setting of array_size 5603 * attribute. 5604 */ 5605 if (raid5_size(mddev, 0, conf->raid_disks + mddev->delta_disks) 5606 < mddev->array_sectors) { 5607 printk(KERN_ERR "md/raid:%s: array size must be reduced " 5608 "before number of disks\n", mdname(mddev)); 5609 return -EINVAL; 5610 } 5611 5612 atomic_set(&conf->reshape_stripes, 0); 5613 spin_lock_irq(&conf->device_lock); 5614 conf->previous_raid_disks = conf->raid_disks; 5615 conf->raid_disks += mddev->delta_disks; 5616 conf->prev_chunk_sectors = conf->chunk_sectors; 5617 conf->chunk_sectors = mddev->new_chunk_sectors; 5618 conf->prev_algo = conf->algorithm; 5619 conf->algorithm = mddev->new_layout; 5620 conf->generation++; 5621 /* Code that selects data_offset needs to see the generation update 5622 * if reshape_progress has been set - so a memory barrier needed. 5623 */ 5624 smp_mb(); 5625 if (mddev->reshape_backwards) 5626 conf->reshape_progress = raid5_size(mddev, 0, 0); 5627 else 5628 conf->reshape_progress = 0; 5629 conf->reshape_safe = conf->reshape_progress; 5630 spin_unlock_irq(&conf->device_lock); 5631 5632 /* Add some new drives, as many as will fit. 5633 * We know there are enough to make the newly sized array work. 5634 * Don't add devices if we are reducing the number of 5635 * devices in the array. This is because it is not possible 5636 * to correctly record the "partially reconstructed" state of 5637 * such devices during the reshape and confusion could result. 5638 */ 5639 if (mddev->delta_disks >= 0) { 5640 rdev_for_each(rdev, mddev) 5641 if (rdev->raid_disk < 0 && 5642 !test_bit(Faulty, &rdev->flags)) { 5643 if (raid5_add_disk(mddev, rdev) == 0) { 5644 if (rdev->raid_disk 5645 >= conf->previous_raid_disks) 5646 set_bit(In_sync, &rdev->flags); 5647 else 5648 rdev->recovery_offset = 0; 5649 5650 if (sysfs_link_rdev(mddev, rdev)) 5651 /* Failure here is OK */; 5652 } 5653 } else if (rdev->raid_disk >= conf->previous_raid_disks 5654 && !test_bit(Faulty, &rdev->flags)) { 5655 /* This is a spare that was manually added */ 5656 set_bit(In_sync, &rdev->flags); 5657 } 5658 5659 /* When a reshape changes the number of devices, 5660 * ->degraded is measured against the larger of the 5661 * pre and post number of devices. 5662 */ 5663 spin_lock_irqsave(&conf->device_lock, flags); 5664 mddev->degraded = calc_degraded(conf); 5665 spin_unlock_irqrestore(&conf->device_lock, flags); 5666 } 5667 mddev->raid_disks = conf->raid_disks; 5668 mddev->reshape_position = conf->reshape_progress; 5669 set_bit(MD_CHANGE_DEVS, &mddev->flags); 5670 5671 clear_bit(MD_RECOVERY_SYNC, &mddev->recovery); 5672 clear_bit(MD_RECOVERY_CHECK, &mddev->recovery); 5673 set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery); 5674 set_bit(MD_RECOVERY_RUNNING, &mddev->recovery); 5675 mddev->sync_thread = md_register_thread(md_do_sync, mddev, 5676 "reshape"); 5677 if (!mddev->sync_thread) { 5678 mddev->recovery = 0; 5679 spin_lock_irq(&conf->device_lock); 5680 mddev->raid_disks = conf->raid_disks = conf->previous_raid_disks; 5681 rdev_for_each(rdev, mddev) 5682 rdev->new_data_offset = rdev->data_offset; 5683 smp_wmb(); 5684 conf->reshape_progress = MaxSector; 5685 mddev->reshape_position = MaxSector; 5686 spin_unlock_irq(&conf->device_lock); 5687 return -EAGAIN; 5688 } 5689 conf->reshape_checkpoint = jiffies; 5690 md_wakeup_thread(mddev->sync_thread); 5691 md_new_event(mddev); 5692 return 0; 5693 } 5694 5695 /* This is called from the reshape thread and should make any 5696 * changes needed in 'conf' 5697 */ 5698 static void end_reshape(struct r5conf *conf) 5699 { 5700 5701 if (!test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery)) { 5702 struct md_rdev *rdev; 5703 5704 spin_lock_irq(&conf->device_lock); 5705 conf->previous_raid_disks = conf->raid_disks; 5706 rdev_for_each(rdev, conf->mddev) 5707 rdev->data_offset = rdev->new_data_offset; 5708 smp_wmb(); 5709 conf->reshape_progress = MaxSector; 5710 spin_unlock_irq(&conf->device_lock); 5711 wake_up(&conf->wait_for_overlap); 5712 5713 /* read-ahead size must cover two whole stripes, which is 5714 * 2 * (datadisks) * chunksize where 'n' is the number of raid devices 5715 */ 5716 if (conf->mddev->queue) { 5717 int data_disks = conf->raid_disks - conf->max_degraded; 5718 int stripe = data_disks * ((conf->chunk_sectors << 9) 5719 / PAGE_SIZE); 5720 if (conf->mddev->queue->backing_dev_info.ra_pages < 2 * stripe) 5721 conf->mddev->queue->backing_dev_info.ra_pages = 2 * stripe; 5722 } 5723 } 5724 } 5725 5726 /* This is called from the raid5d thread with mddev_lock held. 5727 * It makes config changes to the device. 5728 */ 5729 static void raid5_finish_reshape(struct mddev *mddev) 5730 { 5731 struct r5conf *conf = mddev->private; 5732 5733 if (!test_bit(MD_RECOVERY_INTR, &mddev->recovery)) { 5734 5735 if (mddev->delta_disks > 0) { 5736 md_set_array_sectors(mddev, raid5_size(mddev, 0, 0)); 5737 set_capacity(mddev->gendisk, mddev->array_sectors); 5738 revalidate_disk(mddev->gendisk); 5739 } else { 5740 int d; 5741 spin_lock_irq(&conf->device_lock); 5742 mddev->degraded = calc_degraded(conf); 5743 spin_unlock_irq(&conf->device_lock); 5744 for (d = conf->raid_disks ; 5745 d < conf->raid_disks - mddev->delta_disks; 5746 d++) { 5747 struct md_rdev *rdev = conf->disks[d].rdev; 5748 if (rdev && 5749 raid5_remove_disk(mddev, rdev) == 0) { 5750 sysfs_unlink_rdev(mddev, rdev); 5751 rdev->raid_disk = -1; 5752 } 5753 } 5754 } 5755 mddev->layout = conf->algorithm; 5756 mddev->chunk_sectors = conf->chunk_sectors; 5757 mddev->reshape_position = MaxSector; 5758 mddev->delta_disks = 0; 5759 mddev->reshape_backwards = 0; 5760 } 5761 } 5762 5763 static void raid5_quiesce(struct mddev *mddev, int state) 5764 { 5765 struct r5conf *conf = mddev->private; 5766 5767 switch(state) { 5768 case 2: /* resume for a suspend */ 5769 wake_up(&conf->wait_for_overlap); 5770 break; 5771 5772 case 1: /* stop all writes */ 5773 spin_lock_irq(&conf->device_lock); 5774 /* '2' tells resync/reshape to pause so that all 5775 * active stripes can drain 5776 */ 5777 conf->quiesce = 2; 5778 wait_event_lock_irq(conf->wait_for_stripe, 5779 atomic_read(&conf->active_stripes) == 0 && 5780 atomic_read(&conf->active_aligned_reads) == 0, 5781 conf->device_lock, /* nothing */); 5782 conf->quiesce = 1; 5783 spin_unlock_irq(&conf->device_lock); 5784 /* allow reshape to continue */ 5785 wake_up(&conf->wait_for_overlap); 5786 break; 5787 5788 case 0: /* re-enable writes */ 5789 spin_lock_irq(&conf->device_lock); 5790 conf->quiesce = 0; 5791 wake_up(&conf->wait_for_stripe); 5792 wake_up(&conf->wait_for_overlap); 5793 spin_unlock_irq(&conf->device_lock); 5794 break; 5795 } 5796 } 5797 5798 5799 static void *raid45_takeover_raid0(struct mddev *mddev, int level) 5800 { 5801 struct r0conf *raid0_conf = mddev->private; 5802 sector_t sectors; 5803 5804 /* for raid0 takeover only one zone is supported */ 5805 if (raid0_conf->nr_strip_zones > 1) { 5806 printk(KERN_ERR "md/raid:%s: cannot takeover raid0 with more than one zone.\n", 5807 mdname(mddev)); 5808 return ERR_PTR(-EINVAL); 5809 } 5810 5811 sectors = raid0_conf->strip_zone[0].zone_end; 5812 sector_div(sectors, raid0_conf->strip_zone[0].nb_dev); 5813 mddev->dev_sectors = sectors; 5814 mddev->new_level = level; 5815 mddev->new_layout = ALGORITHM_PARITY_N; 5816 mddev->new_chunk_sectors = mddev->chunk_sectors; 5817 mddev->raid_disks += 1; 5818 mddev->delta_disks = 1; 5819 /* make sure it will be not marked as dirty */ 5820 mddev->recovery_cp = MaxSector; 5821 5822 return setup_conf(mddev); 5823 } 5824 5825 5826 static void *raid5_takeover_raid1(struct mddev *mddev) 5827 { 5828 int chunksect; 5829 5830 if (mddev->raid_disks != 2 || 5831 mddev->degraded > 1) 5832 return ERR_PTR(-EINVAL); 5833 5834 /* Should check if there are write-behind devices? */ 5835 5836 chunksect = 64*2; /* 64K by default */ 5837 5838 /* The array must be an exact multiple of chunksize */ 5839 while (chunksect && (mddev->array_sectors & (chunksect-1))) 5840 chunksect >>= 1; 5841 5842 if ((chunksect<<9) < STRIPE_SIZE) 5843 /* array size does not allow a suitable chunk size */ 5844 return ERR_PTR(-EINVAL); 5845 5846 mddev->new_level = 5; 5847 mddev->new_layout = ALGORITHM_LEFT_SYMMETRIC; 5848 mddev->new_chunk_sectors = chunksect; 5849 5850 return setup_conf(mddev); 5851 } 5852 5853 static void *raid5_takeover_raid6(struct mddev *mddev) 5854 { 5855 int new_layout; 5856 5857 switch (mddev->layout) { 5858 case ALGORITHM_LEFT_ASYMMETRIC_6: 5859 new_layout = ALGORITHM_LEFT_ASYMMETRIC; 5860 break; 5861 case ALGORITHM_RIGHT_ASYMMETRIC_6: 5862 new_layout = ALGORITHM_RIGHT_ASYMMETRIC; 5863 break; 5864 case ALGORITHM_LEFT_SYMMETRIC_6: 5865 new_layout = ALGORITHM_LEFT_SYMMETRIC; 5866 break; 5867 case ALGORITHM_RIGHT_SYMMETRIC_6: 5868 new_layout = ALGORITHM_RIGHT_SYMMETRIC; 5869 break; 5870 case ALGORITHM_PARITY_0_6: 5871 new_layout = ALGORITHM_PARITY_0; 5872 break; 5873 case ALGORITHM_PARITY_N: 5874 new_layout = ALGORITHM_PARITY_N; 5875 break; 5876 default: 5877 return ERR_PTR(-EINVAL); 5878 } 5879 mddev->new_level = 5; 5880 mddev->new_layout = new_layout; 5881 mddev->delta_disks = -1; 5882 mddev->raid_disks -= 1; 5883 return setup_conf(mddev); 5884 } 5885 5886 5887 static int raid5_check_reshape(struct mddev *mddev) 5888 { 5889 /* For a 2-drive array, the layout and chunk size can be changed 5890 * immediately as not restriping is needed. 5891 * For larger arrays we record the new value - after validation 5892 * to be used by a reshape pass. 5893 */ 5894 struct r5conf *conf = mddev->private; 5895 int new_chunk = mddev->new_chunk_sectors; 5896 5897 if (mddev->new_layout >= 0 && !algorithm_valid_raid5(mddev->new_layout)) 5898 return -EINVAL; 5899 if (new_chunk > 0) { 5900 if (!is_power_of_2(new_chunk)) 5901 return -EINVAL; 5902 if (new_chunk < (PAGE_SIZE>>9)) 5903 return -EINVAL; 5904 if (mddev->array_sectors & (new_chunk-1)) 5905 /* not factor of array size */ 5906 return -EINVAL; 5907 } 5908 5909 /* They look valid */ 5910 5911 if (mddev->raid_disks == 2) { 5912 /* can make the change immediately */ 5913 if (mddev->new_layout >= 0) { 5914 conf->algorithm = mddev->new_layout; 5915 mddev->layout = mddev->new_layout; 5916 } 5917 if (new_chunk > 0) { 5918 conf->chunk_sectors = new_chunk ; 5919 mddev->chunk_sectors = new_chunk; 5920 } 5921 set_bit(MD_CHANGE_DEVS, &mddev->flags); 5922 md_wakeup_thread(mddev->thread); 5923 } 5924 return check_reshape(mddev); 5925 } 5926 5927 static int raid6_check_reshape(struct mddev *mddev) 5928 { 5929 int new_chunk = mddev->new_chunk_sectors; 5930 5931 if (mddev->new_layout >= 0 && !algorithm_valid_raid6(mddev->new_layout)) 5932 return -EINVAL; 5933 if (new_chunk > 0) { 5934 if (!is_power_of_2(new_chunk)) 5935 return -EINVAL; 5936 if (new_chunk < (PAGE_SIZE >> 9)) 5937 return -EINVAL; 5938 if (mddev->array_sectors & (new_chunk-1)) 5939 /* not factor of array size */ 5940 return -EINVAL; 5941 } 5942 5943 /* They look valid */ 5944 return check_reshape(mddev); 5945 } 5946 5947 static void *raid5_takeover(struct mddev *mddev) 5948 { 5949 /* raid5 can take over: 5950 * raid0 - if there is only one strip zone - make it a raid4 layout 5951 * raid1 - if there are two drives. We need to know the chunk size 5952 * raid4 - trivial - just use a raid4 layout. 5953 * raid6 - Providing it is a *_6 layout 5954 */ 5955 if (mddev->level == 0) 5956 return raid45_takeover_raid0(mddev, 5); 5957 if (mddev->level == 1) 5958 return raid5_takeover_raid1(mddev); 5959 if (mddev->level == 4) { 5960 mddev->new_layout = ALGORITHM_PARITY_N; 5961 mddev->new_level = 5; 5962 return setup_conf(mddev); 5963 } 5964 if (mddev->level == 6) 5965 return raid5_takeover_raid6(mddev); 5966 5967 return ERR_PTR(-EINVAL); 5968 } 5969 5970 static void *raid4_takeover(struct mddev *mddev) 5971 { 5972 /* raid4 can take over: 5973 * raid0 - if there is only one strip zone 5974 * raid5 - if layout is right 5975 */ 5976 if (mddev->level == 0) 5977 return raid45_takeover_raid0(mddev, 4); 5978 if (mddev->level == 5 && 5979 mddev->layout == ALGORITHM_PARITY_N) { 5980 mddev->new_layout = 0; 5981 mddev->new_level = 4; 5982 return setup_conf(mddev); 5983 } 5984 return ERR_PTR(-EINVAL); 5985 } 5986 5987 static struct md_personality raid5_personality; 5988 5989 static void *raid6_takeover(struct mddev *mddev) 5990 { 5991 /* Currently can only take over a raid5. We map the 5992 * personality to an equivalent raid6 personality 5993 * with the Q block at the end. 5994 */ 5995 int new_layout; 5996 5997 if (mddev->pers != &raid5_personality) 5998 return ERR_PTR(-EINVAL); 5999 if (mddev->degraded > 1) 6000 return ERR_PTR(-EINVAL); 6001 if (mddev->raid_disks > 253) 6002 return ERR_PTR(-EINVAL); 6003 if (mddev->raid_disks < 3) 6004 return ERR_PTR(-EINVAL); 6005 6006 switch (mddev->layout) { 6007 case ALGORITHM_LEFT_ASYMMETRIC: 6008 new_layout = ALGORITHM_LEFT_ASYMMETRIC_6; 6009 break; 6010 case ALGORITHM_RIGHT_ASYMMETRIC: 6011 new_layout = ALGORITHM_RIGHT_ASYMMETRIC_6; 6012 break; 6013 case ALGORITHM_LEFT_SYMMETRIC: 6014 new_layout = ALGORITHM_LEFT_SYMMETRIC_6; 6015 break; 6016 case ALGORITHM_RIGHT_SYMMETRIC: 6017 new_layout = ALGORITHM_RIGHT_SYMMETRIC_6; 6018 break; 6019 case ALGORITHM_PARITY_0: 6020 new_layout = ALGORITHM_PARITY_0_6; 6021 break; 6022 case ALGORITHM_PARITY_N: 6023 new_layout = ALGORITHM_PARITY_N; 6024 break; 6025 default: 6026 return ERR_PTR(-EINVAL); 6027 } 6028 mddev->new_level = 6; 6029 mddev->new_layout = new_layout; 6030 mddev->delta_disks = 1; 6031 mddev->raid_disks += 1; 6032 return setup_conf(mddev); 6033 } 6034 6035 6036 static struct md_personality raid6_personality = 6037 { 6038 .name = "raid6", 6039 .level = 6, 6040 .owner = THIS_MODULE, 6041 .make_request = make_request, 6042 .run = run, 6043 .stop = stop, 6044 .status = status, 6045 .error_handler = error, 6046 .hot_add_disk = raid5_add_disk, 6047 .hot_remove_disk= raid5_remove_disk, 6048 .spare_active = raid5_spare_active, 6049 .sync_request = sync_request, 6050 .resize = raid5_resize, 6051 .size = raid5_size, 6052 .check_reshape = raid6_check_reshape, 6053 .start_reshape = raid5_start_reshape, 6054 .finish_reshape = raid5_finish_reshape, 6055 .quiesce = raid5_quiesce, 6056 .takeover = raid6_takeover, 6057 }; 6058 static struct md_personality raid5_personality = 6059 { 6060 .name = "raid5", 6061 .level = 5, 6062 .owner = THIS_MODULE, 6063 .make_request = make_request, 6064 .run = run, 6065 .stop = stop, 6066 .status = status, 6067 .error_handler = error, 6068 .hot_add_disk = raid5_add_disk, 6069 .hot_remove_disk= raid5_remove_disk, 6070 .spare_active = raid5_spare_active, 6071 .sync_request = sync_request, 6072 .resize = raid5_resize, 6073 .size = raid5_size, 6074 .check_reshape = raid5_check_reshape, 6075 .start_reshape = raid5_start_reshape, 6076 .finish_reshape = raid5_finish_reshape, 6077 .quiesce = raid5_quiesce, 6078 .takeover = raid5_takeover, 6079 }; 6080 6081 static struct md_personality raid4_personality = 6082 { 6083 .name = "raid4", 6084 .level = 4, 6085 .owner = THIS_MODULE, 6086 .make_request = make_request, 6087 .run = run, 6088 .stop = stop, 6089 .status = status, 6090 .error_handler = error, 6091 .hot_add_disk = raid5_add_disk, 6092 .hot_remove_disk= raid5_remove_disk, 6093 .spare_active = raid5_spare_active, 6094 .sync_request = sync_request, 6095 .resize = raid5_resize, 6096 .size = raid5_size, 6097 .check_reshape = raid5_check_reshape, 6098 .start_reshape = raid5_start_reshape, 6099 .finish_reshape = raid5_finish_reshape, 6100 .quiesce = raid5_quiesce, 6101 .takeover = raid4_takeover, 6102 }; 6103 6104 static int __init raid5_init(void) 6105 { 6106 register_md_personality(&raid6_personality); 6107 register_md_personality(&raid5_personality); 6108 register_md_personality(&raid4_personality); 6109 return 0; 6110 } 6111 6112 static void raid5_exit(void) 6113 { 6114 unregister_md_personality(&raid6_personality); 6115 unregister_md_personality(&raid5_personality); 6116 unregister_md_personality(&raid4_personality); 6117 } 6118 6119 module_init(raid5_init); 6120 module_exit(raid5_exit); 6121 MODULE_LICENSE("GPL"); 6122 MODULE_DESCRIPTION("RAID4/5/6 (striping with parity) personality for MD"); 6123 MODULE_ALIAS("md-personality-4"); /* RAID5 */ 6124 MODULE_ALIAS("md-raid5"); 6125 MODULE_ALIAS("md-raid4"); 6126 MODULE_ALIAS("md-level-5"); 6127 MODULE_ALIAS("md-level-4"); 6128 MODULE_ALIAS("md-personality-8"); /* RAID6 */ 6129 MODULE_ALIAS("md-raid6"); 6130 MODULE_ALIAS("md-level-6"); 6131 6132 /* This used to be two separate modules, they were: */ 6133 MODULE_ALIAS("raid5"); 6134 MODULE_ALIAS("raid6"); 6135