1 /* 2 * raid5.c : Multiple Devices driver for Linux 3 * Copyright (C) 1996, 1997 Ingo Molnar, Miguel de Icaza, Gadi Oxman 4 * Copyright (C) 1999, 2000 Ingo Molnar 5 * Copyright (C) 2002, 2003 H. Peter Anvin 6 * 7 * RAID-4/5/6 management functions. 8 * Thanks to Penguin Computing for making the RAID-6 development possible 9 * by donating a test server! 10 * 11 * This program is free software; you can redistribute it and/or modify 12 * it under the terms of the GNU General Public License as published by 13 * the Free Software Foundation; either version 2, or (at your option) 14 * any later version. 15 * 16 * You should have received a copy of the GNU General Public License 17 * (for example /usr/src/linux/COPYING); if not, write to the Free 18 * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. 19 */ 20 21 /* 22 * BITMAP UNPLUGGING: 23 * 24 * The sequencing for updating the bitmap reliably is a little 25 * subtle (and I got it wrong the first time) so it deserves some 26 * explanation. 27 * 28 * We group bitmap updates into batches. Each batch has a number. 29 * We may write out several batches at once, but that isn't very important. 30 * conf->bm_write is the number of the last batch successfully written. 31 * conf->bm_flush is the number of the last batch that was closed to 32 * new additions. 33 * When we discover that we will need to write to any block in a stripe 34 * (in add_stripe_bio) we update the in-memory bitmap and record in sh->bm_seq 35 * the number of the batch it will be in. This is bm_flush+1. 36 * When we are ready to do a write, if that batch hasn't been written yet, 37 * we plug the array and queue the stripe for later. 38 * When an unplug happens, we increment bm_flush, thus closing the current 39 * batch. 40 * When we notice that bm_flush > bm_write, we write out all pending updates 41 * to the bitmap, and advance bm_write to where bm_flush was. 42 * This may occasionally write a bit out twice, but is sure never to 43 * miss any bits. 44 */ 45 46 #include <linux/blkdev.h> 47 #include <linux/kthread.h> 48 #include <linux/raid/pq.h> 49 #include <linux/async_tx.h> 50 #include <linux/async.h> 51 #include <linux/seq_file.h> 52 #include <linux/cpu.h> 53 #include "md.h" 54 #include "raid5.h" 55 #include "bitmap.h" 56 57 /* 58 * Stripe cache 59 */ 60 61 #define NR_STRIPES 256 62 #define STRIPE_SIZE PAGE_SIZE 63 #define STRIPE_SHIFT (PAGE_SHIFT - 9) 64 #define STRIPE_SECTORS (STRIPE_SIZE>>9) 65 #define IO_THRESHOLD 1 66 #define BYPASS_THRESHOLD 1 67 #define NR_HASH (PAGE_SIZE / sizeof(struct hlist_head)) 68 #define HASH_MASK (NR_HASH - 1) 69 70 #define stripe_hash(conf, sect) (&((conf)->stripe_hashtbl[((sect) >> STRIPE_SHIFT) & HASH_MASK])) 71 72 /* bio's attached to a stripe+device for I/O are linked together in bi_sector 73 * order without overlap. There may be several bio's per stripe+device, and 74 * a bio could span several devices. 75 * When walking this list for a particular stripe+device, we must never proceed 76 * beyond a bio that extends past this device, as the next bio might no longer 77 * be valid. 78 * This macro is used to determine the 'next' bio in the list, given the sector 79 * of the current stripe+device 80 */ 81 #define r5_next_bio(bio, sect) ( ( (bio)->bi_sector + ((bio)->bi_size>>9) < sect + STRIPE_SECTORS) ? (bio)->bi_next : NULL) 82 /* 83 * The following can be used to debug the driver 84 */ 85 #define RAID5_PARANOIA 1 86 #if RAID5_PARANOIA && defined(CONFIG_SMP) 87 # define CHECK_DEVLOCK() assert_spin_locked(&conf->device_lock) 88 #else 89 # define CHECK_DEVLOCK() 90 #endif 91 92 #ifdef DEBUG 93 #define inline 94 #define __inline__ 95 #endif 96 97 #define printk_rl(args...) ((void) (printk_ratelimit() && printk(args))) 98 99 /* 100 * We maintain a biased count of active stripes in the bottom 16 bits of 101 * bi_phys_segments, and a count of processed stripes in the upper 16 bits 102 */ 103 static inline int raid5_bi_phys_segments(struct bio *bio) 104 { 105 return bio->bi_phys_segments & 0xffff; 106 } 107 108 static inline int raid5_bi_hw_segments(struct bio *bio) 109 { 110 return (bio->bi_phys_segments >> 16) & 0xffff; 111 } 112 113 static inline int raid5_dec_bi_phys_segments(struct bio *bio) 114 { 115 --bio->bi_phys_segments; 116 return raid5_bi_phys_segments(bio); 117 } 118 119 static inline int raid5_dec_bi_hw_segments(struct bio *bio) 120 { 121 unsigned short val = raid5_bi_hw_segments(bio); 122 123 --val; 124 bio->bi_phys_segments = (val << 16) | raid5_bi_phys_segments(bio); 125 return val; 126 } 127 128 static inline void raid5_set_bi_hw_segments(struct bio *bio, unsigned int cnt) 129 { 130 bio->bi_phys_segments = raid5_bi_phys_segments(bio) || (cnt << 16); 131 } 132 133 /* Find first data disk in a raid6 stripe */ 134 static inline int raid6_d0(struct stripe_head *sh) 135 { 136 if (sh->ddf_layout) 137 /* ddf always start from first device */ 138 return 0; 139 /* md starts just after Q block */ 140 if (sh->qd_idx == sh->disks - 1) 141 return 0; 142 else 143 return sh->qd_idx + 1; 144 } 145 static inline int raid6_next_disk(int disk, int raid_disks) 146 { 147 disk++; 148 return (disk < raid_disks) ? disk : 0; 149 } 150 151 /* When walking through the disks in a raid5, starting at raid6_d0, 152 * We need to map each disk to a 'slot', where the data disks are slot 153 * 0 .. raid_disks-3, the parity disk is raid_disks-2 and the Q disk 154 * is raid_disks-1. This help does that mapping. 155 */ 156 static int raid6_idx_to_slot(int idx, struct stripe_head *sh, 157 int *count, int syndrome_disks) 158 { 159 int slot; 160 161 if (idx == sh->pd_idx) 162 return syndrome_disks; 163 if (idx == sh->qd_idx) 164 return syndrome_disks + 1; 165 slot = (*count)++; 166 return slot; 167 } 168 169 static void return_io(struct bio *return_bi) 170 { 171 struct bio *bi = return_bi; 172 while (bi) { 173 174 return_bi = bi->bi_next; 175 bi->bi_next = NULL; 176 bi->bi_size = 0; 177 bio_endio(bi, 0); 178 bi = return_bi; 179 } 180 } 181 182 static void print_raid5_conf (raid5_conf_t *conf); 183 184 static int stripe_operations_active(struct stripe_head *sh) 185 { 186 return sh->check_state || sh->reconstruct_state || 187 test_bit(STRIPE_BIOFILL_RUN, &sh->state) || 188 test_bit(STRIPE_COMPUTE_RUN, &sh->state); 189 } 190 191 static void __release_stripe(raid5_conf_t *conf, struct stripe_head *sh) 192 { 193 if (atomic_dec_and_test(&sh->count)) { 194 BUG_ON(!list_empty(&sh->lru)); 195 BUG_ON(atomic_read(&conf->active_stripes)==0); 196 if (test_bit(STRIPE_HANDLE, &sh->state)) { 197 if (test_bit(STRIPE_DELAYED, &sh->state)) { 198 list_add_tail(&sh->lru, &conf->delayed_list); 199 blk_plug_device(conf->mddev->queue); 200 } else if (test_bit(STRIPE_BIT_DELAY, &sh->state) && 201 sh->bm_seq - conf->seq_write > 0) { 202 list_add_tail(&sh->lru, &conf->bitmap_list); 203 blk_plug_device(conf->mddev->queue); 204 } else { 205 clear_bit(STRIPE_BIT_DELAY, &sh->state); 206 list_add_tail(&sh->lru, &conf->handle_list); 207 } 208 md_wakeup_thread(conf->mddev->thread); 209 } else { 210 BUG_ON(stripe_operations_active(sh)); 211 if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) { 212 atomic_dec(&conf->preread_active_stripes); 213 if (atomic_read(&conf->preread_active_stripes) < IO_THRESHOLD) 214 md_wakeup_thread(conf->mddev->thread); 215 } 216 atomic_dec(&conf->active_stripes); 217 if (!test_bit(STRIPE_EXPANDING, &sh->state)) { 218 list_add_tail(&sh->lru, &conf->inactive_list); 219 wake_up(&conf->wait_for_stripe); 220 if (conf->retry_read_aligned) 221 md_wakeup_thread(conf->mddev->thread); 222 } 223 } 224 } 225 } 226 227 static void release_stripe(struct stripe_head *sh) 228 { 229 raid5_conf_t *conf = sh->raid_conf; 230 unsigned long flags; 231 232 spin_lock_irqsave(&conf->device_lock, flags); 233 __release_stripe(conf, sh); 234 spin_unlock_irqrestore(&conf->device_lock, flags); 235 } 236 237 static inline void remove_hash(struct stripe_head *sh) 238 { 239 pr_debug("remove_hash(), stripe %llu\n", 240 (unsigned long long)sh->sector); 241 242 hlist_del_init(&sh->hash); 243 } 244 245 static inline void insert_hash(raid5_conf_t *conf, struct stripe_head *sh) 246 { 247 struct hlist_head *hp = stripe_hash(conf, sh->sector); 248 249 pr_debug("insert_hash(), stripe %llu\n", 250 (unsigned long long)sh->sector); 251 252 CHECK_DEVLOCK(); 253 hlist_add_head(&sh->hash, hp); 254 } 255 256 257 /* find an idle stripe, make sure it is unhashed, and return it. */ 258 static struct stripe_head *get_free_stripe(raid5_conf_t *conf) 259 { 260 struct stripe_head *sh = NULL; 261 struct list_head *first; 262 263 CHECK_DEVLOCK(); 264 if (list_empty(&conf->inactive_list)) 265 goto out; 266 first = conf->inactive_list.next; 267 sh = list_entry(first, struct stripe_head, lru); 268 list_del_init(first); 269 remove_hash(sh); 270 atomic_inc(&conf->active_stripes); 271 out: 272 return sh; 273 } 274 275 static void shrink_buffers(struct stripe_head *sh, int num) 276 { 277 struct page *p; 278 int i; 279 280 for (i=0; i<num ; i++) { 281 p = sh->dev[i].page; 282 if (!p) 283 continue; 284 sh->dev[i].page = NULL; 285 put_page(p); 286 } 287 } 288 289 static int grow_buffers(struct stripe_head *sh, int num) 290 { 291 int i; 292 293 for (i=0; i<num; i++) { 294 struct page *page; 295 296 if (!(page = alloc_page(GFP_KERNEL))) { 297 return 1; 298 } 299 sh->dev[i].page = page; 300 } 301 return 0; 302 } 303 304 static void raid5_build_block(struct stripe_head *sh, int i, int previous); 305 static void stripe_set_idx(sector_t stripe, raid5_conf_t *conf, int previous, 306 struct stripe_head *sh); 307 308 static void init_stripe(struct stripe_head *sh, sector_t sector, int previous) 309 { 310 raid5_conf_t *conf = sh->raid_conf; 311 int i; 312 313 BUG_ON(atomic_read(&sh->count) != 0); 314 BUG_ON(test_bit(STRIPE_HANDLE, &sh->state)); 315 BUG_ON(stripe_operations_active(sh)); 316 317 CHECK_DEVLOCK(); 318 pr_debug("init_stripe called, stripe %llu\n", 319 (unsigned long long)sh->sector); 320 321 remove_hash(sh); 322 323 sh->generation = conf->generation - previous; 324 sh->disks = previous ? conf->previous_raid_disks : conf->raid_disks; 325 sh->sector = sector; 326 stripe_set_idx(sector, conf, previous, sh); 327 sh->state = 0; 328 329 330 for (i = sh->disks; i--; ) { 331 struct r5dev *dev = &sh->dev[i]; 332 333 if (dev->toread || dev->read || dev->towrite || dev->written || 334 test_bit(R5_LOCKED, &dev->flags)) { 335 printk(KERN_ERR "sector=%llx i=%d %p %p %p %p %d\n", 336 (unsigned long long)sh->sector, i, dev->toread, 337 dev->read, dev->towrite, dev->written, 338 test_bit(R5_LOCKED, &dev->flags)); 339 BUG(); 340 } 341 dev->flags = 0; 342 raid5_build_block(sh, i, previous); 343 } 344 insert_hash(conf, sh); 345 } 346 347 static struct stripe_head *__find_stripe(raid5_conf_t *conf, sector_t sector, 348 short generation) 349 { 350 struct stripe_head *sh; 351 struct hlist_node *hn; 352 353 CHECK_DEVLOCK(); 354 pr_debug("__find_stripe, sector %llu\n", (unsigned long long)sector); 355 hlist_for_each_entry(sh, hn, stripe_hash(conf, sector), hash) 356 if (sh->sector == sector && sh->generation == generation) 357 return sh; 358 pr_debug("__stripe %llu not in cache\n", (unsigned long long)sector); 359 return NULL; 360 } 361 362 static void unplug_slaves(mddev_t *mddev); 363 static void raid5_unplug_device(struct request_queue *q); 364 365 static struct stripe_head * 366 get_active_stripe(raid5_conf_t *conf, sector_t sector, 367 int previous, int noblock, int noquiesce) 368 { 369 struct stripe_head *sh; 370 371 pr_debug("get_stripe, sector %llu\n", (unsigned long long)sector); 372 373 spin_lock_irq(&conf->device_lock); 374 375 do { 376 wait_event_lock_irq(conf->wait_for_stripe, 377 conf->quiesce == 0 || noquiesce, 378 conf->device_lock, /* nothing */); 379 sh = __find_stripe(conf, sector, conf->generation - previous); 380 if (!sh) { 381 if (!conf->inactive_blocked) 382 sh = get_free_stripe(conf); 383 if (noblock && sh == NULL) 384 break; 385 if (!sh) { 386 conf->inactive_blocked = 1; 387 wait_event_lock_irq(conf->wait_for_stripe, 388 !list_empty(&conf->inactive_list) && 389 (atomic_read(&conf->active_stripes) 390 < (conf->max_nr_stripes *3/4) 391 || !conf->inactive_blocked), 392 conf->device_lock, 393 raid5_unplug_device(conf->mddev->queue) 394 ); 395 conf->inactive_blocked = 0; 396 } else 397 init_stripe(sh, sector, previous); 398 } else { 399 if (atomic_read(&sh->count)) { 400 BUG_ON(!list_empty(&sh->lru) 401 && !test_bit(STRIPE_EXPANDING, &sh->state)); 402 } else { 403 if (!test_bit(STRIPE_HANDLE, &sh->state)) 404 atomic_inc(&conf->active_stripes); 405 if (list_empty(&sh->lru) && 406 !test_bit(STRIPE_EXPANDING, &sh->state)) 407 BUG(); 408 list_del_init(&sh->lru); 409 } 410 } 411 } while (sh == NULL); 412 413 if (sh) 414 atomic_inc(&sh->count); 415 416 spin_unlock_irq(&conf->device_lock); 417 return sh; 418 } 419 420 static void 421 raid5_end_read_request(struct bio *bi, int error); 422 static void 423 raid5_end_write_request(struct bio *bi, int error); 424 425 static void ops_run_io(struct stripe_head *sh, struct stripe_head_state *s) 426 { 427 raid5_conf_t *conf = sh->raid_conf; 428 int i, disks = sh->disks; 429 430 might_sleep(); 431 432 for (i = disks; i--; ) { 433 int rw; 434 struct bio *bi; 435 mdk_rdev_t *rdev; 436 if (test_and_clear_bit(R5_Wantwrite, &sh->dev[i].flags)) 437 rw = WRITE; 438 else if (test_and_clear_bit(R5_Wantread, &sh->dev[i].flags)) 439 rw = READ; 440 else 441 continue; 442 443 bi = &sh->dev[i].req; 444 445 bi->bi_rw = rw; 446 if (rw == WRITE) 447 bi->bi_end_io = raid5_end_write_request; 448 else 449 bi->bi_end_io = raid5_end_read_request; 450 451 rcu_read_lock(); 452 rdev = rcu_dereference(conf->disks[i].rdev); 453 if (rdev && test_bit(Faulty, &rdev->flags)) 454 rdev = NULL; 455 if (rdev) 456 atomic_inc(&rdev->nr_pending); 457 rcu_read_unlock(); 458 459 if (rdev) { 460 if (s->syncing || s->expanding || s->expanded) 461 md_sync_acct(rdev->bdev, STRIPE_SECTORS); 462 463 set_bit(STRIPE_IO_STARTED, &sh->state); 464 465 bi->bi_bdev = rdev->bdev; 466 pr_debug("%s: for %llu schedule op %ld on disc %d\n", 467 __func__, (unsigned long long)sh->sector, 468 bi->bi_rw, i); 469 atomic_inc(&sh->count); 470 bi->bi_sector = sh->sector + rdev->data_offset; 471 bi->bi_flags = 1 << BIO_UPTODATE; 472 bi->bi_vcnt = 1; 473 bi->bi_max_vecs = 1; 474 bi->bi_idx = 0; 475 bi->bi_io_vec = &sh->dev[i].vec; 476 bi->bi_io_vec[0].bv_len = STRIPE_SIZE; 477 bi->bi_io_vec[0].bv_offset = 0; 478 bi->bi_size = STRIPE_SIZE; 479 bi->bi_next = NULL; 480 if (rw == WRITE && 481 test_bit(R5_ReWrite, &sh->dev[i].flags)) 482 atomic_add(STRIPE_SECTORS, 483 &rdev->corrected_errors); 484 generic_make_request(bi); 485 } else { 486 if (rw == WRITE) 487 set_bit(STRIPE_DEGRADED, &sh->state); 488 pr_debug("skip op %ld on disc %d for sector %llu\n", 489 bi->bi_rw, i, (unsigned long long)sh->sector); 490 clear_bit(R5_LOCKED, &sh->dev[i].flags); 491 set_bit(STRIPE_HANDLE, &sh->state); 492 } 493 } 494 } 495 496 static struct dma_async_tx_descriptor * 497 async_copy_data(int frombio, struct bio *bio, struct page *page, 498 sector_t sector, struct dma_async_tx_descriptor *tx) 499 { 500 struct bio_vec *bvl; 501 struct page *bio_page; 502 int i; 503 int page_offset; 504 struct async_submit_ctl submit; 505 enum async_tx_flags flags = 0; 506 507 if (bio->bi_sector >= sector) 508 page_offset = (signed)(bio->bi_sector - sector) * 512; 509 else 510 page_offset = (signed)(sector - bio->bi_sector) * -512; 511 512 if (frombio) 513 flags |= ASYNC_TX_FENCE; 514 init_async_submit(&submit, flags, tx, NULL, NULL, NULL); 515 516 bio_for_each_segment(bvl, bio, i) { 517 int len = bio_iovec_idx(bio, i)->bv_len; 518 int clen; 519 int b_offset = 0; 520 521 if (page_offset < 0) { 522 b_offset = -page_offset; 523 page_offset += b_offset; 524 len -= b_offset; 525 } 526 527 if (len > 0 && page_offset + len > STRIPE_SIZE) 528 clen = STRIPE_SIZE - page_offset; 529 else 530 clen = len; 531 532 if (clen > 0) { 533 b_offset += bio_iovec_idx(bio, i)->bv_offset; 534 bio_page = bio_iovec_idx(bio, i)->bv_page; 535 if (frombio) 536 tx = async_memcpy(page, bio_page, page_offset, 537 b_offset, clen, &submit); 538 else 539 tx = async_memcpy(bio_page, page, b_offset, 540 page_offset, clen, &submit); 541 } 542 /* chain the operations */ 543 submit.depend_tx = tx; 544 545 if (clen < len) /* hit end of page */ 546 break; 547 page_offset += len; 548 } 549 550 return tx; 551 } 552 553 static void ops_complete_biofill(void *stripe_head_ref) 554 { 555 struct stripe_head *sh = stripe_head_ref; 556 struct bio *return_bi = NULL; 557 raid5_conf_t *conf = sh->raid_conf; 558 int i; 559 560 pr_debug("%s: stripe %llu\n", __func__, 561 (unsigned long long)sh->sector); 562 563 /* clear completed biofills */ 564 spin_lock_irq(&conf->device_lock); 565 for (i = sh->disks; i--; ) { 566 struct r5dev *dev = &sh->dev[i]; 567 568 /* acknowledge completion of a biofill operation */ 569 /* and check if we need to reply to a read request, 570 * new R5_Wantfill requests are held off until 571 * !STRIPE_BIOFILL_RUN 572 */ 573 if (test_and_clear_bit(R5_Wantfill, &dev->flags)) { 574 struct bio *rbi, *rbi2; 575 576 BUG_ON(!dev->read); 577 rbi = dev->read; 578 dev->read = NULL; 579 while (rbi && rbi->bi_sector < 580 dev->sector + STRIPE_SECTORS) { 581 rbi2 = r5_next_bio(rbi, dev->sector); 582 if (!raid5_dec_bi_phys_segments(rbi)) { 583 rbi->bi_next = return_bi; 584 return_bi = rbi; 585 } 586 rbi = rbi2; 587 } 588 } 589 } 590 spin_unlock_irq(&conf->device_lock); 591 clear_bit(STRIPE_BIOFILL_RUN, &sh->state); 592 593 return_io(return_bi); 594 595 set_bit(STRIPE_HANDLE, &sh->state); 596 release_stripe(sh); 597 } 598 599 static void ops_run_biofill(struct stripe_head *sh) 600 { 601 struct dma_async_tx_descriptor *tx = NULL; 602 raid5_conf_t *conf = sh->raid_conf; 603 struct async_submit_ctl submit; 604 int i; 605 606 pr_debug("%s: stripe %llu\n", __func__, 607 (unsigned long long)sh->sector); 608 609 for (i = sh->disks; i--; ) { 610 struct r5dev *dev = &sh->dev[i]; 611 if (test_bit(R5_Wantfill, &dev->flags)) { 612 struct bio *rbi; 613 spin_lock_irq(&conf->device_lock); 614 dev->read = rbi = dev->toread; 615 dev->toread = NULL; 616 spin_unlock_irq(&conf->device_lock); 617 while (rbi && rbi->bi_sector < 618 dev->sector + STRIPE_SECTORS) { 619 tx = async_copy_data(0, rbi, dev->page, 620 dev->sector, tx); 621 rbi = r5_next_bio(rbi, dev->sector); 622 } 623 } 624 } 625 626 atomic_inc(&sh->count); 627 init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_biofill, sh, NULL); 628 async_trigger_callback(&submit); 629 } 630 631 static void mark_target_uptodate(struct stripe_head *sh, int target) 632 { 633 struct r5dev *tgt; 634 635 if (target < 0) 636 return; 637 638 tgt = &sh->dev[target]; 639 set_bit(R5_UPTODATE, &tgt->flags); 640 BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags)); 641 clear_bit(R5_Wantcompute, &tgt->flags); 642 } 643 644 static void ops_complete_compute(void *stripe_head_ref) 645 { 646 struct stripe_head *sh = stripe_head_ref; 647 648 pr_debug("%s: stripe %llu\n", __func__, 649 (unsigned long long)sh->sector); 650 651 /* mark the computed target(s) as uptodate */ 652 mark_target_uptodate(sh, sh->ops.target); 653 mark_target_uptodate(sh, sh->ops.target2); 654 655 clear_bit(STRIPE_COMPUTE_RUN, &sh->state); 656 if (sh->check_state == check_state_compute_run) 657 sh->check_state = check_state_compute_result; 658 set_bit(STRIPE_HANDLE, &sh->state); 659 release_stripe(sh); 660 } 661 662 /* return a pointer to the address conversion region of the scribble buffer */ 663 static addr_conv_t *to_addr_conv(struct stripe_head *sh, 664 struct raid5_percpu *percpu) 665 { 666 return percpu->scribble + sizeof(struct page *) * (sh->disks + 2); 667 } 668 669 static struct dma_async_tx_descriptor * 670 ops_run_compute5(struct stripe_head *sh, struct raid5_percpu *percpu) 671 { 672 int disks = sh->disks; 673 struct page **xor_srcs = percpu->scribble; 674 int target = sh->ops.target; 675 struct r5dev *tgt = &sh->dev[target]; 676 struct page *xor_dest = tgt->page; 677 int count = 0; 678 struct dma_async_tx_descriptor *tx; 679 struct async_submit_ctl submit; 680 int i; 681 682 pr_debug("%s: stripe %llu block: %d\n", 683 __func__, (unsigned long long)sh->sector, target); 684 BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags)); 685 686 for (i = disks; i--; ) 687 if (i != target) 688 xor_srcs[count++] = sh->dev[i].page; 689 690 atomic_inc(&sh->count); 691 692 init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, NULL, 693 ops_complete_compute, sh, to_addr_conv(sh, percpu)); 694 if (unlikely(count == 1)) 695 tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit); 696 else 697 tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit); 698 699 return tx; 700 } 701 702 /* set_syndrome_sources - populate source buffers for gen_syndrome 703 * @srcs - (struct page *) array of size sh->disks 704 * @sh - stripe_head to parse 705 * 706 * Populates srcs in proper layout order for the stripe and returns the 707 * 'count' of sources to be used in a call to async_gen_syndrome. The P 708 * destination buffer is recorded in srcs[count] and the Q destination 709 * is recorded in srcs[count+1]]. 710 */ 711 static int set_syndrome_sources(struct page **srcs, struct stripe_head *sh) 712 { 713 int disks = sh->disks; 714 int syndrome_disks = sh->ddf_layout ? disks : (disks - 2); 715 int d0_idx = raid6_d0(sh); 716 int count; 717 int i; 718 719 for (i = 0; i < disks; i++) 720 srcs[i] = (void *)raid6_empty_zero_page; 721 722 count = 0; 723 i = d0_idx; 724 do { 725 int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks); 726 727 srcs[slot] = sh->dev[i].page; 728 i = raid6_next_disk(i, disks); 729 } while (i != d0_idx); 730 BUG_ON(count != syndrome_disks); 731 732 return count; 733 } 734 735 static struct dma_async_tx_descriptor * 736 ops_run_compute6_1(struct stripe_head *sh, struct raid5_percpu *percpu) 737 { 738 int disks = sh->disks; 739 struct page **blocks = percpu->scribble; 740 int target; 741 int qd_idx = sh->qd_idx; 742 struct dma_async_tx_descriptor *tx; 743 struct async_submit_ctl submit; 744 struct r5dev *tgt; 745 struct page *dest; 746 int i; 747 int count; 748 749 if (sh->ops.target < 0) 750 target = sh->ops.target2; 751 else if (sh->ops.target2 < 0) 752 target = sh->ops.target; 753 else 754 /* we should only have one valid target */ 755 BUG(); 756 BUG_ON(target < 0); 757 pr_debug("%s: stripe %llu block: %d\n", 758 __func__, (unsigned long long)sh->sector, target); 759 760 tgt = &sh->dev[target]; 761 BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags)); 762 dest = tgt->page; 763 764 atomic_inc(&sh->count); 765 766 if (target == qd_idx) { 767 count = set_syndrome_sources(blocks, sh); 768 blocks[count] = NULL; /* regenerating p is not necessary */ 769 BUG_ON(blocks[count+1] != dest); /* q should already be set */ 770 init_async_submit(&submit, ASYNC_TX_FENCE, NULL, 771 ops_complete_compute, sh, 772 to_addr_conv(sh, percpu)); 773 tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit); 774 } else { 775 /* Compute any data- or p-drive using XOR */ 776 count = 0; 777 for (i = disks; i-- ; ) { 778 if (i == target || i == qd_idx) 779 continue; 780 blocks[count++] = sh->dev[i].page; 781 } 782 783 init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, 784 NULL, ops_complete_compute, sh, 785 to_addr_conv(sh, percpu)); 786 tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE, &submit); 787 } 788 789 return tx; 790 } 791 792 static struct dma_async_tx_descriptor * 793 ops_run_compute6_2(struct stripe_head *sh, struct raid5_percpu *percpu) 794 { 795 int i, count, disks = sh->disks; 796 int syndrome_disks = sh->ddf_layout ? disks : disks-2; 797 int d0_idx = raid6_d0(sh); 798 int faila = -1, failb = -1; 799 int target = sh->ops.target; 800 int target2 = sh->ops.target2; 801 struct r5dev *tgt = &sh->dev[target]; 802 struct r5dev *tgt2 = &sh->dev[target2]; 803 struct dma_async_tx_descriptor *tx; 804 struct page **blocks = percpu->scribble; 805 struct async_submit_ctl submit; 806 807 pr_debug("%s: stripe %llu block1: %d block2: %d\n", 808 __func__, (unsigned long long)sh->sector, target, target2); 809 BUG_ON(target < 0 || target2 < 0); 810 BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags)); 811 BUG_ON(!test_bit(R5_Wantcompute, &tgt2->flags)); 812 813 /* we need to open-code set_syndrome_sources to handle to the 814 * slot number conversion for 'faila' and 'failb' 815 */ 816 for (i = 0; i < disks ; i++) 817 blocks[i] = (void *)raid6_empty_zero_page; 818 count = 0; 819 i = d0_idx; 820 do { 821 int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks); 822 823 blocks[slot] = sh->dev[i].page; 824 825 if (i == target) 826 faila = slot; 827 if (i == target2) 828 failb = slot; 829 i = raid6_next_disk(i, disks); 830 } while (i != d0_idx); 831 BUG_ON(count != syndrome_disks); 832 833 BUG_ON(faila == failb); 834 if (failb < faila) 835 swap(faila, failb); 836 pr_debug("%s: stripe: %llu faila: %d failb: %d\n", 837 __func__, (unsigned long long)sh->sector, faila, failb); 838 839 atomic_inc(&sh->count); 840 841 if (failb == syndrome_disks+1) { 842 /* Q disk is one of the missing disks */ 843 if (faila == syndrome_disks) { 844 /* Missing P+Q, just recompute */ 845 init_async_submit(&submit, ASYNC_TX_FENCE, NULL, 846 ops_complete_compute, sh, 847 to_addr_conv(sh, percpu)); 848 return async_gen_syndrome(blocks, 0, count+2, 849 STRIPE_SIZE, &submit); 850 } else { 851 struct page *dest; 852 int data_target; 853 int qd_idx = sh->qd_idx; 854 855 /* Missing D+Q: recompute D from P, then recompute Q */ 856 if (target == qd_idx) 857 data_target = target2; 858 else 859 data_target = target; 860 861 count = 0; 862 for (i = disks; i-- ; ) { 863 if (i == data_target || i == qd_idx) 864 continue; 865 blocks[count++] = sh->dev[i].page; 866 } 867 dest = sh->dev[data_target].page; 868 init_async_submit(&submit, 869 ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, 870 NULL, NULL, NULL, 871 to_addr_conv(sh, percpu)); 872 tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE, 873 &submit); 874 875 count = set_syndrome_sources(blocks, sh); 876 init_async_submit(&submit, ASYNC_TX_FENCE, tx, 877 ops_complete_compute, sh, 878 to_addr_conv(sh, percpu)); 879 return async_gen_syndrome(blocks, 0, count+2, 880 STRIPE_SIZE, &submit); 881 } 882 } 883 884 init_async_submit(&submit, ASYNC_TX_FENCE, NULL, ops_complete_compute, 885 sh, to_addr_conv(sh, percpu)); 886 if (failb == syndrome_disks) { 887 /* We're missing D+P. */ 888 return async_raid6_datap_recov(syndrome_disks+2, STRIPE_SIZE, 889 faila, blocks, &submit); 890 } else { 891 /* We're missing D+D. */ 892 return async_raid6_2data_recov(syndrome_disks+2, STRIPE_SIZE, 893 faila, failb, blocks, &submit); 894 } 895 } 896 897 898 static void ops_complete_prexor(void *stripe_head_ref) 899 { 900 struct stripe_head *sh = stripe_head_ref; 901 902 pr_debug("%s: stripe %llu\n", __func__, 903 (unsigned long long)sh->sector); 904 } 905 906 static struct dma_async_tx_descriptor * 907 ops_run_prexor(struct stripe_head *sh, struct raid5_percpu *percpu, 908 struct dma_async_tx_descriptor *tx) 909 { 910 int disks = sh->disks; 911 struct page **xor_srcs = percpu->scribble; 912 int count = 0, pd_idx = sh->pd_idx, i; 913 struct async_submit_ctl submit; 914 915 /* existing parity data subtracted */ 916 struct page *xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page; 917 918 pr_debug("%s: stripe %llu\n", __func__, 919 (unsigned long long)sh->sector); 920 921 for (i = disks; i--; ) { 922 struct r5dev *dev = &sh->dev[i]; 923 /* Only process blocks that are known to be uptodate */ 924 if (test_bit(R5_Wantdrain, &dev->flags)) 925 xor_srcs[count++] = dev->page; 926 } 927 928 init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx, 929 ops_complete_prexor, sh, to_addr_conv(sh, percpu)); 930 tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit); 931 932 return tx; 933 } 934 935 static struct dma_async_tx_descriptor * 936 ops_run_biodrain(struct stripe_head *sh, struct dma_async_tx_descriptor *tx) 937 { 938 int disks = sh->disks; 939 int i; 940 941 pr_debug("%s: stripe %llu\n", __func__, 942 (unsigned long long)sh->sector); 943 944 for (i = disks; i--; ) { 945 struct r5dev *dev = &sh->dev[i]; 946 struct bio *chosen; 947 948 if (test_and_clear_bit(R5_Wantdrain, &dev->flags)) { 949 struct bio *wbi; 950 951 spin_lock(&sh->lock); 952 chosen = dev->towrite; 953 dev->towrite = NULL; 954 BUG_ON(dev->written); 955 wbi = dev->written = chosen; 956 spin_unlock(&sh->lock); 957 958 while (wbi && wbi->bi_sector < 959 dev->sector + STRIPE_SECTORS) { 960 tx = async_copy_data(1, wbi, dev->page, 961 dev->sector, tx); 962 wbi = r5_next_bio(wbi, dev->sector); 963 } 964 } 965 } 966 967 return tx; 968 } 969 970 static void ops_complete_reconstruct(void *stripe_head_ref) 971 { 972 struct stripe_head *sh = stripe_head_ref; 973 int disks = sh->disks; 974 int pd_idx = sh->pd_idx; 975 int qd_idx = sh->qd_idx; 976 int i; 977 978 pr_debug("%s: stripe %llu\n", __func__, 979 (unsigned long long)sh->sector); 980 981 for (i = disks; i--; ) { 982 struct r5dev *dev = &sh->dev[i]; 983 984 if (dev->written || i == pd_idx || i == qd_idx) 985 set_bit(R5_UPTODATE, &dev->flags); 986 } 987 988 if (sh->reconstruct_state == reconstruct_state_drain_run) 989 sh->reconstruct_state = reconstruct_state_drain_result; 990 else if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) 991 sh->reconstruct_state = reconstruct_state_prexor_drain_result; 992 else { 993 BUG_ON(sh->reconstruct_state != reconstruct_state_run); 994 sh->reconstruct_state = reconstruct_state_result; 995 } 996 997 set_bit(STRIPE_HANDLE, &sh->state); 998 release_stripe(sh); 999 } 1000 1001 static void 1002 ops_run_reconstruct5(struct stripe_head *sh, struct raid5_percpu *percpu, 1003 struct dma_async_tx_descriptor *tx) 1004 { 1005 int disks = sh->disks; 1006 struct page **xor_srcs = percpu->scribble; 1007 struct async_submit_ctl submit; 1008 int count = 0, pd_idx = sh->pd_idx, i; 1009 struct page *xor_dest; 1010 int prexor = 0; 1011 unsigned long flags; 1012 1013 pr_debug("%s: stripe %llu\n", __func__, 1014 (unsigned long long)sh->sector); 1015 1016 /* check if prexor is active which means only process blocks 1017 * that are part of a read-modify-write (written) 1018 */ 1019 if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) { 1020 prexor = 1; 1021 xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page; 1022 for (i = disks; i--; ) { 1023 struct r5dev *dev = &sh->dev[i]; 1024 if (dev->written) 1025 xor_srcs[count++] = dev->page; 1026 } 1027 } else { 1028 xor_dest = sh->dev[pd_idx].page; 1029 for (i = disks; i--; ) { 1030 struct r5dev *dev = &sh->dev[i]; 1031 if (i != pd_idx) 1032 xor_srcs[count++] = dev->page; 1033 } 1034 } 1035 1036 /* 1/ if we prexor'd then the dest is reused as a source 1037 * 2/ if we did not prexor then we are redoing the parity 1038 * set ASYNC_TX_XOR_DROP_DST and ASYNC_TX_XOR_ZERO_DST 1039 * for the synchronous xor case 1040 */ 1041 flags = ASYNC_TX_ACK | 1042 (prexor ? ASYNC_TX_XOR_DROP_DST : ASYNC_TX_XOR_ZERO_DST); 1043 1044 atomic_inc(&sh->count); 1045 1046 init_async_submit(&submit, flags, tx, ops_complete_reconstruct, sh, 1047 to_addr_conv(sh, percpu)); 1048 if (unlikely(count == 1)) 1049 tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit); 1050 else 1051 tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit); 1052 } 1053 1054 static void 1055 ops_run_reconstruct6(struct stripe_head *sh, struct raid5_percpu *percpu, 1056 struct dma_async_tx_descriptor *tx) 1057 { 1058 struct async_submit_ctl submit; 1059 struct page **blocks = percpu->scribble; 1060 int count; 1061 1062 pr_debug("%s: stripe %llu\n", __func__, (unsigned long long)sh->sector); 1063 1064 count = set_syndrome_sources(blocks, sh); 1065 1066 atomic_inc(&sh->count); 1067 1068 init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_reconstruct, 1069 sh, to_addr_conv(sh, percpu)); 1070 async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit); 1071 } 1072 1073 static void ops_complete_check(void *stripe_head_ref) 1074 { 1075 struct stripe_head *sh = stripe_head_ref; 1076 1077 pr_debug("%s: stripe %llu\n", __func__, 1078 (unsigned long long)sh->sector); 1079 1080 sh->check_state = check_state_check_result; 1081 set_bit(STRIPE_HANDLE, &sh->state); 1082 release_stripe(sh); 1083 } 1084 1085 static void ops_run_check_p(struct stripe_head *sh, struct raid5_percpu *percpu) 1086 { 1087 int disks = sh->disks; 1088 int pd_idx = sh->pd_idx; 1089 int qd_idx = sh->qd_idx; 1090 struct page *xor_dest; 1091 struct page **xor_srcs = percpu->scribble; 1092 struct dma_async_tx_descriptor *tx; 1093 struct async_submit_ctl submit; 1094 int count; 1095 int i; 1096 1097 pr_debug("%s: stripe %llu\n", __func__, 1098 (unsigned long long)sh->sector); 1099 1100 count = 0; 1101 xor_dest = sh->dev[pd_idx].page; 1102 xor_srcs[count++] = xor_dest; 1103 for (i = disks; i--; ) { 1104 if (i == pd_idx || i == qd_idx) 1105 continue; 1106 xor_srcs[count++] = sh->dev[i].page; 1107 } 1108 1109 init_async_submit(&submit, 0, NULL, NULL, NULL, 1110 to_addr_conv(sh, percpu)); 1111 tx = async_xor_val(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, 1112 &sh->ops.zero_sum_result, &submit); 1113 1114 atomic_inc(&sh->count); 1115 init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_check, sh, NULL); 1116 tx = async_trigger_callback(&submit); 1117 } 1118 1119 static void ops_run_check_pq(struct stripe_head *sh, struct raid5_percpu *percpu, int checkp) 1120 { 1121 struct page **srcs = percpu->scribble; 1122 struct async_submit_ctl submit; 1123 int count; 1124 1125 pr_debug("%s: stripe %llu checkp: %d\n", __func__, 1126 (unsigned long long)sh->sector, checkp); 1127 1128 count = set_syndrome_sources(srcs, sh); 1129 if (!checkp) 1130 srcs[count] = NULL; 1131 1132 atomic_inc(&sh->count); 1133 init_async_submit(&submit, ASYNC_TX_ACK, NULL, ops_complete_check, 1134 sh, to_addr_conv(sh, percpu)); 1135 async_syndrome_val(srcs, 0, count+2, STRIPE_SIZE, 1136 &sh->ops.zero_sum_result, percpu->spare_page, &submit); 1137 } 1138 1139 static void raid_run_ops(struct stripe_head *sh, unsigned long ops_request) 1140 { 1141 int overlap_clear = 0, i, disks = sh->disks; 1142 struct dma_async_tx_descriptor *tx = NULL; 1143 raid5_conf_t *conf = sh->raid_conf; 1144 int level = conf->level; 1145 struct raid5_percpu *percpu; 1146 unsigned long cpu; 1147 1148 cpu = get_cpu(); 1149 percpu = per_cpu_ptr(conf->percpu, cpu); 1150 if (test_bit(STRIPE_OP_BIOFILL, &ops_request)) { 1151 ops_run_biofill(sh); 1152 overlap_clear++; 1153 } 1154 1155 if (test_bit(STRIPE_OP_COMPUTE_BLK, &ops_request)) { 1156 if (level < 6) 1157 tx = ops_run_compute5(sh, percpu); 1158 else { 1159 if (sh->ops.target2 < 0 || sh->ops.target < 0) 1160 tx = ops_run_compute6_1(sh, percpu); 1161 else 1162 tx = ops_run_compute6_2(sh, percpu); 1163 } 1164 /* terminate the chain if reconstruct is not set to be run */ 1165 if (tx && !test_bit(STRIPE_OP_RECONSTRUCT, &ops_request)) 1166 async_tx_ack(tx); 1167 } 1168 1169 if (test_bit(STRIPE_OP_PREXOR, &ops_request)) 1170 tx = ops_run_prexor(sh, percpu, tx); 1171 1172 if (test_bit(STRIPE_OP_BIODRAIN, &ops_request)) { 1173 tx = ops_run_biodrain(sh, tx); 1174 overlap_clear++; 1175 } 1176 1177 if (test_bit(STRIPE_OP_RECONSTRUCT, &ops_request)) { 1178 if (level < 6) 1179 ops_run_reconstruct5(sh, percpu, tx); 1180 else 1181 ops_run_reconstruct6(sh, percpu, tx); 1182 } 1183 1184 if (test_bit(STRIPE_OP_CHECK, &ops_request)) { 1185 if (sh->check_state == check_state_run) 1186 ops_run_check_p(sh, percpu); 1187 else if (sh->check_state == check_state_run_q) 1188 ops_run_check_pq(sh, percpu, 0); 1189 else if (sh->check_state == check_state_run_pq) 1190 ops_run_check_pq(sh, percpu, 1); 1191 else 1192 BUG(); 1193 } 1194 1195 if (overlap_clear) 1196 for (i = disks; i--; ) { 1197 struct r5dev *dev = &sh->dev[i]; 1198 if (test_and_clear_bit(R5_Overlap, &dev->flags)) 1199 wake_up(&sh->raid_conf->wait_for_overlap); 1200 } 1201 put_cpu(); 1202 } 1203 1204 static int grow_one_stripe(raid5_conf_t *conf) 1205 { 1206 struct stripe_head *sh; 1207 sh = kmem_cache_alloc(conf->slab_cache, GFP_KERNEL); 1208 if (!sh) 1209 return 0; 1210 memset(sh, 0, sizeof(*sh) + (conf->raid_disks-1)*sizeof(struct r5dev)); 1211 sh->raid_conf = conf; 1212 spin_lock_init(&sh->lock); 1213 1214 if (grow_buffers(sh, conf->raid_disks)) { 1215 shrink_buffers(sh, conf->raid_disks); 1216 kmem_cache_free(conf->slab_cache, sh); 1217 return 0; 1218 } 1219 sh->disks = conf->raid_disks; 1220 /* we just created an active stripe so... */ 1221 atomic_set(&sh->count, 1); 1222 atomic_inc(&conf->active_stripes); 1223 INIT_LIST_HEAD(&sh->lru); 1224 release_stripe(sh); 1225 return 1; 1226 } 1227 1228 static int grow_stripes(raid5_conf_t *conf, int num) 1229 { 1230 struct kmem_cache *sc; 1231 int devs = conf->raid_disks; 1232 1233 sprintf(conf->cache_name[0], 1234 "raid%d-%s", conf->level, mdname(conf->mddev)); 1235 sprintf(conf->cache_name[1], 1236 "raid%d-%s-alt", conf->level, mdname(conf->mddev)); 1237 conf->active_name = 0; 1238 sc = kmem_cache_create(conf->cache_name[conf->active_name], 1239 sizeof(struct stripe_head)+(devs-1)*sizeof(struct r5dev), 1240 0, 0, NULL); 1241 if (!sc) 1242 return 1; 1243 conf->slab_cache = sc; 1244 conf->pool_size = devs; 1245 while (num--) 1246 if (!grow_one_stripe(conf)) 1247 return 1; 1248 return 0; 1249 } 1250 1251 /** 1252 * scribble_len - return the required size of the scribble region 1253 * @num - total number of disks in the array 1254 * 1255 * The size must be enough to contain: 1256 * 1/ a struct page pointer for each device in the array +2 1257 * 2/ room to convert each entry in (1) to its corresponding dma 1258 * (dma_map_page()) or page (page_address()) address. 1259 * 1260 * Note: the +2 is for the destination buffers of the ddf/raid6 case where we 1261 * calculate over all devices (not just the data blocks), using zeros in place 1262 * of the P and Q blocks. 1263 */ 1264 static size_t scribble_len(int num) 1265 { 1266 size_t len; 1267 1268 len = sizeof(struct page *) * (num+2) + sizeof(addr_conv_t) * (num+2); 1269 1270 return len; 1271 } 1272 1273 static int resize_stripes(raid5_conf_t *conf, int newsize) 1274 { 1275 /* Make all the stripes able to hold 'newsize' devices. 1276 * New slots in each stripe get 'page' set to a new page. 1277 * 1278 * This happens in stages: 1279 * 1/ create a new kmem_cache and allocate the required number of 1280 * stripe_heads. 1281 * 2/ gather all the old stripe_heads and tranfer the pages across 1282 * to the new stripe_heads. This will have the side effect of 1283 * freezing the array as once all stripe_heads have been collected, 1284 * no IO will be possible. Old stripe heads are freed once their 1285 * pages have been transferred over, and the old kmem_cache is 1286 * freed when all stripes are done. 1287 * 3/ reallocate conf->disks to be suitable bigger. If this fails, 1288 * we simple return a failre status - no need to clean anything up. 1289 * 4/ allocate new pages for the new slots in the new stripe_heads. 1290 * If this fails, we don't bother trying the shrink the 1291 * stripe_heads down again, we just leave them as they are. 1292 * As each stripe_head is processed the new one is released into 1293 * active service. 1294 * 1295 * Once step2 is started, we cannot afford to wait for a write, 1296 * so we use GFP_NOIO allocations. 1297 */ 1298 struct stripe_head *osh, *nsh; 1299 LIST_HEAD(newstripes); 1300 struct disk_info *ndisks; 1301 unsigned long cpu; 1302 int err; 1303 struct kmem_cache *sc; 1304 int i; 1305 1306 if (newsize <= conf->pool_size) 1307 return 0; /* never bother to shrink */ 1308 1309 err = md_allow_write(conf->mddev); 1310 if (err) 1311 return err; 1312 1313 /* Step 1 */ 1314 sc = kmem_cache_create(conf->cache_name[1-conf->active_name], 1315 sizeof(struct stripe_head)+(newsize-1)*sizeof(struct r5dev), 1316 0, 0, NULL); 1317 if (!sc) 1318 return -ENOMEM; 1319 1320 for (i = conf->max_nr_stripes; i; i--) { 1321 nsh = kmem_cache_alloc(sc, GFP_KERNEL); 1322 if (!nsh) 1323 break; 1324 1325 memset(nsh, 0, sizeof(*nsh) + (newsize-1)*sizeof(struct r5dev)); 1326 1327 nsh->raid_conf = conf; 1328 spin_lock_init(&nsh->lock); 1329 1330 list_add(&nsh->lru, &newstripes); 1331 } 1332 if (i) { 1333 /* didn't get enough, give up */ 1334 while (!list_empty(&newstripes)) { 1335 nsh = list_entry(newstripes.next, struct stripe_head, lru); 1336 list_del(&nsh->lru); 1337 kmem_cache_free(sc, nsh); 1338 } 1339 kmem_cache_destroy(sc); 1340 return -ENOMEM; 1341 } 1342 /* Step 2 - Must use GFP_NOIO now. 1343 * OK, we have enough stripes, start collecting inactive 1344 * stripes and copying them over 1345 */ 1346 list_for_each_entry(nsh, &newstripes, lru) { 1347 spin_lock_irq(&conf->device_lock); 1348 wait_event_lock_irq(conf->wait_for_stripe, 1349 !list_empty(&conf->inactive_list), 1350 conf->device_lock, 1351 unplug_slaves(conf->mddev) 1352 ); 1353 osh = get_free_stripe(conf); 1354 spin_unlock_irq(&conf->device_lock); 1355 atomic_set(&nsh->count, 1); 1356 for(i=0; i<conf->pool_size; i++) 1357 nsh->dev[i].page = osh->dev[i].page; 1358 for( ; i<newsize; i++) 1359 nsh->dev[i].page = NULL; 1360 kmem_cache_free(conf->slab_cache, osh); 1361 } 1362 kmem_cache_destroy(conf->slab_cache); 1363 1364 /* Step 3. 1365 * At this point, we are holding all the stripes so the array 1366 * is completely stalled, so now is a good time to resize 1367 * conf->disks and the scribble region 1368 */ 1369 ndisks = kzalloc(newsize * sizeof(struct disk_info), GFP_NOIO); 1370 if (ndisks) { 1371 for (i=0; i<conf->raid_disks; i++) 1372 ndisks[i] = conf->disks[i]; 1373 kfree(conf->disks); 1374 conf->disks = ndisks; 1375 } else 1376 err = -ENOMEM; 1377 1378 get_online_cpus(); 1379 conf->scribble_len = scribble_len(newsize); 1380 for_each_present_cpu(cpu) { 1381 struct raid5_percpu *percpu; 1382 void *scribble; 1383 1384 percpu = per_cpu_ptr(conf->percpu, cpu); 1385 scribble = kmalloc(conf->scribble_len, GFP_NOIO); 1386 1387 if (scribble) { 1388 kfree(percpu->scribble); 1389 percpu->scribble = scribble; 1390 } else { 1391 err = -ENOMEM; 1392 break; 1393 } 1394 } 1395 put_online_cpus(); 1396 1397 /* Step 4, return new stripes to service */ 1398 while(!list_empty(&newstripes)) { 1399 nsh = list_entry(newstripes.next, struct stripe_head, lru); 1400 list_del_init(&nsh->lru); 1401 1402 for (i=conf->raid_disks; i < newsize; i++) 1403 if (nsh->dev[i].page == NULL) { 1404 struct page *p = alloc_page(GFP_NOIO); 1405 nsh->dev[i].page = p; 1406 if (!p) 1407 err = -ENOMEM; 1408 } 1409 release_stripe(nsh); 1410 } 1411 /* critical section pass, GFP_NOIO no longer needed */ 1412 1413 conf->slab_cache = sc; 1414 conf->active_name = 1-conf->active_name; 1415 conf->pool_size = newsize; 1416 return err; 1417 } 1418 1419 static int drop_one_stripe(raid5_conf_t *conf) 1420 { 1421 struct stripe_head *sh; 1422 1423 spin_lock_irq(&conf->device_lock); 1424 sh = get_free_stripe(conf); 1425 spin_unlock_irq(&conf->device_lock); 1426 if (!sh) 1427 return 0; 1428 BUG_ON(atomic_read(&sh->count)); 1429 shrink_buffers(sh, conf->pool_size); 1430 kmem_cache_free(conf->slab_cache, sh); 1431 atomic_dec(&conf->active_stripes); 1432 return 1; 1433 } 1434 1435 static void shrink_stripes(raid5_conf_t *conf) 1436 { 1437 while (drop_one_stripe(conf)) 1438 ; 1439 1440 if (conf->slab_cache) 1441 kmem_cache_destroy(conf->slab_cache); 1442 conf->slab_cache = NULL; 1443 } 1444 1445 static void raid5_end_read_request(struct bio * bi, int error) 1446 { 1447 struct stripe_head *sh = bi->bi_private; 1448 raid5_conf_t *conf = sh->raid_conf; 1449 int disks = sh->disks, i; 1450 int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags); 1451 char b[BDEVNAME_SIZE]; 1452 mdk_rdev_t *rdev; 1453 1454 1455 for (i=0 ; i<disks; i++) 1456 if (bi == &sh->dev[i].req) 1457 break; 1458 1459 pr_debug("end_read_request %llu/%d, count: %d, uptodate %d.\n", 1460 (unsigned long long)sh->sector, i, atomic_read(&sh->count), 1461 uptodate); 1462 if (i == disks) { 1463 BUG(); 1464 return; 1465 } 1466 1467 if (uptodate) { 1468 set_bit(R5_UPTODATE, &sh->dev[i].flags); 1469 if (test_bit(R5_ReadError, &sh->dev[i].flags)) { 1470 rdev = conf->disks[i].rdev; 1471 printk_rl(KERN_INFO "raid5:%s: read error corrected" 1472 " (%lu sectors at %llu on %s)\n", 1473 mdname(conf->mddev), STRIPE_SECTORS, 1474 (unsigned long long)(sh->sector 1475 + rdev->data_offset), 1476 bdevname(rdev->bdev, b)); 1477 clear_bit(R5_ReadError, &sh->dev[i].flags); 1478 clear_bit(R5_ReWrite, &sh->dev[i].flags); 1479 } 1480 if (atomic_read(&conf->disks[i].rdev->read_errors)) 1481 atomic_set(&conf->disks[i].rdev->read_errors, 0); 1482 } else { 1483 const char *bdn = bdevname(conf->disks[i].rdev->bdev, b); 1484 int retry = 0; 1485 rdev = conf->disks[i].rdev; 1486 1487 clear_bit(R5_UPTODATE, &sh->dev[i].flags); 1488 atomic_inc(&rdev->read_errors); 1489 if (conf->mddev->degraded) 1490 printk_rl(KERN_WARNING 1491 "raid5:%s: read error not correctable " 1492 "(sector %llu on %s).\n", 1493 mdname(conf->mddev), 1494 (unsigned long long)(sh->sector 1495 + rdev->data_offset), 1496 bdn); 1497 else if (test_bit(R5_ReWrite, &sh->dev[i].flags)) 1498 /* Oh, no!!! */ 1499 printk_rl(KERN_WARNING 1500 "raid5:%s: read error NOT corrected!! " 1501 "(sector %llu on %s).\n", 1502 mdname(conf->mddev), 1503 (unsigned long long)(sh->sector 1504 + rdev->data_offset), 1505 bdn); 1506 else if (atomic_read(&rdev->read_errors) 1507 > conf->max_nr_stripes) 1508 printk(KERN_WARNING 1509 "raid5:%s: Too many read errors, failing device %s.\n", 1510 mdname(conf->mddev), bdn); 1511 else 1512 retry = 1; 1513 if (retry) 1514 set_bit(R5_ReadError, &sh->dev[i].flags); 1515 else { 1516 clear_bit(R5_ReadError, &sh->dev[i].flags); 1517 clear_bit(R5_ReWrite, &sh->dev[i].flags); 1518 md_error(conf->mddev, rdev); 1519 } 1520 } 1521 rdev_dec_pending(conf->disks[i].rdev, conf->mddev); 1522 clear_bit(R5_LOCKED, &sh->dev[i].flags); 1523 set_bit(STRIPE_HANDLE, &sh->state); 1524 release_stripe(sh); 1525 } 1526 1527 static void raid5_end_write_request(struct bio *bi, int error) 1528 { 1529 struct stripe_head *sh = bi->bi_private; 1530 raid5_conf_t *conf = sh->raid_conf; 1531 int disks = sh->disks, i; 1532 int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags); 1533 1534 for (i=0 ; i<disks; i++) 1535 if (bi == &sh->dev[i].req) 1536 break; 1537 1538 pr_debug("end_write_request %llu/%d, count %d, uptodate: %d.\n", 1539 (unsigned long long)sh->sector, i, atomic_read(&sh->count), 1540 uptodate); 1541 if (i == disks) { 1542 BUG(); 1543 return; 1544 } 1545 1546 if (!uptodate) 1547 md_error(conf->mddev, conf->disks[i].rdev); 1548 1549 rdev_dec_pending(conf->disks[i].rdev, conf->mddev); 1550 1551 clear_bit(R5_LOCKED, &sh->dev[i].flags); 1552 set_bit(STRIPE_HANDLE, &sh->state); 1553 release_stripe(sh); 1554 } 1555 1556 1557 static sector_t compute_blocknr(struct stripe_head *sh, int i, int previous); 1558 1559 static void raid5_build_block(struct stripe_head *sh, int i, int previous) 1560 { 1561 struct r5dev *dev = &sh->dev[i]; 1562 1563 bio_init(&dev->req); 1564 dev->req.bi_io_vec = &dev->vec; 1565 dev->req.bi_vcnt++; 1566 dev->req.bi_max_vecs++; 1567 dev->vec.bv_page = dev->page; 1568 dev->vec.bv_len = STRIPE_SIZE; 1569 dev->vec.bv_offset = 0; 1570 1571 dev->req.bi_sector = sh->sector; 1572 dev->req.bi_private = sh; 1573 1574 dev->flags = 0; 1575 dev->sector = compute_blocknr(sh, i, previous); 1576 } 1577 1578 static void error(mddev_t *mddev, mdk_rdev_t *rdev) 1579 { 1580 char b[BDEVNAME_SIZE]; 1581 raid5_conf_t *conf = (raid5_conf_t *) mddev->private; 1582 pr_debug("raid5: error called\n"); 1583 1584 if (!test_bit(Faulty, &rdev->flags)) { 1585 set_bit(MD_CHANGE_DEVS, &mddev->flags); 1586 if (test_and_clear_bit(In_sync, &rdev->flags)) { 1587 unsigned long flags; 1588 spin_lock_irqsave(&conf->device_lock, flags); 1589 mddev->degraded++; 1590 spin_unlock_irqrestore(&conf->device_lock, flags); 1591 /* 1592 * if recovery was running, make sure it aborts. 1593 */ 1594 set_bit(MD_RECOVERY_INTR, &mddev->recovery); 1595 } 1596 set_bit(Faulty, &rdev->flags); 1597 printk(KERN_ALERT 1598 "raid5: Disk failure on %s, disabling device.\n" 1599 "raid5: Operation continuing on %d devices.\n", 1600 bdevname(rdev->bdev,b), conf->raid_disks - mddev->degraded); 1601 } 1602 } 1603 1604 /* 1605 * Input: a 'big' sector number, 1606 * Output: index of the data and parity disk, and the sector # in them. 1607 */ 1608 static sector_t raid5_compute_sector(raid5_conf_t *conf, sector_t r_sector, 1609 int previous, int *dd_idx, 1610 struct stripe_head *sh) 1611 { 1612 long stripe; 1613 unsigned long chunk_number; 1614 unsigned int chunk_offset; 1615 int pd_idx, qd_idx; 1616 int ddf_layout = 0; 1617 sector_t new_sector; 1618 int algorithm = previous ? conf->prev_algo 1619 : conf->algorithm; 1620 int sectors_per_chunk = previous ? conf->prev_chunk_sectors 1621 : conf->chunk_sectors; 1622 int raid_disks = previous ? conf->previous_raid_disks 1623 : conf->raid_disks; 1624 int data_disks = raid_disks - conf->max_degraded; 1625 1626 /* First compute the information on this sector */ 1627 1628 /* 1629 * Compute the chunk number and the sector offset inside the chunk 1630 */ 1631 chunk_offset = sector_div(r_sector, sectors_per_chunk); 1632 chunk_number = r_sector; 1633 BUG_ON(r_sector != chunk_number); 1634 1635 /* 1636 * Compute the stripe number 1637 */ 1638 stripe = chunk_number / data_disks; 1639 1640 /* 1641 * Compute the data disk and parity disk indexes inside the stripe 1642 */ 1643 *dd_idx = chunk_number % data_disks; 1644 1645 /* 1646 * Select the parity disk based on the user selected algorithm. 1647 */ 1648 pd_idx = qd_idx = ~0; 1649 switch(conf->level) { 1650 case 4: 1651 pd_idx = data_disks; 1652 break; 1653 case 5: 1654 switch (algorithm) { 1655 case ALGORITHM_LEFT_ASYMMETRIC: 1656 pd_idx = data_disks - stripe % raid_disks; 1657 if (*dd_idx >= pd_idx) 1658 (*dd_idx)++; 1659 break; 1660 case ALGORITHM_RIGHT_ASYMMETRIC: 1661 pd_idx = stripe % raid_disks; 1662 if (*dd_idx >= pd_idx) 1663 (*dd_idx)++; 1664 break; 1665 case ALGORITHM_LEFT_SYMMETRIC: 1666 pd_idx = data_disks - stripe % raid_disks; 1667 *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks; 1668 break; 1669 case ALGORITHM_RIGHT_SYMMETRIC: 1670 pd_idx = stripe % raid_disks; 1671 *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks; 1672 break; 1673 case ALGORITHM_PARITY_0: 1674 pd_idx = 0; 1675 (*dd_idx)++; 1676 break; 1677 case ALGORITHM_PARITY_N: 1678 pd_idx = data_disks; 1679 break; 1680 default: 1681 printk(KERN_ERR "raid5: unsupported algorithm %d\n", 1682 algorithm); 1683 BUG(); 1684 } 1685 break; 1686 case 6: 1687 1688 switch (algorithm) { 1689 case ALGORITHM_LEFT_ASYMMETRIC: 1690 pd_idx = raid_disks - 1 - (stripe % raid_disks); 1691 qd_idx = pd_idx + 1; 1692 if (pd_idx == raid_disks-1) { 1693 (*dd_idx)++; /* Q D D D P */ 1694 qd_idx = 0; 1695 } else if (*dd_idx >= pd_idx) 1696 (*dd_idx) += 2; /* D D P Q D */ 1697 break; 1698 case ALGORITHM_RIGHT_ASYMMETRIC: 1699 pd_idx = stripe % raid_disks; 1700 qd_idx = pd_idx + 1; 1701 if (pd_idx == raid_disks-1) { 1702 (*dd_idx)++; /* Q D D D P */ 1703 qd_idx = 0; 1704 } else if (*dd_idx >= pd_idx) 1705 (*dd_idx) += 2; /* D D P Q D */ 1706 break; 1707 case ALGORITHM_LEFT_SYMMETRIC: 1708 pd_idx = raid_disks - 1 - (stripe % raid_disks); 1709 qd_idx = (pd_idx + 1) % raid_disks; 1710 *dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks; 1711 break; 1712 case ALGORITHM_RIGHT_SYMMETRIC: 1713 pd_idx = stripe % raid_disks; 1714 qd_idx = (pd_idx + 1) % raid_disks; 1715 *dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks; 1716 break; 1717 1718 case ALGORITHM_PARITY_0: 1719 pd_idx = 0; 1720 qd_idx = 1; 1721 (*dd_idx) += 2; 1722 break; 1723 case ALGORITHM_PARITY_N: 1724 pd_idx = data_disks; 1725 qd_idx = data_disks + 1; 1726 break; 1727 1728 case ALGORITHM_ROTATING_ZERO_RESTART: 1729 /* Exactly the same as RIGHT_ASYMMETRIC, but or 1730 * of blocks for computing Q is different. 1731 */ 1732 pd_idx = stripe % raid_disks; 1733 qd_idx = pd_idx + 1; 1734 if (pd_idx == raid_disks-1) { 1735 (*dd_idx)++; /* Q D D D P */ 1736 qd_idx = 0; 1737 } else if (*dd_idx >= pd_idx) 1738 (*dd_idx) += 2; /* D D P Q D */ 1739 ddf_layout = 1; 1740 break; 1741 1742 case ALGORITHM_ROTATING_N_RESTART: 1743 /* Same a left_asymmetric, by first stripe is 1744 * D D D P Q rather than 1745 * Q D D D P 1746 */ 1747 pd_idx = raid_disks - 1 - ((stripe + 1) % raid_disks); 1748 qd_idx = pd_idx + 1; 1749 if (pd_idx == raid_disks-1) { 1750 (*dd_idx)++; /* Q D D D P */ 1751 qd_idx = 0; 1752 } else if (*dd_idx >= pd_idx) 1753 (*dd_idx) += 2; /* D D P Q D */ 1754 ddf_layout = 1; 1755 break; 1756 1757 case ALGORITHM_ROTATING_N_CONTINUE: 1758 /* Same as left_symmetric but Q is before P */ 1759 pd_idx = raid_disks - 1 - (stripe % raid_disks); 1760 qd_idx = (pd_idx + raid_disks - 1) % raid_disks; 1761 *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks; 1762 ddf_layout = 1; 1763 break; 1764 1765 case ALGORITHM_LEFT_ASYMMETRIC_6: 1766 /* RAID5 left_asymmetric, with Q on last device */ 1767 pd_idx = data_disks - stripe % (raid_disks-1); 1768 if (*dd_idx >= pd_idx) 1769 (*dd_idx)++; 1770 qd_idx = raid_disks - 1; 1771 break; 1772 1773 case ALGORITHM_RIGHT_ASYMMETRIC_6: 1774 pd_idx = stripe % (raid_disks-1); 1775 if (*dd_idx >= pd_idx) 1776 (*dd_idx)++; 1777 qd_idx = raid_disks - 1; 1778 break; 1779 1780 case ALGORITHM_LEFT_SYMMETRIC_6: 1781 pd_idx = data_disks - stripe % (raid_disks-1); 1782 *dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1); 1783 qd_idx = raid_disks - 1; 1784 break; 1785 1786 case ALGORITHM_RIGHT_SYMMETRIC_6: 1787 pd_idx = stripe % (raid_disks-1); 1788 *dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1); 1789 qd_idx = raid_disks - 1; 1790 break; 1791 1792 case ALGORITHM_PARITY_0_6: 1793 pd_idx = 0; 1794 (*dd_idx)++; 1795 qd_idx = raid_disks - 1; 1796 break; 1797 1798 1799 default: 1800 printk(KERN_CRIT "raid6: unsupported algorithm %d\n", 1801 algorithm); 1802 BUG(); 1803 } 1804 break; 1805 } 1806 1807 if (sh) { 1808 sh->pd_idx = pd_idx; 1809 sh->qd_idx = qd_idx; 1810 sh->ddf_layout = ddf_layout; 1811 } 1812 /* 1813 * Finally, compute the new sector number 1814 */ 1815 new_sector = (sector_t)stripe * sectors_per_chunk + chunk_offset; 1816 return new_sector; 1817 } 1818 1819 1820 static sector_t compute_blocknr(struct stripe_head *sh, int i, int previous) 1821 { 1822 raid5_conf_t *conf = sh->raid_conf; 1823 int raid_disks = sh->disks; 1824 int data_disks = raid_disks - conf->max_degraded; 1825 sector_t new_sector = sh->sector, check; 1826 int sectors_per_chunk = previous ? conf->prev_chunk_sectors 1827 : conf->chunk_sectors; 1828 int algorithm = previous ? conf->prev_algo 1829 : conf->algorithm; 1830 sector_t stripe; 1831 int chunk_offset; 1832 int chunk_number, dummy1, dd_idx = i; 1833 sector_t r_sector; 1834 struct stripe_head sh2; 1835 1836 1837 chunk_offset = sector_div(new_sector, sectors_per_chunk); 1838 stripe = new_sector; 1839 BUG_ON(new_sector != stripe); 1840 1841 if (i == sh->pd_idx) 1842 return 0; 1843 switch(conf->level) { 1844 case 4: break; 1845 case 5: 1846 switch (algorithm) { 1847 case ALGORITHM_LEFT_ASYMMETRIC: 1848 case ALGORITHM_RIGHT_ASYMMETRIC: 1849 if (i > sh->pd_idx) 1850 i--; 1851 break; 1852 case ALGORITHM_LEFT_SYMMETRIC: 1853 case ALGORITHM_RIGHT_SYMMETRIC: 1854 if (i < sh->pd_idx) 1855 i += raid_disks; 1856 i -= (sh->pd_idx + 1); 1857 break; 1858 case ALGORITHM_PARITY_0: 1859 i -= 1; 1860 break; 1861 case ALGORITHM_PARITY_N: 1862 break; 1863 default: 1864 printk(KERN_ERR "raid5: unsupported algorithm %d\n", 1865 algorithm); 1866 BUG(); 1867 } 1868 break; 1869 case 6: 1870 if (i == sh->qd_idx) 1871 return 0; /* It is the Q disk */ 1872 switch (algorithm) { 1873 case ALGORITHM_LEFT_ASYMMETRIC: 1874 case ALGORITHM_RIGHT_ASYMMETRIC: 1875 case ALGORITHM_ROTATING_ZERO_RESTART: 1876 case ALGORITHM_ROTATING_N_RESTART: 1877 if (sh->pd_idx == raid_disks-1) 1878 i--; /* Q D D D P */ 1879 else if (i > sh->pd_idx) 1880 i -= 2; /* D D P Q D */ 1881 break; 1882 case ALGORITHM_LEFT_SYMMETRIC: 1883 case ALGORITHM_RIGHT_SYMMETRIC: 1884 if (sh->pd_idx == raid_disks-1) 1885 i--; /* Q D D D P */ 1886 else { 1887 /* D D P Q D */ 1888 if (i < sh->pd_idx) 1889 i += raid_disks; 1890 i -= (sh->pd_idx + 2); 1891 } 1892 break; 1893 case ALGORITHM_PARITY_0: 1894 i -= 2; 1895 break; 1896 case ALGORITHM_PARITY_N: 1897 break; 1898 case ALGORITHM_ROTATING_N_CONTINUE: 1899 if (sh->pd_idx == 0) 1900 i--; /* P D D D Q */ 1901 else if (i > sh->pd_idx) 1902 i -= 2; /* D D Q P D */ 1903 break; 1904 case ALGORITHM_LEFT_ASYMMETRIC_6: 1905 case ALGORITHM_RIGHT_ASYMMETRIC_6: 1906 if (i > sh->pd_idx) 1907 i--; 1908 break; 1909 case ALGORITHM_LEFT_SYMMETRIC_6: 1910 case ALGORITHM_RIGHT_SYMMETRIC_6: 1911 if (i < sh->pd_idx) 1912 i += data_disks + 1; 1913 i -= (sh->pd_idx + 1); 1914 break; 1915 case ALGORITHM_PARITY_0_6: 1916 i -= 1; 1917 break; 1918 default: 1919 printk(KERN_CRIT "raid6: unsupported algorithm %d\n", 1920 algorithm); 1921 BUG(); 1922 } 1923 break; 1924 } 1925 1926 chunk_number = stripe * data_disks + i; 1927 r_sector = (sector_t)chunk_number * sectors_per_chunk + chunk_offset; 1928 1929 check = raid5_compute_sector(conf, r_sector, 1930 previous, &dummy1, &sh2); 1931 if (check != sh->sector || dummy1 != dd_idx || sh2.pd_idx != sh->pd_idx 1932 || sh2.qd_idx != sh->qd_idx) { 1933 printk(KERN_ERR "compute_blocknr: map not correct\n"); 1934 return 0; 1935 } 1936 return r_sector; 1937 } 1938 1939 1940 static void 1941 schedule_reconstruction(struct stripe_head *sh, struct stripe_head_state *s, 1942 int rcw, int expand) 1943 { 1944 int i, pd_idx = sh->pd_idx, disks = sh->disks; 1945 raid5_conf_t *conf = sh->raid_conf; 1946 int level = conf->level; 1947 1948 if (rcw) { 1949 /* if we are not expanding this is a proper write request, and 1950 * there will be bios with new data to be drained into the 1951 * stripe cache 1952 */ 1953 if (!expand) { 1954 sh->reconstruct_state = reconstruct_state_drain_run; 1955 set_bit(STRIPE_OP_BIODRAIN, &s->ops_request); 1956 } else 1957 sh->reconstruct_state = reconstruct_state_run; 1958 1959 set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request); 1960 1961 for (i = disks; i--; ) { 1962 struct r5dev *dev = &sh->dev[i]; 1963 1964 if (dev->towrite) { 1965 set_bit(R5_LOCKED, &dev->flags); 1966 set_bit(R5_Wantdrain, &dev->flags); 1967 if (!expand) 1968 clear_bit(R5_UPTODATE, &dev->flags); 1969 s->locked++; 1970 } 1971 } 1972 if (s->locked + conf->max_degraded == disks) 1973 if (!test_and_set_bit(STRIPE_FULL_WRITE, &sh->state)) 1974 atomic_inc(&conf->pending_full_writes); 1975 } else { 1976 BUG_ON(level == 6); 1977 BUG_ON(!(test_bit(R5_UPTODATE, &sh->dev[pd_idx].flags) || 1978 test_bit(R5_Wantcompute, &sh->dev[pd_idx].flags))); 1979 1980 sh->reconstruct_state = reconstruct_state_prexor_drain_run; 1981 set_bit(STRIPE_OP_PREXOR, &s->ops_request); 1982 set_bit(STRIPE_OP_BIODRAIN, &s->ops_request); 1983 set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request); 1984 1985 for (i = disks; i--; ) { 1986 struct r5dev *dev = &sh->dev[i]; 1987 if (i == pd_idx) 1988 continue; 1989 1990 if (dev->towrite && 1991 (test_bit(R5_UPTODATE, &dev->flags) || 1992 test_bit(R5_Wantcompute, &dev->flags))) { 1993 set_bit(R5_Wantdrain, &dev->flags); 1994 set_bit(R5_LOCKED, &dev->flags); 1995 clear_bit(R5_UPTODATE, &dev->flags); 1996 s->locked++; 1997 } 1998 } 1999 } 2000 2001 /* keep the parity disk(s) locked while asynchronous operations 2002 * are in flight 2003 */ 2004 set_bit(R5_LOCKED, &sh->dev[pd_idx].flags); 2005 clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags); 2006 s->locked++; 2007 2008 if (level == 6) { 2009 int qd_idx = sh->qd_idx; 2010 struct r5dev *dev = &sh->dev[qd_idx]; 2011 2012 set_bit(R5_LOCKED, &dev->flags); 2013 clear_bit(R5_UPTODATE, &dev->flags); 2014 s->locked++; 2015 } 2016 2017 pr_debug("%s: stripe %llu locked: %d ops_request: %lx\n", 2018 __func__, (unsigned long long)sh->sector, 2019 s->locked, s->ops_request); 2020 } 2021 2022 /* 2023 * Each stripe/dev can have one or more bion attached. 2024 * toread/towrite point to the first in a chain. 2025 * The bi_next chain must be in order. 2026 */ 2027 static int add_stripe_bio(struct stripe_head *sh, struct bio *bi, int dd_idx, int forwrite) 2028 { 2029 struct bio **bip; 2030 raid5_conf_t *conf = sh->raid_conf; 2031 int firstwrite=0; 2032 2033 pr_debug("adding bh b#%llu to stripe s#%llu\n", 2034 (unsigned long long)bi->bi_sector, 2035 (unsigned long long)sh->sector); 2036 2037 2038 spin_lock(&sh->lock); 2039 spin_lock_irq(&conf->device_lock); 2040 if (forwrite) { 2041 bip = &sh->dev[dd_idx].towrite; 2042 if (*bip == NULL && sh->dev[dd_idx].written == NULL) 2043 firstwrite = 1; 2044 } else 2045 bip = &sh->dev[dd_idx].toread; 2046 while (*bip && (*bip)->bi_sector < bi->bi_sector) { 2047 if ((*bip)->bi_sector + ((*bip)->bi_size >> 9) > bi->bi_sector) 2048 goto overlap; 2049 bip = & (*bip)->bi_next; 2050 } 2051 if (*bip && (*bip)->bi_sector < bi->bi_sector + ((bi->bi_size)>>9)) 2052 goto overlap; 2053 2054 BUG_ON(*bip && bi->bi_next && (*bip) != bi->bi_next); 2055 if (*bip) 2056 bi->bi_next = *bip; 2057 *bip = bi; 2058 bi->bi_phys_segments++; 2059 spin_unlock_irq(&conf->device_lock); 2060 spin_unlock(&sh->lock); 2061 2062 pr_debug("added bi b#%llu to stripe s#%llu, disk %d.\n", 2063 (unsigned long long)bi->bi_sector, 2064 (unsigned long long)sh->sector, dd_idx); 2065 2066 if (conf->mddev->bitmap && firstwrite) { 2067 bitmap_startwrite(conf->mddev->bitmap, sh->sector, 2068 STRIPE_SECTORS, 0); 2069 sh->bm_seq = conf->seq_flush+1; 2070 set_bit(STRIPE_BIT_DELAY, &sh->state); 2071 } 2072 2073 if (forwrite) { 2074 /* check if page is covered */ 2075 sector_t sector = sh->dev[dd_idx].sector; 2076 for (bi=sh->dev[dd_idx].towrite; 2077 sector < sh->dev[dd_idx].sector + STRIPE_SECTORS && 2078 bi && bi->bi_sector <= sector; 2079 bi = r5_next_bio(bi, sh->dev[dd_idx].sector)) { 2080 if (bi->bi_sector + (bi->bi_size>>9) >= sector) 2081 sector = bi->bi_sector + (bi->bi_size>>9); 2082 } 2083 if (sector >= sh->dev[dd_idx].sector + STRIPE_SECTORS) 2084 set_bit(R5_OVERWRITE, &sh->dev[dd_idx].flags); 2085 } 2086 return 1; 2087 2088 overlap: 2089 set_bit(R5_Overlap, &sh->dev[dd_idx].flags); 2090 spin_unlock_irq(&conf->device_lock); 2091 spin_unlock(&sh->lock); 2092 return 0; 2093 } 2094 2095 static void end_reshape(raid5_conf_t *conf); 2096 2097 static void stripe_set_idx(sector_t stripe, raid5_conf_t *conf, int previous, 2098 struct stripe_head *sh) 2099 { 2100 int sectors_per_chunk = 2101 previous ? conf->prev_chunk_sectors : conf->chunk_sectors; 2102 int dd_idx; 2103 int chunk_offset = sector_div(stripe, sectors_per_chunk); 2104 int disks = previous ? conf->previous_raid_disks : conf->raid_disks; 2105 2106 raid5_compute_sector(conf, 2107 stripe * (disks - conf->max_degraded) 2108 *sectors_per_chunk + chunk_offset, 2109 previous, 2110 &dd_idx, sh); 2111 } 2112 2113 static void 2114 handle_failed_stripe(raid5_conf_t *conf, struct stripe_head *sh, 2115 struct stripe_head_state *s, int disks, 2116 struct bio **return_bi) 2117 { 2118 int i; 2119 for (i = disks; i--; ) { 2120 struct bio *bi; 2121 int bitmap_end = 0; 2122 2123 if (test_bit(R5_ReadError, &sh->dev[i].flags)) { 2124 mdk_rdev_t *rdev; 2125 rcu_read_lock(); 2126 rdev = rcu_dereference(conf->disks[i].rdev); 2127 if (rdev && test_bit(In_sync, &rdev->flags)) 2128 /* multiple read failures in one stripe */ 2129 md_error(conf->mddev, rdev); 2130 rcu_read_unlock(); 2131 } 2132 spin_lock_irq(&conf->device_lock); 2133 /* fail all writes first */ 2134 bi = sh->dev[i].towrite; 2135 sh->dev[i].towrite = NULL; 2136 if (bi) { 2137 s->to_write--; 2138 bitmap_end = 1; 2139 } 2140 2141 if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags)) 2142 wake_up(&conf->wait_for_overlap); 2143 2144 while (bi && bi->bi_sector < 2145 sh->dev[i].sector + STRIPE_SECTORS) { 2146 struct bio *nextbi = r5_next_bio(bi, sh->dev[i].sector); 2147 clear_bit(BIO_UPTODATE, &bi->bi_flags); 2148 if (!raid5_dec_bi_phys_segments(bi)) { 2149 md_write_end(conf->mddev); 2150 bi->bi_next = *return_bi; 2151 *return_bi = bi; 2152 } 2153 bi = nextbi; 2154 } 2155 /* and fail all 'written' */ 2156 bi = sh->dev[i].written; 2157 sh->dev[i].written = NULL; 2158 if (bi) bitmap_end = 1; 2159 while (bi && bi->bi_sector < 2160 sh->dev[i].sector + STRIPE_SECTORS) { 2161 struct bio *bi2 = r5_next_bio(bi, sh->dev[i].sector); 2162 clear_bit(BIO_UPTODATE, &bi->bi_flags); 2163 if (!raid5_dec_bi_phys_segments(bi)) { 2164 md_write_end(conf->mddev); 2165 bi->bi_next = *return_bi; 2166 *return_bi = bi; 2167 } 2168 bi = bi2; 2169 } 2170 2171 /* fail any reads if this device is non-operational and 2172 * the data has not reached the cache yet. 2173 */ 2174 if (!test_bit(R5_Wantfill, &sh->dev[i].flags) && 2175 (!test_bit(R5_Insync, &sh->dev[i].flags) || 2176 test_bit(R5_ReadError, &sh->dev[i].flags))) { 2177 bi = sh->dev[i].toread; 2178 sh->dev[i].toread = NULL; 2179 if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags)) 2180 wake_up(&conf->wait_for_overlap); 2181 if (bi) s->to_read--; 2182 while (bi && bi->bi_sector < 2183 sh->dev[i].sector + STRIPE_SECTORS) { 2184 struct bio *nextbi = 2185 r5_next_bio(bi, sh->dev[i].sector); 2186 clear_bit(BIO_UPTODATE, &bi->bi_flags); 2187 if (!raid5_dec_bi_phys_segments(bi)) { 2188 bi->bi_next = *return_bi; 2189 *return_bi = bi; 2190 } 2191 bi = nextbi; 2192 } 2193 } 2194 spin_unlock_irq(&conf->device_lock); 2195 if (bitmap_end) 2196 bitmap_endwrite(conf->mddev->bitmap, sh->sector, 2197 STRIPE_SECTORS, 0, 0); 2198 } 2199 2200 if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state)) 2201 if (atomic_dec_and_test(&conf->pending_full_writes)) 2202 md_wakeup_thread(conf->mddev->thread); 2203 } 2204 2205 /* fetch_block5 - checks the given member device to see if its data needs 2206 * to be read or computed to satisfy a request. 2207 * 2208 * Returns 1 when no more member devices need to be checked, otherwise returns 2209 * 0 to tell the loop in handle_stripe_fill5 to continue 2210 */ 2211 static int fetch_block5(struct stripe_head *sh, struct stripe_head_state *s, 2212 int disk_idx, int disks) 2213 { 2214 struct r5dev *dev = &sh->dev[disk_idx]; 2215 struct r5dev *failed_dev = &sh->dev[s->failed_num]; 2216 2217 /* is the data in this block needed, and can we get it? */ 2218 if (!test_bit(R5_LOCKED, &dev->flags) && 2219 !test_bit(R5_UPTODATE, &dev->flags) && 2220 (dev->toread || 2221 (dev->towrite && !test_bit(R5_OVERWRITE, &dev->flags)) || 2222 s->syncing || s->expanding || 2223 (s->failed && 2224 (failed_dev->toread || 2225 (failed_dev->towrite && 2226 !test_bit(R5_OVERWRITE, &failed_dev->flags)))))) { 2227 /* We would like to get this block, possibly by computing it, 2228 * otherwise read it if the backing disk is insync 2229 */ 2230 if ((s->uptodate == disks - 1) && 2231 (s->failed && disk_idx == s->failed_num)) { 2232 set_bit(STRIPE_COMPUTE_RUN, &sh->state); 2233 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request); 2234 set_bit(R5_Wantcompute, &dev->flags); 2235 sh->ops.target = disk_idx; 2236 sh->ops.target2 = -1; 2237 s->req_compute = 1; 2238 /* Careful: from this point on 'uptodate' is in the eye 2239 * of raid_run_ops which services 'compute' operations 2240 * before writes. R5_Wantcompute flags a block that will 2241 * be R5_UPTODATE by the time it is needed for a 2242 * subsequent operation. 2243 */ 2244 s->uptodate++; 2245 return 1; /* uptodate + compute == disks */ 2246 } else if (test_bit(R5_Insync, &dev->flags)) { 2247 set_bit(R5_LOCKED, &dev->flags); 2248 set_bit(R5_Wantread, &dev->flags); 2249 s->locked++; 2250 pr_debug("Reading block %d (sync=%d)\n", disk_idx, 2251 s->syncing); 2252 } 2253 } 2254 2255 return 0; 2256 } 2257 2258 /** 2259 * handle_stripe_fill5 - read or compute data to satisfy pending requests. 2260 */ 2261 static void handle_stripe_fill5(struct stripe_head *sh, 2262 struct stripe_head_state *s, int disks) 2263 { 2264 int i; 2265 2266 /* look for blocks to read/compute, skip this if a compute 2267 * is already in flight, or if the stripe contents are in the 2268 * midst of changing due to a write 2269 */ 2270 if (!test_bit(STRIPE_COMPUTE_RUN, &sh->state) && !sh->check_state && 2271 !sh->reconstruct_state) 2272 for (i = disks; i--; ) 2273 if (fetch_block5(sh, s, i, disks)) 2274 break; 2275 set_bit(STRIPE_HANDLE, &sh->state); 2276 } 2277 2278 /* fetch_block6 - checks the given member device to see if its data needs 2279 * to be read or computed to satisfy a request. 2280 * 2281 * Returns 1 when no more member devices need to be checked, otherwise returns 2282 * 0 to tell the loop in handle_stripe_fill6 to continue 2283 */ 2284 static int fetch_block6(struct stripe_head *sh, struct stripe_head_state *s, 2285 struct r6_state *r6s, int disk_idx, int disks) 2286 { 2287 struct r5dev *dev = &sh->dev[disk_idx]; 2288 struct r5dev *fdev[2] = { &sh->dev[r6s->failed_num[0]], 2289 &sh->dev[r6s->failed_num[1]] }; 2290 2291 if (!test_bit(R5_LOCKED, &dev->flags) && 2292 !test_bit(R5_UPTODATE, &dev->flags) && 2293 (dev->toread || 2294 (dev->towrite && !test_bit(R5_OVERWRITE, &dev->flags)) || 2295 s->syncing || s->expanding || 2296 (s->failed >= 1 && 2297 (fdev[0]->toread || s->to_write)) || 2298 (s->failed >= 2 && 2299 (fdev[1]->toread || s->to_write)))) { 2300 /* we would like to get this block, possibly by computing it, 2301 * otherwise read it if the backing disk is insync 2302 */ 2303 BUG_ON(test_bit(R5_Wantcompute, &dev->flags)); 2304 BUG_ON(test_bit(R5_Wantread, &dev->flags)); 2305 if ((s->uptodate == disks - 1) && 2306 (s->failed && (disk_idx == r6s->failed_num[0] || 2307 disk_idx == r6s->failed_num[1]))) { 2308 /* have disk failed, and we're requested to fetch it; 2309 * do compute it 2310 */ 2311 pr_debug("Computing stripe %llu block %d\n", 2312 (unsigned long long)sh->sector, disk_idx); 2313 set_bit(STRIPE_COMPUTE_RUN, &sh->state); 2314 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request); 2315 set_bit(R5_Wantcompute, &dev->flags); 2316 sh->ops.target = disk_idx; 2317 sh->ops.target2 = -1; /* no 2nd target */ 2318 s->req_compute = 1; 2319 s->uptodate++; 2320 return 1; 2321 } else if (s->uptodate == disks-2 && s->failed >= 2) { 2322 /* Computing 2-failure is *very* expensive; only 2323 * do it if failed >= 2 2324 */ 2325 int other; 2326 for (other = disks; other--; ) { 2327 if (other == disk_idx) 2328 continue; 2329 if (!test_bit(R5_UPTODATE, 2330 &sh->dev[other].flags)) 2331 break; 2332 } 2333 BUG_ON(other < 0); 2334 pr_debug("Computing stripe %llu blocks %d,%d\n", 2335 (unsigned long long)sh->sector, 2336 disk_idx, other); 2337 set_bit(STRIPE_COMPUTE_RUN, &sh->state); 2338 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request); 2339 set_bit(R5_Wantcompute, &sh->dev[disk_idx].flags); 2340 set_bit(R5_Wantcompute, &sh->dev[other].flags); 2341 sh->ops.target = disk_idx; 2342 sh->ops.target2 = other; 2343 s->uptodate += 2; 2344 s->req_compute = 1; 2345 return 1; 2346 } else if (test_bit(R5_Insync, &dev->flags)) { 2347 set_bit(R5_LOCKED, &dev->flags); 2348 set_bit(R5_Wantread, &dev->flags); 2349 s->locked++; 2350 pr_debug("Reading block %d (sync=%d)\n", 2351 disk_idx, s->syncing); 2352 } 2353 } 2354 2355 return 0; 2356 } 2357 2358 /** 2359 * handle_stripe_fill6 - read or compute data to satisfy pending requests. 2360 */ 2361 static void handle_stripe_fill6(struct stripe_head *sh, 2362 struct stripe_head_state *s, struct r6_state *r6s, 2363 int disks) 2364 { 2365 int i; 2366 2367 /* look for blocks to read/compute, skip this if a compute 2368 * is already in flight, or if the stripe contents are in the 2369 * midst of changing due to a write 2370 */ 2371 if (!test_bit(STRIPE_COMPUTE_RUN, &sh->state) && !sh->check_state && 2372 !sh->reconstruct_state) 2373 for (i = disks; i--; ) 2374 if (fetch_block6(sh, s, r6s, i, disks)) 2375 break; 2376 set_bit(STRIPE_HANDLE, &sh->state); 2377 } 2378 2379 2380 /* handle_stripe_clean_event 2381 * any written block on an uptodate or failed drive can be returned. 2382 * Note that if we 'wrote' to a failed drive, it will be UPTODATE, but 2383 * never LOCKED, so we don't need to test 'failed' directly. 2384 */ 2385 static void handle_stripe_clean_event(raid5_conf_t *conf, 2386 struct stripe_head *sh, int disks, struct bio **return_bi) 2387 { 2388 int i; 2389 struct r5dev *dev; 2390 2391 for (i = disks; i--; ) 2392 if (sh->dev[i].written) { 2393 dev = &sh->dev[i]; 2394 if (!test_bit(R5_LOCKED, &dev->flags) && 2395 test_bit(R5_UPTODATE, &dev->flags)) { 2396 /* We can return any write requests */ 2397 struct bio *wbi, *wbi2; 2398 int bitmap_end = 0; 2399 pr_debug("Return write for disc %d\n", i); 2400 spin_lock_irq(&conf->device_lock); 2401 wbi = dev->written; 2402 dev->written = NULL; 2403 while (wbi && wbi->bi_sector < 2404 dev->sector + STRIPE_SECTORS) { 2405 wbi2 = r5_next_bio(wbi, dev->sector); 2406 if (!raid5_dec_bi_phys_segments(wbi)) { 2407 md_write_end(conf->mddev); 2408 wbi->bi_next = *return_bi; 2409 *return_bi = wbi; 2410 } 2411 wbi = wbi2; 2412 } 2413 if (dev->towrite == NULL) 2414 bitmap_end = 1; 2415 spin_unlock_irq(&conf->device_lock); 2416 if (bitmap_end) 2417 bitmap_endwrite(conf->mddev->bitmap, 2418 sh->sector, 2419 STRIPE_SECTORS, 2420 !test_bit(STRIPE_DEGRADED, &sh->state), 2421 0); 2422 } 2423 } 2424 2425 if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state)) 2426 if (atomic_dec_and_test(&conf->pending_full_writes)) 2427 md_wakeup_thread(conf->mddev->thread); 2428 } 2429 2430 static void handle_stripe_dirtying5(raid5_conf_t *conf, 2431 struct stripe_head *sh, struct stripe_head_state *s, int disks) 2432 { 2433 int rmw = 0, rcw = 0, i; 2434 for (i = disks; i--; ) { 2435 /* would I have to read this buffer for read_modify_write */ 2436 struct r5dev *dev = &sh->dev[i]; 2437 if ((dev->towrite || i == sh->pd_idx) && 2438 !test_bit(R5_LOCKED, &dev->flags) && 2439 !(test_bit(R5_UPTODATE, &dev->flags) || 2440 test_bit(R5_Wantcompute, &dev->flags))) { 2441 if (test_bit(R5_Insync, &dev->flags)) 2442 rmw++; 2443 else 2444 rmw += 2*disks; /* cannot read it */ 2445 } 2446 /* Would I have to read this buffer for reconstruct_write */ 2447 if (!test_bit(R5_OVERWRITE, &dev->flags) && i != sh->pd_idx && 2448 !test_bit(R5_LOCKED, &dev->flags) && 2449 !(test_bit(R5_UPTODATE, &dev->flags) || 2450 test_bit(R5_Wantcompute, &dev->flags))) { 2451 if (test_bit(R5_Insync, &dev->flags)) rcw++; 2452 else 2453 rcw += 2*disks; 2454 } 2455 } 2456 pr_debug("for sector %llu, rmw=%d rcw=%d\n", 2457 (unsigned long long)sh->sector, rmw, rcw); 2458 set_bit(STRIPE_HANDLE, &sh->state); 2459 if (rmw < rcw && rmw > 0) 2460 /* prefer read-modify-write, but need to get some data */ 2461 for (i = disks; i--; ) { 2462 struct r5dev *dev = &sh->dev[i]; 2463 if ((dev->towrite || i == sh->pd_idx) && 2464 !test_bit(R5_LOCKED, &dev->flags) && 2465 !(test_bit(R5_UPTODATE, &dev->flags) || 2466 test_bit(R5_Wantcompute, &dev->flags)) && 2467 test_bit(R5_Insync, &dev->flags)) { 2468 if ( 2469 test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) { 2470 pr_debug("Read_old block " 2471 "%d for r-m-w\n", i); 2472 set_bit(R5_LOCKED, &dev->flags); 2473 set_bit(R5_Wantread, &dev->flags); 2474 s->locked++; 2475 } else { 2476 set_bit(STRIPE_DELAYED, &sh->state); 2477 set_bit(STRIPE_HANDLE, &sh->state); 2478 } 2479 } 2480 } 2481 if (rcw <= rmw && rcw > 0) 2482 /* want reconstruct write, but need to get some data */ 2483 for (i = disks; i--; ) { 2484 struct r5dev *dev = &sh->dev[i]; 2485 if (!test_bit(R5_OVERWRITE, &dev->flags) && 2486 i != sh->pd_idx && 2487 !test_bit(R5_LOCKED, &dev->flags) && 2488 !(test_bit(R5_UPTODATE, &dev->flags) || 2489 test_bit(R5_Wantcompute, &dev->flags)) && 2490 test_bit(R5_Insync, &dev->flags)) { 2491 if ( 2492 test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) { 2493 pr_debug("Read_old block " 2494 "%d for Reconstruct\n", i); 2495 set_bit(R5_LOCKED, &dev->flags); 2496 set_bit(R5_Wantread, &dev->flags); 2497 s->locked++; 2498 } else { 2499 set_bit(STRIPE_DELAYED, &sh->state); 2500 set_bit(STRIPE_HANDLE, &sh->state); 2501 } 2502 } 2503 } 2504 /* now if nothing is locked, and if we have enough data, 2505 * we can start a write request 2506 */ 2507 /* since handle_stripe can be called at any time we need to handle the 2508 * case where a compute block operation has been submitted and then a 2509 * subsequent call wants to start a write request. raid_run_ops only 2510 * handles the case where compute block and reconstruct are requested 2511 * simultaneously. If this is not the case then new writes need to be 2512 * held off until the compute completes. 2513 */ 2514 if ((s->req_compute || !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) && 2515 (s->locked == 0 && (rcw == 0 || rmw == 0) && 2516 !test_bit(STRIPE_BIT_DELAY, &sh->state))) 2517 schedule_reconstruction(sh, s, rcw == 0, 0); 2518 } 2519 2520 static void handle_stripe_dirtying6(raid5_conf_t *conf, 2521 struct stripe_head *sh, struct stripe_head_state *s, 2522 struct r6_state *r6s, int disks) 2523 { 2524 int rcw = 0, pd_idx = sh->pd_idx, i; 2525 int qd_idx = sh->qd_idx; 2526 2527 set_bit(STRIPE_HANDLE, &sh->state); 2528 for (i = disks; i--; ) { 2529 struct r5dev *dev = &sh->dev[i]; 2530 /* check if we haven't enough data */ 2531 if (!test_bit(R5_OVERWRITE, &dev->flags) && 2532 i != pd_idx && i != qd_idx && 2533 !test_bit(R5_LOCKED, &dev->flags) && 2534 !(test_bit(R5_UPTODATE, &dev->flags) || 2535 test_bit(R5_Wantcompute, &dev->flags))) { 2536 rcw++; 2537 if (!test_bit(R5_Insync, &dev->flags)) 2538 continue; /* it's a failed drive */ 2539 2540 if ( 2541 test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) { 2542 pr_debug("Read_old stripe %llu " 2543 "block %d for Reconstruct\n", 2544 (unsigned long long)sh->sector, i); 2545 set_bit(R5_LOCKED, &dev->flags); 2546 set_bit(R5_Wantread, &dev->flags); 2547 s->locked++; 2548 } else { 2549 pr_debug("Request delayed stripe %llu " 2550 "block %d for Reconstruct\n", 2551 (unsigned long long)sh->sector, i); 2552 set_bit(STRIPE_DELAYED, &sh->state); 2553 set_bit(STRIPE_HANDLE, &sh->state); 2554 } 2555 } 2556 } 2557 /* now if nothing is locked, and if we have enough data, we can start a 2558 * write request 2559 */ 2560 if ((s->req_compute || !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) && 2561 s->locked == 0 && rcw == 0 && 2562 !test_bit(STRIPE_BIT_DELAY, &sh->state)) { 2563 schedule_reconstruction(sh, s, 1, 0); 2564 } 2565 } 2566 2567 static void handle_parity_checks5(raid5_conf_t *conf, struct stripe_head *sh, 2568 struct stripe_head_state *s, int disks) 2569 { 2570 struct r5dev *dev = NULL; 2571 2572 set_bit(STRIPE_HANDLE, &sh->state); 2573 2574 switch (sh->check_state) { 2575 case check_state_idle: 2576 /* start a new check operation if there are no failures */ 2577 if (s->failed == 0) { 2578 BUG_ON(s->uptodate != disks); 2579 sh->check_state = check_state_run; 2580 set_bit(STRIPE_OP_CHECK, &s->ops_request); 2581 clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags); 2582 s->uptodate--; 2583 break; 2584 } 2585 dev = &sh->dev[s->failed_num]; 2586 /* fall through */ 2587 case check_state_compute_result: 2588 sh->check_state = check_state_idle; 2589 if (!dev) 2590 dev = &sh->dev[sh->pd_idx]; 2591 2592 /* check that a write has not made the stripe insync */ 2593 if (test_bit(STRIPE_INSYNC, &sh->state)) 2594 break; 2595 2596 /* either failed parity check, or recovery is happening */ 2597 BUG_ON(!test_bit(R5_UPTODATE, &dev->flags)); 2598 BUG_ON(s->uptodate != disks); 2599 2600 set_bit(R5_LOCKED, &dev->flags); 2601 s->locked++; 2602 set_bit(R5_Wantwrite, &dev->flags); 2603 2604 clear_bit(STRIPE_DEGRADED, &sh->state); 2605 set_bit(STRIPE_INSYNC, &sh->state); 2606 break; 2607 case check_state_run: 2608 break; /* we will be called again upon completion */ 2609 case check_state_check_result: 2610 sh->check_state = check_state_idle; 2611 2612 /* if a failure occurred during the check operation, leave 2613 * STRIPE_INSYNC not set and let the stripe be handled again 2614 */ 2615 if (s->failed) 2616 break; 2617 2618 /* handle a successful check operation, if parity is correct 2619 * we are done. Otherwise update the mismatch count and repair 2620 * parity if !MD_RECOVERY_CHECK 2621 */ 2622 if ((sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) == 0) 2623 /* parity is correct (on disc, 2624 * not in buffer any more) 2625 */ 2626 set_bit(STRIPE_INSYNC, &sh->state); 2627 else { 2628 conf->mddev->resync_mismatches += STRIPE_SECTORS; 2629 if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery)) 2630 /* don't try to repair!! */ 2631 set_bit(STRIPE_INSYNC, &sh->state); 2632 else { 2633 sh->check_state = check_state_compute_run; 2634 set_bit(STRIPE_COMPUTE_RUN, &sh->state); 2635 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request); 2636 set_bit(R5_Wantcompute, 2637 &sh->dev[sh->pd_idx].flags); 2638 sh->ops.target = sh->pd_idx; 2639 sh->ops.target2 = -1; 2640 s->uptodate++; 2641 } 2642 } 2643 break; 2644 case check_state_compute_run: 2645 break; 2646 default: 2647 printk(KERN_ERR "%s: unknown check_state: %d sector: %llu\n", 2648 __func__, sh->check_state, 2649 (unsigned long long) sh->sector); 2650 BUG(); 2651 } 2652 } 2653 2654 2655 static void handle_parity_checks6(raid5_conf_t *conf, struct stripe_head *sh, 2656 struct stripe_head_state *s, 2657 struct r6_state *r6s, int disks) 2658 { 2659 int pd_idx = sh->pd_idx; 2660 int qd_idx = sh->qd_idx; 2661 struct r5dev *dev; 2662 2663 set_bit(STRIPE_HANDLE, &sh->state); 2664 2665 BUG_ON(s->failed > 2); 2666 2667 /* Want to check and possibly repair P and Q. 2668 * However there could be one 'failed' device, in which 2669 * case we can only check one of them, possibly using the 2670 * other to generate missing data 2671 */ 2672 2673 switch (sh->check_state) { 2674 case check_state_idle: 2675 /* start a new check operation if there are < 2 failures */ 2676 if (s->failed == r6s->q_failed) { 2677 /* The only possible failed device holds Q, so it 2678 * makes sense to check P (If anything else were failed, 2679 * we would have used P to recreate it). 2680 */ 2681 sh->check_state = check_state_run; 2682 } 2683 if (!r6s->q_failed && s->failed < 2) { 2684 /* Q is not failed, and we didn't use it to generate 2685 * anything, so it makes sense to check it 2686 */ 2687 if (sh->check_state == check_state_run) 2688 sh->check_state = check_state_run_pq; 2689 else 2690 sh->check_state = check_state_run_q; 2691 } 2692 2693 /* discard potentially stale zero_sum_result */ 2694 sh->ops.zero_sum_result = 0; 2695 2696 if (sh->check_state == check_state_run) { 2697 /* async_xor_zero_sum destroys the contents of P */ 2698 clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags); 2699 s->uptodate--; 2700 } 2701 if (sh->check_state >= check_state_run && 2702 sh->check_state <= check_state_run_pq) { 2703 /* async_syndrome_zero_sum preserves P and Q, so 2704 * no need to mark them !uptodate here 2705 */ 2706 set_bit(STRIPE_OP_CHECK, &s->ops_request); 2707 break; 2708 } 2709 2710 /* we have 2-disk failure */ 2711 BUG_ON(s->failed != 2); 2712 /* fall through */ 2713 case check_state_compute_result: 2714 sh->check_state = check_state_idle; 2715 2716 /* check that a write has not made the stripe insync */ 2717 if (test_bit(STRIPE_INSYNC, &sh->state)) 2718 break; 2719 2720 /* now write out any block on a failed drive, 2721 * or P or Q if they were recomputed 2722 */ 2723 BUG_ON(s->uptodate < disks - 1); /* We don't need Q to recover */ 2724 if (s->failed == 2) { 2725 dev = &sh->dev[r6s->failed_num[1]]; 2726 s->locked++; 2727 set_bit(R5_LOCKED, &dev->flags); 2728 set_bit(R5_Wantwrite, &dev->flags); 2729 } 2730 if (s->failed >= 1) { 2731 dev = &sh->dev[r6s->failed_num[0]]; 2732 s->locked++; 2733 set_bit(R5_LOCKED, &dev->flags); 2734 set_bit(R5_Wantwrite, &dev->flags); 2735 } 2736 if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) { 2737 dev = &sh->dev[pd_idx]; 2738 s->locked++; 2739 set_bit(R5_LOCKED, &dev->flags); 2740 set_bit(R5_Wantwrite, &dev->flags); 2741 } 2742 if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) { 2743 dev = &sh->dev[qd_idx]; 2744 s->locked++; 2745 set_bit(R5_LOCKED, &dev->flags); 2746 set_bit(R5_Wantwrite, &dev->flags); 2747 } 2748 clear_bit(STRIPE_DEGRADED, &sh->state); 2749 2750 set_bit(STRIPE_INSYNC, &sh->state); 2751 break; 2752 case check_state_run: 2753 case check_state_run_q: 2754 case check_state_run_pq: 2755 break; /* we will be called again upon completion */ 2756 case check_state_check_result: 2757 sh->check_state = check_state_idle; 2758 2759 /* handle a successful check operation, if parity is correct 2760 * we are done. Otherwise update the mismatch count and repair 2761 * parity if !MD_RECOVERY_CHECK 2762 */ 2763 if (sh->ops.zero_sum_result == 0) { 2764 /* both parities are correct */ 2765 if (!s->failed) 2766 set_bit(STRIPE_INSYNC, &sh->state); 2767 else { 2768 /* in contrast to the raid5 case we can validate 2769 * parity, but still have a failure to write 2770 * back 2771 */ 2772 sh->check_state = check_state_compute_result; 2773 /* Returning at this point means that we may go 2774 * off and bring p and/or q uptodate again so 2775 * we make sure to check zero_sum_result again 2776 * to verify if p or q need writeback 2777 */ 2778 } 2779 } else { 2780 conf->mddev->resync_mismatches += STRIPE_SECTORS; 2781 if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery)) 2782 /* don't try to repair!! */ 2783 set_bit(STRIPE_INSYNC, &sh->state); 2784 else { 2785 int *target = &sh->ops.target; 2786 2787 sh->ops.target = -1; 2788 sh->ops.target2 = -1; 2789 sh->check_state = check_state_compute_run; 2790 set_bit(STRIPE_COMPUTE_RUN, &sh->state); 2791 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request); 2792 if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) { 2793 set_bit(R5_Wantcompute, 2794 &sh->dev[pd_idx].flags); 2795 *target = pd_idx; 2796 target = &sh->ops.target2; 2797 s->uptodate++; 2798 } 2799 if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) { 2800 set_bit(R5_Wantcompute, 2801 &sh->dev[qd_idx].flags); 2802 *target = qd_idx; 2803 s->uptodate++; 2804 } 2805 } 2806 } 2807 break; 2808 case check_state_compute_run: 2809 break; 2810 default: 2811 printk(KERN_ERR "%s: unknown check_state: %d sector: %llu\n", 2812 __func__, sh->check_state, 2813 (unsigned long long) sh->sector); 2814 BUG(); 2815 } 2816 } 2817 2818 static void handle_stripe_expansion(raid5_conf_t *conf, struct stripe_head *sh, 2819 struct r6_state *r6s) 2820 { 2821 int i; 2822 2823 /* We have read all the blocks in this stripe and now we need to 2824 * copy some of them into a target stripe for expand. 2825 */ 2826 struct dma_async_tx_descriptor *tx = NULL; 2827 clear_bit(STRIPE_EXPAND_SOURCE, &sh->state); 2828 for (i = 0; i < sh->disks; i++) 2829 if (i != sh->pd_idx && i != sh->qd_idx) { 2830 int dd_idx, j; 2831 struct stripe_head *sh2; 2832 struct async_submit_ctl submit; 2833 2834 sector_t bn = compute_blocknr(sh, i, 1); 2835 sector_t s = raid5_compute_sector(conf, bn, 0, 2836 &dd_idx, NULL); 2837 sh2 = get_active_stripe(conf, s, 0, 1, 1); 2838 if (sh2 == NULL) 2839 /* so far only the early blocks of this stripe 2840 * have been requested. When later blocks 2841 * get requested, we will try again 2842 */ 2843 continue; 2844 if (!test_bit(STRIPE_EXPANDING, &sh2->state) || 2845 test_bit(R5_Expanded, &sh2->dev[dd_idx].flags)) { 2846 /* must have already done this block */ 2847 release_stripe(sh2); 2848 continue; 2849 } 2850 2851 /* place all the copies on one channel */ 2852 init_async_submit(&submit, 0, tx, NULL, NULL, NULL); 2853 tx = async_memcpy(sh2->dev[dd_idx].page, 2854 sh->dev[i].page, 0, 0, STRIPE_SIZE, 2855 &submit); 2856 2857 set_bit(R5_Expanded, &sh2->dev[dd_idx].flags); 2858 set_bit(R5_UPTODATE, &sh2->dev[dd_idx].flags); 2859 for (j = 0; j < conf->raid_disks; j++) 2860 if (j != sh2->pd_idx && 2861 (!r6s || j != sh2->qd_idx) && 2862 !test_bit(R5_Expanded, &sh2->dev[j].flags)) 2863 break; 2864 if (j == conf->raid_disks) { 2865 set_bit(STRIPE_EXPAND_READY, &sh2->state); 2866 set_bit(STRIPE_HANDLE, &sh2->state); 2867 } 2868 release_stripe(sh2); 2869 2870 } 2871 /* done submitting copies, wait for them to complete */ 2872 if (tx) { 2873 async_tx_ack(tx); 2874 dma_wait_for_async_tx(tx); 2875 } 2876 } 2877 2878 2879 /* 2880 * handle_stripe - do things to a stripe. 2881 * 2882 * We lock the stripe and then examine the state of various bits 2883 * to see what needs to be done. 2884 * Possible results: 2885 * return some read request which now have data 2886 * return some write requests which are safely on disc 2887 * schedule a read on some buffers 2888 * schedule a write of some buffers 2889 * return confirmation of parity correctness 2890 * 2891 * buffers are taken off read_list or write_list, and bh_cache buffers 2892 * get BH_Lock set before the stripe lock is released. 2893 * 2894 */ 2895 2896 static bool handle_stripe5(struct stripe_head *sh) 2897 { 2898 raid5_conf_t *conf = sh->raid_conf; 2899 int disks = sh->disks, i; 2900 struct bio *return_bi = NULL; 2901 struct stripe_head_state s; 2902 struct r5dev *dev; 2903 mdk_rdev_t *blocked_rdev = NULL; 2904 int prexor; 2905 2906 memset(&s, 0, sizeof(s)); 2907 pr_debug("handling stripe %llu, state=%#lx cnt=%d, pd_idx=%d check:%d " 2908 "reconstruct:%d\n", (unsigned long long)sh->sector, sh->state, 2909 atomic_read(&sh->count), sh->pd_idx, sh->check_state, 2910 sh->reconstruct_state); 2911 2912 spin_lock(&sh->lock); 2913 clear_bit(STRIPE_HANDLE, &sh->state); 2914 clear_bit(STRIPE_DELAYED, &sh->state); 2915 2916 s.syncing = test_bit(STRIPE_SYNCING, &sh->state); 2917 s.expanding = test_bit(STRIPE_EXPAND_SOURCE, &sh->state); 2918 s.expanded = test_bit(STRIPE_EXPAND_READY, &sh->state); 2919 2920 /* Now to look around and see what can be done */ 2921 rcu_read_lock(); 2922 for (i=disks; i--; ) { 2923 mdk_rdev_t *rdev; 2924 struct r5dev *dev = &sh->dev[i]; 2925 clear_bit(R5_Insync, &dev->flags); 2926 2927 pr_debug("check %d: state 0x%lx toread %p read %p write %p " 2928 "written %p\n", i, dev->flags, dev->toread, dev->read, 2929 dev->towrite, dev->written); 2930 2931 /* maybe we can request a biofill operation 2932 * 2933 * new wantfill requests are only permitted while 2934 * ops_complete_biofill is guaranteed to be inactive 2935 */ 2936 if (test_bit(R5_UPTODATE, &dev->flags) && dev->toread && 2937 !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) 2938 set_bit(R5_Wantfill, &dev->flags); 2939 2940 /* now count some things */ 2941 if (test_bit(R5_LOCKED, &dev->flags)) s.locked++; 2942 if (test_bit(R5_UPTODATE, &dev->flags)) s.uptodate++; 2943 if (test_bit(R5_Wantcompute, &dev->flags)) s.compute++; 2944 2945 if (test_bit(R5_Wantfill, &dev->flags)) 2946 s.to_fill++; 2947 else if (dev->toread) 2948 s.to_read++; 2949 if (dev->towrite) { 2950 s.to_write++; 2951 if (!test_bit(R5_OVERWRITE, &dev->flags)) 2952 s.non_overwrite++; 2953 } 2954 if (dev->written) 2955 s.written++; 2956 rdev = rcu_dereference(conf->disks[i].rdev); 2957 if (blocked_rdev == NULL && 2958 rdev && unlikely(test_bit(Blocked, &rdev->flags))) { 2959 blocked_rdev = rdev; 2960 atomic_inc(&rdev->nr_pending); 2961 } 2962 if (!rdev || !test_bit(In_sync, &rdev->flags)) { 2963 /* The ReadError flag will just be confusing now */ 2964 clear_bit(R5_ReadError, &dev->flags); 2965 clear_bit(R5_ReWrite, &dev->flags); 2966 } 2967 if (!rdev || !test_bit(In_sync, &rdev->flags) 2968 || test_bit(R5_ReadError, &dev->flags)) { 2969 s.failed++; 2970 s.failed_num = i; 2971 } else 2972 set_bit(R5_Insync, &dev->flags); 2973 } 2974 rcu_read_unlock(); 2975 2976 if (unlikely(blocked_rdev)) { 2977 if (s.syncing || s.expanding || s.expanded || 2978 s.to_write || s.written) { 2979 set_bit(STRIPE_HANDLE, &sh->state); 2980 goto unlock; 2981 } 2982 /* There is nothing for the blocked_rdev to block */ 2983 rdev_dec_pending(blocked_rdev, conf->mddev); 2984 blocked_rdev = NULL; 2985 } 2986 2987 if (s.to_fill && !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) { 2988 set_bit(STRIPE_OP_BIOFILL, &s.ops_request); 2989 set_bit(STRIPE_BIOFILL_RUN, &sh->state); 2990 } 2991 2992 pr_debug("locked=%d uptodate=%d to_read=%d" 2993 " to_write=%d failed=%d failed_num=%d\n", 2994 s.locked, s.uptodate, s.to_read, s.to_write, 2995 s.failed, s.failed_num); 2996 /* check if the array has lost two devices and, if so, some requests might 2997 * need to be failed 2998 */ 2999 if (s.failed > 1 && s.to_read+s.to_write+s.written) 3000 handle_failed_stripe(conf, sh, &s, disks, &return_bi); 3001 if (s.failed > 1 && s.syncing) { 3002 md_done_sync(conf->mddev, STRIPE_SECTORS,0); 3003 clear_bit(STRIPE_SYNCING, &sh->state); 3004 s.syncing = 0; 3005 } 3006 3007 /* might be able to return some write requests if the parity block 3008 * is safe, or on a failed drive 3009 */ 3010 dev = &sh->dev[sh->pd_idx]; 3011 if ( s.written && 3012 ((test_bit(R5_Insync, &dev->flags) && 3013 !test_bit(R5_LOCKED, &dev->flags) && 3014 test_bit(R5_UPTODATE, &dev->flags)) || 3015 (s.failed == 1 && s.failed_num == sh->pd_idx))) 3016 handle_stripe_clean_event(conf, sh, disks, &return_bi); 3017 3018 /* Now we might consider reading some blocks, either to check/generate 3019 * parity, or to satisfy requests 3020 * or to load a block that is being partially written. 3021 */ 3022 if (s.to_read || s.non_overwrite || 3023 (s.syncing && (s.uptodate + s.compute < disks)) || s.expanding) 3024 handle_stripe_fill5(sh, &s, disks); 3025 3026 /* Now we check to see if any write operations have recently 3027 * completed 3028 */ 3029 prexor = 0; 3030 if (sh->reconstruct_state == reconstruct_state_prexor_drain_result) 3031 prexor = 1; 3032 if (sh->reconstruct_state == reconstruct_state_drain_result || 3033 sh->reconstruct_state == reconstruct_state_prexor_drain_result) { 3034 sh->reconstruct_state = reconstruct_state_idle; 3035 3036 /* All the 'written' buffers and the parity block are ready to 3037 * be written back to disk 3038 */ 3039 BUG_ON(!test_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags)); 3040 for (i = disks; i--; ) { 3041 dev = &sh->dev[i]; 3042 if (test_bit(R5_LOCKED, &dev->flags) && 3043 (i == sh->pd_idx || dev->written)) { 3044 pr_debug("Writing block %d\n", i); 3045 set_bit(R5_Wantwrite, &dev->flags); 3046 if (prexor) 3047 continue; 3048 if (!test_bit(R5_Insync, &dev->flags) || 3049 (i == sh->pd_idx && s.failed == 0)) 3050 set_bit(STRIPE_INSYNC, &sh->state); 3051 } 3052 } 3053 if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) { 3054 atomic_dec(&conf->preread_active_stripes); 3055 if (atomic_read(&conf->preread_active_stripes) < 3056 IO_THRESHOLD) 3057 md_wakeup_thread(conf->mddev->thread); 3058 } 3059 } 3060 3061 /* Now to consider new write requests and what else, if anything 3062 * should be read. We do not handle new writes when: 3063 * 1/ A 'write' operation (copy+xor) is already in flight. 3064 * 2/ A 'check' operation is in flight, as it may clobber the parity 3065 * block. 3066 */ 3067 if (s.to_write && !sh->reconstruct_state && !sh->check_state) 3068 handle_stripe_dirtying5(conf, sh, &s, disks); 3069 3070 /* maybe we need to check and possibly fix the parity for this stripe 3071 * Any reads will already have been scheduled, so we just see if enough 3072 * data is available. The parity check is held off while parity 3073 * dependent operations are in flight. 3074 */ 3075 if (sh->check_state || 3076 (s.syncing && s.locked == 0 && 3077 !test_bit(STRIPE_COMPUTE_RUN, &sh->state) && 3078 !test_bit(STRIPE_INSYNC, &sh->state))) 3079 handle_parity_checks5(conf, sh, &s, disks); 3080 3081 if (s.syncing && s.locked == 0 && test_bit(STRIPE_INSYNC, &sh->state)) { 3082 md_done_sync(conf->mddev, STRIPE_SECTORS,1); 3083 clear_bit(STRIPE_SYNCING, &sh->state); 3084 } 3085 3086 /* If the failed drive is just a ReadError, then we might need to progress 3087 * the repair/check process 3088 */ 3089 if (s.failed == 1 && !conf->mddev->ro && 3090 test_bit(R5_ReadError, &sh->dev[s.failed_num].flags) 3091 && !test_bit(R5_LOCKED, &sh->dev[s.failed_num].flags) 3092 && test_bit(R5_UPTODATE, &sh->dev[s.failed_num].flags) 3093 ) { 3094 dev = &sh->dev[s.failed_num]; 3095 if (!test_bit(R5_ReWrite, &dev->flags)) { 3096 set_bit(R5_Wantwrite, &dev->flags); 3097 set_bit(R5_ReWrite, &dev->flags); 3098 set_bit(R5_LOCKED, &dev->flags); 3099 s.locked++; 3100 } else { 3101 /* let's read it back */ 3102 set_bit(R5_Wantread, &dev->flags); 3103 set_bit(R5_LOCKED, &dev->flags); 3104 s.locked++; 3105 } 3106 } 3107 3108 /* Finish reconstruct operations initiated by the expansion process */ 3109 if (sh->reconstruct_state == reconstruct_state_result) { 3110 struct stripe_head *sh2 3111 = get_active_stripe(conf, sh->sector, 1, 1, 1); 3112 if (sh2 && test_bit(STRIPE_EXPAND_SOURCE, &sh2->state)) { 3113 /* sh cannot be written until sh2 has been read. 3114 * so arrange for sh to be delayed a little 3115 */ 3116 set_bit(STRIPE_DELAYED, &sh->state); 3117 set_bit(STRIPE_HANDLE, &sh->state); 3118 if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, 3119 &sh2->state)) 3120 atomic_inc(&conf->preread_active_stripes); 3121 release_stripe(sh2); 3122 goto unlock; 3123 } 3124 if (sh2) 3125 release_stripe(sh2); 3126 3127 sh->reconstruct_state = reconstruct_state_idle; 3128 clear_bit(STRIPE_EXPANDING, &sh->state); 3129 for (i = conf->raid_disks; i--; ) { 3130 set_bit(R5_Wantwrite, &sh->dev[i].flags); 3131 set_bit(R5_LOCKED, &sh->dev[i].flags); 3132 s.locked++; 3133 } 3134 } 3135 3136 if (s.expanded && test_bit(STRIPE_EXPANDING, &sh->state) && 3137 !sh->reconstruct_state) { 3138 /* Need to write out all blocks after computing parity */ 3139 sh->disks = conf->raid_disks; 3140 stripe_set_idx(sh->sector, conf, 0, sh); 3141 schedule_reconstruction(sh, &s, 1, 1); 3142 } else if (s.expanded && !sh->reconstruct_state && s.locked == 0) { 3143 clear_bit(STRIPE_EXPAND_READY, &sh->state); 3144 atomic_dec(&conf->reshape_stripes); 3145 wake_up(&conf->wait_for_overlap); 3146 md_done_sync(conf->mddev, STRIPE_SECTORS, 1); 3147 } 3148 3149 if (s.expanding && s.locked == 0 && 3150 !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) 3151 handle_stripe_expansion(conf, sh, NULL); 3152 3153 unlock: 3154 spin_unlock(&sh->lock); 3155 3156 /* wait for this device to become unblocked */ 3157 if (unlikely(blocked_rdev)) 3158 md_wait_for_blocked_rdev(blocked_rdev, conf->mddev); 3159 3160 if (s.ops_request) 3161 raid_run_ops(sh, s.ops_request); 3162 3163 ops_run_io(sh, &s); 3164 3165 return_io(return_bi); 3166 3167 return blocked_rdev == NULL; 3168 } 3169 3170 static bool handle_stripe6(struct stripe_head *sh) 3171 { 3172 raid5_conf_t *conf = sh->raid_conf; 3173 int disks = sh->disks; 3174 struct bio *return_bi = NULL; 3175 int i, pd_idx = sh->pd_idx, qd_idx = sh->qd_idx; 3176 struct stripe_head_state s; 3177 struct r6_state r6s; 3178 struct r5dev *dev, *pdev, *qdev; 3179 mdk_rdev_t *blocked_rdev = NULL; 3180 3181 pr_debug("handling stripe %llu, state=%#lx cnt=%d, " 3182 "pd_idx=%d, qd_idx=%d\n, check:%d, reconstruct:%d\n", 3183 (unsigned long long)sh->sector, sh->state, 3184 atomic_read(&sh->count), pd_idx, qd_idx, 3185 sh->check_state, sh->reconstruct_state); 3186 memset(&s, 0, sizeof(s)); 3187 3188 spin_lock(&sh->lock); 3189 clear_bit(STRIPE_HANDLE, &sh->state); 3190 clear_bit(STRIPE_DELAYED, &sh->state); 3191 3192 s.syncing = test_bit(STRIPE_SYNCING, &sh->state); 3193 s.expanding = test_bit(STRIPE_EXPAND_SOURCE, &sh->state); 3194 s.expanded = test_bit(STRIPE_EXPAND_READY, &sh->state); 3195 /* Now to look around and see what can be done */ 3196 3197 rcu_read_lock(); 3198 for (i=disks; i--; ) { 3199 mdk_rdev_t *rdev; 3200 dev = &sh->dev[i]; 3201 clear_bit(R5_Insync, &dev->flags); 3202 3203 pr_debug("check %d: state 0x%lx read %p write %p written %p\n", 3204 i, dev->flags, dev->toread, dev->towrite, dev->written); 3205 /* maybe we can reply to a read 3206 * 3207 * new wantfill requests are only permitted while 3208 * ops_complete_biofill is guaranteed to be inactive 3209 */ 3210 if (test_bit(R5_UPTODATE, &dev->flags) && dev->toread && 3211 !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) 3212 set_bit(R5_Wantfill, &dev->flags); 3213 3214 /* now count some things */ 3215 if (test_bit(R5_LOCKED, &dev->flags)) s.locked++; 3216 if (test_bit(R5_UPTODATE, &dev->flags)) s.uptodate++; 3217 if (test_bit(R5_Wantcompute, &dev->flags)) 3218 BUG_ON(++s.compute > 2); 3219 3220 if (test_bit(R5_Wantfill, &dev->flags)) { 3221 s.to_fill++; 3222 } else if (dev->toread) 3223 s.to_read++; 3224 if (dev->towrite) { 3225 s.to_write++; 3226 if (!test_bit(R5_OVERWRITE, &dev->flags)) 3227 s.non_overwrite++; 3228 } 3229 if (dev->written) 3230 s.written++; 3231 rdev = rcu_dereference(conf->disks[i].rdev); 3232 if (blocked_rdev == NULL && 3233 rdev && unlikely(test_bit(Blocked, &rdev->flags))) { 3234 blocked_rdev = rdev; 3235 atomic_inc(&rdev->nr_pending); 3236 } 3237 if (!rdev || !test_bit(In_sync, &rdev->flags)) { 3238 /* The ReadError flag will just be confusing now */ 3239 clear_bit(R5_ReadError, &dev->flags); 3240 clear_bit(R5_ReWrite, &dev->flags); 3241 } 3242 if (!rdev || !test_bit(In_sync, &rdev->flags) 3243 || test_bit(R5_ReadError, &dev->flags)) { 3244 if (s.failed < 2) 3245 r6s.failed_num[s.failed] = i; 3246 s.failed++; 3247 } else 3248 set_bit(R5_Insync, &dev->flags); 3249 } 3250 rcu_read_unlock(); 3251 3252 if (unlikely(blocked_rdev)) { 3253 if (s.syncing || s.expanding || s.expanded || 3254 s.to_write || s.written) { 3255 set_bit(STRIPE_HANDLE, &sh->state); 3256 goto unlock; 3257 } 3258 /* There is nothing for the blocked_rdev to block */ 3259 rdev_dec_pending(blocked_rdev, conf->mddev); 3260 blocked_rdev = NULL; 3261 } 3262 3263 if (s.to_fill && !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) { 3264 set_bit(STRIPE_OP_BIOFILL, &s.ops_request); 3265 set_bit(STRIPE_BIOFILL_RUN, &sh->state); 3266 } 3267 3268 pr_debug("locked=%d uptodate=%d to_read=%d" 3269 " to_write=%d failed=%d failed_num=%d,%d\n", 3270 s.locked, s.uptodate, s.to_read, s.to_write, s.failed, 3271 r6s.failed_num[0], r6s.failed_num[1]); 3272 /* check if the array has lost >2 devices and, if so, some requests 3273 * might need to be failed 3274 */ 3275 if (s.failed > 2 && s.to_read+s.to_write+s.written) 3276 handle_failed_stripe(conf, sh, &s, disks, &return_bi); 3277 if (s.failed > 2 && s.syncing) { 3278 md_done_sync(conf->mddev, STRIPE_SECTORS,0); 3279 clear_bit(STRIPE_SYNCING, &sh->state); 3280 s.syncing = 0; 3281 } 3282 3283 /* 3284 * might be able to return some write requests if the parity blocks 3285 * are safe, or on a failed drive 3286 */ 3287 pdev = &sh->dev[pd_idx]; 3288 r6s.p_failed = (s.failed >= 1 && r6s.failed_num[0] == pd_idx) 3289 || (s.failed >= 2 && r6s.failed_num[1] == pd_idx); 3290 qdev = &sh->dev[qd_idx]; 3291 r6s.q_failed = (s.failed >= 1 && r6s.failed_num[0] == qd_idx) 3292 || (s.failed >= 2 && r6s.failed_num[1] == qd_idx); 3293 3294 if ( s.written && 3295 ( r6s.p_failed || ((test_bit(R5_Insync, &pdev->flags) 3296 && !test_bit(R5_LOCKED, &pdev->flags) 3297 && test_bit(R5_UPTODATE, &pdev->flags)))) && 3298 ( r6s.q_failed || ((test_bit(R5_Insync, &qdev->flags) 3299 && !test_bit(R5_LOCKED, &qdev->flags) 3300 && test_bit(R5_UPTODATE, &qdev->flags))))) 3301 handle_stripe_clean_event(conf, sh, disks, &return_bi); 3302 3303 /* Now we might consider reading some blocks, either to check/generate 3304 * parity, or to satisfy requests 3305 * or to load a block that is being partially written. 3306 */ 3307 if (s.to_read || s.non_overwrite || (s.to_write && s.failed) || 3308 (s.syncing && (s.uptodate + s.compute < disks)) || s.expanding) 3309 handle_stripe_fill6(sh, &s, &r6s, disks); 3310 3311 /* Now we check to see if any write operations have recently 3312 * completed 3313 */ 3314 if (sh->reconstruct_state == reconstruct_state_drain_result) { 3315 int qd_idx = sh->qd_idx; 3316 3317 sh->reconstruct_state = reconstruct_state_idle; 3318 /* All the 'written' buffers and the parity blocks are ready to 3319 * be written back to disk 3320 */ 3321 BUG_ON(!test_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags)); 3322 BUG_ON(!test_bit(R5_UPTODATE, &sh->dev[qd_idx].flags)); 3323 for (i = disks; i--; ) { 3324 dev = &sh->dev[i]; 3325 if (test_bit(R5_LOCKED, &dev->flags) && 3326 (i == sh->pd_idx || i == qd_idx || 3327 dev->written)) { 3328 pr_debug("Writing block %d\n", i); 3329 BUG_ON(!test_bit(R5_UPTODATE, &dev->flags)); 3330 set_bit(R5_Wantwrite, &dev->flags); 3331 if (!test_bit(R5_Insync, &dev->flags) || 3332 ((i == sh->pd_idx || i == qd_idx) && 3333 s.failed == 0)) 3334 set_bit(STRIPE_INSYNC, &sh->state); 3335 } 3336 } 3337 if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) { 3338 atomic_dec(&conf->preread_active_stripes); 3339 if (atomic_read(&conf->preread_active_stripes) < 3340 IO_THRESHOLD) 3341 md_wakeup_thread(conf->mddev->thread); 3342 } 3343 } 3344 3345 /* Now to consider new write requests and what else, if anything 3346 * should be read. We do not handle new writes when: 3347 * 1/ A 'write' operation (copy+gen_syndrome) is already in flight. 3348 * 2/ A 'check' operation is in flight, as it may clobber the parity 3349 * block. 3350 */ 3351 if (s.to_write && !sh->reconstruct_state && !sh->check_state) 3352 handle_stripe_dirtying6(conf, sh, &s, &r6s, disks); 3353 3354 /* maybe we need to check and possibly fix the parity for this stripe 3355 * Any reads will already have been scheduled, so we just see if enough 3356 * data is available. The parity check is held off while parity 3357 * dependent operations are in flight. 3358 */ 3359 if (sh->check_state || 3360 (s.syncing && s.locked == 0 && 3361 !test_bit(STRIPE_COMPUTE_RUN, &sh->state) && 3362 !test_bit(STRIPE_INSYNC, &sh->state))) 3363 handle_parity_checks6(conf, sh, &s, &r6s, disks); 3364 3365 if (s.syncing && s.locked == 0 && test_bit(STRIPE_INSYNC, &sh->state)) { 3366 md_done_sync(conf->mddev, STRIPE_SECTORS,1); 3367 clear_bit(STRIPE_SYNCING, &sh->state); 3368 } 3369 3370 /* If the failed drives are just a ReadError, then we might need 3371 * to progress the repair/check process 3372 */ 3373 if (s.failed <= 2 && !conf->mddev->ro) 3374 for (i = 0; i < s.failed; i++) { 3375 dev = &sh->dev[r6s.failed_num[i]]; 3376 if (test_bit(R5_ReadError, &dev->flags) 3377 && !test_bit(R5_LOCKED, &dev->flags) 3378 && test_bit(R5_UPTODATE, &dev->flags) 3379 ) { 3380 if (!test_bit(R5_ReWrite, &dev->flags)) { 3381 set_bit(R5_Wantwrite, &dev->flags); 3382 set_bit(R5_ReWrite, &dev->flags); 3383 set_bit(R5_LOCKED, &dev->flags); 3384 s.locked++; 3385 } else { 3386 /* let's read it back */ 3387 set_bit(R5_Wantread, &dev->flags); 3388 set_bit(R5_LOCKED, &dev->flags); 3389 s.locked++; 3390 } 3391 } 3392 } 3393 3394 /* Finish reconstruct operations initiated by the expansion process */ 3395 if (sh->reconstruct_state == reconstruct_state_result) { 3396 sh->reconstruct_state = reconstruct_state_idle; 3397 clear_bit(STRIPE_EXPANDING, &sh->state); 3398 for (i = conf->raid_disks; i--; ) { 3399 set_bit(R5_Wantwrite, &sh->dev[i].flags); 3400 set_bit(R5_LOCKED, &sh->dev[i].flags); 3401 s.locked++; 3402 } 3403 } 3404 3405 if (s.expanded && test_bit(STRIPE_EXPANDING, &sh->state) && 3406 !sh->reconstruct_state) { 3407 struct stripe_head *sh2 3408 = get_active_stripe(conf, sh->sector, 1, 1, 1); 3409 if (sh2 && test_bit(STRIPE_EXPAND_SOURCE, &sh2->state)) { 3410 /* sh cannot be written until sh2 has been read. 3411 * so arrange for sh to be delayed a little 3412 */ 3413 set_bit(STRIPE_DELAYED, &sh->state); 3414 set_bit(STRIPE_HANDLE, &sh->state); 3415 if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, 3416 &sh2->state)) 3417 atomic_inc(&conf->preread_active_stripes); 3418 release_stripe(sh2); 3419 goto unlock; 3420 } 3421 if (sh2) 3422 release_stripe(sh2); 3423 3424 /* Need to write out all blocks after computing P&Q */ 3425 sh->disks = conf->raid_disks; 3426 stripe_set_idx(sh->sector, conf, 0, sh); 3427 schedule_reconstruction(sh, &s, 1, 1); 3428 } else if (s.expanded && !sh->reconstruct_state && s.locked == 0) { 3429 clear_bit(STRIPE_EXPAND_READY, &sh->state); 3430 atomic_dec(&conf->reshape_stripes); 3431 wake_up(&conf->wait_for_overlap); 3432 md_done_sync(conf->mddev, STRIPE_SECTORS, 1); 3433 } 3434 3435 if (s.expanding && s.locked == 0 && 3436 !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) 3437 handle_stripe_expansion(conf, sh, &r6s); 3438 3439 unlock: 3440 spin_unlock(&sh->lock); 3441 3442 /* wait for this device to become unblocked */ 3443 if (unlikely(blocked_rdev)) 3444 md_wait_for_blocked_rdev(blocked_rdev, conf->mddev); 3445 3446 if (s.ops_request) 3447 raid_run_ops(sh, s.ops_request); 3448 3449 ops_run_io(sh, &s); 3450 3451 return_io(return_bi); 3452 3453 return blocked_rdev == NULL; 3454 } 3455 3456 /* returns true if the stripe was handled */ 3457 static bool handle_stripe(struct stripe_head *sh) 3458 { 3459 if (sh->raid_conf->level == 6) 3460 return handle_stripe6(sh); 3461 else 3462 return handle_stripe5(sh); 3463 } 3464 3465 static void raid5_activate_delayed(raid5_conf_t *conf) 3466 { 3467 if (atomic_read(&conf->preread_active_stripes) < IO_THRESHOLD) { 3468 while (!list_empty(&conf->delayed_list)) { 3469 struct list_head *l = conf->delayed_list.next; 3470 struct stripe_head *sh; 3471 sh = list_entry(l, struct stripe_head, lru); 3472 list_del_init(l); 3473 clear_bit(STRIPE_DELAYED, &sh->state); 3474 if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) 3475 atomic_inc(&conf->preread_active_stripes); 3476 list_add_tail(&sh->lru, &conf->hold_list); 3477 } 3478 } else 3479 blk_plug_device(conf->mddev->queue); 3480 } 3481 3482 static void activate_bit_delay(raid5_conf_t *conf) 3483 { 3484 /* device_lock is held */ 3485 struct list_head head; 3486 list_add(&head, &conf->bitmap_list); 3487 list_del_init(&conf->bitmap_list); 3488 while (!list_empty(&head)) { 3489 struct stripe_head *sh = list_entry(head.next, struct stripe_head, lru); 3490 list_del_init(&sh->lru); 3491 atomic_inc(&sh->count); 3492 __release_stripe(conf, sh); 3493 } 3494 } 3495 3496 static void unplug_slaves(mddev_t *mddev) 3497 { 3498 raid5_conf_t *conf = mddev->private; 3499 int i; 3500 3501 rcu_read_lock(); 3502 for (i = 0; i < conf->raid_disks; i++) { 3503 mdk_rdev_t *rdev = rcu_dereference(conf->disks[i].rdev); 3504 if (rdev && !test_bit(Faulty, &rdev->flags) && atomic_read(&rdev->nr_pending)) { 3505 struct request_queue *r_queue = bdev_get_queue(rdev->bdev); 3506 3507 atomic_inc(&rdev->nr_pending); 3508 rcu_read_unlock(); 3509 3510 blk_unplug(r_queue); 3511 3512 rdev_dec_pending(rdev, mddev); 3513 rcu_read_lock(); 3514 } 3515 } 3516 rcu_read_unlock(); 3517 } 3518 3519 static void raid5_unplug_device(struct request_queue *q) 3520 { 3521 mddev_t *mddev = q->queuedata; 3522 raid5_conf_t *conf = mddev->private; 3523 unsigned long flags; 3524 3525 spin_lock_irqsave(&conf->device_lock, flags); 3526 3527 if (blk_remove_plug(q)) { 3528 conf->seq_flush++; 3529 raid5_activate_delayed(conf); 3530 } 3531 md_wakeup_thread(mddev->thread); 3532 3533 spin_unlock_irqrestore(&conf->device_lock, flags); 3534 3535 unplug_slaves(mddev); 3536 } 3537 3538 static int raid5_congested(void *data, int bits) 3539 { 3540 mddev_t *mddev = data; 3541 raid5_conf_t *conf = mddev->private; 3542 3543 /* No difference between reads and writes. Just check 3544 * how busy the stripe_cache is 3545 */ 3546 if (conf->inactive_blocked) 3547 return 1; 3548 if (conf->quiesce) 3549 return 1; 3550 if (list_empty_careful(&conf->inactive_list)) 3551 return 1; 3552 3553 return 0; 3554 } 3555 3556 /* We want read requests to align with chunks where possible, 3557 * but write requests don't need to. 3558 */ 3559 static int raid5_mergeable_bvec(struct request_queue *q, 3560 struct bvec_merge_data *bvm, 3561 struct bio_vec *biovec) 3562 { 3563 mddev_t *mddev = q->queuedata; 3564 sector_t sector = bvm->bi_sector + get_start_sect(bvm->bi_bdev); 3565 int max; 3566 unsigned int chunk_sectors = mddev->chunk_sectors; 3567 unsigned int bio_sectors = bvm->bi_size >> 9; 3568 3569 if ((bvm->bi_rw & 1) == WRITE) 3570 return biovec->bv_len; /* always allow writes to be mergeable */ 3571 3572 if (mddev->new_chunk_sectors < mddev->chunk_sectors) 3573 chunk_sectors = mddev->new_chunk_sectors; 3574 max = (chunk_sectors - ((sector & (chunk_sectors - 1)) + bio_sectors)) << 9; 3575 if (max < 0) max = 0; 3576 if (max <= biovec->bv_len && bio_sectors == 0) 3577 return biovec->bv_len; 3578 else 3579 return max; 3580 } 3581 3582 3583 static int in_chunk_boundary(mddev_t *mddev, struct bio *bio) 3584 { 3585 sector_t sector = bio->bi_sector + get_start_sect(bio->bi_bdev); 3586 unsigned int chunk_sectors = mddev->chunk_sectors; 3587 unsigned int bio_sectors = bio->bi_size >> 9; 3588 3589 if (mddev->new_chunk_sectors < mddev->chunk_sectors) 3590 chunk_sectors = mddev->new_chunk_sectors; 3591 return chunk_sectors >= 3592 ((sector & (chunk_sectors - 1)) + bio_sectors); 3593 } 3594 3595 /* 3596 * add bio to the retry LIFO ( in O(1) ... we are in interrupt ) 3597 * later sampled by raid5d. 3598 */ 3599 static void add_bio_to_retry(struct bio *bi,raid5_conf_t *conf) 3600 { 3601 unsigned long flags; 3602 3603 spin_lock_irqsave(&conf->device_lock, flags); 3604 3605 bi->bi_next = conf->retry_read_aligned_list; 3606 conf->retry_read_aligned_list = bi; 3607 3608 spin_unlock_irqrestore(&conf->device_lock, flags); 3609 md_wakeup_thread(conf->mddev->thread); 3610 } 3611 3612 3613 static struct bio *remove_bio_from_retry(raid5_conf_t *conf) 3614 { 3615 struct bio *bi; 3616 3617 bi = conf->retry_read_aligned; 3618 if (bi) { 3619 conf->retry_read_aligned = NULL; 3620 return bi; 3621 } 3622 bi = conf->retry_read_aligned_list; 3623 if(bi) { 3624 conf->retry_read_aligned_list = bi->bi_next; 3625 bi->bi_next = NULL; 3626 /* 3627 * this sets the active strip count to 1 and the processed 3628 * strip count to zero (upper 8 bits) 3629 */ 3630 bi->bi_phys_segments = 1; /* biased count of active stripes */ 3631 } 3632 3633 return bi; 3634 } 3635 3636 3637 /* 3638 * The "raid5_align_endio" should check if the read succeeded and if it 3639 * did, call bio_endio on the original bio (having bio_put the new bio 3640 * first). 3641 * If the read failed.. 3642 */ 3643 static void raid5_align_endio(struct bio *bi, int error) 3644 { 3645 struct bio* raid_bi = bi->bi_private; 3646 mddev_t *mddev; 3647 raid5_conf_t *conf; 3648 int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags); 3649 mdk_rdev_t *rdev; 3650 3651 bio_put(bi); 3652 3653 mddev = raid_bi->bi_bdev->bd_disk->queue->queuedata; 3654 conf = mddev->private; 3655 rdev = (void*)raid_bi->bi_next; 3656 raid_bi->bi_next = NULL; 3657 3658 rdev_dec_pending(rdev, conf->mddev); 3659 3660 if (!error && uptodate) { 3661 bio_endio(raid_bi, 0); 3662 if (atomic_dec_and_test(&conf->active_aligned_reads)) 3663 wake_up(&conf->wait_for_stripe); 3664 return; 3665 } 3666 3667 3668 pr_debug("raid5_align_endio : io error...handing IO for a retry\n"); 3669 3670 add_bio_to_retry(raid_bi, conf); 3671 } 3672 3673 static int bio_fits_rdev(struct bio *bi) 3674 { 3675 struct request_queue *q = bdev_get_queue(bi->bi_bdev); 3676 3677 if ((bi->bi_size>>9) > queue_max_sectors(q)) 3678 return 0; 3679 blk_recount_segments(q, bi); 3680 if (bi->bi_phys_segments > queue_max_phys_segments(q)) 3681 return 0; 3682 3683 if (q->merge_bvec_fn) 3684 /* it's too hard to apply the merge_bvec_fn at this stage, 3685 * just just give up 3686 */ 3687 return 0; 3688 3689 return 1; 3690 } 3691 3692 3693 static int chunk_aligned_read(struct request_queue *q, struct bio * raid_bio) 3694 { 3695 mddev_t *mddev = q->queuedata; 3696 raid5_conf_t *conf = mddev->private; 3697 unsigned int dd_idx; 3698 struct bio* align_bi; 3699 mdk_rdev_t *rdev; 3700 3701 if (!in_chunk_boundary(mddev, raid_bio)) { 3702 pr_debug("chunk_aligned_read : non aligned\n"); 3703 return 0; 3704 } 3705 /* 3706 * use bio_clone to make a copy of the bio 3707 */ 3708 align_bi = bio_clone(raid_bio, GFP_NOIO); 3709 if (!align_bi) 3710 return 0; 3711 /* 3712 * set bi_end_io to a new function, and set bi_private to the 3713 * original bio. 3714 */ 3715 align_bi->bi_end_io = raid5_align_endio; 3716 align_bi->bi_private = raid_bio; 3717 /* 3718 * compute position 3719 */ 3720 align_bi->bi_sector = raid5_compute_sector(conf, raid_bio->bi_sector, 3721 0, 3722 &dd_idx, NULL); 3723 3724 rcu_read_lock(); 3725 rdev = rcu_dereference(conf->disks[dd_idx].rdev); 3726 if (rdev && test_bit(In_sync, &rdev->flags)) { 3727 atomic_inc(&rdev->nr_pending); 3728 rcu_read_unlock(); 3729 raid_bio->bi_next = (void*)rdev; 3730 align_bi->bi_bdev = rdev->bdev; 3731 align_bi->bi_flags &= ~(1 << BIO_SEG_VALID); 3732 align_bi->bi_sector += rdev->data_offset; 3733 3734 if (!bio_fits_rdev(align_bi)) { 3735 /* too big in some way */ 3736 bio_put(align_bi); 3737 rdev_dec_pending(rdev, mddev); 3738 return 0; 3739 } 3740 3741 spin_lock_irq(&conf->device_lock); 3742 wait_event_lock_irq(conf->wait_for_stripe, 3743 conf->quiesce == 0, 3744 conf->device_lock, /* nothing */); 3745 atomic_inc(&conf->active_aligned_reads); 3746 spin_unlock_irq(&conf->device_lock); 3747 3748 generic_make_request(align_bi); 3749 return 1; 3750 } else { 3751 rcu_read_unlock(); 3752 bio_put(align_bi); 3753 return 0; 3754 } 3755 } 3756 3757 /* __get_priority_stripe - get the next stripe to process 3758 * 3759 * Full stripe writes are allowed to pass preread active stripes up until 3760 * the bypass_threshold is exceeded. In general the bypass_count 3761 * increments when the handle_list is handled before the hold_list; however, it 3762 * will not be incremented when STRIPE_IO_STARTED is sampled set signifying a 3763 * stripe with in flight i/o. The bypass_count will be reset when the 3764 * head of the hold_list has changed, i.e. the head was promoted to the 3765 * handle_list. 3766 */ 3767 static struct stripe_head *__get_priority_stripe(raid5_conf_t *conf) 3768 { 3769 struct stripe_head *sh; 3770 3771 pr_debug("%s: handle: %s hold: %s full_writes: %d bypass_count: %d\n", 3772 __func__, 3773 list_empty(&conf->handle_list) ? "empty" : "busy", 3774 list_empty(&conf->hold_list) ? "empty" : "busy", 3775 atomic_read(&conf->pending_full_writes), conf->bypass_count); 3776 3777 if (!list_empty(&conf->handle_list)) { 3778 sh = list_entry(conf->handle_list.next, typeof(*sh), lru); 3779 3780 if (list_empty(&conf->hold_list)) 3781 conf->bypass_count = 0; 3782 else if (!test_bit(STRIPE_IO_STARTED, &sh->state)) { 3783 if (conf->hold_list.next == conf->last_hold) 3784 conf->bypass_count++; 3785 else { 3786 conf->last_hold = conf->hold_list.next; 3787 conf->bypass_count -= conf->bypass_threshold; 3788 if (conf->bypass_count < 0) 3789 conf->bypass_count = 0; 3790 } 3791 } 3792 } else if (!list_empty(&conf->hold_list) && 3793 ((conf->bypass_threshold && 3794 conf->bypass_count > conf->bypass_threshold) || 3795 atomic_read(&conf->pending_full_writes) == 0)) { 3796 sh = list_entry(conf->hold_list.next, 3797 typeof(*sh), lru); 3798 conf->bypass_count -= conf->bypass_threshold; 3799 if (conf->bypass_count < 0) 3800 conf->bypass_count = 0; 3801 } else 3802 return NULL; 3803 3804 list_del_init(&sh->lru); 3805 atomic_inc(&sh->count); 3806 BUG_ON(atomic_read(&sh->count) != 1); 3807 return sh; 3808 } 3809 3810 static int make_request(struct request_queue *q, struct bio * bi) 3811 { 3812 mddev_t *mddev = q->queuedata; 3813 raid5_conf_t *conf = mddev->private; 3814 int dd_idx; 3815 sector_t new_sector; 3816 sector_t logical_sector, last_sector; 3817 struct stripe_head *sh; 3818 const int rw = bio_data_dir(bi); 3819 int cpu, remaining; 3820 3821 if (unlikely(bio_barrier(bi))) { 3822 bio_endio(bi, -EOPNOTSUPP); 3823 return 0; 3824 } 3825 3826 md_write_start(mddev, bi); 3827 3828 cpu = part_stat_lock(); 3829 part_stat_inc(cpu, &mddev->gendisk->part0, ios[rw]); 3830 part_stat_add(cpu, &mddev->gendisk->part0, sectors[rw], 3831 bio_sectors(bi)); 3832 part_stat_unlock(); 3833 3834 if (rw == READ && 3835 mddev->reshape_position == MaxSector && 3836 chunk_aligned_read(q,bi)) 3837 return 0; 3838 3839 logical_sector = bi->bi_sector & ~((sector_t)STRIPE_SECTORS-1); 3840 last_sector = bi->bi_sector + (bi->bi_size>>9); 3841 bi->bi_next = NULL; 3842 bi->bi_phys_segments = 1; /* over-loaded to count active stripes */ 3843 3844 for (;logical_sector < last_sector; logical_sector += STRIPE_SECTORS) { 3845 DEFINE_WAIT(w); 3846 int disks, data_disks; 3847 int previous; 3848 3849 retry: 3850 previous = 0; 3851 disks = conf->raid_disks; 3852 prepare_to_wait(&conf->wait_for_overlap, &w, TASK_UNINTERRUPTIBLE); 3853 if (unlikely(conf->reshape_progress != MaxSector)) { 3854 /* spinlock is needed as reshape_progress may be 3855 * 64bit on a 32bit platform, and so it might be 3856 * possible to see a half-updated value 3857 * Ofcourse reshape_progress could change after 3858 * the lock is dropped, so once we get a reference 3859 * to the stripe that we think it is, we will have 3860 * to check again. 3861 */ 3862 spin_lock_irq(&conf->device_lock); 3863 if (mddev->delta_disks < 0 3864 ? logical_sector < conf->reshape_progress 3865 : logical_sector >= conf->reshape_progress) { 3866 disks = conf->previous_raid_disks; 3867 previous = 1; 3868 } else { 3869 if (mddev->delta_disks < 0 3870 ? logical_sector < conf->reshape_safe 3871 : logical_sector >= conf->reshape_safe) { 3872 spin_unlock_irq(&conf->device_lock); 3873 schedule(); 3874 goto retry; 3875 } 3876 } 3877 spin_unlock_irq(&conf->device_lock); 3878 } 3879 data_disks = disks - conf->max_degraded; 3880 3881 new_sector = raid5_compute_sector(conf, logical_sector, 3882 previous, 3883 &dd_idx, NULL); 3884 pr_debug("raid5: make_request, sector %llu logical %llu\n", 3885 (unsigned long long)new_sector, 3886 (unsigned long long)logical_sector); 3887 3888 sh = get_active_stripe(conf, new_sector, previous, 3889 (bi->bi_rw&RWA_MASK), 0); 3890 if (sh) { 3891 if (unlikely(previous)) { 3892 /* expansion might have moved on while waiting for a 3893 * stripe, so we must do the range check again. 3894 * Expansion could still move past after this 3895 * test, but as we are holding a reference to 3896 * 'sh', we know that if that happens, 3897 * STRIPE_EXPANDING will get set and the expansion 3898 * won't proceed until we finish with the stripe. 3899 */ 3900 int must_retry = 0; 3901 spin_lock_irq(&conf->device_lock); 3902 if (mddev->delta_disks < 0 3903 ? logical_sector >= conf->reshape_progress 3904 : logical_sector < conf->reshape_progress) 3905 /* mismatch, need to try again */ 3906 must_retry = 1; 3907 spin_unlock_irq(&conf->device_lock); 3908 if (must_retry) { 3909 release_stripe(sh); 3910 schedule(); 3911 goto retry; 3912 } 3913 } 3914 3915 if (bio_data_dir(bi) == WRITE && 3916 logical_sector >= mddev->suspend_lo && 3917 logical_sector < mddev->suspend_hi) { 3918 release_stripe(sh); 3919 /* As the suspend_* range is controlled by 3920 * userspace, we want an interruptible 3921 * wait. 3922 */ 3923 flush_signals(current); 3924 prepare_to_wait(&conf->wait_for_overlap, 3925 &w, TASK_INTERRUPTIBLE); 3926 if (logical_sector >= mddev->suspend_lo && 3927 logical_sector < mddev->suspend_hi) 3928 schedule(); 3929 goto retry; 3930 } 3931 3932 if (test_bit(STRIPE_EXPANDING, &sh->state) || 3933 !add_stripe_bio(sh, bi, dd_idx, (bi->bi_rw&RW_MASK))) { 3934 /* Stripe is busy expanding or 3935 * add failed due to overlap. Flush everything 3936 * and wait a while 3937 */ 3938 raid5_unplug_device(mddev->queue); 3939 release_stripe(sh); 3940 schedule(); 3941 goto retry; 3942 } 3943 finish_wait(&conf->wait_for_overlap, &w); 3944 set_bit(STRIPE_HANDLE, &sh->state); 3945 clear_bit(STRIPE_DELAYED, &sh->state); 3946 release_stripe(sh); 3947 } else { 3948 /* cannot get stripe for read-ahead, just give-up */ 3949 clear_bit(BIO_UPTODATE, &bi->bi_flags); 3950 finish_wait(&conf->wait_for_overlap, &w); 3951 break; 3952 } 3953 3954 } 3955 spin_lock_irq(&conf->device_lock); 3956 remaining = raid5_dec_bi_phys_segments(bi); 3957 spin_unlock_irq(&conf->device_lock); 3958 if (remaining == 0) { 3959 3960 if ( rw == WRITE ) 3961 md_write_end(mddev); 3962 3963 bio_endio(bi, 0); 3964 } 3965 return 0; 3966 } 3967 3968 static sector_t raid5_size(mddev_t *mddev, sector_t sectors, int raid_disks); 3969 3970 static sector_t reshape_request(mddev_t *mddev, sector_t sector_nr, int *skipped) 3971 { 3972 /* reshaping is quite different to recovery/resync so it is 3973 * handled quite separately ... here. 3974 * 3975 * On each call to sync_request, we gather one chunk worth of 3976 * destination stripes and flag them as expanding. 3977 * Then we find all the source stripes and request reads. 3978 * As the reads complete, handle_stripe will copy the data 3979 * into the destination stripe and release that stripe. 3980 */ 3981 raid5_conf_t *conf = (raid5_conf_t *) mddev->private; 3982 struct stripe_head *sh; 3983 sector_t first_sector, last_sector; 3984 int raid_disks = conf->previous_raid_disks; 3985 int data_disks = raid_disks - conf->max_degraded; 3986 int new_data_disks = conf->raid_disks - conf->max_degraded; 3987 int i; 3988 int dd_idx; 3989 sector_t writepos, readpos, safepos; 3990 sector_t stripe_addr; 3991 int reshape_sectors; 3992 struct list_head stripes; 3993 3994 if (sector_nr == 0) { 3995 /* If restarting in the middle, skip the initial sectors */ 3996 if (mddev->delta_disks < 0 && 3997 conf->reshape_progress < raid5_size(mddev, 0, 0)) { 3998 sector_nr = raid5_size(mddev, 0, 0) 3999 - conf->reshape_progress; 4000 } else if (mddev->delta_disks >= 0 && 4001 conf->reshape_progress > 0) 4002 sector_nr = conf->reshape_progress; 4003 sector_div(sector_nr, new_data_disks); 4004 if (sector_nr) { 4005 *skipped = 1; 4006 return sector_nr; 4007 } 4008 } 4009 4010 /* We need to process a full chunk at a time. 4011 * If old and new chunk sizes differ, we need to process the 4012 * largest of these 4013 */ 4014 if (mddev->new_chunk_sectors > mddev->chunk_sectors) 4015 reshape_sectors = mddev->new_chunk_sectors; 4016 else 4017 reshape_sectors = mddev->chunk_sectors; 4018 4019 /* we update the metadata when there is more than 3Meg 4020 * in the block range (that is rather arbitrary, should 4021 * probably be time based) or when the data about to be 4022 * copied would over-write the source of the data at 4023 * the front of the range. 4024 * i.e. one new_stripe along from reshape_progress new_maps 4025 * to after where reshape_safe old_maps to 4026 */ 4027 writepos = conf->reshape_progress; 4028 sector_div(writepos, new_data_disks); 4029 readpos = conf->reshape_progress; 4030 sector_div(readpos, data_disks); 4031 safepos = conf->reshape_safe; 4032 sector_div(safepos, data_disks); 4033 if (mddev->delta_disks < 0) { 4034 writepos -= min_t(sector_t, reshape_sectors, writepos); 4035 readpos += reshape_sectors; 4036 safepos += reshape_sectors; 4037 } else { 4038 writepos += reshape_sectors; 4039 readpos -= min_t(sector_t, reshape_sectors, readpos); 4040 safepos -= min_t(sector_t, reshape_sectors, safepos); 4041 } 4042 4043 /* 'writepos' is the most advanced device address we might write. 4044 * 'readpos' is the least advanced device address we might read. 4045 * 'safepos' is the least address recorded in the metadata as having 4046 * been reshaped. 4047 * If 'readpos' is behind 'writepos', then there is no way that we can 4048 * ensure safety in the face of a crash - that must be done by userspace 4049 * making a backup of the data. So in that case there is no particular 4050 * rush to update metadata. 4051 * Otherwise if 'safepos' is behind 'writepos', then we really need to 4052 * update the metadata to advance 'safepos' to match 'readpos' so that 4053 * we can be safe in the event of a crash. 4054 * So we insist on updating metadata if safepos is behind writepos and 4055 * readpos is beyond writepos. 4056 * In any case, update the metadata every 10 seconds. 4057 * Maybe that number should be configurable, but I'm not sure it is 4058 * worth it.... maybe it could be a multiple of safemode_delay??? 4059 */ 4060 if ((mddev->delta_disks < 0 4061 ? (safepos > writepos && readpos < writepos) 4062 : (safepos < writepos && readpos > writepos)) || 4063 time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) { 4064 /* Cannot proceed until we've updated the superblock... */ 4065 wait_event(conf->wait_for_overlap, 4066 atomic_read(&conf->reshape_stripes)==0); 4067 mddev->reshape_position = conf->reshape_progress; 4068 mddev->curr_resync_completed = mddev->curr_resync; 4069 conf->reshape_checkpoint = jiffies; 4070 set_bit(MD_CHANGE_DEVS, &mddev->flags); 4071 md_wakeup_thread(mddev->thread); 4072 wait_event(mddev->sb_wait, mddev->flags == 0 || 4073 kthread_should_stop()); 4074 spin_lock_irq(&conf->device_lock); 4075 conf->reshape_safe = mddev->reshape_position; 4076 spin_unlock_irq(&conf->device_lock); 4077 wake_up(&conf->wait_for_overlap); 4078 sysfs_notify(&mddev->kobj, NULL, "sync_completed"); 4079 } 4080 4081 if (mddev->delta_disks < 0) { 4082 BUG_ON(conf->reshape_progress == 0); 4083 stripe_addr = writepos; 4084 BUG_ON((mddev->dev_sectors & 4085 ~((sector_t)reshape_sectors - 1)) 4086 - reshape_sectors - stripe_addr 4087 != sector_nr); 4088 } else { 4089 BUG_ON(writepos != sector_nr + reshape_sectors); 4090 stripe_addr = sector_nr; 4091 } 4092 INIT_LIST_HEAD(&stripes); 4093 for (i = 0; i < reshape_sectors; i += STRIPE_SECTORS) { 4094 int j; 4095 int skipped = 0; 4096 sh = get_active_stripe(conf, stripe_addr+i, 0, 0, 1); 4097 set_bit(STRIPE_EXPANDING, &sh->state); 4098 atomic_inc(&conf->reshape_stripes); 4099 /* If any of this stripe is beyond the end of the old 4100 * array, then we need to zero those blocks 4101 */ 4102 for (j=sh->disks; j--;) { 4103 sector_t s; 4104 if (j == sh->pd_idx) 4105 continue; 4106 if (conf->level == 6 && 4107 j == sh->qd_idx) 4108 continue; 4109 s = compute_blocknr(sh, j, 0); 4110 if (s < raid5_size(mddev, 0, 0)) { 4111 skipped = 1; 4112 continue; 4113 } 4114 memset(page_address(sh->dev[j].page), 0, STRIPE_SIZE); 4115 set_bit(R5_Expanded, &sh->dev[j].flags); 4116 set_bit(R5_UPTODATE, &sh->dev[j].flags); 4117 } 4118 if (!skipped) { 4119 set_bit(STRIPE_EXPAND_READY, &sh->state); 4120 set_bit(STRIPE_HANDLE, &sh->state); 4121 } 4122 list_add(&sh->lru, &stripes); 4123 } 4124 spin_lock_irq(&conf->device_lock); 4125 if (mddev->delta_disks < 0) 4126 conf->reshape_progress -= reshape_sectors * new_data_disks; 4127 else 4128 conf->reshape_progress += reshape_sectors * new_data_disks; 4129 spin_unlock_irq(&conf->device_lock); 4130 /* Ok, those stripe are ready. We can start scheduling 4131 * reads on the source stripes. 4132 * The source stripes are determined by mapping the first and last 4133 * block on the destination stripes. 4134 */ 4135 first_sector = 4136 raid5_compute_sector(conf, stripe_addr*(new_data_disks), 4137 1, &dd_idx, NULL); 4138 last_sector = 4139 raid5_compute_sector(conf, ((stripe_addr+reshape_sectors) 4140 * new_data_disks - 1), 4141 1, &dd_idx, NULL); 4142 if (last_sector >= mddev->dev_sectors) 4143 last_sector = mddev->dev_sectors - 1; 4144 while (first_sector <= last_sector) { 4145 sh = get_active_stripe(conf, first_sector, 1, 0, 1); 4146 set_bit(STRIPE_EXPAND_SOURCE, &sh->state); 4147 set_bit(STRIPE_HANDLE, &sh->state); 4148 release_stripe(sh); 4149 first_sector += STRIPE_SECTORS; 4150 } 4151 /* Now that the sources are clearly marked, we can release 4152 * the destination stripes 4153 */ 4154 while (!list_empty(&stripes)) { 4155 sh = list_entry(stripes.next, struct stripe_head, lru); 4156 list_del_init(&sh->lru); 4157 release_stripe(sh); 4158 } 4159 /* If this takes us to the resync_max point where we have to pause, 4160 * then we need to write out the superblock. 4161 */ 4162 sector_nr += reshape_sectors; 4163 if ((sector_nr - mddev->curr_resync_completed) * 2 4164 >= mddev->resync_max - mddev->curr_resync_completed) { 4165 /* Cannot proceed until we've updated the superblock... */ 4166 wait_event(conf->wait_for_overlap, 4167 atomic_read(&conf->reshape_stripes) == 0); 4168 mddev->reshape_position = conf->reshape_progress; 4169 mddev->curr_resync_completed = mddev->curr_resync + reshape_sectors; 4170 conf->reshape_checkpoint = jiffies; 4171 set_bit(MD_CHANGE_DEVS, &mddev->flags); 4172 md_wakeup_thread(mddev->thread); 4173 wait_event(mddev->sb_wait, 4174 !test_bit(MD_CHANGE_DEVS, &mddev->flags) 4175 || kthread_should_stop()); 4176 spin_lock_irq(&conf->device_lock); 4177 conf->reshape_safe = mddev->reshape_position; 4178 spin_unlock_irq(&conf->device_lock); 4179 wake_up(&conf->wait_for_overlap); 4180 sysfs_notify(&mddev->kobj, NULL, "sync_completed"); 4181 } 4182 return reshape_sectors; 4183 } 4184 4185 /* FIXME go_faster isn't used */ 4186 static inline sector_t sync_request(mddev_t *mddev, sector_t sector_nr, int *skipped, int go_faster) 4187 { 4188 raid5_conf_t *conf = (raid5_conf_t *) mddev->private; 4189 struct stripe_head *sh; 4190 sector_t max_sector = mddev->dev_sectors; 4191 int sync_blocks; 4192 int still_degraded = 0; 4193 int i; 4194 4195 if (sector_nr >= max_sector) { 4196 /* just being told to finish up .. nothing much to do */ 4197 unplug_slaves(mddev); 4198 4199 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) { 4200 end_reshape(conf); 4201 return 0; 4202 } 4203 4204 if (mddev->curr_resync < max_sector) /* aborted */ 4205 bitmap_end_sync(mddev->bitmap, mddev->curr_resync, 4206 &sync_blocks, 1); 4207 else /* completed sync */ 4208 conf->fullsync = 0; 4209 bitmap_close_sync(mddev->bitmap); 4210 4211 return 0; 4212 } 4213 4214 /* Allow raid5_quiesce to complete */ 4215 wait_event(conf->wait_for_overlap, conf->quiesce != 2); 4216 4217 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) 4218 return reshape_request(mddev, sector_nr, skipped); 4219 4220 /* No need to check resync_max as we never do more than one 4221 * stripe, and as resync_max will always be on a chunk boundary, 4222 * if the check in md_do_sync didn't fire, there is no chance 4223 * of overstepping resync_max here 4224 */ 4225 4226 /* if there is too many failed drives and we are trying 4227 * to resync, then assert that we are finished, because there is 4228 * nothing we can do. 4229 */ 4230 if (mddev->degraded >= conf->max_degraded && 4231 test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) { 4232 sector_t rv = mddev->dev_sectors - sector_nr; 4233 *skipped = 1; 4234 return rv; 4235 } 4236 if (!bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, 1) && 4237 !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) && 4238 !conf->fullsync && sync_blocks >= STRIPE_SECTORS) { 4239 /* we can skip this block, and probably more */ 4240 sync_blocks /= STRIPE_SECTORS; 4241 *skipped = 1; 4242 return sync_blocks * STRIPE_SECTORS; /* keep things rounded to whole stripes */ 4243 } 4244 4245 4246 bitmap_cond_end_sync(mddev->bitmap, sector_nr); 4247 4248 sh = get_active_stripe(conf, sector_nr, 0, 1, 0); 4249 if (sh == NULL) { 4250 sh = get_active_stripe(conf, sector_nr, 0, 0, 0); 4251 /* make sure we don't swamp the stripe cache if someone else 4252 * is trying to get access 4253 */ 4254 schedule_timeout_uninterruptible(1); 4255 } 4256 /* Need to check if array will still be degraded after recovery/resync 4257 * We don't need to check the 'failed' flag as when that gets set, 4258 * recovery aborts. 4259 */ 4260 for (i = 0; i < conf->raid_disks; i++) 4261 if (conf->disks[i].rdev == NULL) 4262 still_degraded = 1; 4263 4264 bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, still_degraded); 4265 4266 spin_lock(&sh->lock); 4267 set_bit(STRIPE_SYNCING, &sh->state); 4268 clear_bit(STRIPE_INSYNC, &sh->state); 4269 spin_unlock(&sh->lock); 4270 4271 /* wait for any blocked device to be handled */ 4272 while (unlikely(!handle_stripe(sh))) 4273 ; 4274 release_stripe(sh); 4275 4276 return STRIPE_SECTORS; 4277 } 4278 4279 static int retry_aligned_read(raid5_conf_t *conf, struct bio *raid_bio) 4280 { 4281 /* We may not be able to submit a whole bio at once as there 4282 * may not be enough stripe_heads available. 4283 * We cannot pre-allocate enough stripe_heads as we may need 4284 * more than exist in the cache (if we allow ever large chunks). 4285 * So we do one stripe head at a time and record in 4286 * ->bi_hw_segments how many have been done. 4287 * 4288 * We *know* that this entire raid_bio is in one chunk, so 4289 * it will be only one 'dd_idx' and only need one call to raid5_compute_sector. 4290 */ 4291 struct stripe_head *sh; 4292 int dd_idx; 4293 sector_t sector, logical_sector, last_sector; 4294 int scnt = 0; 4295 int remaining; 4296 int handled = 0; 4297 4298 logical_sector = raid_bio->bi_sector & ~((sector_t)STRIPE_SECTORS-1); 4299 sector = raid5_compute_sector(conf, logical_sector, 4300 0, &dd_idx, NULL); 4301 last_sector = raid_bio->bi_sector + (raid_bio->bi_size>>9); 4302 4303 for (; logical_sector < last_sector; 4304 logical_sector += STRIPE_SECTORS, 4305 sector += STRIPE_SECTORS, 4306 scnt++) { 4307 4308 if (scnt < raid5_bi_hw_segments(raid_bio)) 4309 /* already done this stripe */ 4310 continue; 4311 4312 sh = get_active_stripe(conf, sector, 0, 1, 0); 4313 4314 if (!sh) { 4315 /* failed to get a stripe - must wait */ 4316 raid5_set_bi_hw_segments(raid_bio, scnt); 4317 conf->retry_read_aligned = raid_bio; 4318 return handled; 4319 } 4320 4321 set_bit(R5_ReadError, &sh->dev[dd_idx].flags); 4322 if (!add_stripe_bio(sh, raid_bio, dd_idx, 0)) { 4323 release_stripe(sh); 4324 raid5_set_bi_hw_segments(raid_bio, scnt); 4325 conf->retry_read_aligned = raid_bio; 4326 return handled; 4327 } 4328 4329 handle_stripe(sh); 4330 release_stripe(sh); 4331 handled++; 4332 } 4333 spin_lock_irq(&conf->device_lock); 4334 remaining = raid5_dec_bi_phys_segments(raid_bio); 4335 spin_unlock_irq(&conf->device_lock); 4336 if (remaining == 0) 4337 bio_endio(raid_bio, 0); 4338 if (atomic_dec_and_test(&conf->active_aligned_reads)) 4339 wake_up(&conf->wait_for_stripe); 4340 return handled; 4341 } 4342 4343 #ifdef CONFIG_MULTICORE_RAID456 4344 static void __process_stripe(void *param, async_cookie_t cookie) 4345 { 4346 struct stripe_head *sh = param; 4347 4348 handle_stripe(sh); 4349 release_stripe(sh); 4350 } 4351 4352 static void process_stripe(struct stripe_head *sh, struct list_head *domain) 4353 { 4354 async_schedule_domain(__process_stripe, sh, domain); 4355 } 4356 4357 static void synchronize_stripe_processing(struct list_head *domain) 4358 { 4359 async_synchronize_full_domain(domain); 4360 } 4361 #else 4362 static void process_stripe(struct stripe_head *sh, struct list_head *domain) 4363 { 4364 handle_stripe(sh); 4365 release_stripe(sh); 4366 cond_resched(); 4367 } 4368 4369 static void synchronize_stripe_processing(struct list_head *domain) 4370 { 4371 } 4372 #endif 4373 4374 4375 /* 4376 * This is our raid5 kernel thread. 4377 * 4378 * We scan the hash table for stripes which can be handled now. 4379 * During the scan, completed stripes are saved for us by the interrupt 4380 * handler, so that they will not have to wait for our next wakeup. 4381 */ 4382 static void raid5d(mddev_t *mddev) 4383 { 4384 struct stripe_head *sh; 4385 raid5_conf_t *conf = mddev->private; 4386 int handled; 4387 LIST_HEAD(raid_domain); 4388 4389 pr_debug("+++ raid5d active\n"); 4390 4391 md_check_recovery(mddev); 4392 4393 handled = 0; 4394 spin_lock_irq(&conf->device_lock); 4395 while (1) { 4396 struct bio *bio; 4397 4398 if (conf->seq_flush != conf->seq_write) { 4399 int seq = conf->seq_flush; 4400 spin_unlock_irq(&conf->device_lock); 4401 bitmap_unplug(mddev->bitmap); 4402 spin_lock_irq(&conf->device_lock); 4403 conf->seq_write = seq; 4404 activate_bit_delay(conf); 4405 } 4406 4407 while ((bio = remove_bio_from_retry(conf))) { 4408 int ok; 4409 spin_unlock_irq(&conf->device_lock); 4410 ok = retry_aligned_read(conf, bio); 4411 spin_lock_irq(&conf->device_lock); 4412 if (!ok) 4413 break; 4414 handled++; 4415 } 4416 4417 sh = __get_priority_stripe(conf); 4418 4419 if (!sh) 4420 break; 4421 spin_unlock_irq(&conf->device_lock); 4422 4423 handled++; 4424 process_stripe(sh, &raid_domain); 4425 4426 spin_lock_irq(&conf->device_lock); 4427 } 4428 pr_debug("%d stripes handled\n", handled); 4429 4430 spin_unlock_irq(&conf->device_lock); 4431 4432 synchronize_stripe_processing(&raid_domain); 4433 async_tx_issue_pending_all(); 4434 unplug_slaves(mddev); 4435 4436 pr_debug("--- raid5d inactive\n"); 4437 } 4438 4439 static ssize_t 4440 raid5_show_stripe_cache_size(mddev_t *mddev, char *page) 4441 { 4442 raid5_conf_t *conf = mddev->private; 4443 if (conf) 4444 return sprintf(page, "%d\n", conf->max_nr_stripes); 4445 else 4446 return 0; 4447 } 4448 4449 static ssize_t 4450 raid5_store_stripe_cache_size(mddev_t *mddev, const char *page, size_t len) 4451 { 4452 raid5_conf_t *conf = mddev->private; 4453 unsigned long new; 4454 int err; 4455 4456 if (len >= PAGE_SIZE) 4457 return -EINVAL; 4458 if (!conf) 4459 return -ENODEV; 4460 4461 if (strict_strtoul(page, 10, &new)) 4462 return -EINVAL; 4463 if (new <= 16 || new > 32768) 4464 return -EINVAL; 4465 while (new < conf->max_nr_stripes) { 4466 if (drop_one_stripe(conf)) 4467 conf->max_nr_stripes--; 4468 else 4469 break; 4470 } 4471 err = md_allow_write(mddev); 4472 if (err) 4473 return err; 4474 while (new > conf->max_nr_stripes) { 4475 if (grow_one_stripe(conf)) 4476 conf->max_nr_stripes++; 4477 else break; 4478 } 4479 return len; 4480 } 4481 4482 static struct md_sysfs_entry 4483 raid5_stripecache_size = __ATTR(stripe_cache_size, S_IRUGO | S_IWUSR, 4484 raid5_show_stripe_cache_size, 4485 raid5_store_stripe_cache_size); 4486 4487 static ssize_t 4488 raid5_show_preread_threshold(mddev_t *mddev, char *page) 4489 { 4490 raid5_conf_t *conf = mddev->private; 4491 if (conf) 4492 return sprintf(page, "%d\n", conf->bypass_threshold); 4493 else 4494 return 0; 4495 } 4496 4497 static ssize_t 4498 raid5_store_preread_threshold(mddev_t *mddev, const char *page, size_t len) 4499 { 4500 raid5_conf_t *conf = mddev->private; 4501 unsigned long new; 4502 if (len >= PAGE_SIZE) 4503 return -EINVAL; 4504 if (!conf) 4505 return -ENODEV; 4506 4507 if (strict_strtoul(page, 10, &new)) 4508 return -EINVAL; 4509 if (new > conf->max_nr_stripes) 4510 return -EINVAL; 4511 conf->bypass_threshold = new; 4512 return len; 4513 } 4514 4515 static struct md_sysfs_entry 4516 raid5_preread_bypass_threshold = __ATTR(preread_bypass_threshold, 4517 S_IRUGO | S_IWUSR, 4518 raid5_show_preread_threshold, 4519 raid5_store_preread_threshold); 4520 4521 static ssize_t 4522 stripe_cache_active_show(mddev_t *mddev, char *page) 4523 { 4524 raid5_conf_t *conf = mddev->private; 4525 if (conf) 4526 return sprintf(page, "%d\n", atomic_read(&conf->active_stripes)); 4527 else 4528 return 0; 4529 } 4530 4531 static struct md_sysfs_entry 4532 raid5_stripecache_active = __ATTR_RO(stripe_cache_active); 4533 4534 static struct attribute *raid5_attrs[] = { 4535 &raid5_stripecache_size.attr, 4536 &raid5_stripecache_active.attr, 4537 &raid5_preread_bypass_threshold.attr, 4538 NULL, 4539 }; 4540 static struct attribute_group raid5_attrs_group = { 4541 .name = NULL, 4542 .attrs = raid5_attrs, 4543 }; 4544 4545 static sector_t 4546 raid5_size(mddev_t *mddev, sector_t sectors, int raid_disks) 4547 { 4548 raid5_conf_t *conf = mddev->private; 4549 4550 if (!sectors) 4551 sectors = mddev->dev_sectors; 4552 if (!raid_disks) { 4553 /* size is defined by the smallest of previous and new size */ 4554 if (conf->raid_disks < conf->previous_raid_disks) 4555 raid_disks = conf->raid_disks; 4556 else 4557 raid_disks = conf->previous_raid_disks; 4558 } 4559 4560 sectors &= ~((sector_t)mddev->chunk_sectors - 1); 4561 sectors &= ~((sector_t)mddev->new_chunk_sectors - 1); 4562 return sectors * (raid_disks - conf->max_degraded); 4563 } 4564 4565 static void raid5_free_percpu(raid5_conf_t *conf) 4566 { 4567 struct raid5_percpu *percpu; 4568 unsigned long cpu; 4569 4570 if (!conf->percpu) 4571 return; 4572 4573 get_online_cpus(); 4574 for_each_possible_cpu(cpu) { 4575 percpu = per_cpu_ptr(conf->percpu, cpu); 4576 safe_put_page(percpu->spare_page); 4577 kfree(percpu->scribble); 4578 } 4579 #ifdef CONFIG_HOTPLUG_CPU 4580 unregister_cpu_notifier(&conf->cpu_notify); 4581 #endif 4582 put_online_cpus(); 4583 4584 free_percpu(conf->percpu); 4585 } 4586 4587 static void free_conf(raid5_conf_t *conf) 4588 { 4589 shrink_stripes(conf); 4590 raid5_free_percpu(conf); 4591 kfree(conf->disks); 4592 kfree(conf->stripe_hashtbl); 4593 kfree(conf); 4594 } 4595 4596 #ifdef CONFIG_HOTPLUG_CPU 4597 static int raid456_cpu_notify(struct notifier_block *nfb, unsigned long action, 4598 void *hcpu) 4599 { 4600 raid5_conf_t *conf = container_of(nfb, raid5_conf_t, cpu_notify); 4601 long cpu = (long)hcpu; 4602 struct raid5_percpu *percpu = per_cpu_ptr(conf->percpu, cpu); 4603 4604 switch (action) { 4605 case CPU_UP_PREPARE: 4606 case CPU_UP_PREPARE_FROZEN: 4607 if (conf->level == 6 && !percpu->spare_page) 4608 percpu->spare_page = alloc_page(GFP_KERNEL); 4609 if (!percpu->scribble) 4610 percpu->scribble = kmalloc(conf->scribble_len, GFP_KERNEL); 4611 4612 if (!percpu->scribble || 4613 (conf->level == 6 && !percpu->spare_page)) { 4614 safe_put_page(percpu->spare_page); 4615 kfree(percpu->scribble); 4616 pr_err("%s: failed memory allocation for cpu%ld\n", 4617 __func__, cpu); 4618 return NOTIFY_BAD; 4619 } 4620 break; 4621 case CPU_DEAD: 4622 case CPU_DEAD_FROZEN: 4623 safe_put_page(percpu->spare_page); 4624 kfree(percpu->scribble); 4625 percpu->spare_page = NULL; 4626 percpu->scribble = NULL; 4627 break; 4628 default: 4629 break; 4630 } 4631 return NOTIFY_OK; 4632 } 4633 #endif 4634 4635 static int raid5_alloc_percpu(raid5_conf_t *conf) 4636 { 4637 unsigned long cpu; 4638 struct page *spare_page; 4639 struct raid5_percpu *allcpus; 4640 void *scribble; 4641 int err; 4642 4643 allcpus = alloc_percpu(struct raid5_percpu); 4644 if (!allcpus) 4645 return -ENOMEM; 4646 conf->percpu = allcpus; 4647 4648 get_online_cpus(); 4649 err = 0; 4650 for_each_present_cpu(cpu) { 4651 if (conf->level == 6) { 4652 spare_page = alloc_page(GFP_KERNEL); 4653 if (!spare_page) { 4654 err = -ENOMEM; 4655 break; 4656 } 4657 per_cpu_ptr(conf->percpu, cpu)->spare_page = spare_page; 4658 } 4659 scribble = kmalloc(scribble_len(conf->raid_disks), GFP_KERNEL); 4660 if (!scribble) { 4661 err = -ENOMEM; 4662 break; 4663 } 4664 per_cpu_ptr(conf->percpu, cpu)->scribble = scribble; 4665 } 4666 #ifdef CONFIG_HOTPLUG_CPU 4667 conf->cpu_notify.notifier_call = raid456_cpu_notify; 4668 conf->cpu_notify.priority = 0; 4669 if (err == 0) 4670 err = register_cpu_notifier(&conf->cpu_notify); 4671 #endif 4672 put_online_cpus(); 4673 4674 return err; 4675 } 4676 4677 static raid5_conf_t *setup_conf(mddev_t *mddev) 4678 { 4679 raid5_conf_t *conf; 4680 int raid_disk, memory; 4681 mdk_rdev_t *rdev; 4682 struct disk_info *disk; 4683 4684 if (mddev->new_level != 5 4685 && mddev->new_level != 4 4686 && mddev->new_level != 6) { 4687 printk(KERN_ERR "raid5: %s: raid level not set to 4/5/6 (%d)\n", 4688 mdname(mddev), mddev->new_level); 4689 return ERR_PTR(-EIO); 4690 } 4691 if ((mddev->new_level == 5 4692 && !algorithm_valid_raid5(mddev->new_layout)) || 4693 (mddev->new_level == 6 4694 && !algorithm_valid_raid6(mddev->new_layout))) { 4695 printk(KERN_ERR "raid5: %s: layout %d not supported\n", 4696 mdname(mddev), mddev->new_layout); 4697 return ERR_PTR(-EIO); 4698 } 4699 if (mddev->new_level == 6 && mddev->raid_disks < 4) { 4700 printk(KERN_ERR "raid6: not enough configured devices for %s (%d, minimum 4)\n", 4701 mdname(mddev), mddev->raid_disks); 4702 return ERR_PTR(-EINVAL); 4703 } 4704 4705 if (!mddev->new_chunk_sectors || 4706 (mddev->new_chunk_sectors << 9) % PAGE_SIZE || 4707 !is_power_of_2(mddev->new_chunk_sectors)) { 4708 printk(KERN_ERR "raid5: invalid chunk size %d for %s\n", 4709 mddev->new_chunk_sectors << 9, mdname(mddev)); 4710 return ERR_PTR(-EINVAL); 4711 } 4712 4713 conf = kzalloc(sizeof(raid5_conf_t), GFP_KERNEL); 4714 if (conf == NULL) 4715 goto abort; 4716 4717 conf->raid_disks = mddev->raid_disks; 4718 conf->scribble_len = scribble_len(conf->raid_disks); 4719 if (mddev->reshape_position == MaxSector) 4720 conf->previous_raid_disks = mddev->raid_disks; 4721 else 4722 conf->previous_raid_disks = mddev->raid_disks - mddev->delta_disks; 4723 4724 conf->disks = kzalloc(conf->raid_disks * sizeof(struct disk_info), 4725 GFP_KERNEL); 4726 if (!conf->disks) 4727 goto abort; 4728 4729 conf->mddev = mddev; 4730 4731 if ((conf->stripe_hashtbl = kzalloc(PAGE_SIZE, GFP_KERNEL)) == NULL) 4732 goto abort; 4733 4734 conf->level = mddev->new_level; 4735 if (raid5_alloc_percpu(conf) != 0) 4736 goto abort; 4737 4738 spin_lock_init(&conf->device_lock); 4739 init_waitqueue_head(&conf->wait_for_stripe); 4740 init_waitqueue_head(&conf->wait_for_overlap); 4741 INIT_LIST_HEAD(&conf->handle_list); 4742 INIT_LIST_HEAD(&conf->hold_list); 4743 INIT_LIST_HEAD(&conf->delayed_list); 4744 INIT_LIST_HEAD(&conf->bitmap_list); 4745 INIT_LIST_HEAD(&conf->inactive_list); 4746 atomic_set(&conf->active_stripes, 0); 4747 atomic_set(&conf->preread_active_stripes, 0); 4748 atomic_set(&conf->active_aligned_reads, 0); 4749 conf->bypass_threshold = BYPASS_THRESHOLD; 4750 4751 pr_debug("raid5: run(%s) called.\n", mdname(mddev)); 4752 4753 list_for_each_entry(rdev, &mddev->disks, same_set) { 4754 raid_disk = rdev->raid_disk; 4755 if (raid_disk >= conf->raid_disks 4756 || raid_disk < 0) 4757 continue; 4758 disk = conf->disks + raid_disk; 4759 4760 disk->rdev = rdev; 4761 4762 if (test_bit(In_sync, &rdev->flags)) { 4763 char b[BDEVNAME_SIZE]; 4764 printk(KERN_INFO "raid5: device %s operational as raid" 4765 " disk %d\n", bdevname(rdev->bdev,b), 4766 raid_disk); 4767 } else 4768 /* Cannot rely on bitmap to complete recovery */ 4769 conf->fullsync = 1; 4770 } 4771 4772 conf->chunk_sectors = mddev->new_chunk_sectors; 4773 conf->level = mddev->new_level; 4774 if (conf->level == 6) 4775 conf->max_degraded = 2; 4776 else 4777 conf->max_degraded = 1; 4778 conf->algorithm = mddev->new_layout; 4779 conf->max_nr_stripes = NR_STRIPES; 4780 conf->reshape_progress = mddev->reshape_position; 4781 if (conf->reshape_progress != MaxSector) { 4782 conf->prev_chunk_sectors = mddev->chunk_sectors; 4783 conf->prev_algo = mddev->layout; 4784 } 4785 4786 memory = conf->max_nr_stripes * (sizeof(struct stripe_head) + 4787 conf->raid_disks * ((sizeof(struct bio) + PAGE_SIZE))) / 1024; 4788 if (grow_stripes(conf, conf->max_nr_stripes)) { 4789 printk(KERN_ERR 4790 "raid5: couldn't allocate %dkB for buffers\n", memory); 4791 goto abort; 4792 } else 4793 printk(KERN_INFO "raid5: allocated %dkB for %s\n", 4794 memory, mdname(mddev)); 4795 4796 conf->thread = md_register_thread(raid5d, mddev, "%s_raid5"); 4797 if (!conf->thread) { 4798 printk(KERN_ERR 4799 "raid5: couldn't allocate thread for %s\n", 4800 mdname(mddev)); 4801 goto abort; 4802 } 4803 4804 return conf; 4805 4806 abort: 4807 if (conf) { 4808 free_conf(conf); 4809 return ERR_PTR(-EIO); 4810 } else 4811 return ERR_PTR(-ENOMEM); 4812 } 4813 4814 static int run(mddev_t *mddev) 4815 { 4816 raid5_conf_t *conf; 4817 int working_disks = 0, chunk_size; 4818 mdk_rdev_t *rdev; 4819 4820 if (mddev->recovery_cp != MaxSector) 4821 printk(KERN_NOTICE "raid5: %s is not clean" 4822 " -- starting background reconstruction\n", 4823 mdname(mddev)); 4824 if (mddev->reshape_position != MaxSector) { 4825 /* Check that we can continue the reshape. 4826 * Currently only disks can change, it must 4827 * increase, and we must be past the point where 4828 * a stripe over-writes itself 4829 */ 4830 sector_t here_new, here_old; 4831 int old_disks; 4832 int max_degraded = (mddev->level == 6 ? 2 : 1); 4833 4834 if (mddev->new_level != mddev->level) { 4835 printk(KERN_ERR "raid5: %s: unsupported reshape " 4836 "required - aborting.\n", 4837 mdname(mddev)); 4838 return -EINVAL; 4839 } 4840 old_disks = mddev->raid_disks - mddev->delta_disks; 4841 /* reshape_position must be on a new-stripe boundary, and one 4842 * further up in new geometry must map after here in old 4843 * geometry. 4844 */ 4845 here_new = mddev->reshape_position; 4846 if (sector_div(here_new, mddev->new_chunk_sectors * 4847 (mddev->raid_disks - max_degraded))) { 4848 printk(KERN_ERR "raid5: reshape_position not " 4849 "on a stripe boundary\n"); 4850 return -EINVAL; 4851 } 4852 /* here_new is the stripe we will write to */ 4853 here_old = mddev->reshape_position; 4854 sector_div(here_old, mddev->chunk_sectors * 4855 (old_disks-max_degraded)); 4856 /* here_old is the first stripe that we might need to read 4857 * from */ 4858 if (mddev->delta_disks == 0) { 4859 /* We cannot be sure it is safe to start an in-place 4860 * reshape. It is only safe if user-space if monitoring 4861 * and taking constant backups. 4862 * mdadm always starts a situation like this in 4863 * readonly mode so it can take control before 4864 * allowing any writes. So just check for that. 4865 */ 4866 if ((here_new * mddev->new_chunk_sectors != 4867 here_old * mddev->chunk_sectors) || 4868 mddev->ro == 0) { 4869 printk(KERN_ERR "raid5: in-place reshape must be started" 4870 " in read-only mode - aborting\n"); 4871 return -EINVAL; 4872 } 4873 } else if (mddev->delta_disks < 0 4874 ? (here_new * mddev->new_chunk_sectors <= 4875 here_old * mddev->chunk_sectors) 4876 : (here_new * mddev->new_chunk_sectors >= 4877 here_old * mddev->chunk_sectors)) { 4878 /* Reading from the same stripe as writing to - bad */ 4879 printk(KERN_ERR "raid5: reshape_position too early for " 4880 "auto-recovery - aborting.\n"); 4881 return -EINVAL; 4882 } 4883 printk(KERN_INFO "raid5: reshape will continue\n"); 4884 /* OK, we should be able to continue; */ 4885 } else { 4886 BUG_ON(mddev->level != mddev->new_level); 4887 BUG_ON(mddev->layout != mddev->new_layout); 4888 BUG_ON(mddev->chunk_sectors != mddev->new_chunk_sectors); 4889 BUG_ON(mddev->delta_disks != 0); 4890 } 4891 4892 if (mddev->private == NULL) 4893 conf = setup_conf(mddev); 4894 else 4895 conf = mddev->private; 4896 4897 if (IS_ERR(conf)) 4898 return PTR_ERR(conf); 4899 4900 mddev->thread = conf->thread; 4901 conf->thread = NULL; 4902 mddev->private = conf; 4903 4904 /* 4905 * 0 for a fully functional array, 1 or 2 for a degraded array. 4906 */ 4907 list_for_each_entry(rdev, &mddev->disks, same_set) 4908 if (rdev->raid_disk >= 0 && 4909 test_bit(In_sync, &rdev->flags)) 4910 working_disks++; 4911 4912 mddev->degraded = conf->raid_disks - working_disks; 4913 4914 if (mddev->degraded > conf->max_degraded) { 4915 printk(KERN_ERR "raid5: not enough operational devices for %s" 4916 " (%d/%d failed)\n", 4917 mdname(mddev), mddev->degraded, conf->raid_disks); 4918 goto abort; 4919 } 4920 4921 /* device size must be a multiple of chunk size */ 4922 mddev->dev_sectors &= ~(mddev->chunk_sectors - 1); 4923 mddev->resync_max_sectors = mddev->dev_sectors; 4924 4925 if (mddev->degraded > 0 && 4926 mddev->recovery_cp != MaxSector) { 4927 if (mddev->ok_start_degraded) 4928 printk(KERN_WARNING 4929 "raid5: starting dirty degraded array: %s" 4930 "- data corruption possible.\n", 4931 mdname(mddev)); 4932 else { 4933 printk(KERN_ERR 4934 "raid5: cannot start dirty degraded array for %s\n", 4935 mdname(mddev)); 4936 goto abort; 4937 } 4938 } 4939 4940 if (mddev->degraded == 0) 4941 printk("raid5: raid level %d set %s active with %d out of %d" 4942 " devices, algorithm %d\n", conf->level, mdname(mddev), 4943 mddev->raid_disks-mddev->degraded, mddev->raid_disks, 4944 mddev->new_layout); 4945 else 4946 printk(KERN_ALERT "raid5: raid level %d set %s active with %d" 4947 " out of %d devices, algorithm %d\n", conf->level, 4948 mdname(mddev), mddev->raid_disks - mddev->degraded, 4949 mddev->raid_disks, mddev->new_layout); 4950 4951 print_raid5_conf(conf); 4952 4953 if (conf->reshape_progress != MaxSector) { 4954 printk("...ok start reshape thread\n"); 4955 conf->reshape_safe = conf->reshape_progress; 4956 atomic_set(&conf->reshape_stripes, 0); 4957 clear_bit(MD_RECOVERY_SYNC, &mddev->recovery); 4958 clear_bit(MD_RECOVERY_CHECK, &mddev->recovery); 4959 set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery); 4960 set_bit(MD_RECOVERY_RUNNING, &mddev->recovery); 4961 mddev->sync_thread = md_register_thread(md_do_sync, mddev, 4962 "%s_reshape"); 4963 } 4964 4965 /* read-ahead size must cover two whole stripes, which is 4966 * 2 * (datadisks) * chunksize where 'n' is the number of raid devices 4967 */ 4968 { 4969 int data_disks = conf->previous_raid_disks - conf->max_degraded; 4970 int stripe = data_disks * 4971 ((mddev->chunk_sectors << 9) / PAGE_SIZE); 4972 if (mddev->queue->backing_dev_info.ra_pages < 2 * stripe) 4973 mddev->queue->backing_dev_info.ra_pages = 2 * stripe; 4974 } 4975 4976 /* Ok, everything is just fine now */ 4977 if (sysfs_create_group(&mddev->kobj, &raid5_attrs_group)) 4978 printk(KERN_WARNING 4979 "raid5: failed to create sysfs attributes for %s\n", 4980 mdname(mddev)); 4981 4982 mddev->queue->queue_lock = &conf->device_lock; 4983 4984 mddev->queue->unplug_fn = raid5_unplug_device; 4985 mddev->queue->backing_dev_info.congested_data = mddev; 4986 mddev->queue->backing_dev_info.congested_fn = raid5_congested; 4987 4988 md_set_array_sectors(mddev, raid5_size(mddev, 0, 0)); 4989 4990 blk_queue_merge_bvec(mddev->queue, raid5_mergeable_bvec); 4991 chunk_size = mddev->chunk_sectors << 9; 4992 blk_queue_io_min(mddev->queue, chunk_size); 4993 blk_queue_io_opt(mddev->queue, chunk_size * 4994 (conf->raid_disks - conf->max_degraded)); 4995 4996 list_for_each_entry(rdev, &mddev->disks, same_set) 4997 disk_stack_limits(mddev->gendisk, rdev->bdev, 4998 rdev->data_offset << 9); 4999 5000 return 0; 5001 abort: 5002 md_unregister_thread(mddev->thread); 5003 mddev->thread = NULL; 5004 if (conf) { 5005 print_raid5_conf(conf); 5006 free_conf(conf); 5007 } 5008 mddev->private = NULL; 5009 printk(KERN_ALERT "raid5: failed to run raid set %s\n", mdname(mddev)); 5010 return -EIO; 5011 } 5012 5013 5014 5015 static int stop(mddev_t *mddev) 5016 { 5017 raid5_conf_t *conf = (raid5_conf_t *) mddev->private; 5018 5019 md_unregister_thread(mddev->thread); 5020 mddev->thread = NULL; 5021 mddev->queue->backing_dev_info.congested_fn = NULL; 5022 blk_sync_queue(mddev->queue); /* the unplug fn references 'conf'*/ 5023 sysfs_remove_group(&mddev->kobj, &raid5_attrs_group); 5024 free_conf(conf); 5025 mddev->private = NULL; 5026 return 0; 5027 } 5028 5029 #ifdef DEBUG 5030 static void print_sh(struct seq_file *seq, struct stripe_head *sh) 5031 { 5032 int i; 5033 5034 seq_printf(seq, "sh %llu, pd_idx %d, state %ld.\n", 5035 (unsigned long long)sh->sector, sh->pd_idx, sh->state); 5036 seq_printf(seq, "sh %llu, count %d.\n", 5037 (unsigned long long)sh->sector, atomic_read(&sh->count)); 5038 seq_printf(seq, "sh %llu, ", (unsigned long long)sh->sector); 5039 for (i = 0; i < sh->disks; i++) { 5040 seq_printf(seq, "(cache%d: %p %ld) ", 5041 i, sh->dev[i].page, sh->dev[i].flags); 5042 } 5043 seq_printf(seq, "\n"); 5044 } 5045 5046 static void printall(struct seq_file *seq, raid5_conf_t *conf) 5047 { 5048 struct stripe_head *sh; 5049 struct hlist_node *hn; 5050 int i; 5051 5052 spin_lock_irq(&conf->device_lock); 5053 for (i = 0; i < NR_HASH; i++) { 5054 hlist_for_each_entry(sh, hn, &conf->stripe_hashtbl[i], hash) { 5055 if (sh->raid_conf != conf) 5056 continue; 5057 print_sh(seq, sh); 5058 } 5059 } 5060 spin_unlock_irq(&conf->device_lock); 5061 } 5062 #endif 5063 5064 static void status(struct seq_file *seq, mddev_t *mddev) 5065 { 5066 raid5_conf_t *conf = (raid5_conf_t *) mddev->private; 5067 int i; 5068 5069 seq_printf(seq, " level %d, %dk chunk, algorithm %d", mddev->level, 5070 mddev->chunk_sectors / 2, mddev->layout); 5071 seq_printf (seq, " [%d/%d] [", conf->raid_disks, conf->raid_disks - mddev->degraded); 5072 for (i = 0; i < conf->raid_disks; i++) 5073 seq_printf (seq, "%s", 5074 conf->disks[i].rdev && 5075 test_bit(In_sync, &conf->disks[i].rdev->flags) ? "U" : "_"); 5076 seq_printf (seq, "]"); 5077 #ifdef DEBUG 5078 seq_printf (seq, "\n"); 5079 printall(seq, conf); 5080 #endif 5081 } 5082 5083 static void print_raid5_conf (raid5_conf_t *conf) 5084 { 5085 int i; 5086 struct disk_info *tmp; 5087 5088 printk("RAID5 conf printout:\n"); 5089 if (!conf) { 5090 printk("(conf==NULL)\n"); 5091 return; 5092 } 5093 printk(" --- rd:%d wd:%d\n", conf->raid_disks, 5094 conf->raid_disks - conf->mddev->degraded); 5095 5096 for (i = 0; i < conf->raid_disks; i++) { 5097 char b[BDEVNAME_SIZE]; 5098 tmp = conf->disks + i; 5099 if (tmp->rdev) 5100 printk(" disk %d, o:%d, dev:%s\n", 5101 i, !test_bit(Faulty, &tmp->rdev->flags), 5102 bdevname(tmp->rdev->bdev,b)); 5103 } 5104 } 5105 5106 static int raid5_spare_active(mddev_t *mddev) 5107 { 5108 int i; 5109 raid5_conf_t *conf = mddev->private; 5110 struct disk_info *tmp; 5111 5112 for (i = 0; i < conf->raid_disks; i++) { 5113 tmp = conf->disks + i; 5114 if (tmp->rdev 5115 && !test_bit(Faulty, &tmp->rdev->flags) 5116 && !test_and_set_bit(In_sync, &tmp->rdev->flags)) { 5117 unsigned long flags; 5118 spin_lock_irqsave(&conf->device_lock, flags); 5119 mddev->degraded--; 5120 spin_unlock_irqrestore(&conf->device_lock, flags); 5121 } 5122 } 5123 print_raid5_conf(conf); 5124 return 0; 5125 } 5126 5127 static int raid5_remove_disk(mddev_t *mddev, int number) 5128 { 5129 raid5_conf_t *conf = mddev->private; 5130 int err = 0; 5131 mdk_rdev_t *rdev; 5132 struct disk_info *p = conf->disks + number; 5133 5134 print_raid5_conf(conf); 5135 rdev = p->rdev; 5136 if (rdev) { 5137 if (number >= conf->raid_disks && 5138 conf->reshape_progress == MaxSector) 5139 clear_bit(In_sync, &rdev->flags); 5140 5141 if (test_bit(In_sync, &rdev->flags) || 5142 atomic_read(&rdev->nr_pending)) { 5143 err = -EBUSY; 5144 goto abort; 5145 } 5146 /* Only remove non-faulty devices if recovery 5147 * isn't possible. 5148 */ 5149 if (!test_bit(Faulty, &rdev->flags) && 5150 mddev->degraded <= conf->max_degraded && 5151 number < conf->raid_disks) { 5152 err = -EBUSY; 5153 goto abort; 5154 } 5155 p->rdev = NULL; 5156 synchronize_rcu(); 5157 if (atomic_read(&rdev->nr_pending)) { 5158 /* lost the race, try later */ 5159 err = -EBUSY; 5160 p->rdev = rdev; 5161 } 5162 } 5163 abort: 5164 5165 print_raid5_conf(conf); 5166 return err; 5167 } 5168 5169 static int raid5_add_disk(mddev_t *mddev, mdk_rdev_t *rdev) 5170 { 5171 raid5_conf_t *conf = mddev->private; 5172 int err = -EEXIST; 5173 int disk; 5174 struct disk_info *p; 5175 int first = 0; 5176 int last = conf->raid_disks - 1; 5177 5178 if (mddev->degraded > conf->max_degraded) 5179 /* no point adding a device */ 5180 return -EINVAL; 5181 5182 if (rdev->raid_disk >= 0) 5183 first = last = rdev->raid_disk; 5184 5185 /* 5186 * find the disk ... but prefer rdev->saved_raid_disk 5187 * if possible. 5188 */ 5189 if (rdev->saved_raid_disk >= 0 && 5190 rdev->saved_raid_disk >= first && 5191 conf->disks[rdev->saved_raid_disk].rdev == NULL) 5192 disk = rdev->saved_raid_disk; 5193 else 5194 disk = first; 5195 for ( ; disk <= last ; disk++) 5196 if ((p=conf->disks + disk)->rdev == NULL) { 5197 clear_bit(In_sync, &rdev->flags); 5198 rdev->raid_disk = disk; 5199 err = 0; 5200 if (rdev->saved_raid_disk != disk) 5201 conf->fullsync = 1; 5202 rcu_assign_pointer(p->rdev, rdev); 5203 break; 5204 } 5205 print_raid5_conf(conf); 5206 return err; 5207 } 5208 5209 static int raid5_resize(mddev_t *mddev, sector_t sectors) 5210 { 5211 /* no resync is happening, and there is enough space 5212 * on all devices, so we can resize. 5213 * We need to make sure resync covers any new space. 5214 * If the array is shrinking we should possibly wait until 5215 * any io in the removed space completes, but it hardly seems 5216 * worth it. 5217 */ 5218 sectors &= ~((sector_t)mddev->chunk_sectors - 1); 5219 md_set_array_sectors(mddev, raid5_size(mddev, sectors, 5220 mddev->raid_disks)); 5221 if (mddev->array_sectors > 5222 raid5_size(mddev, sectors, mddev->raid_disks)) 5223 return -EINVAL; 5224 set_capacity(mddev->gendisk, mddev->array_sectors); 5225 mddev->changed = 1; 5226 revalidate_disk(mddev->gendisk); 5227 if (sectors > mddev->dev_sectors && mddev->recovery_cp == MaxSector) { 5228 mddev->recovery_cp = mddev->dev_sectors; 5229 set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); 5230 } 5231 mddev->dev_sectors = sectors; 5232 mddev->resync_max_sectors = sectors; 5233 return 0; 5234 } 5235 5236 static int check_stripe_cache(mddev_t *mddev) 5237 { 5238 /* Can only proceed if there are plenty of stripe_heads. 5239 * We need a minimum of one full stripe,, and for sensible progress 5240 * it is best to have about 4 times that. 5241 * If we require 4 times, then the default 256 4K stripe_heads will 5242 * allow for chunk sizes up to 256K, which is probably OK. 5243 * If the chunk size is greater, user-space should request more 5244 * stripe_heads first. 5245 */ 5246 raid5_conf_t *conf = mddev->private; 5247 if (((mddev->chunk_sectors << 9) / STRIPE_SIZE) * 4 5248 > conf->max_nr_stripes || 5249 ((mddev->new_chunk_sectors << 9) / STRIPE_SIZE) * 4 5250 > conf->max_nr_stripes) { 5251 printk(KERN_WARNING "raid5: reshape: not enough stripes. Needed %lu\n", 5252 ((max(mddev->chunk_sectors, mddev->new_chunk_sectors) << 9) 5253 / STRIPE_SIZE)*4); 5254 return 0; 5255 } 5256 return 1; 5257 } 5258 5259 static int check_reshape(mddev_t *mddev) 5260 { 5261 raid5_conf_t *conf = mddev->private; 5262 5263 if (mddev->delta_disks == 0 && 5264 mddev->new_layout == mddev->layout && 5265 mddev->new_chunk_sectors == mddev->chunk_sectors) 5266 return 0; /* nothing to do */ 5267 if (mddev->bitmap) 5268 /* Cannot grow a bitmap yet */ 5269 return -EBUSY; 5270 if (mddev->degraded > conf->max_degraded) 5271 return -EINVAL; 5272 if (mddev->delta_disks < 0) { 5273 /* We might be able to shrink, but the devices must 5274 * be made bigger first. 5275 * For raid6, 4 is the minimum size. 5276 * Otherwise 2 is the minimum 5277 */ 5278 int min = 2; 5279 if (mddev->level == 6) 5280 min = 4; 5281 if (mddev->raid_disks + mddev->delta_disks < min) 5282 return -EINVAL; 5283 } 5284 5285 if (!check_stripe_cache(mddev)) 5286 return -ENOSPC; 5287 5288 return resize_stripes(conf, conf->raid_disks + mddev->delta_disks); 5289 } 5290 5291 static int raid5_start_reshape(mddev_t *mddev) 5292 { 5293 raid5_conf_t *conf = mddev->private; 5294 mdk_rdev_t *rdev; 5295 int spares = 0; 5296 int added_devices = 0; 5297 unsigned long flags; 5298 5299 if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery)) 5300 return -EBUSY; 5301 5302 if (!check_stripe_cache(mddev)) 5303 return -ENOSPC; 5304 5305 list_for_each_entry(rdev, &mddev->disks, same_set) 5306 if (rdev->raid_disk < 0 && 5307 !test_bit(Faulty, &rdev->flags)) 5308 spares++; 5309 5310 if (spares - mddev->degraded < mddev->delta_disks - conf->max_degraded) 5311 /* Not enough devices even to make a degraded array 5312 * of that size 5313 */ 5314 return -EINVAL; 5315 5316 /* Refuse to reduce size of the array. Any reductions in 5317 * array size must be through explicit setting of array_size 5318 * attribute. 5319 */ 5320 if (raid5_size(mddev, 0, conf->raid_disks + mddev->delta_disks) 5321 < mddev->array_sectors) { 5322 printk(KERN_ERR "md: %s: array size must be reduced " 5323 "before number of disks\n", mdname(mddev)); 5324 return -EINVAL; 5325 } 5326 5327 atomic_set(&conf->reshape_stripes, 0); 5328 spin_lock_irq(&conf->device_lock); 5329 conf->previous_raid_disks = conf->raid_disks; 5330 conf->raid_disks += mddev->delta_disks; 5331 conf->prev_chunk_sectors = conf->chunk_sectors; 5332 conf->chunk_sectors = mddev->new_chunk_sectors; 5333 conf->prev_algo = conf->algorithm; 5334 conf->algorithm = mddev->new_layout; 5335 if (mddev->delta_disks < 0) 5336 conf->reshape_progress = raid5_size(mddev, 0, 0); 5337 else 5338 conf->reshape_progress = 0; 5339 conf->reshape_safe = conf->reshape_progress; 5340 conf->generation++; 5341 spin_unlock_irq(&conf->device_lock); 5342 5343 /* Add some new drives, as many as will fit. 5344 * We know there are enough to make the newly sized array work. 5345 */ 5346 list_for_each_entry(rdev, &mddev->disks, same_set) 5347 if (rdev->raid_disk < 0 && 5348 !test_bit(Faulty, &rdev->flags)) { 5349 if (raid5_add_disk(mddev, rdev) == 0) { 5350 char nm[20]; 5351 set_bit(In_sync, &rdev->flags); 5352 added_devices++; 5353 rdev->recovery_offset = 0; 5354 sprintf(nm, "rd%d", rdev->raid_disk); 5355 if (sysfs_create_link(&mddev->kobj, 5356 &rdev->kobj, nm)) 5357 printk(KERN_WARNING 5358 "raid5: failed to create " 5359 " link %s for %s\n", 5360 nm, mdname(mddev)); 5361 } else 5362 break; 5363 } 5364 5365 if (mddev->delta_disks > 0) { 5366 spin_lock_irqsave(&conf->device_lock, flags); 5367 mddev->degraded = (conf->raid_disks - conf->previous_raid_disks) 5368 - added_devices; 5369 spin_unlock_irqrestore(&conf->device_lock, flags); 5370 } 5371 mddev->raid_disks = conf->raid_disks; 5372 mddev->reshape_position = conf->reshape_progress; 5373 set_bit(MD_CHANGE_DEVS, &mddev->flags); 5374 5375 clear_bit(MD_RECOVERY_SYNC, &mddev->recovery); 5376 clear_bit(MD_RECOVERY_CHECK, &mddev->recovery); 5377 set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery); 5378 set_bit(MD_RECOVERY_RUNNING, &mddev->recovery); 5379 mddev->sync_thread = md_register_thread(md_do_sync, mddev, 5380 "%s_reshape"); 5381 if (!mddev->sync_thread) { 5382 mddev->recovery = 0; 5383 spin_lock_irq(&conf->device_lock); 5384 mddev->raid_disks = conf->raid_disks = conf->previous_raid_disks; 5385 conf->reshape_progress = MaxSector; 5386 spin_unlock_irq(&conf->device_lock); 5387 return -EAGAIN; 5388 } 5389 conf->reshape_checkpoint = jiffies; 5390 md_wakeup_thread(mddev->sync_thread); 5391 md_new_event(mddev); 5392 return 0; 5393 } 5394 5395 /* This is called from the reshape thread and should make any 5396 * changes needed in 'conf' 5397 */ 5398 static void end_reshape(raid5_conf_t *conf) 5399 { 5400 5401 if (!test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery)) { 5402 5403 spin_lock_irq(&conf->device_lock); 5404 conf->previous_raid_disks = conf->raid_disks; 5405 conf->reshape_progress = MaxSector; 5406 spin_unlock_irq(&conf->device_lock); 5407 wake_up(&conf->wait_for_overlap); 5408 5409 /* read-ahead size must cover two whole stripes, which is 5410 * 2 * (datadisks) * chunksize where 'n' is the number of raid devices 5411 */ 5412 { 5413 int data_disks = conf->raid_disks - conf->max_degraded; 5414 int stripe = data_disks * ((conf->chunk_sectors << 9) 5415 / PAGE_SIZE); 5416 if (conf->mddev->queue->backing_dev_info.ra_pages < 2 * stripe) 5417 conf->mddev->queue->backing_dev_info.ra_pages = 2 * stripe; 5418 } 5419 } 5420 } 5421 5422 /* This is called from the raid5d thread with mddev_lock held. 5423 * It makes config changes to the device. 5424 */ 5425 static void raid5_finish_reshape(mddev_t *mddev) 5426 { 5427 raid5_conf_t *conf = mddev->private; 5428 5429 if (!test_bit(MD_RECOVERY_INTR, &mddev->recovery)) { 5430 5431 if (mddev->delta_disks > 0) { 5432 md_set_array_sectors(mddev, raid5_size(mddev, 0, 0)); 5433 set_capacity(mddev->gendisk, mddev->array_sectors); 5434 mddev->changed = 1; 5435 revalidate_disk(mddev->gendisk); 5436 } else { 5437 int d; 5438 mddev->degraded = conf->raid_disks; 5439 for (d = 0; d < conf->raid_disks ; d++) 5440 if (conf->disks[d].rdev && 5441 test_bit(In_sync, 5442 &conf->disks[d].rdev->flags)) 5443 mddev->degraded--; 5444 for (d = conf->raid_disks ; 5445 d < conf->raid_disks - mddev->delta_disks; 5446 d++) { 5447 mdk_rdev_t *rdev = conf->disks[d].rdev; 5448 if (rdev && raid5_remove_disk(mddev, d) == 0) { 5449 char nm[20]; 5450 sprintf(nm, "rd%d", rdev->raid_disk); 5451 sysfs_remove_link(&mddev->kobj, nm); 5452 rdev->raid_disk = -1; 5453 } 5454 } 5455 } 5456 mddev->layout = conf->algorithm; 5457 mddev->chunk_sectors = conf->chunk_sectors; 5458 mddev->reshape_position = MaxSector; 5459 mddev->delta_disks = 0; 5460 } 5461 } 5462 5463 static void raid5_quiesce(mddev_t *mddev, int state) 5464 { 5465 raid5_conf_t *conf = mddev->private; 5466 5467 switch(state) { 5468 case 2: /* resume for a suspend */ 5469 wake_up(&conf->wait_for_overlap); 5470 break; 5471 5472 case 1: /* stop all writes */ 5473 spin_lock_irq(&conf->device_lock); 5474 /* '2' tells resync/reshape to pause so that all 5475 * active stripes can drain 5476 */ 5477 conf->quiesce = 2; 5478 wait_event_lock_irq(conf->wait_for_stripe, 5479 atomic_read(&conf->active_stripes) == 0 && 5480 atomic_read(&conf->active_aligned_reads) == 0, 5481 conf->device_lock, /* nothing */); 5482 conf->quiesce = 1; 5483 spin_unlock_irq(&conf->device_lock); 5484 /* allow reshape to continue */ 5485 wake_up(&conf->wait_for_overlap); 5486 break; 5487 5488 case 0: /* re-enable writes */ 5489 spin_lock_irq(&conf->device_lock); 5490 conf->quiesce = 0; 5491 wake_up(&conf->wait_for_stripe); 5492 wake_up(&conf->wait_for_overlap); 5493 spin_unlock_irq(&conf->device_lock); 5494 break; 5495 } 5496 } 5497 5498 5499 static void *raid5_takeover_raid1(mddev_t *mddev) 5500 { 5501 int chunksect; 5502 5503 if (mddev->raid_disks != 2 || 5504 mddev->degraded > 1) 5505 return ERR_PTR(-EINVAL); 5506 5507 /* Should check if there are write-behind devices? */ 5508 5509 chunksect = 64*2; /* 64K by default */ 5510 5511 /* The array must be an exact multiple of chunksize */ 5512 while (chunksect && (mddev->array_sectors & (chunksect-1))) 5513 chunksect >>= 1; 5514 5515 if ((chunksect<<9) < STRIPE_SIZE) 5516 /* array size does not allow a suitable chunk size */ 5517 return ERR_PTR(-EINVAL); 5518 5519 mddev->new_level = 5; 5520 mddev->new_layout = ALGORITHM_LEFT_SYMMETRIC; 5521 mddev->new_chunk_sectors = chunksect; 5522 5523 return setup_conf(mddev); 5524 } 5525 5526 static void *raid5_takeover_raid6(mddev_t *mddev) 5527 { 5528 int new_layout; 5529 5530 switch (mddev->layout) { 5531 case ALGORITHM_LEFT_ASYMMETRIC_6: 5532 new_layout = ALGORITHM_LEFT_ASYMMETRIC; 5533 break; 5534 case ALGORITHM_RIGHT_ASYMMETRIC_6: 5535 new_layout = ALGORITHM_RIGHT_ASYMMETRIC; 5536 break; 5537 case ALGORITHM_LEFT_SYMMETRIC_6: 5538 new_layout = ALGORITHM_LEFT_SYMMETRIC; 5539 break; 5540 case ALGORITHM_RIGHT_SYMMETRIC_6: 5541 new_layout = ALGORITHM_RIGHT_SYMMETRIC; 5542 break; 5543 case ALGORITHM_PARITY_0_6: 5544 new_layout = ALGORITHM_PARITY_0; 5545 break; 5546 case ALGORITHM_PARITY_N: 5547 new_layout = ALGORITHM_PARITY_N; 5548 break; 5549 default: 5550 return ERR_PTR(-EINVAL); 5551 } 5552 mddev->new_level = 5; 5553 mddev->new_layout = new_layout; 5554 mddev->delta_disks = -1; 5555 mddev->raid_disks -= 1; 5556 return setup_conf(mddev); 5557 } 5558 5559 5560 static int raid5_check_reshape(mddev_t *mddev) 5561 { 5562 /* For a 2-drive array, the layout and chunk size can be changed 5563 * immediately as not restriping is needed. 5564 * For larger arrays we record the new value - after validation 5565 * to be used by a reshape pass. 5566 */ 5567 raid5_conf_t *conf = mddev->private; 5568 int new_chunk = mddev->new_chunk_sectors; 5569 5570 if (mddev->new_layout >= 0 && !algorithm_valid_raid5(mddev->new_layout)) 5571 return -EINVAL; 5572 if (new_chunk > 0) { 5573 if (!is_power_of_2(new_chunk)) 5574 return -EINVAL; 5575 if (new_chunk < (PAGE_SIZE>>9)) 5576 return -EINVAL; 5577 if (mddev->array_sectors & (new_chunk-1)) 5578 /* not factor of array size */ 5579 return -EINVAL; 5580 } 5581 5582 /* They look valid */ 5583 5584 if (mddev->raid_disks == 2) { 5585 /* can make the change immediately */ 5586 if (mddev->new_layout >= 0) { 5587 conf->algorithm = mddev->new_layout; 5588 mddev->layout = mddev->new_layout; 5589 } 5590 if (new_chunk > 0) { 5591 conf->chunk_sectors = new_chunk ; 5592 mddev->chunk_sectors = new_chunk; 5593 } 5594 set_bit(MD_CHANGE_DEVS, &mddev->flags); 5595 md_wakeup_thread(mddev->thread); 5596 } 5597 return check_reshape(mddev); 5598 } 5599 5600 static int raid6_check_reshape(mddev_t *mddev) 5601 { 5602 int new_chunk = mddev->new_chunk_sectors; 5603 5604 if (mddev->new_layout >= 0 && !algorithm_valid_raid6(mddev->new_layout)) 5605 return -EINVAL; 5606 if (new_chunk > 0) { 5607 if (!is_power_of_2(new_chunk)) 5608 return -EINVAL; 5609 if (new_chunk < (PAGE_SIZE >> 9)) 5610 return -EINVAL; 5611 if (mddev->array_sectors & (new_chunk-1)) 5612 /* not factor of array size */ 5613 return -EINVAL; 5614 } 5615 5616 /* They look valid */ 5617 return check_reshape(mddev); 5618 } 5619 5620 static void *raid5_takeover(mddev_t *mddev) 5621 { 5622 /* raid5 can take over: 5623 * raid0 - if all devices are the same - make it a raid4 layout 5624 * raid1 - if there are two drives. We need to know the chunk size 5625 * raid4 - trivial - just use a raid4 layout. 5626 * raid6 - Providing it is a *_6 layout 5627 */ 5628 5629 if (mddev->level == 1) 5630 return raid5_takeover_raid1(mddev); 5631 if (mddev->level == 4) { 5632 mddev->new_layout = ALGORITHM_PARITY_N; 5633 mddev->new_level = 5; 5634 return setup_conf(mddev); 5635 } 5636 if (mddev->level == 6) 5637 return raid5_takeover_raid6(mddev); 5638 5639 return ERR_PTR(-EINVAL); 5640 } 5641 5642 5643 static struct mdk_personality raid5_personality; 5644 5645 static void *raid6_takeover(mddev_t *mddev) 5646 { 5647 /* Currently can only take over a raid5. We map the 5648 * personality to an equivalent raid6 personality 5649 * with the Q block at the end. 5650 */ 5651 int new_layout; 5652 5653 if (mddev->pers != &raid5_personality) 5654 return ERR_PTR(-EINVAL); 5655 if (mddev->degraded > 1) 5656 return ERR_PTR(-EINVAL); 5657 if (mddev->raid_disks > 253) 5658 return ERR_PTR(-EINVAL); 5659 if (mddev->raid_disks < 3) 5660 return ERR_PTR(-EINVAL); 5661 5662 switch (mddev->layout) { 5663 case ALGORITHM_LEFT_ASYMMETRIC: 5664 new_layout = ALGORITHM_LEFT_ASYMMETRIC_6; 5665 break; 5666 case ALGORITHM_RIGHT_ASYMMETRIC: 5667 new_layout = ALGORITHM_RIGHT_ASYMMETRIC_6; 5668 break; 5669 case ALGORITHM_LEFT_SYMMETRIC: 5670 new_layout = ALGORITHM_LEFT_SYMMETRIC_6; 5671 break; 5672 case ALGORITHM_RIGHT_SYMMETRIC: 5673 new_layout = ALGORITHM_RIGHT_SYMMETRIC_6; 5674 break; 5675 case ALGORITHM_PARITY_0: 5676 new_layout = ALGORITHM_PARITY_0_6; 5677 break; 5678 case ALGORITHM_PARITY_N: 5679 new_layout = ALGORITHM_PARITY_N; 5680 break; 5681 default: 5682 return ERR_PTR(-EINVAL); 5683 } 5684 mddev->new_level = 6; 5685 mddev->new_layout = new_layout; 5686 mddev->delta_disks = 1; 5687 mddev->raid_disks += 1; 5688 return setup_conf(mddev); 5689 } 5690 5691 5692 static struct mdk_personality raid6_personality = 5693 { 5694 .name = "raid6", 5695 .level = 6, 5696 .owner = THIS_MODULE, 5697 .make_request = make_request, 5698 .run = run, 5699 .stop = stop, 5700 .status = status, 5701 .error_handler = error, 5702 .hot_add_disk = raid5_add_disk, 5703 .hot_remove_disk= raid5_remove_disk, 5704 .spare_active = raid5_spare_active, 5705 .sync_request = sync_request, 5706 .resize = raid5_resize, 5707 .size = raid5_size, 5708 .check_reshape = raid6_check_reshape, 5709 .start_reshape = raid5_start_reshape, 5710 .finish_reshape = raid5_finish_reshape, 5711 .quiesce = raid5_quiesce, 5712 .takeover = raid6_takeover, 5713 }; 5714 static struct mdk_personality raid5_personality = 5715 { 5716 .name = "raid5", 5717 .level = 5, 5718 .owner = THIS_MODULE, 5719 .make_request = make_request, 5720 .run = run, 5721 .stop = stop, 5722 .status = status, 5723 .error_handler = error, 5724 .hot_add_disk = raid5_add_disk, 5725 .hot_remove_disk= raid5_remove_disk, 5726 .spare_active = raid5_spare_active, 5727 .sync_request = sync_request, 5728 .resize = raid5_resize, 5729 .size = raid5_size, 5730 .check_reshape = raid5_check_reshape, 5731 .start_reshape = raid5_start_reshape, 5732 .finish_reshape = raid5_finish_reshape, 5733 .quiesce = raid5_quiesce, 5734 .takeover = raid5_takeover, 5735 }; 5736 5737 static struct mdk_personality raid4_personality = 5738 { 5739 .name = "raid4", 5740 .level = 4, 5741 .owner = THIS_MODULE, 5742 .make_request = make_request, 5743 .run = run, 5744 .stop = stop, 5745 .status = status, 5746 .error_handler = error, 5747 .hot_add_disk = raid5_add_disk, 5748 .hot_remove_disk= raid5_remove_disk, 5749 .spare_active = raid5_spare_active, 5750 .sync_request = sync_request, 5751 .resize = raid5_resize, 5752 .size = raid5_size, 5753 .check_reshape = raid5_check_reshape, 5754 .start_reshape = raid5_start_reshape, 5755 .finish_reshape = raid5_finish_reshape, 5756 .quiesce = raid5_quiesce, 5757 }; 5758 5759 static int __init raid5_init(void) 5760 { 5761 register_md_personality(&raid6_personality); 5762 register_md_personality(&raid5_personality); 5763 register_md_personality(&raid4_personality); 5764 return 0; 5765 } 5766 5767 static void raid5_exit(void) 5768 { 5769 unregister_md_personality(&raid6_personality); 5770 unregister_md_personality(&raid5_personality); 5771 unregister_md_personality(&raid4_personality); 5772 } 5773 5774 module_init(raid5_init); 5775 module_exit(raid5_exit); 5776 MODULE_LICENSE("GPL"); 5777 MODULE_ALIAS("md-personality-4"); /* RAID5 */ 5778 MODULE_ALIAS("md-raid5"); 5779 MODULE_ALIAS("md-raid4"); 5780 MODULE_ALIAS("md-level-5"); 5781 MODULE_ALIAS("md-level-4"); 5782 MODULE_ALIAS("md-personality-8"); /* RAID6 */ 5783 MODULE_ALIAS("md-raid6"); 5784 MODULE_ALIAS("md-level-6"); 5785 5786 /* This used to be two separate modules, they were: */ 5787 MODULE_ALIAS("raid5"); 5788 MODULE_ALIAS("raid6"); 5789