1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * raid1.c : Multiple Devices driver for Linux 4 * 5 * Copyright (C) 1999, 2000, 2001 Ingo Molnar, Red Hat 6 * 7 * Copyright (C) 1996, 1997, 1998 Ingo Molnar, Miguel de Icaza, Gadi Oxman 8 * 9 * RAID-1 management functions. 10 * 11 * Better read-balancing code written by Mika Kuoppala <miku@iki.fi>, 2000 12 * 13 * Fixes to reconstruction by Jakob Østergaard" <jakob@ostenfeld.dk> 14 * Various fixes by Neil Brown <neilb@cse.unsw.edu.au> 15 * 16 * Changes by Peter T. Breuer <ptb@it.uc3m.es> 31/1/2003 to support 17 * bitmapped intelligence in resync: 18 * 19 * - bitmap marked during normal i/o 20 * - bitmap used to skip nondirty blocks during sync 21 * 22 * Additions to bitmap code, (C) 2003-2004 Paul Clements, SteelEye Technology: 23 * - persistent bitmap code 24 */ 25 26 #include <linux/slab.h> 27 #include <linux/delay.h> 28 #include <linux/blkdev.h> 29 #include <linux/module.h> 30 #include <linux/seq_file.h> 31 #include <linux/ratelimit.h> 32 #include <linux/interval_tree_generic.h> 33 34 #include <trace/events/block.h> 35 36 #include "md.h" 37 #include "raid1.h" 38 #include "md-bitmap.h" 39 40 #define UNSUPPORTED_MDDEV_FLAGS \ 41 ((1L << MD_HAS_JOURNAL) | \ 42 (1L << MD_JOURNAL_CLEAN) | \ 43 (1L << MD_HAS_PPL) | \ 44 (1L << MD_HAS_MULTIPLE_PPLS)) 45 46 static void allow_barrier(struct r1conf *conf, sector_t sector_nr); 47 static void lower_barrier(struct r1conf *conf, sector_t sector_nr); 48 49 #define RAID_1_10_NAME "raid1" 50 #include "raid1-10.c" 51 52 #define START(node) ((node)->start) 53 #define LAST(node) ((node)->last) 54 INTERVAL_TREE_DEFINE(struct serial_info, node, sector_t, _subtree_last, 55 START, LAST, static inline, raid1_rb); 56 57 static int check_and_add_serial(struct md_rdev *rdev, struct r1bio *r1_bio, 58 struct serial_info *si, int idx) 59 { 60 unsigned long flags; 61 int ret = 0; 62 sector_t lo = r1_bio->sector; 63 sector_t hi = lo + r1_bio->sectors; 64 struct serial_in_rdev *serial = &rdev->serial[idx]; 65 66 spin_lock_irqsave(&serial->serial_lock, flags); 67 /* collision happened */ 68 if (raid1_rb_iter_first(&serial->serial_rb, lo, hi)) 69 ret = -EBUSY; 70 else { 71 si->start = lo; 72 si->last = hi; 73 raid1_rb_insert(si, &serial->serial_rb); 74 } 75 spin_unlock_irqrestore(&serial->serial_lock, flags); 76 77 return ret; 78 } 79 80 static void wait_for_serialization(struct md_rdev *rdev, struct r1bio *r1_bio) 81 { 82 struct mddev *mddev = rdev->mddev; 83 struct serial_info *si; 84 int idx = sector_to_idx(r1_bio->sector); 85 struct serial_in_rdev *serial = &rdev->serial[idx]; 86 87 if (WARN_ON(!mddev->serial_info_pool)) 88 return; 89 si = mempool_alloc(mddev->serial_info_pool, GFP_NOIO); 90 wait_event(serial->serial_io_wait, 91 check_and_add_serial(rdev, r1_bio, si, idx) == 0); 92 } 93 94 static void remove_serial(struct md_rdev *rdev, sector_t lo, sector_t hi) 95 { 96 struct serial_info *si; 97 unsigned long flags; 98 int found = 0; 99 struct mddev *mddev = rdev->mddev; 100 int idx = sector_to_idx(lo); 101 struct serial_in_rdev *serial = &rdev->serial[idx]; 102 103 spin_lock_irqsave(&serial->serial_lock, flags); 104 for (si = raid1_rb_iter_first(&serial->serial_rb, lo, hi); 105 si; si = raid1_rb_iter_next(si, lo, hi)) { 106 if (si->start == lo && si->last == hi) { 107 raid1_rb_remove(si, &serial->serial_rb); 108 mempool_free(si, mddev->serial_info_pool); 109 found = 1; 110 break; 111 } 112 } 113 if (!found) 114 WARN(1, "The write IO is not recorded for serialization\n"); 115 spin_unlock_irqrestore(&serial->serial_lock, flags); 116 wake_up(&serial->serial_io_wait); 117 } 118 119 /* 120 * for resync bio, r1bio pointer can be retrieved from the per-bio 121 * 'struct resync_pages'. 122 */ 123 static inline struct r1bio *get_resync_r1bio(struct bio *bio) 124 { 125 return get_resync_pages(bio)->raid_bio; 126 } 127 128 static void * r1bio_pool_alloc(gfp_t gfp_flags, void *data) 129 { 130 struct pool_info *pi = data; 131 int size = offsetof(struct r1bio, bios[pi->raid_disks]); 132 133 /* allocate a r1bio with room for raid_disks entries in the bios array */ 134 return kzalloc(size, gfp_flags); 135 } 136 137 #define RESYNC_DEPTH 32 138 #define RESYNC_SECTORS (RESYNC_BLOCK_SIZE >> 9) 139 #define RESYNC_WINDOW (RESYNC_BLOCK_SIZE * RESYNC_DEPTH) 140 #define RESYNC_WINDOW_SECTORS (RESYNC_WINDOW >> 9) 141 #define CLUSTER_RESYNC_WINDOW (16 * RESYNC_WINDOW) 142 #define CLUSTER_RESYNC_WINDOW_SECTORS (CLUSTER_RESYNC_WINDOW >> 9) 143 144 static void * r1buf_pool_alloc(gfp_t gfp_flags, void *data) 145 { 146 struct pool_info *pi = data; 147 struct r1bio *r1_bio; 148 struct bio *bio; 149 int need_pages; 150 int j; 151 struct resync_pages *rps; 152 153 r1_bio = r1bio_pool_alloc(gfp_flags, pi); 154 if (!r1_bio) 155 return NULL; 156 157 rps = kmalloc_array(pi->raid_disks, sizeof(struct resync_pages), 158 gfp_flags); 159 if (!rps) 160 goto out_free_r1bio; 161 162 /* 163 * Allocate bios : 1 for reading, n-1 for writing 164 */ 165 for (j = pi->raid_disks ; j-- ; ) { 166 bio = bio_kmalloc(RESYNC_PAGES, gfp_flags); 167 if (!bio) 168 goto out_free_bio; 169 bio_init(bio, NULL, bio->bi_inline_vecs, RESYNC_PAGES, 0); 170 r1_bio->bios[j] = bio; 171 } 172 /* 173 * Allocate RESYNC_PAGES data pages and attach them to 174 * the first bio. 175 * If this is a user-requested check/repair, allocate 176 * RESYNC_PAGES for each bio. 177 */ 178 if (test_bit(MD_RECOVERY_REQUESTED, &pi->mddev->recovery)) 179 need_pages = pi->raid_disks; 180 else 181 need_pages = 1; 182 for (j = 0; j < pi->raid_disks; j++) { 183 struct resync_pages *rp = &rps[j]; 184 185 bio = r1_bio->bios[j]; 186 187 if (j < need_pages) { 188 if (resync_alloc_pages(rp, gfp_flags)) 189 goto out_free_pages; 190 } else { 191 memcpy(rp, &rps[0], sizeof(*rp)); 192 resync_get_all_pages(rp); 193 } 194 195 rp->raid_bio = r1_bio; 196 bio->bi_private = rp; 197 } 198 199 r1_bio->master_bio = NULL; 200 201 return r1_bio; 202 203 out_free_pages: 204 while (--j >= 0) 205 resync_free_pages(&rps[j]); 206 207 out_free_bio: 208 while (++j < pi->raid_disks) { 209 bio_uninit(r1_bio->bios[j]); 210 kfree(r1_bio->bios[j]); 211 } 212 kfree(rps); 213 214 out_free_r1bio: 215 rbio_pool_free(r1_bio, data); 216 return NULL; 217 } 218 219 static void r1buf_pool_free(void *__r1_bio, void *data) 220 { 221 struct pool_info *pi = data; 222 int i; 223 struct r1bio *r1bio = __r1_bio; 224 struct resync_pages *rp = NULL; 225 226 for (i = pi->raid_disks; i--; ) { 227 rp = get_resync_pages(r1bio->bios[i]); 228 resync_free_pages(rp); 229 bio_uninit(r1bio->bios[i]); 230 kfree(r1bio->bios[i]); 231 } 232 233 /* resync pages array stored in the 1st bio's .bi_private */ 234 kfree(rp); 235 236 rbio_pool_free(r1bio, data); 237 } 238 239 static void put_all_bios(struct r1conf *conf, struct r1bio *r1_bio) 240 { 241 int i; 242 243 for (i = 0; i < conf->raid_disks * 2; i++) { 244 struct bio **bio = r1_bio->bios + i; 245 if (!BIO_SPECIAL(*bio)) 246 bio_put(*bio); 247 *bio = NULL; 248 } 249 } 250 251 static void free_r1bio(struct r1bio *r1_bio) 252 { 253 struct r1conf *conf = r1_bio->mddev->private; 254 255 put_all_bios(conf, r1_bio); 256 mempool_free(r1_bio, &conf->r1bio_pool); 257 } 258 259 static void put_buf(struct r1bio *r1_bio) 260 { 261 struct r1conf *conf = r1_bio->mddev->private; 262 sector_t sect = r1_bio->sector; 263 int i; 264 265 for (i = 0; i < conf->raid_disks * 2; i++) { 266 struct bio *bio = r1_bio->bios[i]; 267 if (bio->bi_end_io) 268 rdev_dec_pending(conf->mirrors[i].rdev, r1_bio->mddev); 269 } 270 271 mempool_free(r1_bio, &conf->r1buf_pool); 272 273 lower_barrier(conf, sect); 274 } 275 276 static void reschedule_retry(struct r1bio *r1_bio) 277 { 278 unsigned long flags; 279 struct mddev *mddev = r1_bio->mddev; 280 struct r1conf *conf = mddev->private; 281 int idx; 282 283 idx = sector_to_idx(r1_bio->sector); 284 spin_lock_irqsave(&conf->device_lock, flags); 285 list_add(&r1_bio->retry_list, &conf->retry_list); 286 atomic_inc(&conf->nr_queued[idx]); 287 spin_unlock_irqrestore(&conf->device_lock, flags); 288 289 wake_up(&conf->wait_barrier); 290 md_wakeup_thread(mddev->thread); 291 } 292 293 /* 294 * raid_end_bio_io() is called when we have finished servicing a mirrored 295 * operation and are ready to return a success/failure code to the buffer 296 * cache layer. 297 */ 298 static void call_bio_endio(struct r1bio *r1_bio) 299 { 300 struct bio *bio = r1_bio->master_bio; 301 302 if (!test_bit(R1BIO_Uptodate, &r1_bio->state)) 303 bio->bi_status = BLK_STS_IOERR; 304 305 bio_endio(bio); 306 } 307 308 static void raid_end_bio_io(struct r1bio *r1_bio) 309 { 310 struct bio *bio = r1_bio->master_bio; 311 struct r1conf *conf = r1_bio->mddev->private; 312 sector_t sector = r1_bio->sector; 313 314 /* if nobody has done the final endio yet, do it now */ 315 if (!test_and_set_bit(R1BIO_Returned, &r1_bio->state)) { 316 pr_debug("raid1: sync end %s on sectors %llu-%llu\n", 317 (bio_data_dir(bio) == WRITE) ? "write" : "read", 318 (unsigned long long) bio->bi_iter.bi_sector, 319 (unsigned long long) bio_end_sector(bio) - 1); 320 321 call_bio_endio(r1_bio); 322 } 323 324 free_r1bio(r1_bio); 325 /* 326 * Wake up any possible resync thread that waits for the device 327 * to go idle. All I/Os, even write-behind writes, are done. 328 */ 329 allow_barrier(conf, sector); 330 } 331 332 /* 333 * Update disk head position estimator based on IRQ completion info. 334 */ 335 static inline void update_head_pos(int disk, struct r1bio *r1_bio) 336 { 337 struct r1conf *conf = r1_bio->mddev->private; 338 339 conf->mirrors[disk].head_position = 340 r1_bio->sector + (r1_bio->sectors); 341 } 342 343 /* 344 * Find the disk number which triggered given bio 345 */ 346 static int find_bio_disk(struct r1bio *r1_bio, struct bio *bio) 347 { 348 int mirror; 349 struct r1conf *conf = r1_bio->mddev->private; 350 int raid_disks = conf->raid_disks; 351 352 for (mirror = 0; mirror < raid_disks * 2; mirror++) 353 if (r1_bio->bios[mirror] == bio) 354 break; 355 356 BUG_ON(mirror == raid_disks * 2); 357 update_head_pos(mirror, r1_bio); 358 359 return mirror; 360 } 361 362 static void raid1_end_read_request(struct bio *bio) 363 { 364 int uptodate = !bio->bi_status; 365 struct r1bio *r1_bio = bio->bi_private; 366 struct r1conf *conf = r1_bio->mddev->private; 367 struct md_rdev *rdev = conf->mirrors[r1_bio->read_disk].rdev; 368 369 /* 370 * this branch is our 'one mirror IO has finished' event handler: 371 */ 372 update_head_pos(r1_bio->read_disk, r1_bio); 373 374 if (uptodate) 375 set_bit(R1BIO_Uptodate, &r1_bio->state); 376 else if (test_bit(FailFast, &rdev->flags) && 377 test_bit(R1BIO_FailFast, &r1_bio->state)) 378 /* This was a fail-fast read so we definitely 379 * want to retry */ 380 ; 381 else { 382 /* If all other devices have failed, we want to return 383 * the error upwards rather than fail the last device. 384 * Here we redefine "uptodate" to mean "Don't want to retry" 385 */ 386 unsigned long flags; 387 spin_lock_irqsave(&conf->device_lock, flags); 388 if (r1_bio->mddev->degraded == conf->raid_disks || 389 (r1_bio->mddev->degraded == conf->raid_disks-1 && 390 test_bit(In_sync, &rdev->flags))) 391 uptodate = 1; 392 spin_unlock_irqrestore(&conf->device_lock, flags); 393 } 394 395 if (uptodate) { 396 raid_end_bio_io(r1_bio); 397 rdev_dec_pending(rdev, conf->mddev); 398 } else { 399 /* 400 * oops, read error: 401 */ 402 pr_err_ratelimited("md/raid1:%s: %pg: rescheduling sector %llu\n", 403 mdname(conf->mddev), 404 rdev->bdev, 405 (unsigned long long)r1_bio->sector); 406 set_bit(R1BIO_ReadError, &r1_bio->state); 407 reschedule_retry(r1_bio); 408 /* don't drop the reference on read_disk yet */ 409 } 410 } 411 412 static void close_write(struct r1bio *r1_bio) 413 { 414 /* it really is the end of this request */ 415 if (test_bit(R1BIO_BehindIO, &r1_bio->state)) { 416 bio_free_pages(r1_bio->behind_master_bio); 417 bio_put(r1_bio->behind_master_bio); 418 r1_bio->behind_master_bio = NULL; 419 } 420 /* clear the bitmap if all writes complete successfully */ 421 md_bitmap_endwrite(r1_bio->mddev->bitmap, r1_bio->sector, 422 r1_bio->sectors, 423 !test_bit(R1BIO_Degraded, &r1_bio->state), 424 test_bit(R1BIO_BehindIO, &r1_bio->state)); 425 md_write_end(r1_bio->mddev); 426 } 427 428 static void r1_bio_write_done(struct r1bio *r1_bio) 429 { 430 if (!atomic_dec_and_test(&r1_bio->remaining)) 431 return; 432 433 if (test_bit(R1BIO_WriteError, &r1_bio->state)) 434 reschedule_retry(r1_bio); 435 else { 436 close_write(r1_bio); 437 if (test_bit(R1BIO_MadeGood, &r1_bio->state)) 438 reschedule_retry(r1_bio); 439 else 440 raid_end_bio_io(r1_bio); 441 } 442 } 443 444 static void raid1_end_write_request(struct bio *bio) 445 { 446 struct r1bio *r1_bio = bio->bi_private; 447 int behind = test_bit(R1BIO_BehindIO, &r1_bio->state); 448 struct r1conf *conf = r1_bio->mddev->private; 449 struct bio *to_put = NULL; 450 int mirror = find_bio_disk(r1_bio, bio); 451 struct md_rdev *rdev = conf->mirrors[mirror].rdev; 452 bool discard_error; 453 sector_t lo = r1_bio->sector; 454 sector_t hi = r1_bio->sector + r1_bio->sectors; 455 456 discard_error = bio->bi_status && bio_op(bio) == REQ_OP_DISCARD; 457 458 /* 459 * 'one mirror IO has finished' event handler: 460 */ 461 if (bio->bi_status && !discard_error) { 462 set_bit(WriteErrorSeen, &rdev->flags); 463 if (!test_and_set_bit(WantReplacement, &rdev->flags)) 464 set_bit(MD_RECOVERY_NEEDED, & 465 conf->mddev->recovery); 466 467 if (test_bit(FailFast, &rdev->flags) && 468 (bio->bi_opf & MD_FAILFAST) && 469 /* We never try FailFast to WriteMostly devices */ 470 !test_bit(WriteMostly, &rdev->flags)) { 471 md_error(r1_bio->mddev, rdev); 472 } 473 474 /* 475 * When the device is faulty, it is not necessary to 476 * handle write error. 477 */ 478 if (!test_bit(Faulty, &rdev->flags)) 479 set_bit(R1BIO_WriteError, &r1_bio->state); 480 else { 481 /* Fail the request */ 482 set_bit(R1BIO_Degraded, &r1_bio->state); 483 /* Finished with this branch */ 484 r1_bio->bios[mirror] = NULL; 485 to_put = bio; 486 } 487 } else { 488 /* 489 * Set R1BIO_Uptodate in our master bio, so that we 490 * will return a good error code for to the higher 491 * levels even if IO on some other mirrored buffer 492 * fails. 493 * 494 * The 'master' represents the composite IO operation 495 * to user-side. So if something waits for IO, then it 496 * will wait for the 'master' bio. 497 */ 498 r1_bio->bios[mirror] = NULL; 499 to_put = bio; 500 /* 501 * Do not set R1BIO_Uptodate if the current device is 502 * rebuilding or Faulty. This is because we cannot use 503 * such device for properly reading the data back (we could 504 * potentially use it, if the current write would have felt 505 * before rdev->recovery_offset, but for simplicity we don't 506 * check this here. 507 */ 508 if (test_bit(In_sync, &rdev->flags) && 509 !test_bit(Faulty, &rdev->flags)) 510 set_bit(R1BIO_Uptodate, &r1_bio->state); 511 512 /* Maybe we can clear some bad blocks. */ 513 if (rdev_has_badblock(rdev, r1_bio->sector, r1_bio->sectors) && 514 !discard_error) { 515 r1_bio->bios[mirror] = IO_MADE_GOOD; 516 set_bit(R1BIO_MadeGood, &r1_bio->state); 517 } 518 } 519 520 if (behind) { 521 if (test_bit(CollisionCheck, &rdev->flags)) 522 remove_serial(rdev, lo, hi); 523 if (test_bit(WriteMostly, &rdev->flags)) 524 atomic_dec(&r1_bio->behind_remaining); 525 526 /* 527 * In behind mode, we ACK the master bio once the I/O 528 * has safely reached all non-writemostly 529 * disks. Setting the Returned bit ensures that this 530 * gets done only once -- we don't ever want to return 531 * -EIO here, instead we'll wait 532 */ 533 if (atomic_read(&r1_bio->behind_remaining) >= (atomic_read(&r1_bio->remaining)-1) && 534 test_bit(R1BIO_Uptodate, &r1_bio->state)) { 535 /* Maybe we can return now */ 536 if (!test_and_set_bit(R1BIO_Returned, &r1_bio->state)) { 537 struct bio *mbio = r1_bio->master_bio; 538 pr_debug("raid1: behind end write sectors" 539 " %llu-%llu\n", 540 (unsigned long long) mbio->bi_iter.bi_sector, 541 (unsigned long long) bio_end_sector(mbio) - 1); 542 call_bio_endio(r1_bio); 543 } 544 } 545 } else if (rdev->mddev->serialize_policy) 546 remove_serial(rdev, lo, hi); 547 if (r1_bio->bios[mirror] == NULL) 548 rdev_dec_pending(rdev, conf->mddev); 549 550 /* 551 * Let's see if all mirrored write operations have finished 552 * already. 553 */ 554 r1_bio_write_done(r1_bio); 555 556 if (to_put) 557 bio_put(to_put); 558 } 559 560 static sector_t align_to_barrier_unit_end(sector_t start_sector, 561 sector_t sectors) 562 { 563 sector_t len; 564 565 WARN_ON(sectors == 0); 566 /* 567 * len is the number of sectors from start_sector to end of the 568 * barrier unit which start_sector belongs to. 569 */ 570 len = round_up(start_sector + 1, BARRIER_UNIT_SECTOR_SIZE) - 571 start_sector; 572 573 if (len > sectors) 574 len = sectors; 575 576 return len; 577 } 578 579 static void update_read_sectors(struct r1conf *conf, int disk, 580 sector_t this_sector, int len) 581 { 582 struct raid1_info *info = &conf->mirrors[disk]; 583 584 atomic_inc(&info->rdev->nr_pending); 585 if (info->next_seq_sect != this_sector) 586 info->seq_start = this_sector; 587 info->next_seq_sect = this_sector + len; 588 } 589 590 static int choose_first_rdev(struct r1conf *conf, struct r1bio *r1_bio, 591 int *max_sectors) 592 { 593 sector_t this_sector = r1_bio->sector; 594 int len = r1_bio->sectors; 595 int disk; 596 597 for (disk = 0 ; disk < conf->raid_disks * 2 ; disk++) { 598 struct md_rdev *rdev; 599 int read_len; 600 601 if (r1_bio->bios[disk] == IO_BLOCKED) 602 continue; 603 604 rdev = conf->mirrors[disk].rdev; 605 if (!rdev || test_bit(Faulty, &rdev->flags)) 606 continue; 607 608 /* choose the first disk even if it has some bad blocks. */ 609 read_len = raid1_check_read_range(rdev, this_sector, &len); 610 if (read_len > 0) { 611 update_read_sectors(conf, disk, this_sector, read_len); 612 *max_sectors = read_len; 613 return disk; 614 } 615 } 616 617 return -1; 618 } 619 620 static bool rdev_in_recovery(struct md_rdev *rdev, struct r1bio *r1_bio) 621 { 622 return !test_bit(In_sync, &rdev->flags) && 623 rdev->recovery_offset < r1_bio->sector + r1_bio->sectors; 624 } 625 626 static int choose_bb_rdev(struct r1conf *conf, struct r1bio *r1_bio, 627 int *max_sectors) 628 { 629 sector_t this_sector = r1_bio->sector; 630 int best_disk = -1; 631 int best_len = 0; 632 int disk; 633 634 for (disk = 0 ; disk < conf->raid_disks * 2 ; disk++) { 635 struct md_rdev *rdev; 636 int len; 637 int read_len; 638 639 if (r1_bio->bios[disk] == IO_BLOCKED) 640 continue; 641 642 rdev = conf->mirrors[disk].rdev; 643 if (!rdev || test_bit(Faulty, &rdev->flags) || 644 rdev_in_recovery(rdev, r1_bio) || 645 test_bit(WriteMostly, &rdev->flags)) 646 continue; 647 648 /* keep track of the disk with the most readable sectors. */ 649 len = r1_bio->sectors; 650 read_len = raid1_check_read_range(rdev, this_sector, &len); 651 if (read_len > best_len) { 652 best_disk = disk; 653 best_len = read_len; 654 } 655 } 656 657 if (best_disk != -1) { 658 *max_sectors = best_len; 659 update_read_sectors(conf, best_disk, this_sector, best_len); 660 } 661 662 return best_disk; 663 } 664 665 static int choose_slow_rdev(struct r1conf *conf, struct r1bio *r1_bio, 666 int *max_sectors) 667 { 668 sector_t this_sector = r1_bio->sector; 669 int bb_disk = -1; 670 int bb_read_len = 0; 671 int disk; 672 673 for (disk = 0 ; disk < conf->raid_disks * 2 ; disk++) { 674 struct md_rdev *rdev; 675 int len; 676 int read_len; 677 678 if (r1_bio->bios[disk] == IO_BLOCKED) 679 continue; 680 681 rdev = conf->mirrors[disk].rdev; 682 if (!rdev || test_bit(Faulty, &rdev->flags) || 683 !test_bit(WriteMostly, &rdev->flags) || 684 rdev_in_recovery(rdev, r1_bio)) 685 continue; 686 687 /* there are no bad blocks, we can use this disk */ 688 len = r1_bio->sectors; 689 read_len = raid1_check_read_range(rdev, this_sector, &len); 690 if (read_len == r1_bio->sectors) { 691 *max_sectors = read_len; 692 update_read_sectors(conf, disk, this_sector, read_len); 693 return disk; 694 } 695 696 /* 697 * there are partial bad blocks, choose the rdev with largest 698 * read length. 699 */ 700 if (read_len > bb_read_len) { 701 bb_disk = disk; 702 bb_read_len = read_len; 703 } 704 } 705 706 if (bb_disk != -1) { 707 *max_sectors = bb_read_len; 708 update_read_sectors(conf, bb_disk, this_sector, bb_read_len); 709 } 710 711 return bb_disk; 712 } 713 714 static bool is_sequential(struct r1conf *conf, int disk, struct r1bio *r1_bio) 715 { 716 /* TODO: address issues with this check and concurrency. */ 717 return conf->mirrors[disk].next_seq_sect == r1_bio->sector || 718 conf->mirrors[disk].head_position == r1_bio->sector; 719 } 720 721 /* 722 * If buffered sequential IO size exceeds optimal iosize, check if there is idle 723 * disk. If yes, choose the idle disk. 724 */ 725 static bool should_choose_next(struct r1conf *conf, int disk) 726 { 727 struct raid1_info *mirror = &conf->mirrors[disk]; 728 int opt_iosize; 729 730 if (!test_bit(Nonrot, &mirror->rdev->flags)) 731 return false; 732 733 opt_iosize = bdev_io_opt(mirror->rdev->bdev) >> 9; 734 return opt_iosize > 0 && mirror->seq_start != MaxSector && 735 mirror->next_seq_sect > opt_iosize && 736 mirror->next_seq_sect - opt_iosize >= mirror->seq_start; 737 } 738 739 static bool rdev_readable(struct md_rdev *rdev, struct r1bio *r1_bio) 740 { 741 if (!rdev || test_bit(Faulty, &rdev->flags)) 742 return false; 743 744 if (rdev_in_recovery(rdev, r1_bio)) 745 return false; 746 747 /* don't read from slow disk unless have to */ 748 if (test_bit(WriteMostly, &rdev->flags)) 749 return false; 750 751 /* don't split IO for bad blocks unless have to */ 752 if (rdev_has_badblock(rdev, r1_bio->sector, r1_bio->sectors)) 753 return false; 754 755 return true; 756 } 757 758 struct read_balance_ctl { 759 sector_t closest_dist; 760 int closest_dist_disk; 761 int min_pending; 762 int min_pending_disk; 763 int sequential_disk; 764 int readable_disks; 765 }; 766 767 static int choose_best_rdev(struct r1conf *conf, struct r1bio *r1_bio) 768 { 769 int disk; 770 struct read_balance_ctl ctl = { 771 .closest_dist_disk = -1, 772 .closest_dist = MaxSector, 773 .min_pending_disk = -1, 774 .min_pending = UINT_MAX, 775 .sequential_disk = -1, 776 }; 777 778 for (disk = 0 ; disk < conf->raid_disks * 2 ; disk++) { 779 struct md_rdev *rdev; 780 sector_t dist; 781 unsigned int pending; 782 783 if (r1_bio->bios[disk] == IO_BLOCKED) 784 continue; 785 786 rdev = conf->mirrors[disk].rdev; 787 if (!rdev_readable(rdev, r1_bio)) 788 continue; 789 790 /* At least two disks to choose from so failfast is OK */ 791 if (ctl.readable_disks++ == 1) 792 set_bit(R1BIO_FailFast, &r1_bio->state); 793 794 pending = atomic_read(&rdev->nr_pending); 795 dist = abs(r1_bio->sector - conf->mirrors[disk].head_position); 796 797 /* Don't change to another disk for sequential reads */ 798 if (is_sequential(conf, disk, r1_bio)) { 799 if (!should_choose_next(conf, disk)) 800 return disk; 801 802 /* 803 * Add 'pending' to avoid choosing this disk if 804 * there is other idle disk. 805 */ 806 pending++; 807 /* 808 * If there is no other idle disk, this disk 809 * will be chosen. 810 */ 811 ctl.sequential_disk = disk; 812 } 813 814 if (ctl.min_pending > pending) { 815 ctl.min_pending = pending; 816 ctl.min_pending_disk = disk; 817 } 818 819 if (ctl.closest_dist > dist) { 820 ctl.closest_dist = dist; 821 ctl.closest_dist_disk = disk; 822 } 823 } 824 825 /* 826 * sequential IO size exceeds optimal iosize, however, there is no other 827 * idle disk, so choose the sequential disk. 828 */ 829 if (ctl.sequential_disk != -1 && ctl.min_pending != 0) 830 return ctl.sequential_disk; 831 832 /* 833 * If all disks are rotational, choose the closest disk. If any disk is 834 * non-rotational, choose the disk with less pending request even the 835 * disk is rotational, which might/might not be optimal for raids with 836 * mixed ratation/non-rotational disks depending on workload. 837 */ 838 if (ctl.min_pending_disk != -1 && 839 (READ_ONCE(conf->nonrot_disks) || ctl.min_pending == 0)) 840 return ctl.min_pending_disk; 841 else 842 return ctl.closest_dist_disk; 843 } 844 845 /* 846 * This routine returns the disk from which the requested read should be done. 847 * 848 * 1) If resync is in progress, find the first usable disk and use it even if it 849 * has some bad blocks. 850 * 851 * 2) Now that there is no resync, loop through all disks and skipping slow 852 * disks and disks with bad blocks for now. Only pay attention to key disk 853 * choice. 854 * 855 * 3) If we've made it this far, now look for disks with bad blocks and choose 856 * the one with most number of sectors. 857 * 858 * 4) If we are all the way at the end, we have no choice but to use a disk even 859 * if it is write mostly. 860 * 861 * The rdev for the device selected will have nr_pending incremented. 862 */ 863 static int read_balance(struct r1conf *conf, struct r1bio *r1_bio, 864 int *max_sectors) 865 { 866 int disk; 867 868 clear_bit(R1BIO_FailFast, &r1_bio->state); 869 870 if (raid1_should_read_first(conf->mddev, r1_bio->sector, 871 r1_bio->sectors)) 872 return choose_first_rdev(conf, r1_bio, max_sectors); 873 874 disk = choose_best_rdev(conf, r1_bio); 875 if (disk >= 0) { 876 *max_sectors = r1_bio->sectors; 877 update_read_sectors(conf, disk, r1_bio->sector, 878 r1_bio->sectors); 879 return disk; 880 } 881 882 /* 883 * If we are here it means we didn't find a perfectly good disk so 884 * now spend a bit more time trying to find one with the most good 885 * sectors. 886 */ 887 disk = choose_bb_rdev(conf, r1_bio, max_sectors); 888 if (disk >= 0) 889 return disk; 890 891 return choose_slow_rdev(conf, r1_bio, max_sectors); 892 } 893 894 static void wake_up_barrier(struct r1conf *conf) 895 { 896 if (wq_has_sleeper(&conf->wait_barrier)) 897 wake_up(&conf->wait_barrier); 898 } 899 900 static void flush_bio_list(struct r1conf *conf, struct bio *bio) 901 { 902 /* flush any pending bitmap writes to disk before proceeding w/ I/O */ 903 raid1_prepare_flush_writes(conf->mddev->bitmap); 904 wake_up_barrier(conf); 905 906 while (bio) { /* submit pending writes */ 907 struct bio *next = bio->bi_next; 908 909 raid1_submit_write(bio); 910 bio = next; 911 cond_resched(); 912 } 913 } 914 915 static void flush_pending_writes(struct r1conf *conf) 916 { 917 /* Any writes that have been queued but are awaiting 918 * bitmap updates get flushed here. 919 */ 920 spin_lock_irq(&conf->device_lock); 921 922 if (conf->pending_bio_list.head) { 923 struct blk_plug plug; 924 struct bio *bio; 925 926 bio = bio_list_get(&conf->pending_bio_list); 927 spin_unlock_irq(&conf->device_lock); 928 929 /* 930 * As this is called in a wait_event() loop (see freeze_array), 931 * current->state might be TASK_UNINTERRUPTIBLE which will 932 * cause a warning when we prepare to wait again. As it is 933 * rare that this path is taken, it is perfectly safe to force 934 * us to go around the wait_event() loop again, so the warning 935 * is a false-positive. Silence the warning by resetting 936 * thread state 937 */ 938 __set_current_state(TASK_RUNNING); 939 blk_start_plug(&plug); 940 flush_bio_list(conf, bio); 941 blk_finish_plug(&plug); 942 } else 943 spin_unlock_irq(&conf->device_lock); 944 } 945 946 /* Barriers.... 947 * Sometimes we need to suspend IO while we do something else, 948 * either some resync/recovery, or reconfigure the array. 949 * To do this we raise a 'barrier'. 950 * The 'barrier' is a counter that can be raised multiple times 951 * to count how many activities are happening which preclude 952 * normal IO. 953 * We can only raise the barrier if there is no pending IO. 954 * i.e. if nr_pending == 0. 955 * We choose only to raise the barrier if no-one is waiting for the 956 * barrier to go down. This means that as soon as an IO request 957 * is ready, no other operations which require a barrier will start 958 * until the IO request has had a chance. 959 * 960 * So: regular IO calls 'wait_barrier'. When that returns there 961 * is no backgroup IO happening, It must arrange to call 962 * allow_barrier when it has finished its IO. 963 * backgroup IO calls must call raise_barrier. Once that returns 964 * there is no normal IO happeing. It must arrange to call 965 * lower_barrier when the particular background IO completes. 966 * 967 * If resync/recovery is interrupted, returns -EINTR; 968 * Otherwise, returns 0. 969 */ 970 static int raise_barrier(struct r1conf *conf, sector_t sector_nr) 971 { 972 int idx = sector_to_idx(sector_nr); 973 974 spin_lock_irq(&conf->resync_lock); 975 976 /* Wait until no block IO is waiting */ 977 wait_event_lock_irq(conf->wait_barrier, 978 !atomic_read(&conf->nr_waiting[idx]), 979 conf->resync_lock); 980 981 /* block any new IO from starting */ 982 atomic_inc(&conf->barrier[idx]); 983 /* 984 * In raise_barrier() we firstly increase conf->barrier[idx] then 985 * check conf->nr_pending[idx]. In _wait_barrier() we firstly 986 * increase conf->nr_pending[idx] then check conf->barrier[idx]. 987 * A memory barrier here to make sure conf->nr_pending[idx] won't 988 * be fetched before conf->barrier[idx] is increased. Otherwise 989 * there will be a race between raise_barrier() and _wait_barrier(). 990 */ 991 smp_mb__after_atomic(); 992 993 /* For these conditions we must wait: 994 * A: while the array is in frozen state 995 * B: while conf->nr_pending[idx] is not 0, meaning regular I/O 996 * existing in corresponding I/O barrier bucket. 997 * C: while conf->barrier[idx] >= RESYNC_DEPTH, meaning reaches 998 * max resync count which allowed on current I/O barrier bucket. 999 */ 1000 wait_event_lock_irq(conf->wait_barrier, 1001 (!conf->array_frozen && 1002 !atomic_read(&conf->nr_pending[idx]) && 1003 atomic_read(&conf->barrier[idx]) < RESYNC_DEPTH) || 1004 test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery), 1005 conf->resync_lock); 1006 1007 if (test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery)) { 1008 atomic_dec(&conf->barrier[idx]); 1009 spin_unlock_irq(&conf->resync_lock); 1010 wake_up(&conf->wait_barrier); 1011 return -EINTR; 1012 } 1013 1014 atomic_inc(&conf->nr_sync_pending); 1015 spin_unlock_irq(&conf->resync_lock); 1016 1017 return 0; 1018 } 1019 1020 static void lower_barrier(struct r1conf *conf, sector_t sector_nr) 1021 { 1022 int idx = sector_to_idx(sector_nr); 1023 1024 BUG_ON(atomic_read(&conf->barrier[idx]) <= 0); 1025 1026 atomic_dec(&conf->barrier[idx]); 1027 atomic_dec(&conf->nr_sync_pending); 1028 wake_up(&conf->wait_barrier); 1029 } 1030 1031 static bool _wait_barrier(struct r1conf *conf, int idx, bool nowait) 1032 { 1033 bool ret = true; 1034 1035 /* 1036 * We need to increase conf->nr_pending[idx] very early here, 1037 * then raise_barrier() can be blocked when it waits for 1038 * conf->nr_pending[idx] to be 0. Then we can avoid holding 1039 * conf->resync_lock when there is no barrier raised in same 1040 * barrier unit bucket. Also if the array is frozen, I/O 1041 * should be blocked until array is unfrozen. 1042 */ 1043 atomic_inc(&conf->nr_pending[idx]); 1044 /* 1045 * In _wait_barrier() we firstly increase conf->nr_pending[idx], then 1046 * check conf->barrier[idx]. In raise_barrier() we firstly increase 1047 * conf->barrier[idx], then check conf->nr_pending[idx]. A memory 1048 * barrier is necessary here to make sure conf->barrier[idx] won't be 1049 * fetched before conf->nr_pending[idx] is increased. Otherwise there 1050 * will be a race between _wait_barrier() and raise_barrier(). 1051 */ 1052 smp_mb__after_atomic(); 1053 1054 /* 1055 * Don't worry about checking two atomic_t variables at same time 1056 * here. If during we check conf->barrier[idx], the array is 1057 * frozen (conf->array_frozen is 1), and chonf->barrier[idx] is 1058 * 0, it is safe to return and make the I/O continue. Because the 1059 * array is frozen, all I/O returned here will eventually complete 1060 * or be queued, no race will happen. See code comment in 1061 * frozen_array(). 1062 */ 1063 if (!READ_ONCE(conf->array_frozen) && 1064 !atomic_read(&conf->barrier[idx])) 1065 return ret; 1066 1067 /* 1068 * After holding conf->resync_lock, conf->nr_pending[idx] 1069 * should be decreased before waiting for barrier to drop. 1070 * Otherwise, we may encounter a race condition because 1071 * raise_barrer() might be waiting for conf->nr_pending[idx] 1072 * to be 0 at same time. 1073 */ 1074 spin_lock_irq(&conf->resync_lock); 1075 atomic_inc(&conf->nr_waiting[idx]); 1076 atomic_dec(&conf->nr_pending[idx]); 1077 /* 1078 * In case freeze_array() is waiting for 1079 * get_unqueued_pending() == extra 1080 */ 1081 wake_up_barrier(conf); 1082 /* Wait for the barrier in same barrier unit bucket to drop. */ 1083 1084 /* Return false when nowait flag is set */ 1085 if (nowait) { 1086 ret = false; 1087 } else { 1088 wait_event_lock_irq(conf->wait_barrier, 1089 !conf->array_frozen && 1090 !atomic_read(&conf->barrier[idx]), 1091 conf->resync_lock); 1092 atomic_inc(&conf->nr_pending[idx]); 1093 } 1094 1095 atomic_dec(&conf->nr_waiting[idx]); 1096 spin_unlock_irq(&conf->resync_lock); 1097 return ret; 1098 } 1099 1100 static bool wait_read_barrier(struct r1conf *conf, sector_t sector_nr, bool nowait) 1101 { 1102 int idx = sector_to_idx(sector_nr); 1103 bool ret = true; 1104 1105 /* 1106 * Very similar to _wait_barrier(). The difference is, for read 1107 * I/O we don't need wait for sync I/O, but if the whole array 1108 * is frozen, the read I/O still has to wait until the array is 1109 * unfrozen. Since there is no ordering requirement with 1110 * conf->barrier[idx] here, memory barrier is unnecessary as well. 1111 */ 1112 atomic_inc(&conf->nr_pending[idx]); 1113 1114 if (!READ_ONCE(conf->array_frozen)) 1115 return ret; 1116 1117 spin_lock_irq(&conf->resync_lock); 1118 atomic_inc(&conf->nr_waiting[idx]); 1119 atomic_dec(&conf->nr_pending[idx]); 1120 /* 1121 * In case freeze_array() is waiting for 1122 * get_unqueued_pending() == extra 1123 */ 1124 wake_up_barrier(conf); 1125 /* Wait for array to be unfrozen */ 1126 1127 /* Return false when nowait flag is set */ 1128 if (nowait) { 1129 /* Return false when nowait flag is set */ 1130 ret = false; 1131 } else { 1132 wait_event_lock_irq(conf->wait_barrier, 1133 !conf->array_frozen, 1134 conf->resync_lock); 1135 atomic_inc(&conf->nr_pending[idx]); 1136 } 1137 1138 atomic_dec(&conf->nr_waiting[idx]); 1139 spin_unlock_irq(&conf->resync_lock); 1140 return ret; 1141 } 1142 1143 static bool wait_barrier(struct r1conf *conf, sector_t sector_nr, bool nowait) 1144 { 1145 int idx = sector_to_idx(sector_nr); 1146 1147 return _wait_barrier(conf, idx, nowait); 1148 } 1149 1150 static void _allow_barrier(struct r1conf *conf, int idx) 1151 { 1152 atomic_dec(&conf->nr_pending[idx]); 1153 wake_up_barrier(conf); 1154 } 1155 1156 static void allow_barrier(struct r1conf *conf, sector_t sector_nr) 1157 { 1158 int idx = sector_to_idx(sector_nr); 1159 1160 _allow_barrier(conf, idx); 1161 } 1162 1163 /* conf->resync_lock should be held */ 1164 static int get_unqueued_pending(struct r1conf *conf) 1165 { 1166 int idx, ret; 1167 1168 ret = atomic_read(&conf->nr_sync_pending); 1169 for (idx = 0; idx < BARRIER_BUCKETS_NR; idx++) 1170 ret += atomic_read(&conf->nr_pending[idx]) - 1171 atomic_read(&conf->nr_queued[idx]); 1172 1173 return ret; 1174 } 1175 1176 static void freeze_array(struct r1conf *conf, int extra) 1177 { 1178 /* Stop sync I/O and normal I/O and wait for everything to 1179 * go quiet. 1180 * This is called in two situations: 1181 * 1) management command handlers (reshape, remove disk, quiesce). 1182 * 2) one normal I/O request failed. 1183 1184 * After array_frozen is set to 1, new sync IO will be blocked at 1185 * raise_barrier(), and new normal I/O will blocked at _wait_barrier() 1186 * or wait_read_barrier(). The flying I/Os will either complete or be 1187 * queued. When everything goes quite, there are only queued I/Os left. 1188 1189 * Every flying I/O contributes to a conf->nr_pending[idx], idx is the 1190 * barrier bucket index which this I/O request hits. When all sync and 1191 * normal I/O are queued, sum of all conf->nr_pending[] will match sum 1192 * of all conf->nr_queued[]. But normal I/O failure is an exception, 1193 * in handle_read_error(), we may call freeze_array() before trying to 1194 * fix the read error. In this case, the error read I/O is not queued, 1195 * so get_unqueued_pending() == 1. 1196 * 1197 * Therefore before this function returns, we need to wait until 1198 * get_unqueued_pendings(conf) gets equal to extra. For 1199 * normal I/O context, extra is 1, in rested situations extra is 0. 1200 */ 1201 spin_lock_irq(&conf->resync_lock); 1202 conf->array_frozen = 1; 1203 mddev_add_trace_msg(conf->mddev, "raid1 wait freeze"); 1204 wait_event_lock_irq_cmd( 1205 conf->wait_barrier, 1206 get_unqueued_pending(conf) == extra, 1207 conf->resync_lock, 1208 flush_pending_writes(conf)); 1209 spin_unlock_irq(&conf->resync_lock); 1210 } 1211 static void unfreeze_array(struct r1conf *conf) 1212 { 1213 /* reverse the effect of the freeze */ 1214 spin_lock_irq(&conf->resync_lock); 1215 conf->array_frozen = 0; 1216 spin_unlock_irq(&conf->resync_lock); 1217 wake_up(&conf->wait_barrier); 1218 } 1219 1220 static void alloc_behind_master_bio(struct r1bio *r1_bio, 1221 struct bio *bio) 1222 { 1223 int size = bio->bi_iter.bi_size; 1224 unsigned vcnt = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; 1225 int i = 0; 1226 struct bio *behind_bio = NULL; 1227 1228 behind_bio = bio_alloc_bioset(NULL, vcnt, 0, GFP_NOIO, 1229 &r1_bio->mddev->bio_set); 1230 1231 /* discard op, we don't support writezero/writesame yet */ 1232 if (!bio_has_data(bio)) { 1233 behind_bio->bi_iter.bi_size = size; 1234 goto skip_copy; 1235 } 1236 1237 while (i < vcnt && size) { 1238 struct page *page; 1239 int len = min_t(int, PAGE_SIZE, size); 1240 1241 page = alloc_page(GFP_NOIO); 1242 if (unlikely(!page)) 1243 goto free_pages; 1244 1245 if (!bio_add_page(behind_bio, page, len, 0)) { 1246 put_page(page); 1247 goto free_pages; 1248 } 1249 1250 size -= len; 1251 i++; 1252 } 1253 1254 bio_copy_data(behind_bio, bio); 1255 skip_copy: 1256 r1_bio->behind_master_bio = behind_bio; 1257 set_bit(R1BIO_BehindIO, &r1_bio->state); 1258 1259 return; 1260 1261 free_pages: 1262 pr_debug("%dB behind alloc failed, doing sync I/O\n", 1263 bio->bi_iter.bi_size); 1264 bio_free_pages(behind_bio); 1265 bio_put(behind_bio); 1266 } 1267 1268 static void raid1_unplug(struct blk_plug_cb *cb, bool from_schedule) 1269 { 1270 struct raid1_plug_cb *plug = container_of(cb, struct raid1_plug_cb, 1271 cb); 1272 struct mddev *mddev = plug->cb.data; 1273 struct r1conf *conf = mddev->private; 1274 struct bio *bio; 1275 1276 if (from_schedule) { 1277 spin_lock_irq(&conf->device_lock); 1278 bio_list_merge(&conf->pending_bio_list, &plug->pending); 1279 spin_unlock_irq(&conf->device_lock); 1280 wake_up_barrier(conf); 1281 md_wakeup_thread(mddev->thread); 1282 kfree(plug); 1283 return; 1284 } 1285 1286 /* we aren't scheduling, so we can do the write-out directly. */ 1287 bio = bio_list_get(&plug->pending); 1288 flush_bio_list(conf, bio); 1289 kfree(plug); 1290 } 1291 1292 static void init_r1bio(struct r1bio *r1_bio, struct mddev *mddev, struct bio *bio) 1293 { 1294 r1_bio->master_bio = bio; 1295 r1_bio->sectors = bio_sectors(bio); 1296 r1_bio->state = 0; 1297 r1_bio->mddev = mddev; 1298 r1_bio->sector = bio->bi_iter.bi_sector; 1299 } 1300 1301 static inline struct r1bio * 1302 alloc_r1bio(struct mddev *mddev, struct bio *bio) 1303 { 1304 struct r1conf *conf = mddev->private; 1305 struct r1bio *r1_bio; 1306 1307 r1_bio = mempool_alloc(&conf->r1bio_pool, GFP_NOIO); 1308 /* Ensure no bio records IO_BLOCKED */ 1309 memset(r1_bio->bios, 0, conf->raid_disks * sizeof(r1_bio->bios[0])); 1310 init_r1bio(r1_bio, mddev, bio); 1311 return r1_bio; 1312 } 1313 1314 static void raid1_read_request(struct mddev *mddev, struct bio *bio, 1315 int max_read_sectors, struct r1bio *r1_bio) 1316 { 1317 struct r1conf *conf = mddev->private; 1318 struct raid1_info *mirror; 1319 struct bio *read_bio; 1320 struct bitmap *bitmap = mddev->bitmap; 1321 const enum req_op op = bio_op(bio); 1322 const blk_opf_t do_sync = bio->bi_opf & REQ_SYNC; 1323 int max_sectors; 1324 int rdisk; 1325 bool r1bio_existed = !!r1_bio; 1326 char b[BDEVNAME_SIZE]; 1327 1328 /* 1329 * If r1_bio is set, we are blocking the raid1d thread 1330 * so there is a tiny risk of deadlock. So ask for 1331 * emergency memory if needed. 1332 */ 1333 gfp_t gfp = r1_bio ? (GFP_NOIO | __GFP_HIGH) : GFP_NOIO; 1334 1335 if (r1bio_existed) { 1336 /* Need to get the block device name carefully */ 1337 struct md_rdev *rdev = conf->mirrors[r1_bio->read_disk].rdev; 1338 1339 if (rdev) 1340 snprintf(b, sizeof(b), "%pg", rdev->bdev); 1341 else 1342 strcpy(b, "???"); 1343 } 1344 1345 /* 1346 * Still need barrier for READ in case that whole 1347 * array is frozen. 1348 */ 1349 if (!wait_read_barrier(conf, bio->bi_iter.bi_sector, 1350 bio->bi_opf & REQ_NOWAIT)) { 1351 bio_wouldblock_error(bio); 1352 return; 1353 } 1354 1355 if (!r1_bio) 1356 r1_bio = alloc_r1bio(mddev, bio); 1357 else 1358 init_r1bio(r1_bio, mddev, bio); 1359 r1_bio->sectors = max_read_sectors; 1360 1361 /* 1362 * make_request() can abort the operation when read-ahead is being 1363 * used and no empty request is available. 1364 */ 1365 rdisk = read_balance(conf, r1_bio, &max_sectors); 1366 1367 if (rdisk < 0) { 1368 /* couldn't find anywhere to read from */ 1369 if (r1bio_existed) { 1370 pr_crit_ratelimited("md/raid1:%s: %s: unrecoverable I/O read error for block %llu\n", 1371 mdname(mddev), 1372 b, 1373 (unsigned long long)r1_bio->sector); 1374 } 1375 raid_end_bio_io(r1_bio); 1376 return; 1377 } 1378 mirror = conf->mirrors + rdisk; 1379 1380 if (r1bio_existed) 1381 pr_info_ratelimited("md/raid1:%s: redirecting sector %llu to other mirror: %pg\n", 1382 mdname(mddev), 1383 (unsigned long long)r1_bio->sector, 1384 mirror->rdev->bdev); 1385 1386 if (test_bit(WriteMostly, &mirror->rdev->flags) && 1387 bitmap) { 1388 /* 1389 * Reading from a write-mostly device must take care not to 1390 * over-take any writes that are 'behind' 1391 */ 1392 mddev_add_trace_msg(mddev, "raid1 wait behind writes"); 1393 wait_event(bitmap->behind_wait, 1394 atomic_read(&bitmap->behind_writes) == 0); 1395 } 1396 1397 if (max_sectors < bio_sectors(bio)) { 1398 struct bio *split = bio_split(bio, max_sectors, 1399 gfp, &conf->bio_split); 1400 bio_chain(split, bio); 1401 submit_bio_noacct(bio); 1402 bio = split; 1403 r1_bio->master_bio = bio; 1404 r1_bio->sectors = max_sectors; 1405 } 1406 1407 r1_bio->read_disk = rdisk; 1408 if (!r1bio_existed) { 1409 md_account_bio(mddev, &bio); 1410 r1_bio->master_bio = bio; 1411 } 1412 read_bio = bio_alloc_clone(mirror->rdev->bdev, bio, gfp, 1413 &mddev->bio_set); 1414 1415 r1_bio->bios[rdisk] = read_bio; 1416 1417 read_bio->bi_iter.bi_sector = r1_bio->sector + 1418 mirror->rdev->data_offset; 1419 read_bio->bi_end_io = raid1_end_read_request; 1420 read_bio->bi_opf = op | do_sync; 1421 if (test_bit(FailFast, &mirror->rdev->flags) && 1422 test_bit(R1BIO_FailFast, &r1_bio->state)) 1423 read_bio->bi_opf |= MD_FAILFAST; 1424 read_bio->bi_private = r1_bio; 1425 mddev_trace_remap(mddev, read_bio, r1_bio->sector); 1426 submit_bio_noacct(read_bio); 1427 } 1428 1429 static void raid1_write_request(struct mddev *mddev, struct bio *bio, 1430 int max_write_sectors) 1431 { 1432 struct r1conf *conf = mddev->private; 1433 struct r1bio *r1_bio; 1434 int i, disks; 1435 struct bitmap *bitmap = mddev->bitmap; 1436 unsigned long flags; 1437 struct md_rdev *blocked_rdev; 1438 int first_clone; 1439 int max_sectors; 1440 bool write_behind = false; 1441 bool is_discard = (bio_op(bio) == REQ_OP_DISCARD); 1442 1443 if (mddev_is_clustered(mddev) && 1444 md_cluster_ops->area_resyncing(mddev, WRITE, 1445 bio->bi_iter.bi_sector, bio_end_sector(bio))) { 1446 1447 DEFINE_WAIT(w); 1448 if (bio->bi_opf & REQ_NOWAIT) { 1449 bio_wouldblock_error(bio); 1450 return; 1451 } 1452 for (;;) { 1453 prepare_to_wait(&conf->wait_barrier, 1454 &w, TASK_IDLE); 1455 if (!md_cluster_ops->area_resyncing(mddev, WRITE, 1456 bio->bi_iter.bi_sector, 1457 bio_end_sector(bio))) 1458 break; 1459 schedule(); 1460 } 1461 finish_wait(&conf->wait_barrier, &w); 1462 } 1463 1464 /* 1465 * Register the new request and wait if the reconstruction 1466 * thread has put up a bar for new requests. 1467 * Continue immediately if no resync is active currently. 1468 */ 1469 if (!wait_barrier(conf, bio->bi_iter.bi_sector, 1470 bio->bi_opf & REQ_NOWAIT)) { 1471 bio_wouldblock_error(bio); 1472 return; 1473 } 1474 1475 retry_write: 1476 r1_bio = alloc_r1bio(mddev, bio); 1477 r1_bio->sectors = max_write_sectors; 1478 1479 /* first select target devices under rcu_lock and 1480 * inc refcount on their rdev. Record them by setting 1481 * bios[x] to bio 1482 * If there are known/acknowledged bad blocks on any device on 1483 * which we have seen a write error, we want to avoid writing those 1484 * blocks. 1485 * This potentially requires several writes to write around 1486 * the bad blocks. Each set of writes gets it's own r1bio 1487 * with a set of bios attached. 1488 */ 1489 1490 disks = conf->raid_disks * 2; 1491 blocked_rdev = NULL; 1492 max_sectors = r1_bio->sectors; 1493 for (i = 0; i < disks; i++) { 1494 struct md_rdev *rdev = conf->mirrors[i].rdev; 1495 1496 /* 1497 * The write-behind io is only attempted on drives marked as 1498 * write-mostly, which means we could allocate write behind 1499 * bio later. 1500 */ 1501 if (!is_discard && rdev && test_bit(WriteMostly, &rdev->flags)) 1502 write_behind = true; 1503 1504 if (rdev && unlikely(test_bit(Blocked, &rdev->flags))) { 1505 atomic_inc(&rdev->nr_pending); 1506 blocked_rdev = rdev; 1507 break; 1508 } 1509 r1_bio->bios[i] = NULL; 1510 if (!rdev || test_bit(Faulty, &rdev->flags)) { 1511 if (i < conf->raid_disks) 1512 set_bit(R1BIO_Degraded, &r1_bio->state); 1513 continue; 1514 } 1515 1516 atomic_inc(&rdev->nr_pending); 1517 if (test_bit(WriteErrorSeen, &rdev->flags)) { 1518 sector_t first_bad; 1519 int bad_sectors; 1520 int is_bad; 1521 1522 is_bad = is_badblock(rdev, r1_bio->sector, max_sectors, 1523 &first_bad, &bad_sectors); 1524 if (is_bad < 0) { 1525 /* mustn't write here until the bad block is 1526 * acknowledged*/ 1527 set_bit(BlockedBadBlocks, &rdev->flags); 1528 blocked_rdev = rdev; 1529 break; 1530 } 1531 if (is_bad && first_bad <= r1_bio->sector) { 1532 /* Cannot write here at all */ 1533 bad_sectors -= (r1_bio->sector - first_bad); 1534 if (bad_sectors < max_sectors) 1535 /* mustn't write more than bad_sectors 1536 * to other devices yet 1537 */ 1538 max_sectors = bad_sectors; 1539 rdev_dec_pending(rdev, mddev); 1540 /* We don't set R1BIO_Degraded as that 1541 * only applies if the disk is 1542 * missing, so it might be re-added, 1543 * and we want to know to recover this 1544 * chunk. 1545 * In this case the device is here, 1546 * and the fact that this chunk is not 1547 * in-sync is recorded in the bad 1548 * block log 1549 */ 1550 continue; 1551 } 1552 if (is_bad) { 1553 int good_sectors = first_bad - r1_bio->sector; 1554 if (good_sectors < max_sectors) 1555 max_sectors = good_sectors; 1556 } 1557 } 1558 r1_bio->bios[i] = bio; 1559 } 1560 1561 if (unlikely(blocked_rdev)) { 1562 /* Wait for this device to become unblocked */ 1563 int j; 1564 1565 for (j = 0; j < i; j++) 1566 if (r1_bio->bios[j]) 1567 rdev_dec_pending(conf->mirrors[j].rdev, mddev); 1568 mempool_free(r1_bio, &conf->r1bio_pool); 1569 allow_barrier(conf, bio->bi_iter.bi_sector); 1570 1571 if (bio->bi_opf & REQ_NOWAIT) { 1572 bio_wouldblock_error(bio); 1573 return; 1574 } 1575 mddev_add_trace_msg(mddev, "raid1 wait rdev %d blocked", 1576 blocked_rdev->raid_disk); 1577 md_wait_for_blocked_rdev(blocked_rdev, mddev); 1578 wait_barrier(conf, bio->bi_iter.bi_sector, false); 1579 goto retry_write; 1580 } 1581 1582 /* 1583 * When using a bitmap, we may call alloc_behind_master_bio below. 1584 * alloc_behind_master_bio allocates a copy of the data payload a page 1585 * at a time and thus needs a new bio that can fit the whole payload 1586 * this bio in page sized chunks. 1587 */ 1588 if (write_behind && bitmap) 1589 max_sectors = min_t(int, max_sectors, 1590 BIO_MAX_VECS * (PAGE_SIZE >> 9)); 1591 if (max_sectors < bio_sectors(bio)) { 1592 struct bio *split = bio_split(bio, max_sectors, 1593 GFP_NOIO, &conf->bio_split); 1594 bio_chain(split, bio); 1595 submit_bio_noacct(bio); 1596 bio = split; 1597 r1_bio->master_bio = bio; 1598 r1_bio->sectors = max_sectors; 1599 } 1600 1601 md_account_bio(mddev, &bio); 1602 r1_bio->master_bio = bio; 1603 atomic_set(&r1_bio->remaining, 1); 1604 atomic_set(&r1_bio->behind_remaining, 0); 1605 1606 first_clone = 1; 1607 1608 for (i = 0; i < disks; i++) { 1609 struct bio *mbio = NULL; 1610 struct md_rdev *rdev = conf->mirrors[i].rdev; 1611 if (!r1_bio->bios[i]) 1612 continue; 1613 1614 if (first_clone) { 1615 /* do behind I/O ? 1616 * Not if there are too many, or cannot 1617 * allocate memory, or a reader on WriteMostly 1618 * is waiting for behind writes to flush */ 1619 if (bitmap && write_behind && 1620 (atomic_read(&bitmap->behind_writes) 1621 < mddev->bitmap_info.max_write_behind) && 1622 !waitqueue_active(&bitmap->behind_wait)) { 1623 alloc_behind_master_bio(r1_bio, bio); 1624 } 1625 1626 md_bitmap_startwrite(bitmap, r1_bio->sector, r1_bio->sectors, 1627 test_bit(R1BIO_BehindIO, &r1_bio->state)); 1628 first_clone = 0; 1629 } 1630 1631 if (r1_bio->behind_master_bio) { 1632 mbio = bio_alloc_clone(rdev->bdev, 1633 r1_bio->behind_master_bio, 1634 GFP_NOIO, &mddev->bio_set); 1635 if (test_bit(CollisionCheck, &rdev->flags)) 1636 wait_for_serialization(rdev, r1_bio); 1637 if (test_bit(WriteMostly, &rdev->flags)) 1638 atomic_inc(&r1_bio->behind_remaining); 1639 } else { 1640 mbio = bio_alloc_clone(rdev->bdev, bio, GFP_NOIO, 1641 &mddev->bio_set); 1642 1643 if (mddev->serialize_policy) 1644 wait_for_serialization(rdev, r1_bio); 1645 } 1646 1647 r1_bio->bios[i] = mbio; 1648 1649 mbio->bi_iter.bi_sector = (r1_bio->sector + rdev->data_offset); 1650 mbio->bi_end_io = raid1_end_write_request; 1651 mbio->bi_opf = bio_op(bio) | (bio->bi_opf & (REQ_SYNC | REQ_FUA)); 1652 if (test_bit(FailFast, &rdev->flags) && 1653 !test_bit(WriteMostly, &rdev->flags) && 1654 conf->raid_disks - mddev->degraded > 1) 1655 mbio->bi_opf |= MD_FAILFAST; 1656 mbio->bi_private = r1_bio; 1657 1658 atomic_inc(&r1_bio->remaining); 1659 mddev_trace_remap(mddev, mbio, r1_bio->sector); 1660 /* flush_pending_writes() needs access to the rdev so...*/ 1661 mbio->bi_bdev = (void *)rdev; 1662 if (!raid1_add_bio_to_plug(mddev, mbio, raid1_unplug, disks)) { 1663 spin_lock_irqsave(&conf->device_lock, flags); 1664 bio_list_add(&conf->pending_bio_list, mbio); 1665 spin_unlock_irqrestore(&conf->device_lock, flags); 1666 md_wakeup_thread(mddev->thread); 1667 } 1668 } 1669 1670 r1_bio_write_done(r1_bio); 1671 1672 /* In case raid1d snuck in to freeze_array */ 1673 wake_up_barrier(conf); 1674 } 1675 1676 static bool raid1_make_request(struct mddev *mddev, struct bio *bio) 1677 { 1678 sector_t sectors; 1679 1680 if (unlikely(bio->bi_opf & REQ_PREFLUSH) 1681 && md_flush_request(mddev, bio)) 1682 return true; 1683 1684 /* 1685 * There is a limit to the maximum size, but 1686 * the read/write handler might find a lower limit 1687 * due to bad blocks. To avoid multiple splits, 1688 * we pass the maximum number of sectors down 1689 * and let the lower level perform the split. 1690 */ 1691 sectors = align_to_barrier_unit_end( 1692 bio->bi_iter.bi_sector, bio_sectors(bio)); 1693 1694 if (bio_data_dir(bio) == READ) 1695 raid1_read_request(mddev, bio, sectors, NULL); 1696 else { 1697 md_write_start(mddev,bio); 1698 raid1_write_request(mddev, bio, sectors); 1699 } 1700 return true; 1701 } 1702 1703 static void raid1_status(struct seq_file *seq, struct mddev *mddev) 1704 { 1705 struct r1conf *conf = mddev->private; 1706 int i; 1707 1708 lockdep_assert_held(&mddev->lock); 1709 1710 seq_printf(seq, " [%d/%d] [", conf->raid_disks, 1711 conf->raid_disks - mddev->degraded); 1712 for (i = 0; i < conf->raid_disks; i++) { 1713 struct md_rdev *rdev = READ_ONCE(conf->mirrors[i].rdev); 1714 1715 seq_printf(seq, "%s", 1716 rdev && test_bit(In_sync, &rdev->flags) ? "U" : "_"); 1717 } 1718 seq_printf(seq, "]"); 1719 } 1720 1721 /** 1722 * raid1_error() - RAID1 error handler. 1723 * @mddev: affected md device. 1724 * @rdev: member device to fail. 1725 * 1726 * The routine acknowledges &rdev failure and determines new @mddev state. 1727 * If it failed, then: 1728 * - &MD_BROKEN flag is set in &mddev->flags. 1729 * - recovery is disabled. 1730 * Otherwise, it must be degraded: 1731 * - recovery is interrupted. 1732 * - &mddev->degraded is bumped. 1733 * 1734 * @rdev is marked as &Faulty excluding case when array is failed and 1735 * &mddev->fail_last_dev is off. 1736 */ 1737 static void raid1_error(struct mddev *mddev, struct md_rdev *rdev) 1738 { 1739 struct r1conf *conf = mddev->private; 1740 unsigned long flags; 1741 1742 spin_lock_irqsave(&conf->device_lock, flags); 1743 1744 if (test_bit(In_sync, &rdev->flags) && 1745 (conf->raid_disks - mddev->degraded) == 1) { 1746 set_bit(MD_BROKEN, &mddev->flags); 1747 1748 if (!mddev->fail_last_dev) { 1749 conf->recovery_disabled = mddev->recovery_disabled; 1750 spin_unlock_irqrestore(&conf->device_lock, flags); 1751 return; 1752 } 1753 } 1754 set_bit(Blocked, &rdev->flags); 1755 if (test_and_clear_bit(In_sync, &rdev->flags)) 1756 mddev->degraded++; 1757 set_bit(Faulty, &rdev->flags); 1758 spin_unlock_irqrestore(&conf->device_lock, flags); 1759 /* 1760 * if recovery is running, make sure it aborts. 1761 */ 1762 set_bit(MD_RECOVERY_INTR, &mddev->recovery); 1763 set_mask_bits(&mddev->sb_flags, 0, 1764 BIT(MD_SB_CHANGE_DEVS) | BIT(MD_SB_CHANGE_PENDING)); 1765 pr_crit("md/raid1:%s: Disk failure on %pg, disabling device.\n" 1766 "md/raid1:%s: Operation continuing on %d devices.\n", 1767 mdname(mddev), rdev->bdev, 1768 mdname(mddev), conf->raid_disks - mddev->degraded); 1769 } 1770 1771 static void print_conf(struct r1conf *conf) 1772 { 1773 int i; 1774 1775 pr_debug("RAID1 conf printout:\n"); 1776 if (!conf) { 1777 pr_debug("(!conf)\n"); 1778 return; 1779 } 1780 pr_debug(" --- wd:%d rd:%d\n", conf->raid_disks - conf->mddev->degraded, 1781 conf->raid_disks); 1782 1783 lockdep_assert_held(&conf->mddev->reconfig_mutex); 1784 for (i = 0; i < conf->raid_disks; i++) { 1785 struct md_rdev *rdev = conf->mirrors[i].rdev; 1786 if (rdev) 1787 pr_debug(" disk %d, wo:%d, o:%d, dev:%pg\n", 1788 i, !test_bit(In_sync, &rdev->flags), 1789 !test_bit(Faulty, &rdev->flags), 1790 rdev->bdev); 1791 } 1792 } 1793 1794 static void close_sync(struct r1conf *conf) 1795 { 1796 int idx; 1797 1798 for (idx = 0; idx < BARRIER_BUCKETS_NR; idx++) { 1799 _wait_barrier(conf, idx, false); 1800 _allow_barrier(conf, idx); 1801 } 1802 1803 mempool_exit(&conf->r1buf_pool); 1804 } 1805 1806 static int raid1_spare_active(struct mddev *mddev) 1807 { 1808 int i; 1809 struct r1conf *conf = mddev->private; 1810 int count = 0; 1811 unsigned long flags; 1812 1813 /* 1814 * Find all failed disks within the RAID1 configuration 1815 * and mark them readable. 1816 * Called under mddev lock, so rcu protection not needed. 1817 * device_lock used to avoid races with raid1_end_read_request 1818 * which expects 'In_sync' flags and ->degraded to be consistent. 1819 */ 1820 spin_lock_irqsave(&conf->device_lock, flags); 1821 for (i = 0; i < conf->raid_disks; i++) { 1822 struct md_rdev *rdev = conf->mirrors[i].rdev; 1823 struct md_rdev *repl = conf->mirrors[conf->raid_disks + i].rdev; 1824 if (repl 1825 && !test_bit(Candidate, &repl->flags) 1826 && repl->recovery_offset == MaxSector 1827 && !test_bit(Faulty, &repl->flags) 1828 && !test_and_set_bit(In_sync, &repl->flags)) { 1829 /* replacement has just become active */ 1830 if (!rdev || 1831 !test_and_clear_bit(In_sync, &rdev->flags)) 1832 count++; 1833 if (rdev) { 1834 /* Replaced device not technically 1835 * faulty, but we need to be sure 1836 * it gets removed and never re-added 1837 */ 1838 set_bit(Faulty, &rdev->flags); 1839 sysfs_notify_dirent_safe( 1840 rdev->sysfs_state); 1841 } 1842 } 1843 if (rdev 1844 && rdev->recovery_offset == MaxSector 1845 && !test_bit(Faulty, &rdev->flags) 1846 && !test_and_set_bit(In_sync, &rdev->flags)) { 1847 count++; 1848 sysfs_notify_dirent_safe(rdev->sysfs_state); 1849 } 1850 } 1851 mddev->degraded -= count; 1852 spin_unlock_irqrestore(&conf->device_lock, flags); 1853 1854 print_conf(conf); 1855 return count; 1856 } 1857 1858 static bool raid1_add_conf(struct r1conf *conf, struct md_rdev *rdev, int disk, 1859 bool replacement) 1860 { 1861 struct raid1_info *info = conf->mirrors + disk; 1862 1863 if (replacement) 1864 info += conf->raid_disks; 1865 1866 if (info->rdev) 1867 return false; 1868 1869 if (bdev_nonrot(rdev->bdev)) { 1870 set_bit(Nonrot, &rdev->flags); 1871 WRITE_ONCE(conf->nonrot_disks, conf->nonrot_disks + 1); 1872 } 1873 1874 rdev->raid_disk = disk; 1875 info->head_position = 0; 1876 info->seq_start = MaxSector; 1877 WRITE_ONCE(info->rdev, rdev); 1878 1879 return true; 1880 } 1881 1882 static bool raid1_remove_conf(struct r1conf *conf, int disk) 1883 { 1884 struct raid1_info *info = conf->mirrors + disk; 1885 struct md_rdev *rdev = info->rdev; 1886 1887 if (!rdev || test_bit(In_sync, &rdev->flags) || 1888 atomic_read(&rdev->nr_pending)) 1889 return false; 1890 1891 /* Only remove non-faulty devices if recovery is not possible. */ 1892 if (!test_bit(Faulty, &rdev->flags) && 1893 rdev->mddev->recovery_disabled != conf->recovery_disabled && 1894 rdev->mddev->degraded < conf->raid_disks) 1895 return false; 1896 1897 if (test_and_clear_bit(Nonrot, &rdev->flags)) 1898 WRITE_ONCE(conf->nonrot_disks, conf->nonrot_disks - 1); 1899 1900 WRITE_ONCE(info->rdev, NULL); 1901 return true; 1902 } 1903 1904 static int raid1_add_disk(struct mddev *mddev, struct md_rdev *rdev) 1905 { 1906 struct r1conf *conf = mddev->private; 1907 int err = -EEXIST; 1908 int mirror = 0, repl_slot = -1; 1909 struct raid1_info *p; 1910 int first = 0; 1911 int last = conf->raid_disks - 1; 1912 1913 if (mddev->recovery_disabled == conf->recovery_disabled) 1914 return -EBUSY; 1915 1916 if (rdev->raid_disk >= 0) 1917 first = last = rdev->raid_disk; 1918 1919 /* 1920 * find the disk ... but prefer rdev->saved_raid_disk 1921 * if possible. 1922 */ 1923 if (rdev->saved_raid_disk >= 0 && 1924 rdev->saved_raid_disk >= first && 1925 rdev->saved_raid_disk < conf->raid_disks && 1926 conf->mirrors[rdev->saved_raid_disk].rdev == NULL) 1927 first = last = rdev->saved_raid_disk; 1928 1929 for (mirror = first; mirror <= last; mirror++) { 1930 p = conf->mirrors + mirror; 1931 if (!p->rdev) { 1932 err = mddev_stack_new_rdev(mddev, rdev); 1933 if (err) 1934 return err; 1935 1936 raid1_add_conf(conf, rdev, mirror, false); 1937 /* As all devices are equivalent, we don't need a full recovery 1938 * if this was recently any drive of the array 1939 */ 1940 if (rdev->saved_raid_disk < 0) 1941 conf->fullsync = 1; 1942 break; 1943 } 1944 if (test_bit(WantReplacement, &p->rdev->flags) && 1945 p[conf->raid_disks].rdev == NULL && repl_slot < 0) 1946 repl_slot = mirror; 1947 } 1948 1949 if (err && repl_slot >= 0) { 1950 /* Add this device as a replacement */ 1951 clear_bit(In_sync, &rdev->flags); 1952 set_bit(Replacement, &rdev->flags); 1953 raid1_add_conf(conf, rdev, repl_slot, true); 1954 err = 0; 1955 conf->fullsync = 1; 1956 } 1957 1958 print_conf(conf); 1959 return err; 1960 } 1961 1962 static int raid1_remove_disk(struct mddev *mddev, struct md_rdev *rdev) 1963 { 1964 struct r1conf *conf = mddev->private; 1965 int err = 0; 1966 int number = rdev->raid_disk; 1967 struct raid1_info *p = conf->mirrors + number; 1968 1969 if (unlikely(number >= conf->raid_disks)) 1970 goto abort; 1971 1972 if (rdev != p->rdev) { 1973 number += conf->raid_disks; 1974 p = conf->mirrors + number; 1975 } 1976 1977 print_conf(conf); 1978 if (rdev == p->rdev) { 1979 if (!raid1_remove_conf(conf, number)) { 1980 err = -EBUSY; 1981 goto abort; 1982 } 1983 1984 if (number < conf->raid_disks && 1985 conf->mirrors[conf->raid_disks + number].rdev) { 1986 /* We just removed a device that is being replaced. 1987 * Move down the replacement. We drain all IO before 1988 * doing this to avoid confusion. 1989 */ 1990 struct md_rdev *repl = 1991 conf->mirrors[conf->raid_disks + number].rdev; 1992 freeze_array(conf, 0); 1993 if (atomic_read(&repl->nr_pending)) { 1994 /* It means that some queued IO of retry_list 1995 * hold repl. Thus, we cannot set replacement 1996 * as NULL, avoiding rdev NULL pointer 1997 * dereference in sync_request_write and 1998 * handle_write_finished. 1999 */ 2000 err = -EBUSY; 2001 unfreeze_array(conf); 2002 goto abort; 2003 } 2004 clear_bit(Replacement, &repl->flags); 2005 WRITE_ONCE(p->rdev, repl); 2006 conf->mirrors[conf->raid_disks + number].rdev = NULL; 2007 unfreeze_array(conf); 2008 } 2009 2010 clear_bit(WantReplacement, &rdev->flags); 2011 err = md_integrity_register(mddev); 2012 } 2013 abort: 2014 2015 print_conf(conf); 2016 return err; 2017 } 2018 2019 static void end_sync_read(struct bio *bio) 2020 { 2021 struct r1bio *r1_bio = get_resync_r1bio(bio); 2022 2023 update_head_pos(r1_bio->read_disk, r1_bio); 2024 2025 /* 2026 * we have read a block, now it needs to be re-written, 2027 * or re-read if the read failed. 2028 * We don't do much here, just schedule handling by raid1d 2029 */ 2030 if (!bio->bi_status) 2031 set_bit(R1BIO_Uptodate, &r1_bio->state); 2032 2033 if (atomic_dec_and_test(&r1_bio->remaining)) 2034 reschedule_retry(r1_bio); 2035 } 2036 2037 static void abort_sync_write(struct mddev *mddev, struct r1bio *r1_bio) 2038 { 2039 sector_t sync_blocks = 0; 2040 sector_t s = r1_bio->sector; 2041 long sectors_to_go = r1_bio->sectors; 2042 2043 /* make sure these bits don't get cleared. */ 2044 do { 2045 md_bitmap_end_sync(mddev->bitmap, s, &sync_blocks, 1); 2046 s += sync_blocks; 2047 sectors_to_go -= sync_blocks; 2048 } while (sectors_to_go > 0); 2049 } 2050 2051 static void put_sync_write_buf(struct r1bio *r1_bio, int uptodate) 2052 { 2053 if (atomic_dec_and_test(&r1_bio->remaining)) { 2054 struct mddev *mddev = r1_bio->mddev; 2055 int s = r1_bio->sectors; 2056 2057 if (test_bit(R1BIO_MadeGood, &r1_bio->state) || 2058 test_bit(R1BIO_WriteError, &r1_bio->state)) 2059 reschedule_retry(r1_bio); 2060 else { 2061 put_buf(r1_bio); 2062 md_done_sync(mddev, s, uptodate); 2063 } 2064 } 2065 } 2066 2067 static void end_sync_write(struct bio *bio) 2068 { 2069 int uptodate = !bio->bi_status; 2070 struct r1bio *r1_bio = get_resync_r1bio(bio); 2071 struct mddev *mddev = r1_bio->mddev; 2072 struct r1conf *conf = mddev->private; 2073 struct md_rdev *rdev = conf->mirrors[find_bio_disk(r1_bio, bio)].rdev; 2074 2075 if (!uptodate) { 2076 abort_sync_write(mddev, r1_bio); 2077 set_bit(WriteErrorSeen, &rdev->flags); 2078 if (!test_and_set_bit(WantReplacement, &rdev->flags)) 2079 set_bit(MD_RECOVERY_NEEDED, & 2080 mddev->recovery); 2081 set_bit(R1BIO_WriteError, &r1_bio->state); 2082 } else if (rdev_has_badblock(rdev, r1_bio->sector, r1_bio->sectors) && 2083 !rdev_has_badblock(conf->mirrors[r1_bio->read_disk].rdev, 2084 r1_bio->sector, r1_bio->sectors)) { 2085 set_bit(R1BIO_MadeGood, &r1_bio->state); 2086 } 2087 2088 put_sync_write_buf(r1_bio, uptodate); 2089 } 2090 2091 static int r1_sync_page_io(struct md_rdev *rdev, sector_t sector, 2092 int sectors, struct page *page, blk_opf_t rw) 2093 { 2094 if (sync_page_io(rdev, sector, sectors << 9, page, rw, false)) 2095 /* success */ 2096 return 1; 2097 if (rw == REQ_OP_WRITE) { 2098 set_bit(WriteErrorSeen, &rdev->flags); 2099 if (!test_and_set_bit(WantReplacement, 2100 &rdev->flags)) 2101 set_bit(MD_RECOVERY_NEEDED, & 2102 rdev->mddev->recovery); 2103 } 2104 /* need to record an error - either for the block or the device */ 2105 if (!rdev_set_badblocks(rdev, sector, sectors, 0)) 2106 md_error(rdev->mddev, rdev); 2107 return 0; 2108 } 2109 2110 static int fix_sync_read_error(struct r1bio *r1_bio) 2111 { 2112 /* Try some synchronous reads of other devices to get 2113 * good data, much like with normal read errors. Only 2114 * read into the pages we already have so we don't 2115 * need to re-issue the read request. 2116 * We don't need to freeze the array, because being in an 2117 * active sync request, there is no normal IO, and 2118 * no overlapping syncs. 2119 * We don't need to check is_badblock() again as we 2120 * made sure that anything with a bad block in range 2121 * will have bi_end_io clear. 2122 */ 2123 struct mddev *mddev = r1_bio->mddev; 2124 struct r1conf *conf = mddev->private; 2125 struct bio *bio = r1_bio->bios[r1_bio->read_disk]; 2126 struct page **pages = get_resync_pages(bio)->pages; 2127 sector_t sect = r1_bio->sector; 2128 int sectors = r1_bio->sectors; 2129 int idx = 0; 2130 struct md_rdev *rdev; 2131 2132 rdev = conf->mirrors[r1_bio->read_disk].rdev; 2133 if (test_bit(FailFast, &rdev->flags)) { 2134 /* Don't try recovering from here - just fail it 2135 * ... unless it is the last working device of course */ 2136 md_error(mddev, rdev); 2137 if (test_bit(Faulty, &rdev->flags)) 2138 /* Don't try to read from here, but make sure 2139 * put_buf does it's thing 2140 */ 2141 bio->bi_end_io = end_sync_write; 2142 } 2143 2144 while(sectors) { 2145 int s = sectors; 2146 int d = r1_bio->read_disk; 2147 int success = 0; 2148 int start; 2149 2150 if (s > (PAGE_SIZE>>9)) 2151 s = PAGE_SIZE >> 9; 2152 do { 2153 if (r1_bio->bios[d]->bi_end_io == end_sync_read) { 2154 /* No rcu protection needed here devices 2155 * can only be removed when no resync is 2156 * active, and resync is currently active 2157 */ 2158 rdev = conf->mirrors[d].rdev; 2159 if (sync_page_io(rdev, sect, s<<9, 2160 pages[idx], 2161 REQ_OP_READ, false)) { 2162 success = 1; 2163 break; 2164 } 2165 } 2166 d++; 2167 if (d == conf->raid_disks * 2) 2168 d = 0; 2169 } while (!success && d != r1_bio->read_disk); 2170 2171 if (!success) { 2172 int abort = 0; 2173 /* Cannot read from anywhere, this block is lost. 2174 * Record a bad block on each device. If that doesn't 2175 * work just disable and interrupt the recovery. 2176 * Don't fail devices as that won't really help. 2177 */ 2178 pr_crit_ratelimited("md/raid1:%s: %pg: unrecoverable I/O read error for block %llu\n", 2179 mdname(mddev), bio->bi_bdev, 2180 (unsigned long long)r1_bio->sector); 2181 for (d = 0; d < conf->raid_disks * 2; d++) { 2182 rdev = conf->mirrors[d].rdev; 2183 if (!rdev || test_bit(Faulty, &rdev->flags)) 2184 continue; 2185 if (!rdev_set_badblocks(rdev, sect, s, 0)) 2186 abort = 1; 2187 } 2188 if (abort) { 2189 conf->recovery_disabled = 2190 mddev->recovery_disabled; 2191 set_bit(MD_RECOVERY_INTR, &mddev->recovery); 2192 md_done_sync(mddev, r1_bio->sectors, 0); 2193 put_buf(r1_bio); 2194 return 0; 2195 } 2196 /* Try next page */ 2197 sectors -= s; 2198 sect += s; 2199 idx++; 2200 continue; 2201 } 2202 2203 start = d; 2204 /* write it back and re-read */ 2205 while (d != r1_bio->read_disk) { 2206 if (d == 0) 2207 d = conf->raid_disks * 2; 2208 d--; 2209 if (r1_bio->bios[d]->bi_end_io != end_sync_read) 2210 continue; 2211 rdev = conf->mirrors[d].rdev; 2212 if (r1_sync_page_io(rdev, sect, s, 2213 pages[idx], 2214 REQ_OP_WRITE) == 0) { 2215 r1_bio->bios[d]->bi_end_io = NULL; 2216 rdev_dec_pending(rdev, mddev); 2217 } 2218 } 2219 d = start; 2220 while (d != r1_bio->read_disk) { 2221 if (d == 0) 2222 d = conf->raid_disks * 2; 2223 d--; 2224 if (r1_bio->bios[d]->bi_end_io != end_sync_read) 2225 continue; 2226 rdev = conf->mirrors[d].rdev; 2227 if (r1_sync_page_io(rdev, sect, s, 2228 pages[idx], 2229 REQ_OP_READ) != 0) 2230 atomic_add(s, &rdev->corrected_errors); 2231 } 2232 sectors -= s; 2233 sect += s; 2234 idx ++; 2235 } 2236 set_bit(R1BIO_Uptodate, &r1_bio->state); 2237 bio->bi_status = 0; 2238 return 1; 2239 } 2240 2241 static void process_checks(struct r1bio *r1_bio) 2242 { 2243 /* We have read all readable devices. If we haven't 2244 * got the block, then there is no hope left. 2245 * If we have, then we want to do a comparison 2246 * and skip the write if everything is the same. 2247 * If any blocks failed to read, then we need to 2248 * attempt an over-write 2249 */ 2250 struct mddev *mddev = r1_bio->mddev; 2251 struct r1conf *conf = mddev->private; 2252 int primary; 2253 int i; 2254 int vcnt; 2255 2256 /* Fix variable parts of all bios */ 2257 vcnt = (r1_bio->sectors + PAGE_SIZE / 512 - 1) >> (PAGE_SHIFT - 9); 2258 for (i = 0; i < conf->raid_disks * 2; i++) { 2259 blk_status_t status; 2260 struct bio *b = r1_bio->bios[i]; 2261 struct resync_pages *rp = get_resync_pages(b); 2262 if (b->bi_end_io != end_sync_read) 2263 continue; 2264 /* fixup the bio for reuse, but preserve errno */ 2265 status = b->bi_status; 2266 bio_reset(b, conf->mirrors[i].rdev->bdev, REQ_OP_READ); 2267 b->bi_status = status; 2268 b->bi_iter.bi_sector = r1_bio->sector + 2269 conf->mirrors[i].rdev->data_offset; 2270 b->bi_end_io = end_sync_read; 2271 rp->raid_bio = r1_bio; 2272 b->bi_private = rp; 2273 2274 /* initialize bvec table again */ 2275 md_bio_reset_resync_pages(b, rp, r1_bio->sectors << 9); 2276 } 2277 for (primary = 0; primary < conf->raid_disks * 2; primary++) 2278 if (r1_bio->bios[primary]->bi_end_io == end_sync_read && 2279 !r1_bio->bios[primary]->bi_status) { 2280 r1_bio->bios[primary]->bi_end_io = NULL; 2281 rdev_dec_pending(conf->mirrors[primary].rdev, mddev); 2282 break; 2283 } 2284 r1_bio->read_disk = primary; 2285 for (i = 0; i < conf->raid_disks * 2; i++) { 2286 int j = 0; 2287 struct bio *pbio = r1_bio->bios[primary]; 2288 struct bio *sbio = r1_bio->bios[i]; 2289 blk_status_t status = sbio->bi_status; 2290 struct page **ppages = get_resync_pages(pbio)->pages; 2291 struct page **spages = get_resync_pages(sbio)->pages; 2292 struct bio_vec *bi; 2293 int page_len[RESYNC_PAGES] = { 0 }; 2294 struct bvec_iter_all iter_all; 2295 2296 if (sbio->bi_end_io != end_sync_read) 2297 continue; 2298 /* Now we can 'fixup' the error value */ 2299 sbio->bi_status = 0; 2300 2301 bio_for_each_segment_all(bi, sbio, iter_all) 2302 page_len[j++] = bi->bv_len; 2303 2304 if (!status) { 2305 for (j = vcnt; j-- ; ) { 2306 if (memcmp(page_address(ppages[j]), 2307 page_address(spages[j]), 2308 page_len[j])) 2309 break; 2310 } 2311 } else 2312 j = 0; 2313 if (j >= 0) 2314 atomic64_add(r1_bio->sectors, &mddev->resync_mismatches); 2315 if (j < 0 || (test_bit(MD_RECOVERY_CHECK, &mddev->recovery) 2316 && !status)) { 2317 /* No need to write to this device. */ 2318 sbio->bi_end_io = NULL; 2319 rdev_dec_pending(conf->mirrors[i].rdev, mddev); 2320 continue; 2321 } 2322 2323 bio_copy_data(sbio, pbio); 2324 } 2325 } 2326 2327 static void sync_request_write(struct mddev *mddev, struct r1bio *r1_bio) 2328 { 2329 struct r1conf *conf = mddev->private; 2330 int i; 2331 int disks = conf->raid_disks * 2; 2332 struct bio *wbio; 2333 2334 if (!test_bit(R1BIO_Uptodate, &r1_bio->state)) 2335 /* ouch - failed to read all of that. */ 2336 if (!fix_sync_read_error(r1_bio)) 2337 return; 2338 2339 if (test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery)) 2340 process_checks(r1_bio); 2341 2342 /* 2343 * schedule writes 2344 */ 2345 atomic_set(&r1_bio->remaining, 1); 2346 for (i = 0; i < disks ; i++) { 2347 wbio = r1_bio->bios[i]; 2348 if (wbio->bi_end_io == NULL || 2349 (wbio->bi_end_io == end_sync_read && 2350 (i == r1_bio->read_disk || 2351 !test_bit(MD_RECOVERY_SYNC, &mddev->recovery)))) 2352 continue; 2353 if (test_bit(Faulty, &conf->mirrors[i].rdev->flags)) { 2354 abort_sync_write(mddev, r1_bio); 2355 continue; 2356 } 2357 2358 wbio->bi_opf = REQ_OP_WRITE; 2359 if (test_bit(FailFast, &conf->mirrors[i].rdev->flags)) 2360 wbio->bi_opf |= MD_FAILFAST; 2361 2362 wbio->bi_end_io = end_sync_write; 2363 atomic_inc(&r1_bio->remaining); 2364 md_sync_acct(conf->mirrors[i].rdev->bdev, bio_sectors(wbio)); 2365 2366 submit_bio_noacct(wbio); 2367 } 2368 2369 put_sync_write_buf(r1_bio, 1); 2370 } 2371 2372 /* 2373 * This is a kernel thread which: 2374 * 2375 * 1. Retries failed read operations on working mirrors. 2376 * 2. Updates the raid superblock when problems encounter. 2377 * 3. Performs writes following reads for array synchronising. 2378 */ 2379 2380 static void fix_read_error(struct r1conf *conf, struct r1bio *r1_bio) 2381 { 2382 sector_t sect = r1_bio->sector; 2383 int sectors = r1_bio->sectors; 2384 int read_disk = r1_bio->read_disk; 2385 struct mddev *mddev = conf->mddev; 2386 struct md_rdev *rdev = conf->mirrors[read_disk].rdev; 2387 2388 if (exceed_read_errors(mddev, rdev)) { 2389 r1_bio->bios[r1_bio->read_disk] = IO_BLOCKED; 2390 return; 2391 } 2392 2393 while(sectors) { 2394 int s = sectors; 2395 int d = read_disk; 2396 int success = 0; 2397 int start; 2398 2399 if (s > (PAGE_SIZE>>9)) 2400 s = PAGE_SIZE >> 9; 2401 2402 do { 2403 rdev = conf->mirrors[d].rdev; 2404 if (rdev && 2405 (test_bit(In_sync, &rdev->flags) || 2406 (!test_bit(Faulty, &rdev->flags) && 2407 rdev->recovery_offset >= sect + s)) && 2408 rdev_has_badblock(rdev, sect, s) == 0) { 2409 atomic_inc(&rdev->nr_pending); 2410 if (sync_page_io(rdev, sect, s<<9, 2411 conf->tmppage, REQ_OP_READ, false)) 2412 success = 1; 2413 rdev_dec_pending(rdev, mddev); 2414 if (success) 2415 break; 2416 } 2417 2418 d++; 2419 if (d == conf->raid_disks * 2) 2420 d = 0; 2421 } while (d != read_disk); 2422 2423 if (!success) { 2424 /* Cannot read from anywhere - mark it bad */ 2425 struct md_rdev *rdev = conf->mirrors[read_disk].rdev; 2426 if (!rdev_set_badblocks(rdev, sect, s, 0)) 2427 md_error(mddev, rdev); 2428 break; 2429 } 2430 /* write it back and re-read */ 2431 start = d; 2432 while (d != read_disk) { 2433 if (d==0) 2434 d = conf->raid_disks * 2; 2435 d--; 2436 rdev = conf->mirrors[d].rdev; 2437 if (rdev && 2438 !test_bit(Faulty, &rdev->flags)) { 2439 atomic_inc(&rdev->nr_pending); 2440 r1_sync_page_io(rdev, sect, s, 2441 conf->tmppage, REQ_OP_WRITE); 2442 rdev_dec_pending(rdev, mddev); 2443 } 2444 } 2445 d = start; 2446 while (d != read_disk) { 2447 if (d==0) 2448 d = conf->raid_disks * 2; 2449 d--; 2450 rdev = conf->mirrors[d].rdev; 2451 if (rdev && 2452 !test_bit(Faulty, &rdev->flags)) { 2453 atomic_inc(&rdev->nr_pending); 2454 if (r1_sync_page_io(rdev, sect, s, 2455 conf->tmppage, REQ_OP_READ)) { 2456 atomic_add(s, &rdev->corrected_errors); 2457 pr_info("md/raid1:%s: read error corrected (%d sectors at %llu on %pg)\n", 2458 mdname(mddev), s, 2459 (unsigned long long)(sect + 2460 rdev->data_offset), 2461 rdev->bdev); 2462 } 2463 rdev_dec_pending(rdev, mddev); 2464 } 2465 } 2466 sectors -= s; 2467 sect += s; 2468 } 2469 } 2470 2471 static int narrow_write_error(struct r1bio *r1_bio, int i) 2472 { 2473 struct mddev *mddev = r1_bio->mddev; 2474 struct r1conf *conf = mddev->private; 2475 struct md_rdev *rdev = conf->mirrors[i].rdev; 2476 2477 /* bio has the data to be written to device 'i' where 2478 * we just recently had a write error. 2479 * We repeatedly clone the bio and trim down to one block, 2480 * then try the write. Where the write fails we record 2481 * a bad block. 2482 * It is conceivable that the bio doesn't exactly align with 2483 * blocks. We must handle this somehow. 2484 * 2485 * We currently own a reference on the rdev. 2486 */ 2487 2488 int block_sectors; 2489 sector_t sector; 2490 int sectors; 2491 int sect_to_write = r1_bio->sectors; 2492 int ok = 1; 2493 2494 if (rdev->badblocks.shift < 0) 2495 return 0; 2496 2497 block_sectors = roundup(1 << rdev->badblocks.shift, 2498 bdev_logical_block_size(rdev->bdev) >> 9); 2499 sector = r1_bio->sector; 2500 sectors = ((sector + block_sectors) 2501 & ~(sector_t)(block_sectors - 1)) 2502 - sector; 2503 2504 while (sect_to_write) { 2505 struct bio *wbio; 2506 if (sectors > sect_to_write) 2507 sectors = sect_to_write; 2508 /* Write at 'sector' for 'sectors'*/ 2509 2510 if (test_bit(R1BIO_BehindIO, &r1_bio->state)) { 2511 wbio = bio_alloc_clone(rdev->bdev, 2512 r1_bio->behind_master_bio, 2513 GFP_NOIO, &mddev->bio_set); 2514 } else { 2515 wbio = bio_alloc_clone(rdev->bdev, r1_bio->master_bio, 2516 GFP_NOIO, &mddev->bio_set); 2517 } 2518 2519 wbio->bi_opf = REQ_OP_WRITE; 2520 wbio->bi_iter.bi_sector = r1_bio->sector; 2521 wbio->bi_iter.bi_size = r1_bio->sectors << 9; 2522 2523 bio_trim(wbio, sector - r1_bio->sector, sectors); 2524 wbio->bi_iter.bi_sector += rdev->data_offset; 2525 2526 if (submit_bio_wait(wbio) < 0) 2527 /* failure! */ 2528 ok = rdev_set_badblocks(rdev, sector, 2529 sectors, 0) 2530 && ok; 2531 2532 bio_put(wbio); 2533 sect_to_write -= sectors; 2534 sector += sectors; 2535 sectors = block_sectors; 2536 } 2537 return ok; 2538 } 2539 2540 static void handle_sync_write_finished(struct r1conf *conf, struct r1bio *r1_bio) 2541 { 2542 int m; 2543 int s = r1_bio->sectors; 2544 for (m = 0; m < conf->raid_disks * 2 ; m++) { 2545 struct md_rdev *rdev = conf->mirrors[m].rdev; 2546 struct bio *bio = r1_bio->bios[m]; 2547 if (bio->bi_end_io == NULL) 2548 continue; 2549 if (!bio->bi_status && 2550 test_bit(R1BIO_MadeGood, &r1_bio->state)) { 2551 rdev_clear_badblocks(rdev, r1_bio->sector, s, 0); 2552 } 2553 if (bio->bi_status && 2554 test_bit(R1BIO_WriteError, &r1_bio->state)) { 2555 if (!rdev_set_badblocks(rdev, r1_bio->sector, s, 0)) 2556 md_error(conf->mddev, rdev); 2557 } 2558 } 2559 put_buf(r1_bio); 2560 md_done_sync(conf->mddev, s, 1); 2561 } 2562 2563 static void handle_write_finished(struct r1conf *conf, struct r1bio *r1_bio) 2564 { 2565 int m, idx; 2566 bool fail = false; 2567 2568 for (m = 0; m < conf->raid_disks * 2 ; m++) 2569 if (r1_bio->bios[m] == IO_MADE_GOOD) { 2570 struct md_rdev *rdev = conf->mirrors[m].rdev; 2571 rdev_clear_badblocks(rdev, 2572 r1_bio->sector, 2573 r1_bio->sectors, 0); 2574 rdev_dec_pending(rdev, conf->mddev); 2575 } else if (r1_bio->bios[m] != NULL) { 2576 /* This drive got a write error. We need to 2577 * narrow down and record precise write 2578 * errors. 2579 */ 2580 fail = true; 2581 if (!narrow_write_error(r1_bio, m)) { 2582 md_error(conf->mddev, 2583 conf->mirrors[m].rdev); 2584 /* an I/O failed, we can't clear the bitmap */ 2585 set_bit(R1BIO_Degraded, &r1_bio->state); 2586 } 2587 rdev_dec_pending(conf->mirrors[m].rdev, 2588 conf->mddev); 2589 } 2590 if (fail) { 2591 spin_lock_irq(&conf->device_lock); 2592 list_add(&r1_bio->retry_list, &conf->bio_end_io_list); 2593 idx = sector_to_idx(r1_bio->sector); 2594 atomic_inc(&conf->nr_queued[idx]); 2595 spin_unlock_irq(&conf->device_lock); 2596 /* 2597 * In case freeze_array() is waiting for condition 2598 * get_unqueued_pending() == extra to be true. 2599 */ 2600 wake_up(&conf->wait_barrier); 2601 md_wakeup_thread(conf->mddev->thread); 2602 } else { 2603 if (test_bit(R1BIO_WriteError, &r1_bio->state)) 2604 close_write(r1_bio); 2605 raid_end_bio_io(r1_bio); 2606 } 2607 } 2608 2609 static void handle_read_error(struct r1conf *conf, struct r1bio *r1_bio) 2610 { 2611 struct mddev *mddev = conf->mddev; 2612 struct bio *bio; 2613 struct md_rdev *rdev; 2614 sector_t sector; 2615 2616 clear_bit(R1BIO_ReadError, &r1_bio->state); 2617 /* we got a read error. Maybe the drive is bad. Maybe just 2618 * the block and we can fix it. 2619 * We freeze all other IO, and try reading the block from 2620 * other devices. When we find one, we re-write 2621 * and check it that fixes the read error. 2622 * This is all done synchronously while the array is 2623 * frozen 2624 */ 2625 2626 bio = r1_bio->bios[r1_bio->read_disk]; 2627 bio_put(bio); 2628 r1_bio->bios[r1_bio->read_disk] = NULL; 2629 2630 rdev = conf->mirrors[r1_bio->read_disk].rdev; 2631 if (mddev->ro == 0 2632 && !test_bit(FailFast, &rdev->flags)) { 2633 freeze_array(conf, 1); 2634 fix_read_error(conf, r1_bio); 2635 unfreeze_array(conf); 2636 } else if (mddev->ro == 0 && test_bit(FailFast, &rdev->flags)) { 2637 md_error(mddev, rdev); 2638 } else { 2639 r1_bio->bios[r1_bio->read_disk] = IO_BLOCKED; 2640 } 2641 2642 rdev_dec_pending(rdev, conf->mddev); 2643 sector = r1_bio->sector; 2644 bio = r1_bio->master_bio; 2645 2646 /* Reuse the old r1_bio so that the IO_BLOCKED settings are preserved */ 2647 r1_bio->state = 0; 2648 raid1_read_request(mddev, bio, r1_bio->sectors, r1_bio); 2649 allow_barrier(conf, sector); 2650 } 2651 2652 static void raid1d(struct md_thread *thread) 2653 { 2654 struct mddev *mddev = thread->mddev; 2655 struct r1bio *r1_bio; 2656 unsigned long flags; 2657 struct r1conf *conf = mddev->private; 2658 struct list_head *head = &conf->retry_list; 2659 struct blk_plug plug; 2660 int idx; 2661 2662 md_check_recovery(mddev); 2663 2664 if (!list_empty_careful(&conf->bio_end_io_list) && 2665 !test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)) { 2666 LIST_HEAD(tmp); 2667 spin_lock_irqsave(&conf->device_lock, flags); 2668 if (!test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)) 2669 list_splice_init(&conf->bio_end_io_list, &tmp); 2670 spin_unlock_irqrestore(&conf->device_lock, flags); 2671 while (!list_empty(&tmp)) { 2672 r1_bio = list_first_entry(&tmp, struct r1bio, 2673 retry_list); 2674 list_del(&r1_bio->retry_list); 2675 idx = sector_to_idx(r1_bio->sector); 2676 atomic_dec(&conf->nr_queued[idx]); 2677 if (mddev->degraded) 2678 set_bit(R1BIO_Degraded, &r1_bio->state); 2679 if (test_bit(R1BIO_WriteError, &r1_bio->state)) 2680 close_write(r1_bio); 2681 raid_end_bio_io(r1_bio); 2682 } 2683 } 2684 2685 blk_start_plug(&plug); 2686 for (;;) { 2687 2688 flush_pending_writes(conf); 2689 2690 spin_lock_irqsave(&conf->device_lock, flags); 2691 if (list_empty(head)) { 2692 spin_unlock_irqrestore(&conf->device_lock, flags); 2693 break; 2694 } 2695 r1_bio = list_entry(head->prev, struct r1bio, retry_list); 2696 list_del(head->prev); 2697 idx = sector_to_idx(r1_bio->sector); 2698 atomic_dec(&conf->nr_queued[idx]); 2699 spin_unlock_irqrestore(&conf->device_lock, flags); 2700 2701 mddev = r1_bio->mddev; 2702 conf = mddev->private; 2703 if (test_bit(R1BIO_IsSync, &r1_bio->state)) { 2704 if (test_bit(R1BIO_MadeGood, &r1_bio->state) || 2705 test_bit(R1BIO_WriteError, &r1_bio->state)) 2706 handle_sync_write_finished(conf, r1_bio); 2707 else 2708 sync_request_write(mddev, r1_bio); 2709 } else if (test_bit(R1BIO_MadeGood, &r1_bio->state) || 2710 test_bit(R1BIO_WriteError, &r1_bio->state)) 2711 handle_write_finished(conf, r1_bio); 2712 else if (test_bit(R1BIO_ReadError, &r1_bio->state)) 2713 handle_read_error(conf, r1_bio); 2714 else 2715 WARN_ON_ONCE(1); 2716 2717 cond_resched(); 2718 if (mddev->sb_flags & ~(1<<MD_SB_CHANGE_PENDING)) 2719 md_check_recovery(mddev); 2720 } 2721 blk_finish_plug(&plug); 2722 } 2723 2724 static int init_resync(struct r1conf *conf) 2725 { 2726 int buffs; 2727 2728 buffs = RESYNC_WINDOW / RESYNC_BLOCK_SIZE; 2729 BUG_ON(mempool_initialized(&conf->r1buf_pool)); 2730 2731 return mempool_init(&conf->r1buf_pool, buffs, r1buf_pool_alloc, 2732 r1buf_pool_free, conf->poolinfo); 2733 } 2734 2735 static struct r1bio *raid1_alloc_init_r1buf(struct r1conf *conf) 2736 { 2737 struct r1bio *r1bio = mempool_alloc(&conf->r1buf_pool, GFP_NOIO); 2738 struct resync_pages *rps; 2739 struct bio *bio; 2740 int i; 2741 2742 for (i = conf->poolinfo->raid_disks; i--; ) { 2743 bio = r1bio->bios[i]; 2744 rps = bio->bi_private; 2745 bio_reset(bio, NULL, 0); 2746 bio->bi_private = rps; 2747 } 2748 r1bio->master_bio = NULL; 2749 return r1bio; 2750 } 2751 2752 /* 2753 * perform a "sync" on one "block" 2754 * 2755 * We need to make sure that no normal I/O request - particularly write 2756 * requests - conflict with active sync requests. 2757 * 2758 * This is achieved by tracking pending requests and a 'barrier' concept 2759 * that can be installed to exclude normal IO requests. 2760 */ 2761 2762 static sector_t raid1_sync_request(struct mddev *mddev, sector_t sector_nr, 2763 sector_t max_sector, int *skipped) 2764 { 2765 struct r1conf *conf = mddev->private; 2766 struct r1bio *r1_bio; 2767 struct bio *bio; 2768 sector_t nr_sectors; 2769 int disk = -1; 2770 int i; 2771 int wonly = -1; 2772 int write_targets = 0, read_targets = 0; 2773 sector_t sync_blocks; 2774 int still_degraded = 0; 2775 int good_sectors = RESYNC_SECTORS; 2776 int min_bad = 0; /* number of sectors that are bad in all devices */ 2777 int idx = sector_to_idx(sector_nr); 2778 int page_idx = 0; 2779 2780 if (!mempool_initialized(&conf->r1buf_pool)) 2781 if (init_resync(conf)) 2782 return 0; 2783 2784 if (sector_nr >= max_sector) { 2785 /* If we aborted, we need to abort the 2786 * sync on the 'current' bitmap chunk (there will 2787 * only be one in raid1 resync. 2788 * We can find the current addess in mddev->curr_resync 2789 */ 2790 if (mddev->curr_resync < max_sector) /* aborted */ 2791 md_bitmap_end_sync(mddev->bitmap, mddev->curr_resync, 2792 &sync_blocks, 1); 2793 else /* completed sync */ 2794 conf->fullsync = 0; 2795 2796 md_bitmap_close_sync(mddev->bitmap); 2797 close_sync(conf); 2798 2799 if (mddev_is_clustered(mddev)) { 2800 conf->cluster_sync_low = 0; 2801 conf->cluster_sync_high = 0; 2802 } 2803 return 0; 2804 } 2805 2806 if (mddev->bitmap == NULL && 2807 mddev->recovery_cp == MaxSector && 2808 !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) && 2809 conf->fullsync == 0) { 2810 *skipped = 1; 2811 return max_sector - sector_nr; 2812 } 2813 /* before building a request, check if we can skip these blocks.. 2814 * This call the bitmap_start_sync doesn't actually record anything 2815 */ 2816 if (!md_bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, 1) && 2817 !conf->fullsync && !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery)) { 2818 /* We can skip this block, and probably several more */ 2819 *skipped = 1; 2820 return sync_blocks; 2821 } 2822 2823 /* 2824 * If there is non-resync activity waiting for a turn, then let it 2825 * though before starting on this new sync request. 2826 */ 2827 if (atomic_read(&conf->nr_waiting[idx])) 2828 schedule_timeout_uninterruptible(1); 2829 2830 /* we are incrementing sector_nr below. To be safe, we check against 2831 * sector_nr + two times RESYNC_SECTORS 2832 */ 2833 2834 md_bitmap_cond_end_sync(mddev->bitmap, sector_nr, 2835 mddev_is_clustered(mddev) && (sector_nr + 2 * RESYNC_SECTORS > conf->cluster_sync_high)); 2836 2837 2838 if (raise_barrier(conf, sector_nr)) 2839 return 0; 2840 2841 r1_bio = raid1_alloc_init_r1buf(conf); 2842 2843 /* 2844 * If we get a correctably read error during resync or recovery, 2845 * we might want to read from a different device. So we 2846 * flag all drives that could conceivably be read from for READ, 2847 * and any others (which will be non-In_sync devices) for WRITE. 2848 * If a read fails, we try reading from something else for which READ 2849 * is OK. 2850 */ 2851 2852 r1_bio->mddev = mddev; 2853 r1_bio->sector = sector_nr; 2854 r1_bio->state = 0; 2855 set_bit(R1BIO_IsSync, &r1_bio->state); 2856 /* make sure good_sectors won't go across barrier unit boundary */ 2857 good_sectors = align_to_barrier_unit_end(sector_nr, good_sectors); 2858 2859 for (i = 0; i < conf->raid_disks * 2; i++) { 2860 struct md_rdev *rdev; 2861 bio = r1_bio->bios[i]; 2862 2863 rdev = conf->mirrors[i].rdev; 2864 if (rdev == NULL || 2865 test_bit(Faulty, &rdev->flags)) { 2866 if (i < conf->raid_disks) 2867 still_degraded = 1; 2868 } else if (!test_bit(In_sync, &rdev->flags)) { 2869 bio->bi_opf = REQ_OP_WRITE; 2870 bio->bi_end_io = end_sync_write; 2871 write_targets ++; 2872 } else { 2873 /* may need to read from here */ 2874 sector_t first_bad = MaxSector; 2875 int bad_sectors; 2876 2877 if (is_badblock(rdev, sector_nr, good_sectors, 2878 &first_bad, &bad_sectors)) { 2879 if (first_bad > sector_nr) 2880 good_sectors = first_bad - sector_nr; 2881 else { 2882 bad_sectors -= (sector_nr - first_bad); 2883 if (min_bad == 0 || 2884 min_bad > bad_sectors) 2885 min_bad = bad_sectors; 2886 } 2887 } 2888 if (sector_nr < first_bad) { 2889 if (test_bit(WriteMostly, &rdev->flags)) { 2890 if (wonly < 0) 2891 wonly = i; 2892 } else { 2893 if (disk < 0) 2894 disk = i; 2895 } 2896 bio->bi_opf = REQ_OP_READ; 2897 bio->bi_end_io = end_sync_read; 2898 read_targets++; 2899 } else if (!test_bit(WriteErrorSeen, &rdev->flags) && 2900 test_bit(MD_RECOVERY_SYNC, &mddev->recovery) && 2901 !test_bit(MD_RECOVERY_CHECK, &mddev->recovery)) { 2902 /* 2903 * The device is suitable for reading (InSync), 2904 * but has bad block(s) here. Let's try to correct them, 2905 * if we are doing resync or repair. Otherwise, leave 2906 * this device alone for this sync request. 2907 */ 2908 bio->bi_opf = REQ_OP_WRITE; 2909 bio->bi_end_io = end_sync_write; 2910 write_targets++; 2911 } 2912 } 2913 if (rdev && bio->bi_end_io) { 2914 atomic_inc(&rdev->nr_pending); 2915 bio->bi_iter.bi_sector = sector_nr + rdev->data_offset; 2916 bio_set_dev(bio, rdev->bdev); 2917 if (test_bit(FailFast, &rdev->flags)) 2918 bio->bi_opf |= MD_FAILFAST; 2919 } 2920 } 2921 if (disk < 0) 2922 disk = wonly; 2923 r1_bio->read_disk = disk; 2924 2925 if (read_targets == 0 && min_bad > 0) { 2926 /* These sectors are bad on all InSync devices, so we 2927 * need to mark them bad on all write targets 2928 */ 2929 int ok = 1; 2930 for (i = 0 ; i < conf->raid_disks * 2 ; i++) 2931 if (r1_bio->bios[i]->bi_end_io == end_sync_write) { 2932 struct md_rdev *rdev = conf->mirrors[i].rdev; 2933 ok = rdev_set_badblocks(rdev, sector_nr, 2934 min_bad, 0 2935 ) && ok; 2936 } 2937 set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags); 2938 *skipped = 1; 2939 put_buf(r1_bio); 2940 2941 if (!ok) { 2942 /* Cannot record the badblocks, so need to 2943 * abort the resync. 2944 * If there are multiple read targets, could just 2945 * fail the really bad ones ??? 2946 */ 2947 conf->recovery_disabled = mddev->recovery_disabled; 2948 set_bit(MD_RECOVERY_INTR, &mddev->recovery); 2949 return 0; 2950 } else 2951 return min_bad; 2952 2953 } 2954 if (min_bad > 0 && min_bad < good_sectors) { 2955 /* only resync enough to reach the next bad->good 2956 * transition */ 2957 good_sectors = min_bad; 2958 } 2959 2960 if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery) && read_targets > 0) 2961 /* extra read targets are also write targets */ 2962 write_targets += read_targets-1; 2963 2964 if (write_targets == 0 || read_targets == 0) { 2965 /* There is nowhere to write, so all non-sync 2966 * drives must be failed - so we are finished 2967 */ 2968 sector_t rv; 2969 if (min_bad > 0) 2970 max_sector = sector_nr + min_bad; 2971 rv = max_sector - sector_nr; 2972 *skipped = 1; 2973 put_buf(r1_bio); 2974 return rv; 2975 } 2976 2977 if (max_sector > mddev->resync_max) 2978 max_sector = mddev->resync_max; /* Don't do IO beyond here */ 2979 if (max_sector > sector_nr + good_sectors) 2980 max_sector = sector_nr + good_sectors; 2981 nr_sectors = 0; 2982 sync_blocks = 0; 2983 do { 2984 struct page *page; 2985 int len = PAGE_SIZE; 2986 if (sector_nr + (len>>9) > max_sector) 2987 len = (max_sector - sector_nr) << 9; 2988 if (len == 0) 2989 break; 2990 if (sync_blocks == 0) { 2991 if (!md_bitmap_start_sync(mddev->bitmap, sector_nr, 2992 &sync_blocks, still_degraded) && 2993 !conf->fullsync && 2994 !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery)) 2995 break; 2996 if ((len >> 9) > sync_blocks) 2997 len = sync_blocks<<9; 2998 } 2999 3000 for (i = 0 ; i < conf->raid_disks * 2; i++) { 3001 struct resync_pages *rp; 3002 3003 bio = r1_bio->bios[i]; 3004 rp = get_resync_pages(bio); 3005 if (bio->bi_end_io) { 3006 page = resync_fetch_page(rp, page_idx); 3007 3008 /* 3009 * won't fail because the vec table is big 3010 * enough to hold all these pages 3011 */ 3012 __bio_add_page(bio, page, len, 0); 3013 } 3014 } 3015 nr_sectors += len>>9; 3016 sector_nr += len>>9; 3017 sync_blocks -= (len>>9); 3018 } while (++page_idx < RESYNC_PAGES); 3019 3020 r1_bio->sectors = nr_sectors; 3021 3022 if (mddev_is_clustered(mddev) && 3023 conf->cluster_sync_high < sector_nr + nr_sectors) { 3024 conf->cluster_sync_low = mddev->curr_resync_completed; 3025 conf->cluster_sync_high = conf->cluster_sync_low + CLUSTER_RESYNC_WINDOW_SECTORS; 3026 /* Send resync message */ 3027 md_cluster_ops->resync_info_update(mddev, 3028 conf->cluster_sync_low, 3029 conf->cluster_sync_high); 3030 } 3031 3032 /* For a user-requested sync, we read all readable devices and do a 3033 * compare 3034 */ 3035 if (test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery)) { 3036 atomic_set(&r1_bio->remaining, read_targets); 3037 for (i = 0; i < conf->raid_disks * 2 && read_targets; i++) { 3038 bio = r1_bio->bios[i]; 3039 if (bio->bi_end_io == end_sync_read) { 3040 read_targets--; 3041 md_sync_acct_bio(bio, nr_sectors); 3042 if (read_targets == 1) 3043 bio->bi_opf &= ~MD_FAILFAST; 3044 submit_bio_noacct(bio); 3045 } 3046 } 3047 } else { 3048 atomic_set(&r1_bio->remaining, 1); 3049 bio = r1_bio->bios[r1_bio->read_disk]; 3050 md_sync_acct_bio(bio, nr_sectors); 3051 if (read_targets == 1) 3052 bio->bi_opf &= ~MD_FAILFAST; 3053 submit_bio_noacct(bio); 3054 } 3055 return nr_sectors; 3056 } 3057 3058 static sector_t raid1_size(struct mddev *mddev, sector_t sectors, int raid_disks) 3059 { 3060 if (sectors) 3061 return sectors; 3062 3063 return mddev->dev_sectors; 3064 } 3065 3066 static struct r1conf *setup_conf(struct mddev *mddev) 3067 { 3068 struct r1conf *conf; 3069 int i; 3070 struct raid1_info *disk; 3071 struct md_rdev *rdev; 3072 int err = -ENOMEM; 3073 3074 conf = kzalloc(sizeof(struct r1conf), GFP_KERNEL); 3075 if (!conf) 3076 goto abort; 3077 3078 conf->nr_pending = kcalloc(BARRIER_BUCKETS_NR, 3079 sizeof(atomic_t), GFP_KERNEL); 3080 if (!conf->nr_pending) 3081 goto abort; 3082 3083 conf->nr_waiting = kcalloc(BARRIER_BUCKETS_NR, 3084 sizeof(atomic_t), GFP_KERNEL); 3085 if (!conf->nr_waiting) 3086 goto abort; 3087 3088 conf->nr_queued = kcalloc(BARRIER_BUCKETS_NR, 3089 sizeof(atomic_t), GFP_KERNEL); 3090 if (!conf->nr_queued) 3091 goto abort; 3092 3093 conf->barrier = kcalloc(BARRIER_BUCKETS_NR, 3094 sizeof(atomic_t), GFP_KERNEL); 3095 if (!conf->barrier) 3096 goto abort; 3097 3098 conf->mirrors = kzalloc(array3_size(sizeof(struct raid1_info), 3099 mddev->raid_disks, 2), 3100 GFP_KERNEL); 3101 if (!conf->mirrors) 3102 goto abort; 3103 3104 conf->tmppage = alloc_page(GFP_KERNEL); 3105 if (!conf->tmppage) 3106 goto abort; 3107 3108 conf->poolinfo = kzalloc(sizeof(*conf->poolinfo), GFP_KERNEL); 3109 if (!conf->poolinfo) 3110 goto abort; 3111 conf->poolinfo->raid_disks = mddev->raid_disks * 2; 3112 err = mempool_init(&conf->r1bio_pool, NR_RAID_BIOS, r1bio_pool_alloc, 3113 rbio_pool_free, conf->poolinfo); 3114 if (err) 3115 goto abort; 3116 3117 err = bioset_init(&conf->bio_split, BIO_POOL_SIZE, 0, 0); 3118 if (err) 3119 goto abort; 3120 3121 conf->poolinfo->mddev = mddev; 3122 3123 err = -EINVAL; 3124 spin_lock_init(&conf->device_lock); 3125 conf->raid_disks = mddev->raid_disks; 3126 rdev_for_each(rdev, mddev) { 3127 int disk_idx = rdev->raid_disk; 3128 3129 if (disk_idx >= conf->raid_disks || disk_idx < 0) 3130 continue; 3131 3132 if (!raid1_add_conf(conf, rdev, disk_idx, 3133 test_bit(Replacement, &rdev->flags))) 3134 goto abort; 3135 } 3136 conf->mddev = mddev; 3137 INIT_LIST_HEAD(&conf->retry_list); 3138 INIT_LIST_HEAD(&conf->bio_end_io_list); 3139 3140 spin_lock_init(&conf->resync_lock); 3141 init_waitqueue_head(&conf->wait_barrier); 3142 3143 bio_list_init(&conf->pending_bio_list); 3144 conf->recovery_disabled = mddev->recovery_disabled - 1; 3145 3146 err = -EIO; 3147 for (i = 0; i < conf->raid_disks * 2; i++) { 3148 3149 disk = conf->mirrors + i; 3150 3151 if (i < conf->raid_disks && 3152 disk[conf->raid_disks].rdev) { 3153 /* This slot has a replacement. */ 3154 if (!disk->rdev) { 3155 /* No original, just make the replacement 3156 * a recovering spare 3157 */ 3158 disk->rdev = 3159 disk[conf->raid_disks].rdev; 3160 disk[conf->raid_disks].rdev = NULL; 3161 } else if (!test_bit(In_sync, &disk->rdev->flags)) 3162 /* Original is not in_sync - bad */ 3163 goto abort; 3164 } 3165 3166 if (!disk->rdev || 3167 !test_bit(In_sync, &disk->rdev->flags)) { 3168 disk->head_position = 0; 3169 if (disk->rdev && 3170 (disk->rdev->saved_raid_disk < 0)) 3171 conf->fullsync = 1; 3172 } 3173 } 3174 3175 err = -ENOMEM; 3176 rcu_assign_pointer(conf->thread, 3177 md_register_thread(raid1d, mddev, "raid1")); 3178 if (!conf->thread) 3179 goto abort; 3180 3181 return conf; 3182 3183 abort: 3184 if (conf) { 3185 mempool_exit(&conf->r1bio_pool); 3186 kfree(conf->mirrors); 3187 safe_put_page(conf->tmppage); 3188 kfree(conf->poolinfo); 3189 kfree(conf->nr_pending); 3190 kfree(conf->nr_waiting); 3191 kfree(conf->nr_queued); 3192 kfree(conf->barrier); 3193 bioset_exit(&conf->bio_split); 3194 kfree(conf); 3195 } 3196 return ERR_PTR(err); 3197 } 3198 3199 static int raid1_set_limits(struct mddev *mddev) 3200 { 3201 struct queue_limits lim; 3202 int err; 3203 3204 md_init_stacking_limits(&lim); 3205 lim.max_write_zeroes_sectors = 0; 3206 err = mddev_stack_rdev_limits(mddev, &lim, MDDEV_STACK_INTEGRITY); 3207 if (err) { 3208 queue_limits_cancel_update(mddev->gendisk->queue); 3209 return err; 3210 } 3211 return queue_limits_set(mddev->gendisk->queue, &lim); 3212 } 3213 3214 static int raid1_run(struct mddev *mddev) 3215 { 3216 struct r1conf *conf; 3217 int i; 3218 int ret; 3219 3220 if (mddev->level != 1) { 3221 pr_warn("md/raid1:%s: raid level not set to mirroring (%d)\n", 3222 mdname(mddev), mddev->level); 3223 return -EIO; 3224 } 3225 if (mddev->reshape_position != MaxSector) { 3226 pr_warn("md/raid1:%s: reshape_position set but not supported\n", 3227 mdname(mddev)); 3228 return -EIO; 3229 } 3230 3231 /* 3232 * copy the already verified devices into our private RAID1 3233 * bookkeeping area. [whatever we allocate in run(), 3234 * should be freed in raid1_free()] 3235 */ 3236 if (mddev->private == NULL) 3237 conf = setup_conf(mddev); 3238 else 3239 conf = mddev->private; 3240 3241 if (IS_ERR(conf)) 3242 return PTR_ERR(conf); 3243 3244 if (!mddev_is_dm(mddev)) { 3245 ret = raid1_set_limits(mddev); 3246 if (ret) 3247 return ret; 3248 } 3249 3250 mddev->degraded = 0; 3251 for (i = 0; i < conf->raid_disks; i++) 3252 if (conf->mirrors[i].rdev == NULL || 3253 !test_bit(In_sync, &conf->mirrors[i].rdev->flags) || 3254 test_bit(Faulty, &conf->mirrors[i].rdev->flags)) 3255 mddev->degraded++; 3256 /* 3257 * RAID1 needs at least one disk in active 3258 */ 3259 if (conf->raid_disks - mddev->degraded < 1) { 3260 md_unregister_thread(mddev, &conf->thread); 3261 return -EINVAL; 3262 } 3263 3264 if (conf->raid_disks - mddev->degraded == 1) 3265 mddev->recovery_cp = MaxSector; 3266 3267 if (mddev->recovery_cp != MaxSector) 3268 pr_info("md/raid1:%s: not clean -- starting background reconstruction\n", 3269 mdname(mddev)); 3270 pr_info("md/raid1:%s: active with %d out of %d mirrors\n", 3271 mdname(mddev), mddev->raid_disks - mddev->degraded, 3272 mddev->raid_disks); 3273 3274 /* 3275 * Ok, everything is just fine now 3276 */ 3277 rcu_assign_pointer(mddev->thread, conf->thread); 3278 rcu_assign_pointer(conf->thread, NULL); 3279 mddev->private = conf; 3280 set_bit(MD_FAILFAST_SUPPORTED, &mddev->flags); 3281 3282 md_set_array_sectors(mddev, raid1_size(mddev, 0, 0)); 3283 3284 ret = md_integrity_register(mddev); 3285 if (ret) 3286 md_unregister_thread(mddev, &mddev->thread); 3287 return ret; 3288 } 3289 3290 static void raid1_free(struct mddev *mddev, void *priv) 3291 { 3292 struct r1conf *conf = priv; 3293 3294 mempool_exit(&conf->r1bio_pool); 3295 kfree(conf->mirrors); 3296 safe_put_page(conf->tmppage); 3297 kfree(conf->poolinfo); 3298 kfree(conf->nr_pending); 3299 kfree(conf->nr_waiting); 3300 kfree(conf->nr_queued); 3301 kfree(conf->barrier); 3302 bioset_exit(&conf->bio_split); 3303 kfree(conf); 3304 } 3305 3306 static int raid1_resize(struct mddev *mddev, sector_t sectors) 3307 { 3308 /* no resync is happening, and there is enough space 3309 * on all devices, so we can resize. 3310 * We need to make sure resync covers any new space. 3311 * If the array is shrinking we should possibly wait until 3312 * any io in the removed space completes, but it hardly seems 3313 * worth it. 3314 */ 3315 sector_t newsize = raid1_size(mddev, sectors, 0); 3316 if (mddev->external_size && 3317 mddev->array_sectors > newsize) 3318 return -EINVAL; 3319 if (mddev->bitmap) { 3320 int ret = md_bitmap_resize(mddev->bitmap, newsize, 0, 0); 3321 if (ret) 3322 return ret; 3323 } 3324 md_set_array_sectors(mddev, newsize); 3325 if (sectors > mddev->dev_sectors && 3326 mddev->recovery_cp > mddev->dev_sectors) { 3327 mddev->recovery_cp = mddev->dev_sectors; 3328 set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); 3329 } 3330 mddev->dev_sectors = sectors; 3331 mddev->resync_max_sectors = sectors; 3332 return 0; 3333 } 3334 3335 static int raid1_reshape(struct mddev *mddev) 3336 { 3337 /* We need to: 3338 * 1/ resize the r1bio_pool 3339 * 2/ resize conf->mirrors 3340 * 3341 * We allocate a new r1bio_pool if we can. 3342 * Then raise a device barrier and wait until all IO stops. 3343 * Then resize conf->mirrors and swap in the new r1bio pool. 3344 * 3345 * At the same time, we "pack" the devices so that all the missing 3346 * devices have the higher raid_disk numbers. 3347 */ 3348 mempool_t newpool, oldpool; 3349 struct pool_info *newpoolinfo; 3350 struct raid1_info *newmirrors; 3351 struct r1conf *conf = mddev->private; 3352 int cnt, raid_disks; 3353 unsigned long flags; 3354 int d, d2; 3355 int ret; 3356 3357 memset(&newpool, 0, sizeof(newpool)); 3358 memset(&oldpool, 0, sizeof(oldpool)); 3359 3360 /* Cannot change chunk_size, layout, or level */ 3361 if (mddev->chunk_sectors != mddev->new_chunk_sectors || 3362 mddev->layout != mddev->new_layout || 3363 mddev->level != mddev->new_level) { 3364 mddev->new_chunk_sectors = mddev->chunk_sectors; 3365 mddev->new_layout = mddev->layout; 3366 mddev->new_level = mddev->level; 3367 return -EINVAL; 3368 } 3369 3370 if (!mddev_is_clustered(mddev)) 3371 md_allow_write(mddev); 3372 3373 raid_disks = mddev->raid_disks + mddev->delta_disks; 3374 3375 if (raid_disks < conf->raid_disks) { 3376 cnt=0; 3377 for (d= 0; d < conf->raid_disks; d++) 3378 if (conf->mirrors[d].rdev) 3379 cnt++; 3380 if (cnt > raid_disks) 3381 return -EBUSY; 3382 } 3383 3384 newpoolinfo = kmalloc(sizeof(*newpoolinfo), GFP_KERNEL); 3385 if (!newpoolinfo) 3386 return -ENOMEM; 3387 newpoolinfo->mddev = mddev; 3388 newpoolinfo->raid_disks = raid_disks * 2; 3389 3390 ret = mempool_init(&newpool, NR_RAID_BIOS, r1bio_pool_alloc, 3391 rbio_pool_free, newpoolinfo); 3392 if (ret) { 3393 kfree(newpoolinfo); 3394 return ret; 3395 } 3396 newmirrors = kzalloc(array3_size(sizeof(struct raid1_info), 3397 raid_disks, 2), 3398 GFP_KERNEL); 3399 if (!newmirrors) { 3400 kfree(newpoolinfo); 3401 mempool_exit(&newpool); 3402 return -ENOMEM; 3403 } 3404 3405 freeze_array(conf, 0); 3406 3407 /* ok, everything is stopped */ 3408 oldpool = conf->r1bio_pool; 3409 conf->r1bio_pool = newpool; 3410 3411 for (d = d2 = 0; d < conf->raid_disks; d++) { 3412 struct md_rdev *rdev = conf->mirrors[d].rdev; 3413 if (rdev && rdev->raid_disk != d2) { 3414 sysfs_unlink_rdev(mddev, rdev); 3415 rdev->raid_disk = d2; 3416 sysfs_unlink_rdev(mddev, rdev); 3417 if (sysfs_link_rdev(mddev, rdev)) 3418 pr_warn("md/raid1:%s: cannot register rd%d\n", 3419 mdname(mddev), rdev->raid_disk); 3420 } 3421 if (rdev) 3422 newmirrors[d2++].rdev = rdev; 3423 } 3424 kfree(conf->mirrors); 3425 conf->mirrors = newmirrors; 3426 kfree(conf->poolinfo); 3427 conf->poolinfo = newpoolinfo; 3428 3429 spin_lock_irqsave(&conf->device_lock, flags); 3430 mddev->degraded += (raid_disks - conf->raid_disks); 3431 spin_unlock_irqrestore(&conf->device_lock, flags); 3432 conf->raid_disks = mddev->raid_disks = raid_disks; 3433 mddev->delta_disks = 0; 3434 3435 unfreeze_array(conf); 3436 3437 set_bit(MD_RECOVERY_RECOVER, &mddev->recovery); 3438 set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); 3439 md_wakeup_thread(mddev->thread); 3440 3441 mempool_exit(&oldpool); 3442 return 0; 3443 } 3444 3445 static void raid1_quiesce(struct mddev *mddev, int quiesce) 3446 { 3447 struct r1conf *conf = mddev->private; 3448 3449 if (quiesce) 3450 freeze_array(conf, 0); 3451 else 3452 unfreeze_array(conf); 3453 } 3454 3455 static void *raid1_takeover(struct mddev *mddev) 3456 { 3457 /* raid1 can take over: 3458 * raid5 with 2 devices, any layout or chunk size 3459 */ 3460 if (mddev->level == 5 && mddev->raid_disks == 2) { 3461 struct r1conf *conf; 3462 mddev->new_level = 1; 3463 mddev->new_layout = 0; 3464 mddev->new_chunk_sectors = 0; 3465 conf = setup_conf(mddev); 3466 if (!IS_ERR(conf)) { 3467 /* Array must appear to be quiesced */ 3468 conf->array_frozen = 1; 3469 mddev_clear_unsupported_flags(mddev, 3470 UNSUPPORTED_MDDEV_FLAGS); 3471 } 3472 return conf; 3473 } 3474 return ERR_PTR(-EINVAL); 3475 } 3476 3477 static struct md_personality raid1_personality = 3478 { 3479 .name = "raid1", 3480 .level = 1, 3481 .owner = THIS_MODULE, 3482 .make_request = raid1_make_request, 3483 .run = raid1_run, 3484 .free = raid1_free, 3485 .status = raid1_status, 3486 .error_handler = raid1_error, 3487 .hot_add_disk = raid1_add_disk, 3488 .hot_remove_disk= raid1_remove_disk, 3489 .spare_active = raid1_spare_active, 3490 .sync_request = raid1_sync_request, 3491 .resize = raid1_resize, 3492 .size = raid1_size, 3493 .check_reshape = raid1_reshape, 3494 .quiesce = raid1_quiesce, 3495 .takeover = raid1_takeover, 3496 }; 3497 3498 static int __init raid_init(void) 3499 { 3500 return register_md_personality(&raid1_personality); 3501 } 3502 3503 static void raid_exit(void) 3504 { 3505 unregister_md_personality(&raid1_personality); 3506 } 3507 3508 module_init(raid_init); 3509 module_exit(raid_exit); 3510 MODULE_LICENSE("GPL"); 3511 MODULE_DESCRIPTION("RAID1 (mirroring) personality for MD"); 3512 MODULE_ALIAS("md-personality-3"); /* RAID1 */ 3513 MODULE_ALIAS("md-raid1"); 3514 MODULE_ALIAS("md-level-1"); 3515