xref: /linux/drivers/md/raid1.c (revision bcb63314e2c23f1ed622418b65f9409512659c73)
1 /*
2  * raid1.c : Multiple Devices driver for Linux
3  *
4  * Copyright (C) 1999, 2000, 2001 Ingo Molnar, Red Hat
5  *
6  * Copyright (C) 1996, 1997, 1998 Ingo Molnar, Miguel de Icaza, Gadi Oxman
7  *
8  * RAID-1 management functions.
9  *
10  * Better read-balancing code written by Mika Kuoppala <miku@iki.fi>, 2000
11  *
12  * Fixes to reconstruction by Jakob Østergaard" <jakob@ostenfeld.dk>
13  * Various fixes by Neil Brown <neilb@cse.unsw.edu.au>
14  *
15  * Changes by Peter T. Breuer <ptb@it.uc3m.es> 31/1/2003 to support
16  * bitmapped intelligence in resync:
17  *
18  *      - bitmap marked during normal i/o
19  *      - bitmap used to skip nondirty blocks during sync
20  *
21  * Additions to bitmap code, (C) 2003-2004 Paul Clements, SteelEye Technology:
22  * - persistent bitmap code
23  *
24  * This program is free software; you can redistribute it and/or modify
25  * it under the terms of the GNU General Public License as published by
26  * the Free Software Foundation; either version 2, or (at your option)
27  * any later version.
28  *
29  * You should have received a copy of the GNU General Public License
30  * (for example /usr/src/linux/COPYING); if not, write to the Free
31  * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
32  */
33 
34 #include <linux/slab.h>
35 #include <linux/delay.h>
36 #include <linux/blkdev.h>
37 #include <linux/module.h>
38 #include <linux/seq_file.h>
39 #include <linux/ratelimit.h>
40 #include <trace/events/block.h>
41 #include "md.h"
42 #include "raid1.h"
43 #include "bitmap.h"
44 
45 /*
46  * Number of guaranteed r1bios in case of extreme VM load:
47  */
48 #define	NR_RAID1_BIOS 256
49 
50 /* when we get a read error on a read-only array, we redirect to another
51  * device without failing the first device, or trying to over-write to
52  * correct the read error.  To keep track of bad blocks on a per-bio
53  * level, we store IO_BLOCKED in the appropriate 'bios' pointer
54  */
55 #define IO_BLOCKED ((struct bio *)1)
56 /* When we successfully write to a known bad-block, we need to remove the
57  * bad-block marking which must be done from process context.  So we record
58  * the success by setting devs[n].bio to IO_MADE_GOOD
59  */
60 #define IO_MADE_GOOD ((struct bio *)2)
61 
62 #define BIO_SPECIAL(bio) ((unsigned long)bio <= 2)
63 
64 /* When there are this many requests queue to be written by
65  * the raid1 thread, we become 'congested' to provide back-pressure
66  * for writeback.
67  */
68 static int max_queued_requests = 1024;
69 
70 static void allow_barrier(struct r1conf *conf, sector_t start_next_window,
71 			  sector_t bi_sector);
72 static void lower_barrier(struct r1conf *conf);
73 
74 #define raid1_log(md, fmt, args...)				\
75 	do { if ((md)->queue) blk_add_trace_msg((md)->queue, "raid1 " fmt, ##args); } while (0)
76 
77 static void * r1bio_pool_alloc(gfp_t gfp_flags, void *data)
78 {
79 	struct pool_info *pi = data;
80 	int size = offsetof(struct r1bio, bios[pi->raid_disks]);
81 
82 	/* allocate a r1bio with room for raid_disks entries in the bios array */
83 	return kzalloc(size, gfp_flags);
84 }
85 
86 static void r1bio_pool_free(void *r1_bio, void *data)
87 {
88 	kfree(r1_bio);
89 }
90 
91 #define RESYNC_BLOCK_SIZE (64*1024)
92 #define RESYNC_DEPTH 32
93 #define RESYNC_SECTORS (RESYNC_BLOCK_SIZE >> 9)
94 #define RESYNC_PAGES ((RESYNC_BLOCK_SIZE + PAGE_SIZE-1) / PAGE_SIZE)
95 #define RESYNC_WINDOW (RESYNC_BLOCK_SIZE * RESYNC_DEPTH)
96 #define RESYNC_WINDOW_SECTORS (RESYNC_WINDOW >> 9)
97 #define CLUSTER_RESYNC_WINDOW (16 * RESYNC_WINDOW)
98 #define CLUSTER_RESYNC_WINDOW_SECTORS (CLUSTER_RESYNC_WINDOW >> 9)
99 #define NEXT_NORMALIO_DISTANCE (3 * RESYNC_WINDOW_SECTORS)
100 
101 static void * r1buf_pool_alloc(gfp_t gfp_flags, void *data)
102 {
103 	struct pool_info *pi = data;
104 	struct r1bio *r1_bio;
105 	struct bio *bio;
106 	int need_pages;
107 	int i, j;
108 
109 	r1_bio = r1bio_pool_alloc(gfp_flags, pi);
110 	if (!r1_bio)
111 		return NULL;
112 
113 	/*
114 	 * Allocate bios : 1 for reading, n-1 for writing
115 	 */
116 	for (j = pi->raid_disks ; j-- ; ) {
117 		bio = bio_kmalloc(gfp_flags, RESYNC_PAGES);
118 		if (!bio)
119 			goto out_free_bio;
120 		r1_bio->bios[j] = bio;
121 	}
122 	/*
123 	 * Allocate RESYNC_PAGES data pages and attach them to
124 	 * the first bio.
125 	 * If this is a user-requested check/repair, allocate
126 	 * RESYNC_PAGES for each bio.
127 	 */
128 	if (test_bit(MD_RECOVERY_REQUESTED, &pi->mddev->recovery))
129 		need_pages = pi->raid_disks;
130 	else
131 		need_pages = 1;
132 	for (j = 0; j < need_pages; j++) {
133 		bio = r1_bio->bios[j];
134 		bio->bi_vcnt = RESYNC_PAGES;
135 
136 		if (bio_alloc_pages(bio, gfp_flags))
137 			goto out_free_pages;
138 	}
139 	/* If not user-requests, copy the page pointers to all bios */
140 	if (!test_bit(MD_RECOVERY_REQUESTED, &pi->mddev->recovery)) {
141 		for (i=0; i<RESYNC_PAGES ; i++)
142 			for (j=1; j<pi->raid_disks; j++)
143 				r1_bio->bios[j]->bi_io_vec[i].bv_page =
144 					r1_bio->bios[0]->bi_io_vec[i].bv_page;
145 	}
146 
147 	r1_bio->master_bio = NULL;
148 
149 	return r1_bio;
150 
151 out_free_pages:
152 	while (--j >= 0)
153 		bio_free_pages(r1_bio->bios[j]);
154 
155 out_free_bio:
156 	while (++j < pi->raid_disks)
157 		bio_put(r1_bio->bios[j]);
158 	r1bio_pool_free(r1_bio, data);
159 	return NULL;
160 }
161 
162 static void r1buf_pool_free(void *__r1_bio, void *data)
163 {
164 	struct pool_info *pi = data;
165 	int i,j;
166 	struct r1bio *r1bio = __r1_bio;
167 
168 	for (i = 0; i < RESYNC_PAGES; i++)
169 		for (j = pi->raid_disks; j-- ;) {
170 			if (j == 0 ||
171 			    r1bio->bios[j]->bi_io_vec[i].bv_page !=
172 			    r1bio->bios[0]->bi_io_vec[i].bv_page)
173 				safe_put_page(r1bio->bios[j]->bi_io_vec[i].bv_page);
174 		}
175 	for (i=0 ; i < pi->raid_disks; i++)
176 		bio_put(r1bio->bios[i]);
177 
178 	r1bio_pool_free(r1bio, data);
179 }
180 
181 static void put_all_bios(struct r1conf *conf, struct r1bio *r1_bio)
182 {
183 	int i;
184 
185 	for (i = 0; i < conf->raid_disks * 2; i++) {
186 		struct bio **bio = r1_bio->bios + i;
187 		if (!BIO_SPECIAL(*bio))
188 			bio_put(*bio);
189 		*bio = NULL;
190 	}
191 }
192 
193 static void free_r1bio(struct r1bio *r1_bio)
194 {
195 	struct r1conf *conf = r1_bio->mddev->private;
196 
197 	put_all_bios(conf, r1_bio);
198 	mempool_free(r1_bio, conf->r1bio_pool);
199 }
200 
201 static void put_buf(struct r1bio *r1_bio)
202 {
203 	struct r1conf *conf = r1_bio->mddev->private;
204 	int i;
205 
206 	for (i = 0; i < conf->raid_disks * 2; i++) {
207 		struct bio *bio = r1_bio->bios[i];
208 		if (bio->bi_end_io)
209 			rdev_dec_pending(conf->mirrors[i].rdev, r1_bio->mddev);
210 	}
211 
212 	mempool_free(r1_bio, conf->r1buf_pool);
213 
214 	lower_barrier(conf);
215 }
216 
217 static void reschedule_retry(struct r1bio *r1_bio)
218 {
219 	unsigned long flags;
220 	struct mddev *mddev = r1_bio->mddev;
221 	struct r1conf *conf = mddev->private;
222 
223 	spin_lock_irqsave(&conf->device_lock, flags);
224 	list_add(&r1_bio->retry_list, &conf->retry_list);
225 	conf->nr_queued ++;
226 	spin_unlock_irqrestore(&conf->device_lock, flags);
227 
228 	wake_up(&conf->wait_barrier);
229 	md_wakeup_thread(mddev->thread);
230 }
231 
232 /*
233  * raid_end_bio_io() is called when we have finished servicing a mirrored
234  * operation and are ready to return a success/failure code to the buffer
235  * cache layer.
236  */
237 static void call_bio_endio(struct r1bio *r1_bio)
238 {
239 	struct bio *bio = r1_bio->master_bio;
240 	int done;
241 	struct r1conf *conf = r1_bio->mddev->private;
242 	sector_t start_next_window = r1_bio->start_next_window;
243 	sector_t bi_sector = bio->bi_iter.bi_sector;
244 
245 	if (bio->bi_phys_segments) {
246 		unsigned long flags;
247 		spin_lock_irqsave(&conf->device_lock, flags);
248 		bio->bi_phys_segments--;
249 		done = (bio->bi_phys_segments == 0);
250 		spin_unlock_irqrestore(&conf->device_lock, flags);
251 		/*
252 		 * make_request() might be waiting for
253 		 * bi_phys_segments to decrease
254 		 */
255 		wake_up(&conf->wait_barrier);
256 	} else
257 		done = 1;
258 
259 	if (!test_bit(R1BIO_Uptodate, &r1_bio->state))
260 		bio->bi_error = -EIO;
261 
262 	if (done) {
263 		bio_endio(bio);
264 		/*
265 		 * Wake up any possible resync thread that waits for the device
266 		 * to go idle.
267 		 */
268 		allow_barrier(conf, start_next_window, bi_sector);
269 	}
270 }
271 
272 static void raid_end_bio_io(struct r1bio *r1_bio)
273 {
274 	struct bio *bio = r1_bio->master_bio;
275 
276 	/* if nobody has done the final endio yet, do it now */
277 	if (!test_and_set_bit(R1BIO_Returned, &r1_bio->state)) {
278 		pr_debug("raid1: sync end %s on sectors %llu-%llu\n",
279 			 (bio_data_dir(bio) == WRITE) ? "write" : "read",
280 			 (unsigned long long) bio->bi_iter.bi_sector,
281 			 (unsigned long long) bio_end_sector(bio) - 1);
282 
283 		call_bio_endio(r1_bio);
284 	}
285 	free_r1bio(r1_bio);
286 }
287 
288 /*
289  * Update disk head position estimator based on IRQ completion info.
290  */
291 static inline void update_head_pos(int disk, struct r1bio *r1_bio)
292 {
293 	struct r1conf *conf = r1_bio->mddev->private;
294 
295 	conf->mirrors[disk].head_position =
296 		r1_bio->sector + (r1_bio->sectors);
297 }
298 
299 /*
300  * Find the disk number which triggered given bio
301  */
302 static int find_bio_disk(struct r1bio *r1_bio, struct bio *bio)
303 {
304 	int mirror;
305 	struct r1conf *conf = r1_bio->mddev->private;
306 	int raid_disks = conf->raid_disks;
307 
308 	for (mirror = 0; mirror < raid_disks * 2; mirror++)
309 		if (r1_bio->bios[mirror] == bio)
310 			break;
311 
312 	BUG_ON(mirror == raid_disks * 2);
313 	update_head_pos(mirror, r1_bio);
314 
315 	return mirror;
316 }
317 
318 static void raid1_end_read_request(struct bio *bio)
319 {
320 	int uptodate = !bio->bi_error;
321 	struct r1bio *r1_bio = bio->bi_private;
322 	struct r1conf *conf = r1_bio->mddev->private;
323 	struct md_rdev *rdev = conf->mirrors[r1_bio->read_disk].rdev;
324 
325 	/*
326 	 * this branch is our 'one mirror IO has finished' event handler:
327 	 */
328 	update_head_pos(r1_bio->read_disk, r1_bio);
329 
330 	if (uptodate)
331 		set_bit(R1BIO_Uptodate, &r1_bio->state);
332 	else if (test_bit(FailFast, &rdev->flags) &&
333 		 test_bit(R1BIO_FailFast, &r1_bio->state))
334 		/* This was a fail-fast read so we definitely
335 		 * want to retry */
336 		;
337 	else {
338 		/* If all other devices have failed, we want to return
339 		 * the error upwards rather than fail the last device.
340 		 * Here we redefine "uptodate" to mean "Don't want to retry"
341 		 */
342 		unsigned long flags;
343 		spin_lock_irqsave(&conf->device_lock, flags);
344 		if (r1_bio->mddev->degraded == conf->raid_disks ||
345 		    (r1_bio->mddev->degraded == conf->raid_disks-1 &&
346 		     test_bit(In_sync, &rdev->flags)))
347 			uptodate = 1;
348 		spin_unlock_irqrestore(&conf->device_lock, flags);
349 	}
350 
351 	if (uptodate) {
352 		raid_end_bio_io(r1_bio);
353 		rdev_dec_pending(rdev, conf->mddev);
354 	} else {
355 		/*
356 		 * oops, read error:
357 		 */
358 		char b[BDEVNAME_SIZE];
359 		pr_err_ratelimited("md/raid1:%s: %s: rescheduling sector %llu\n",
360 				   mdname(conf->mddev),
361 				   bdevname(rdev->bdev, b),
362 				   (unsigned long long)r1_bio->sector);
363 		set_bit(R1BIO_ReadError, &r1_bio->state);
364 		reschedule_retry(r1_bio);
365 		/* don't drop the reference on read_disk yet */
366 	}
367 }
368 
369 static void close_write(struct r1bio *r1_bio)
370 {
371 	/* it really is the end of this request */
372 	if (test_bit(R1BIO_BehindIO, &r1_bio->state)) {
373 		/* free extra copy of the data pages */
374 		int i = r1_bio->behind_page_count;
375 		while (i--)
376 			safe_put_page(r1_bio->behind_bvecs[i].bv_page);
377 		kfree(r1_bio->behind_bvecs);
378 		r1_bio->behind_bvecs = NULL;
379 	}
380 	/* clear the bitmap if all writes complete successfully */
381 	bitmap_endwrite(r1_bio->mddev->bitmap, r1_bio->sector,
382 			r1_bio->sectors,
383 			!test_bit(R1BIO_Degraded, &r1_bio->state),
384 			test_bit(R1BIO_BehindIO, &r1_bio->state));
385 	md_write_end(r1_bio->mddev);
386 }
387 
388 static void r1_bio_write_done(struct r1bio *r1_bio)
389 {
390 	if (!atomic_dec_and_test(&r1_bio->remaining))
391 		return;
392 
393 	if (test_bit(R1BIO_WriteError, &r1_bio->state))
394 		reschedule_retry(r1_bio);
395 	else {
396 		close_write(r1_bio);
397 		if (test_bit(R1BIO_MadeGood, &r1_bio->state))
398 			reschedule_retry(r1_bio);
399 		else
400 			raid_end_bio_io(r1_bio);
401 	}
402 }
403 
404 static void raid1_end_write_request(struct bio *bio)
405 {
406 	struct r1bio *r1_bio = bio->bi_private;
407 	int behind = test_bit(R1BIO_BehindIO, &r1_bio->state);
408 	struct r1conf *conf = r1_bio->mddev->private;
409 	struct bio *to_put = NULL;
410 	int mirror = find_bio_disk(r1_bio, bio);
411 	struct md_rdev *rdev = conf->mirrors[mirror].rdev;
412 	bool discard_error;
413 
414 	discard_error = bio->bi_error && bio_op(bio) == REQ_OP_DISCARD;
415 
416 	/*
417 	 * 'one mirror IO has finished' event handler:
418 	 */
419 	if (bio->bi_error && !discard_error) {
420 		set_bit(WriteErrorSeen,	&rdev->flags);
421 		if (!test_and_set_bit(WantReplacement, &rdev->flags))
422 			set_bit(MD_RECOVERY_NEEDED, &
423 				conf->mddev->recovery);
424 
425 		if (test_bit(FailFast, &rdev->flags) &&
426 		    (bio->bi_opf & MD_FAILFAST) &&
427 		    /* We never try FailFast to WriteMostly devices */
428 		    !test_bit(WriteMostly, &rdev->flags)) {
429 			md_error(r1_bio->mddev, rdev);
430 			if (!test_bit(Faulty, &rdev->flags))
431 				/* This is the only remaining device,
432 				 * We need to retry the write without
433 				 * FailFast
434 				 */
435 				set_bit(R1BIO_WriteError, &r1_bio->state);
436 			else {
437 				/* Finished with this branch */
438 				r1_bio->bios[mirror] = NULL;
439 				to_put = bio;
440 			}
441 		} else
442 			set_bit(R1BIO_WriteError, &r1_bio->state);
443 	} else {
444 		/*
445 		 * Set R1BIO_Uptodate in our master bio, so that we
446 		 * will return a good error code for to the higher
447 		 * levels even if IO on some other mirrored buffer
448 		 * fails.
449 		 *
450 		 * The 'master' represents the composite IO operation
451 		 * to user-side. So if something waits for IO, then it
452 		 * will wait for the 'master' bio.
453 		 */
454 		sector_t first_bad;
455 		int bad_sectors;
456 
457 		r1_bio->bios[mirror] = NULL;
458 		to_put = bio;
459 		/*
460 		 * Do not set R1BIO_Uptodate if the current device is
461 		 * rebuilding or Faulty. This is because we cannot use
462 		 * such device for properly reading the data back (we could
463 		 * potentially use it, if the current write would have felt
464 		 * before rdev->recovery_offset, but for simplicity we don't
465 		 * check this here.
466 		 */
467 		if (test_bit(In_sync, &rdev->flags) &&
468 		    !test_bit(Faulty, &rdev->flags))
469 			set_bit(R1BIO_Uptodate, &r1_bio->state);
470 
471 		/* Maybe we can clear some bad blocks. */
472 		if (is_badblock(rdev, r1_bio->sector, r1_bio->sectors,
473 				&first_bad, &bad_sectors) && !discard_error) {
474 			r1_bio->bios[mirror] = IO_MADE_GOOD;
475 			set_bit(R1BIO_MadeGood, &r1_bio->state);
476 		}
477 	}
478 
479 	if (behind) {
480 		if (test_bit(WriteMostly, &rdev->flags))
481 			atomic_dec(&r1_bio->behind_remaining);
482 
483 		/*
484 		 * In behind mode, we ACK the master bio once the I/O
485 		 * has safely reached all non-writemostly
486 		 * disks. Setting the Returned bit ensures that this
487 		 * gets done only once -- we don't ever want to return
488 		 * -EIO here, instead we'll wait
489 		 */
490 		if (atomic_read(&r1_bio->behind_remaining) >= (atomic_read(&r1_bio->remaining)-1) &&
491 		    test_bit(R1BIO_Uptodate, &r1_bio->state)) {
492 			/* Maybe we can return now */
493 			if (!test_and_set_bit(R1BIO_Returned, &r1_bio->state)) {
494 				struct bio *mbio = r1_bio->master_bio;
495 				pr_debug("raid1: behind end write sectors"
496 					 " %llu-%llu\n",
497 					 (unsigned long long) mbio->bi_iter.bi_sector,
498 					 (unsigned long long) bio_end_sector(mbio) - 1);
499 				call_bio_endio(r1_bio);
500 			}
501 		}
502 	}
503 	if (r1_bio->bios[mirror] == NULL)
504 		rdev_dec_pending(rdev, conf->mddev);
505 
506 	/*
507 	 * Let's see if all mirrored write operations have finished
508 	 * already.
509 	 */
510 	r1_bio_write_done(r1_bio);
511 
512 	if (to_put)
513 		bio_put(to_put);
514 }
515 
516 /*
517  * This routine returns the disk from which the requested read should
518  * be done. There is a per-array 'next expected sequential IO' sector
519  * number - if this matches on the next IO then we use the last disk.
520  * There is also a per-disk 'last know head position' sector that is
521  * maintained from IRQ contexts, both the normal and the resync IO
522  * completion handlers update this position correctly. If there is no
523  * perfect sequential match then we pick the disk whose head is closest.
524  *
525  * If there are 2 mirrors in the same 2 devices, performance degrades
526  * because position is mirror, not device based.
527  *
528  * The rdev for the device selected will have nr_pending incremented.
529  */
530 static int read_balance(struct r1conf *conf, struct r1bio *r1_bio, int *max_sectors)
531 {
532 	const sector_t this_sector = r1_bio->sector;
533 	int sectors;
534 	int best_good_sectors;
535 	int best_disk, best_dist_disk, best_pending_disk;
536 	int has_nonrot_disk;
537 	int disk;
538 	sector_t best_dist;
539 	unsigned int min_pending;
540 	struct md_rdev *rdev;
541 	int choose_first;
542 	int choose_next_idle;
543 
544 	rcu_read_lock();
545 	/*
546 	 * Check if we can balance. We can balance on the whole
547 	 * device if no resync is going on, or below the resync window.
548 	 * We take the first readable disk when above the resync window.
549 	 */
550  retry:
551 	sectors = r1_bio->sectors;
552 	best_disk = -1;
553 	best_dist_disk = -1;
554 	best_dist = MaxSector;
555 	best_pending_disk = -1;
556 	min_pending = UINT_MAX;
557 	best_good_sectors = 0;
558 	has_nonrot_disk = 0;
559 	choose_next_idle = 0;
560 	clear_bit(R1BIO_FailFast, &r1_bio->state);
561 
562 	if ((conf->mddev->recovery_cp < this_sector + sectors) ||
563 	    (mddev_is_clustered(conf->mddev) &&
564 	    md_cluster_ops->area_resyncing(conf->mddev, READ, this_sector,
565 		    this_sector + sectors)))
566 		choose_first = 1;
567 	else
568 		choose_first = 0;
569 
570 	for (disk = 0 ; disk < conf->raid_disks * 2 ; disk++) {
571 		sector_t dist;
572 		sector_t first_bad;
573 		int bad_sectors;
574 		unsigned int pending;
575 		bool nonrot;
576 
577 		rdev = rcu_dereference(conf->mirrors[disk].rdev);
578 		if (r1_bio->bios[disk] == IO_BLOCKED
579 		    || rdev == NULL
580 		    || test_bit(Faulty, &rdev->flags))
581 			continue;
582 		if (!test_bit(In_sync, &rdev->flags) &&
583 		    rdev->recovery_offset < this_sector + sectors)
584 			continue;
585 		if (test_bit(WriteMostly, &rdev->flags)) {
586 			/* Don't balance among write-mostly, just
587 			 * use the first as a last resort */
588 			if (best_dist_disk < 0) {
589 				if (is_badblock(rdev, this_sector, sectors,
590 						&first_bad, &bad_sectors)) {
591 					if (first_bad <= this_sector)
592 						/* Cannot use this */
593 						continue;
594 					best_good_sectors = first_bad - this_sector;
595 				} else
596 					best_good_sectors = sectors;
597 				best_dist_disk = disk;
598 				best_pending_disk = disk;
599 			}
600 			continue;
601 		}
602 		/* This is a reasonable device to use.  It might
603 		 * even be best.
604 		 */
605 		if (is_badblock(rdev, this_sector, sectors,
606 				&first_bad, &bad_sectors)) {
607 			if (best_dist < MaxSector)
608 				/* already have a better device */
609 				continue;
610 			if (first_bad <= this_sector) {
611 				/* cannot read here. If this is the 'primary'
612 				 * device, then we must not read beyond
613 				 * bad_sectors from another device..
614 				 */
615 				bad_sectors -= (this_sector - first_bad);
616 				if (choose_first && sectors > bad_sectors)
617 					sectors = bad_sectors;
618 				if (best_good_sectors > sectors)
619 					best_good_sectors = sectors;
620 
621 			} else {
622 				sector_t good_sectors = first_bad - this_sector;
623 				if (good_sectors > best_good_sectors) {
624 					best_good_sectors = good_sectors;
625 					best_disk = disk;
626 				}
627 				if (choose_first)
628 					break;
629 			}
630 			continue;
631 		} else
632 			best_good_sectors = sectors;
633 
634 		if (best_disk >= 0)
635 			/* At least two disks to choose from so failfast is OK */
636 			set_bit(R1BIO_FailFast, &r1_bio->state);
637 
638 		nonrot = blk_queue_nonrot(bdev_get_queue(rdev->bdev));
639 		has_nonrot_disk |= nonrot;
640 		pending = atomic_read(&rdev->nr_pending);
641 		dist = abs(this_sector - conf->mirrors[disk].head_position);
642 		if (choose_first) {
643 			best_disk = disk;
644 			break;
645 		}
646 		/* Don't change to another disk for sequential reads */
647 		if (conf->mirrors[disk].next_seq_sect == this_sector
648 		    || dist == 0) {
649 			int opt_iosize = bdev_io_opt(rdev->bdev) >> 9;
650 			struct raid1_info *mirror = &conf->mirrors[disk];
651 
652 			best_disk = disk;
653 			/*
654 			 * If buffered sequential IO size exceeds optimal
655 			 * iosize, check if there is idle disk. If yes, choose
656 			 * the idle disk. read_balance could already choose an
657 			 * idle disk before noticing it's a sequential IO in
658 			 * this disk. This doesn't matter because this disk
659 			 * will idle, next time it will be utilized after the
660 			 * first disk has IO size exceeds optimal iosize. In
661 			 * this way, iosize of the first disk will be optimal
662 			 * iosize at least. iosize of the second disk might be
663 			 * small, but not a big deal since when the second disk
664 			 * starts IO, the first disk is likely still busy.
665 			 */
666 			if (nonrot && opt_iosize > 0 &&
667 			    mirror->seq_start != MaxSector &&
668 			    mirror->next_seq_sect > opt_iosize &&
669 			    mirror->next_seq_sect - opt_iosize >=
670 			    mirror->seq_start) {
671 				choose_next_idle = 1;
672 				continue;
673 			}
674 			break;
675 		}
676 
677 		if (choose_next_idle)
678 			continue;
679 
680 		if (min_pending > pending) {
681 			min_pending = pending;
682 			best_pending_disk = disk;
683 		}
684 
685 		if (dist < best_dist) {
686 			best_dist = dist;
687 			best_dist_disk = disk;
688 		}
689 	}
690 
691 	/*
692 	 * If all disks are rotational, choose the closest disk. If any disk is
693 	 * non-rotational, choose the disk with less pending request even the
694 	 * disk is rotational, which might/might not be optimal for raids with
695 	 * mixed ratation/non-rotational disks depending on workload.
696 	 */
697 	if (best_disk == -1) {
698 		if (has_nonrot_disk || min_pending == 0)
699 			best_disk = best_pending_disk;
700 		else
701 			best_disk = best_dist_disk;
702 	}
703 
704 	if (best_disk >= 0) {
705 		rdev = rcu_dereference(conf->mirrors[best_disk].rdev);
706 		if (!rdev)
707 			goto retry;
708 		atomic_inc(&rdev->nr_pending);
709 		sectors = best_good_sectors;
710 
711 		if (conf->mirrors[best_disk].next_seq_sect != this_sector)
712 			conf->mirrors[best_disk].seq_start = this_sector;
713 
714 		conf->mirrors[best_disk].next_seq_sect = this_sector + sectors;
715 	}
716 	rcu_read_unlock();
717 	*max_sectors = sectors;
718 
719 	return best_disk;
720 }
721 
722 static int raid1_congested(struct mddev *mddev, int bits)
723 {
724 	struct r1conf *conf = mddev->private;
725 	int i, ret = 0;
726 
727 	if ((bits & (1 << WB_async_congested)) &&
728 	    conf->pending_count >= max_queued_requests)
729 		return 1;
730 
731 	rcu_read_lock();
732 	for (i = 0; i < conf->raid_disks * 2; i++) {
733 		struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev);
734 		if (rdev && !test_bit(Faulty, &rdev->flags)) {
735 			struct request_queue *q = bdev_get_queue(rdev->bdev);
736 
737 			BUG_ON(!q);
738 
739 			/* Note the '|| 1' - when read_balance prefers
740 			 * non-congested targets, it can be removed
741 			 */
742 			if ((bits & (1 << WB_async_congested)) || 1)
743 				ret |= bdi_congested(&q->backing_dev_info, bits);
744 			else
745 				ret &= bdi_congested(&q->backing_dev_info, bits);
746 		}
747 	}
748 	rcu_read_unlock();
749 	return ret;
750 }
751 
752 static void flush_pending_writes(struct r1conf *conf)
753 {
754 	/* Any writes that have been queued but are awaiting
755 	 * bitmap updates get flushed here.
756 	 */
757 	spin_lock_irq(&conf->device_lock);
758 
759 	if (conf->pending_bio_list.head) {
760 		struct bio *bio;
761 		bio = bio_list_get(&conf->pending_bio_list);
762 		conf->pending_count = 0;
763 		spin_unlock_irq(&conf->device_lock);
764 		/* flush any pending bitmap writes to
765 		 * disk before proceeding w/ I/O */
766 		bitmap_unplug(conf->mddev->bitmap);
767 		wake_up(&conf->wait_barrier);
768 
769 		while (bio) { /* submit pending writes */
770 			struct bio *next = bio->bi_next;
771 			struct md_rdev *rdev = (void*)bio->bi_bdev;
772 			bio->bi_next = NULL;
773 			bio->bi_bdev = rdev->bdev;
774 			if (test_bit(Faulty, &rdev->flags)) {
775 				bio->bi_error = -EIO;
776 				bio_endio(bio);
777 			} else if (unlikely((bio_op(bio) == REQ_OP_DISCARD) &&
778 					    !blk_queue_discard(bdev_get_queue(bio->bi_bdev))))
779 				/* Just ignore it */
780 				bio_endio(bio);
781 			else
782 				generic_make_request(bio);
783 			bio = next;
784 		}
785 	} else
786 		spin_unlock_irq(&conf->device_lock);
787 }
788 
789 /* Barriers....
790  * Sometimes we need to suspend IO while we do something else,
791  * either some resync/recovery, or reconfigure the array.
792  * To do this we raise a 'barrier'.
793  * The 'barrier' is a counter that can be raised multiple times
794  * to count how many activities are happening which preclude
795  * normal IO.
796  * We can only raise the barrier if there is no pending IO.
797  * i.e. if nr_pending == 0.
798  * We choose only to raise the barrier if no-one is waiting for the
799  * barrier to go down.  This means that as soon as an IO request
800  * is ready, no other operations which require a barrier will start
801  * until the IO request has had a chance.
802  *
803  * So: regular IO calls 'wait_barrier'.  When that returns there
804  *    is no backgroup IO happening,  It must arrange to call
805  *    allow_barrier when it has finished its IO.
806  * backgroup IO calls must call raise_barrier.  Once that returns
807  *    there is no normal IO happeing.  It must arrange to call
808  *    lower_barrier when the particular background IO completes.
809  */
810 static void raise_barrier(struct r1conf *conf, sector_t sector_nr)
811 {
812 	spin_lock_irq(&conf->resync_lock);
813 
814 	/* Wait until no block IO is waiting */
815 	wait_event_lock_irq(conf->wait_barrier, !conf->nr_waiting,
816 			    conf->resync_lock);
817 
818 	/* block any new IO from starting */
819 	conf->barrier++;
820 	conf->next_resync = sector_nr;
821 
822 	/* For these conditions we must wait:
823 	 * A: while the array is in frozen state
824 	 * B: while barrier >= RESYNC_DEPTH, meaning resync reach
825 	 *    the max count which allowed.
826 	 * C: next_resync + RESYNC_SECTORS > start_next_window, meaning
827 	 *    next resync will reach to the window which normal bios are
828 	 *    handling.
829 	 * D: while there are any active requests in the current window.
830 	 */
831 	wait_event_lock_irq(conf->wait_barrier,
832 			    !conf->array_frozen &&
833 			    conf->barrier < RESYNC_DEPTH &&
834 			    conf->current_window_requests == 0 &&
835 			    (conf->start_next_window >=
836 			     conf->next_resync + RESYNC_SECTORS),
837 			    conf->resync_lock);
838 
839 	conf->nr_pending++;
840 	spin_unlock_irq(&conf->resync_lock);
841 }
842 
843 static void lower_barrier(struct r1conf *conf)
844 {
845 	unsigned long flags;
846 	BUG_ON(conf->barrier <= 0);
847 	spin_lock_irqsave(&conf->resync_lock, flags);
848 	conf->barrier--;
849 	conf->nr_pending--;
850 	spin_unlock_irqrestore(&conf->resync_lock, flags);
851 	wake_up(&conf->wait_barrier);
852 }
853 
854 static bool need_to_wait_for_sync(struct r1conf *conf, struct bio *bio)
855 {
856 	bool wait = false;
857 
858 	if (conf->array_frozen || !bio)
859 		wait = true;
860 	else if (conf->barrier && bio_data_dir(bio) == WRITE) {
861 		if ((conf->mddev->curr_resync_completed
862 		     >= bio_end_sector(bio)) ||
863 		    (conf->start_next_window + NEXT_NORMALIO_DISTANCE
864 		     <= bio->bi_iter.bi_sector))
865 			wait = false;
866 		else
867 			wait = true;
868 	}
869 
870 	return wait;
871 }
872 
873 static sector_t wait_barrier(struct r1conf *conf, struct bio *bio)
874 {
875 	sector_t sector = 0;
876 
877 	spin_lock_irq(&conf->resync_lock);
878 	if (need_to_wait_for_sync(conf, bio)) {
879 		conf->nr_waiting++;
880 		/* Wait for the barrier to drop.
881 		 * However if there are already pending
882 		 * requests (preventing the barrier from
883 		 * rising completely), and the
884 		 * per-process bio queue isn't empty,
885 		 * then don't wait, as we need to empty
886 		 * that queue to allow conf->start_next_window
887 		 * to increase.
888 		 */
889 		raid1_log(conf->mddev, "wait barrier");
890 		wait_event_lock_irq(conf->wait_barrier,
891 				    !conf->array_frozen &&
892 				    (!conf->barrier ||
893 				     ((conf->start_next_window <
894 				       conf->next_resync + RESYNC_SECTORS) &&
895 				      current->bio_list &&
896 				      !bio_list_empty(current->bio_list))),
897 				    conf->resync_lock);
898 		conf->nr_waiting--;
899 	}
900 
901 	if (bio && bio_data_dir(bio) == WRITE) {
902 		if (bio->bi_iter.bi_sector >= conf->next_resync) {
903 			if (conf->start_next_window == MaxSector)
904 				conf->start_next_window =
905 					conf->next_resync +
906 					NEXT_NORMALIO_DISTANCE;
907 
908 			if ((conf->start_next_window + NEXT_NORMALIO_DISTANCE)
909 			    <= bio->bi_iter.bi_sector)
910 				conf->next_window_requests++;
911 			else
912 				conf->current_window_requests++;
913 			sector = conf->start_next_window;
914 		}
915 	}
916 
917 	conf->nr_pending++;
918 	spin_unlock_irq(&conf->resync_lock);
919 	return sector;
920 }
921 
922 static void allow_barrier(struct r1conf *conf, sector_t start_next_window,
923 			  sector_t bi_sector)
924 {
925 	unsigned long flags;
926 
927 	spin_lock_irqsave(&conf->resync_lock, flags);
928 	conf->nr_pending--;
929 	if (start_next_window) {
930 		if (start_next_window == conf->start_next_window) {
931 			if (conf->start_next_window + NEXT_NORMALIO_DISTANCE
932 			    <= bi_sector)
933 				conf->next_window_requests--;
934 			else
935 				conf->current_window_requests--;
936 		} else
937 			conf->current_window_requests--;
938 
939 		if (!conf->current_window_requests) {
940 			if (conf->next_window_requests) {
941 				conf->current_window_requests =
942 					conf->next_window_requests;
943 				conf->next_window_requests = 0;
944 				conf->start_next_window +=
945 					NEXT_NORMALIO_DISTANCE;
946 			} else
947 				conf->start_next_window = MaxSector;
948 		}
949 	}
950 	spin_unlock_irqrestore(&conf->resync_lock, flags);
951 	wake_up(&conf->wait_barrier);
952 }
953 
954 static void freeze_array(struct r1conf *conf, int extra)
955 {
956 	/* stop syncio and normal IO and wait for everything to
957 	 * go quite.
958 	 * We wait until nr_pending match nr_queued+extra
959 	 * This is called in the context of one normal IO request
960 	 * that has failed. Thus any sync request that might be pending
961 	 * will be blocked by nr_pending, and we need to wait for
962 	 * pending IO requests to complete or be queued for re-try.
963 	 * Thus the number queued (nr_queued) plus this request (extra)
964 	 * must match the number of pending IOs (nr_pending) before
965 	 * we continue.
966 	 */
967 	spin_lock_irq(&conf->resync_lock);
968 	conf->array_frozen = 1;
969 	raid1_log(conf->mddev, "wait freeze");
970 	wait_event_lock_irq_cmd(conf->wait_barrier,
971 				conf->nr_pending == conf->nr_queued+extra,
972 				conf->resync_lock,
973 				flush_pending_writes(conf));
974 	spin_unlock_irq(&conf->resync_lock);
975 }
976 static void unfreeze_array(struct r1conf *conf)
977 {
978 	/* reverse the effect of the freeze */
979 	spin_lock_irq(&conf->resync_lock);
980 	conf->array_frozen = 0;
981 	wake_up(&conf->wait_barrier);
982 	spin_unlock_irq(&conf->resync_lock);
983 }
984 
985 /* duplicate the data pages for behind I/O
986  */
987 static void alloc_behind_pages(struct bio *bio, struct r1bio *r1_bio)
988 {
989 	int i;
990 	struct bio_vec *bvec;
991 	struct bio_vec *bvecs = kzalloc(bio->bi_vcnt * sizeof(struct bio_vec),
992 					GFP_NOIO);
993 	if (unlikely(!bvecs))
994 		return;
995 
996 	bio_for_each_segment_all(bvec, bio, i) {
997 		bvecs[i] = *bvec;
998 		bvecs[i].bv_page = alloc_page(GFP_NOIO);
999 		if (unlikely(!bvecs[i].bv_page))
1000 			goto do_sync_io;
1001 		memcpy(kmap(bvecs[i].bv_page) + bvec->bv_offset,
1002 		       kmap(bvec->bv_page) + bvec->bv_offset, bvec->bv_len);
1003 		kunmap(bvecs[i].bv_page);
1004 		kunmap(bvec->bv_page);
1005 	}
1006 	r1_bio->behind_bvecs = bvecs;
1007 	r1_bio->behind_page_count = bio->bi_vcnt;
1008 	set_bit(R1BIO_BehindIO, &r1_bio->state);
1009 	return;
1010 
1011 do_sync_io:
1012 	for (i = 0; i < bio->bi_vcnt; i++)
1013 		if (bvecs[i].bv_page)
1014 			put_page(bvecs[i].bv_page);
1015 	kfree(bvecs);
1016 	pr_debug("%dB behind alloc failed, doing sync I/O\n",
1017 		 bio->bi_iter.bi_size);
1018 }
1019 
1020 struct raid1_plug_cb {
1021 	struct blk_plug_cb	cb;
1022 	struct bio_list		pending;
1023 	int			pending_cnt;
1024 };
1025 
1026 static void raid1_unplug(struct blk_plug_cb *cb, bool from_schedule)
1027 {
1028 	struct raid1_plug_cb *plug = container_of(cb, struct raid1_plug_cb,
1029 						  cb);
1030 	struct mddev *mddev = plug->cb.data;
1031 	struct r1conf *conf = mddev->private;
1032 	struct bio *bio;
1033 
1034 	if (from_schedule || current->bio_list) {
1035 		spin_lock_irq(&conf->device_lock);
1036 		bio_list_merge(&conf->pending_bio_list, &plug->pending);
1037 		conf->pending_count += plug->pending_cnt;
1038 		spin_unlock_irq(&conf->device_lock);
1039 		wake_up(&conf->wait_barrier);
1040 		md_wakeup_thread(mddev->thread);
1041 		kfree(plug);
1042 		return;
1043 	}
1044 
1045 	/* we aren't scheduling, so we can do the write-out directly. */
1046 	bio = bio_list_get(&plug->pending);
1047 	bitmap_unplug(mddev->bitmap);
1048 	wake_up(&conf->wait_barrier);
1049 
1050 	while (bio) { /* submit pending writes */
1051 		struct bio *next = bio->bi_next;
1052 		struct md_rdev *rdev = (void*)bio->bi_bdev;
1053 		bio->bi_next = NULL;
1054 		bio->bi_bdev = rdev->bdev;
1055 		if (test_bit(Faulty, &rdev->flags)) {
1056 			bio->bi_error = -EIO;
1057 			bio_endio(bio);
1058 		} else if (unlikely((bio_op(bio) == REQ_OP_DISCARD) &&
1059 				    !blk_queue_discard(bdev_get_queue(bio->bi_bdev))))
1060 			/* Just ignore it */
1061 			bio_endio(bio);
1062 		else
1063 			generic_make_request(bio);
1064 		bio = next;
1065 	}
1066 	kfree(plug);
1067 }
1068 
1069 static void raid1_make_request(struct mddev *mddev, struct bio * bio)
1070 {
1071 	struct r1conf *conf = mddev->private;
1072 	struct raid1_info *mirror;
1073 	struct r1bio *r1_bio;
1074 	struct bio *read_bio;
1075 	int i, disks;
1076 	struct bitmap *bitmap;
1077 	unsigned long flags;
1078 	const int op = bio_op(bio);
1079 	const int rw = bio_data_dir(bio);
1080 	const unsigned long do_sync = (bio->bi_opf & REQ_SYNC);
1081 	const unsigned long do_flush_fua = (bio->bi_opf &
1082 						(REQ_PREFLUSH | REQ_FUA));
1083 	struct md_rdev *blocked_rdev;
1084 	struct blk_plug_cb *cb;
1085 	struct raid1_plug_cb *plug = NULL;
1086 	int first_clone;
1087 	int sectors_handled;
1088 	int max_sectors;
1089 	sector_t start_next_window;
1090 
1091 	/*
1092 	 * Register the new request and wait if the reconstruction
1093 	 * thread has put up a bar for new requests.
1094 	 * Continue immediately if no resync is active currently.
1095 	 */
1096 
1097 	md_write_start(mddev, bio); /* wait on superblock update early */
1098 
1099 	if (bio_data_dir(bio) == WRITE &&
1100 	    ((bio_end_sector(bio) > mddev->suspend_lo &&
1101 	    bio->bi_iter.bi_sector < mddev->suspend_hi) ||
1102 	    (mddev_is_clustered(mddev) &&
1103 	     md_cluster_ops->area_resyncing(mddev, WRITE,
1104 		     bio->bi_iter.bi_sector, bio_end_sector(bio))))) {
1105 		/* As the suspend_* range is controlled by
1106 		 * userspace, we want an interruptible
1107 		 * wait.
1108 		 */
1109 		DEFINE_WAIT(w);
1110 		for (;;) {
1111 			flush_signals(current);
1112 			prepare_to_wait(&conf->wait_barrier,
1113 					&w, TASK_INTERRUPTIBLE);
1114 			if (bio_end_sector(bio) <= mddev->suspend_lo ||
1115 			    bio->bi_iter.bi_sector >= mddev->suspend_hi ||
1116 			    (mddev_is_clustered(mddev) &&
1117 			     !md_cluster_ops->area_resyncing(mddev, WRITE,
1118 				     bio->bi_iter.bi_sector, bio_end_sector(bio))))
1119 				break;
1120 			schedule();
1121 		}
1122 		finish_wait(&conf->wait_barrier, &w);
1123 	}
1124 
1125 	start_next_window = wait_barrier(conf, bio);
1126 
1127 	bitmap = mddev->bitmap;
1128 
1129 	/*
1130 	 * make_request() can abort the operation when read-ahead is being
1131 	 * used and no empty request is available.
1132 	 *
1133 	 */
1134 	r1_bio = mempool_alloc(conf->r1bio_pool, GFP_NOIO);
1135 
1136 	r1_bio->master_bio = bio;
1137 	r1_bio->sectors = bio_sectors(bio);
1138 	r1_bio->state = 0;
1139 	r1_bio->mddev = mddev;
1140 	r1_bio->sector = bio->bi_iter.bi_sector;
1141 
1142 	/* We might need to issue multiple reads to different
1143 	 * devices if there are bad blocks around, so we keep
1144 	 * track of the number of reads in bio->bi_phys_segments.
1145 	 * If this is 0, there is only one r1_bio and no locking
1146 	 * will be needed when requests complete.  If it is
1147 	 * non-zero, then it is the number of not-completed requests.
1148 	 */
1149 	bio->bi_phys_segments = 0;
1150 	bio_clear_flag(bio, BIO_SEG_VALID);
1151 
1152 	if (rw == READ) {
1153 		/*
1154 		 * read balancing logic:
1155 		 */
1156 		int rdisk;
1157 
1158 read_again:
1159 		rdisk = read_balance(conf, r1_bio, &max_sectors);
1160 
1161 		if (rdisk < 0) {
1162 			/* couldn't find anywhere to read from */
1163 			raid_end_bio_io(r1_bio);
1164 			return;
1165 		}
1166 		mirror = conf->mirrors + rdisk;
1167 
1168 		if (test_bit(WriteMostly, &mirror->rdev->flags) &&
1169 		    bitmap) {
1170 			/* Reading from a write-mostly device must
1171 			 * take care not to over-take any writes
1172 			 * that are 'behind'
1173 			 */
1174 			raid1_log(mddev, "wait behind writes");
1175 			wait_event(bitmap->behind_wait,
1176 				   atomic_read(&bitmap->behind_writes) == 0);
1177 		}
1178 		r1_bio->read_disk = rdisk;
1179 		r1_bio->start_next_window = 0;
1180 
1181 		read_bio = bio_clone_mddev(bio, GFP_NOIO, mddev);
1182 		bio_trim(read_bio, r1_bio->sector - bio->bi_iter.bi_sector,
1183 			 max_sectors);
1184 
1185 		r1_bio->bios[rdisk] = read_bio;
1186 
1187 		read_bio->bi_iter.bi_sector = r1_bio->sector +
1188 			mirror->rdev->data_offset;
1189 		read_bio->bi_bdev = mirror->rdev->bdev;
1190 		read_bio->bi_end_io = raid1_end_read_request;
1191 		bio_set_op_attrs(read_bio, op, do_sync);
1192 		if (test_bit(FailFast, &mirror->rdev->flags) &&
1193 		    test_bit(R1BIO_FailFast, &r1_bio->state))
1194 			read_bio->bi_opf |= MD_FAILFAST;
1195 		read_bio->bi_private = r1_bio;
1196 
1197 		if (mddev->gendisk)
1198 			trace_block_bio_remap(bdev_get_queue(read_bio->bi_bdev),
1199 					      read_bio, disk_devt(mddev->gendisk),
1200 					      r1_bio->sector);
1201 
1202 		if (max_sectors < r1_bio->sectors) {
1203 			/* could not read all from this device, so we will
1204 			 * need another r1_bio.
1205 			 */
1206 
1207 			sectors_handled = (r1_bio->sector + max_sectors
1208 					   - bio->bi_iter.bi_sector);
1209 			r1_bio->sectors = max_sectors;
1210 			spin_lock_irq(&conf->device_lock);
1211 			if (bio->bi_phys_segments == 0)
1212 				bio->bi_phys_segments = 2;
1213 			else
1214 				bio->bi_phys_segments++;
1215 			spin_unlock_irq(&conf->device_lock);
1216 			/* Cannot call generic_make_request directly
1217 			 * as that will be queued in __make_request
1218 			 * and subsequent mempool_alloc might block waiting
1219 			 * for it.  So hand bio over to raid1d.
1220 			 */
1221 			reschedule_retry(r1_bio);
1222 
1223 			r1_bio = mempool_alloc(conf->r1bio_pool, GFP_NOIO);
1224 
1225 			r1_bio->master_bio = bio;
1226 			r1_bio->sectors = bio_sectors(bio) - sectors_handled;
1227 			r1_bio->state = 0;
1228 			r1_bio->mddev = mddev;
1229 			r1_bio->sector = bio->bi_iter.bi_sector +
1230 				sectors_handled;
1231 			goto read_again;
1232 		} else
1233 			generic_make_request(read_bio);
1234 		return;
1235 	}
1236 
1237 	/*
1238 	 * WRITE:
1239 	 */
1240 	if (conf->pending_count >= max_queued_requests) {
1241 		md_wakeup_thread(mddev->thread);
1242 		raid1_log(mddev, "wait queued");
1243 		wait_event(conf->wait_barrier,
1244 			   conf->pending_count < max_queued_requests);
1245 	}
1246 	/* first select target devices under rcu_lock and
1247 	 * inc refcount on their rdev.  Record them by setting
1248 	 * bios[x] to bio
1249 	 * If there are known/acknowledged bad blocks on any device on
1250 	 * which we have seen a write error, we want to avoid writing those
1251 	 * blocks.
1252 	 * This potentially requires several writes to write around
1253 	 * the bad blocks.  Each set of writes gets it's own r1bio
1254 	 * with a set of bios attached.
1255 	 */
1256 
1257 	disks = conf->raid_disks * 2;
1258  retry_write:
1259 	r1_bio->start_next_window = start_next_window;
1260 	blocked_rdev = NULL;
1261 	rcu_read_lock();
1262 	max_sectors = r1_bio->sectors;
1263 	for (i = 0;  i < disks; i++) {
1264 		struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev);
1265 		if (rdev && unlikely(test_bit(Blocked, &rdev->flags))) {
1266 			atomic_inc(&rdev->nr_pending);
1267 			blocked_rdev = rdev;
1268 			break;
1269 		}
1270 		r1_bio->bios[i] = NULL;
1271 		if (!rdev || test_bit(Faulty, &rdev->flags)) {
1272 			if (i < conf->raid_disks)
1273 				set_bit(R1BIO_Degraded, &r1_bio->state);
1274 			continue;
1275 		}
1276 
1277 		atomic_inc(&rdev->nr_pending);
1278 		if (test_bit(WriteErrorSeen, &rdev->flags)) {
1279 			sector_t first_bad;
1280 			int bad_sectors;
1281 			int is_bad;
1282 
1283 			is_bad = is_badblock(rdev, r1_bio->sector,
1284 					     max_sectors,
1285 					     &first_bad, &bad_sectors);
1286 			if (is_bad < 0) {
1287 				/* mustn't write here until the bad block is
1288 				 * acknowledged*/
1289 				set_bit(BlockedBadBlocks, &rdev->flags);
1290 				blocked_rdev = rdev;
1291 				break;
1292 			}
1293 			if (is_bad && first_bad <= r1_bio->sector) {
1294 				/* Cannot write here at all */
1295 				bad_sectors -= (r1_bio->sector - first_bad);
1296 				if (bad_sectors < max_sectors)
1297 					/* mustn't write more than bad_sectors
1298 					 * to other devices yet
1299 					 */
1300 					max_sectors = bad_sectors;
1301 				rdev_dec_pending(rdev, mddev);
1302 				/* We don't set R1BIO_Degraded as that
1303 				 * only applies if the disk is
1304 				 * missing, so it might be re-added,
1305 				 * and we want to know to recover this
1306 				 * chunk.
1307 				 * In this case the device is here,
1308 				 * and the fact that this chunk is not
1309 				 * in-sync is recorded in the bad
1310 				 * block log
1311 				 */
1312 				continue;
1313 			}
1314 			if (is_bad) {
1315 				int good_sectors = first_bad - r1_bio->sector;
1316 				if (good_sectors < max_sectors)
1317 					max_sectors = good_sectors;
1318 			}
1319 		}
1320 		r1_bio->bios[i] = bio;
1321 	}
1322 	rcu_read_unlock();
1323 
1324 	if (unlikely(blocked_rdev)) {
1325 		/* Wait for this device to become unblocked */
1326 		int j;
1327 		sector_t old = start_next_window;
1328 
1329 		for (j = 0; j < i; j++)
1330 			if (r1_bio->bios[j])
1331 				rdev_dec_pending(conf->mirrors[j].rdev, mddev);
1332 		r1_bio->state = 0;
1333 		allow_barrier(conf, start_next_window, bio->bi_iter.bi_sector);
1334 		raid1_log(mddev, "wait rdev %d blocked", blocked_rdev->raid_disk);
1335 		md_wait_for_blocked_rdev(blocked_rdev, mddev);
1336 		start_next_window = wait_barrier(conf, bio);
1337 		/*
1338 		 * We must make sure the multi r1bios of bio have
1339 		 * the same value of bi_phys_segments
1340 		 */
1341 		if (bio->bi_phys_segments && old &&
1342 		    old != start_next_window)
1343 			/* Wait for the former r1bio(s) to complete */
1344 			wait_event(conf->wait_barrier,
1345 				   bio->bi_phys_segments == 1);
1346 		goto retry_write;
1347 	}
1348 
1349 	if (max_sectors < r1_bio->sectors) {
1350 		/* We are splitting this write into multiple parts, so
1351 		 * we need to prepare for allocating another r1_bio.
1352 		 */
1353 		r1_bio->sectors = max_sectors;
1354 		spin_lock_irq(&conf->device_lock);
1355 		if (bio->bi_phys_segments == 0)
1356 			bio->bi_phys_segments = 2;
1357 		else
1358 			bio->bi_phys_segments++;
1359 		spin_unlock_irq(&conf->device_lock);
1360 	}
1361 	sectors_handled = r1_bio->sector + max_sectors - bio->bi_iter.bi_sector;
1362 
1363 	atomic_set(&r1_bio->remaining, 1);
1364 	atomic_set(&r1_bio->behind_remaining, 0);
1365 
1366 	first_clone = 1;
1367 	for (i = 0; i < disks; i++) {
1368 		struct bio *mbio;
1369 		if (!r1_bio->bios[i])
1370 			continue;
1371 
1372 		mbio = bio_clone_mddev(bio, GFP_NOIO, mddev);
1373 		bio_trim(mbio, r1_bio->sector - bio->bi_iter.bi_sector, max_sectors);
1374 
1375 		if (first_clone) {
1376 			/* do behind I/O ?
1377 			 * Not if there are too many, or cannot
1378 			 * allocate memory, or a reader on WriteMostly
1379 			 * is waiting for behind writes to flush */
1380 			if (bitmap &&
1381 			    (atomic_read(&bitmap->behind_writes)
1382 			     < mddev->bitmap_info.max_write_behind) &&
1383 			    !waitqueue_active(&bitmap->behind_wait))
1384 				alloc_behind_pages(mbio, r1_bio);
1385 
1386 			bitmap_startwrite(bitmap, r1_bio->sector,
1387 					  r1_bio->sectors,
1388 					  test_bit(R1BIO_BehindIO,
1389 						   &r1_bio->state));
1390 			first_clone = 0;
1391 		}
1392 		if (r1_bio->behind_bvecs) {
1393 			struct bio_vec *bvec;
1394 			int j;
1395 
1396 			/*
1397 			 * We trimmed the bio, so _all is legit
1398 			 */
1399 			bio_for_each_segment_all(bvec, mbio, j)
1400 				bvec->bv_page = r1_bio->behind_bvecs[j].bv_page;
1401 			if (test_bit(WriteMostly, &conf->mirrors[i].rdev->flags))
1402 				atomic_inc(&r1_bio->behind_remaining);
1403 		}
1404 
1405 		r1_bio->bios[i] = mbio;
1406 
1407 		mbio->bi_iter.bi_sector	= (r1_bio->sector +
1408 				   conf->mirrors[i].rdev->data_offset);
1409 		mbio->bi_bdev = conf->mirrors[i].rdev->bdev;
1410 		mbio->bi_end_io	= raid1_end_write_request;
1411 		bio_set_op_attrs(mbio, op, do_flush_fua | do_sync);
1412 		if (test_bit(FailFast, &conf->mirrors[i].rdev->flags) &&
1413 		    !test_bit(WriteMostly, &conf->mirrors[i].rdev->flags) &&
1414 		    conf->raid_disks - mddev->degraded > 1)
1415 			mbio->bi_opf |= MD_FAILFAST;
1416 		mbio->bi_private = r1_bio;
1417 
1418 		atomic_inc(&r1_bio->remaining);
1419 
1420 		if (mddev->gendisk)
1421 			trace_block_bio_remap(bdev_get_queue(mbio->bi_bdev),
1422 					      mbio, disk_devt(mddev->gendisk),
1423 					      r1_bio->sector);
1424 		/* flush_pending_writes() needs access to the rdev so...*/
1425 		mbio->bi_bdev = (void*)conf->mirrors[i].rdev;
1426 
1427 		cb = blk_check_plugged(raid1_unplug, mddev, sizeof(*plug));
1428 		if (cb)
1429 			plug = container_of(cb, struct raid1_plug_cb, cb);
1430 		else
1431 			plug = NULL;
1432 		spin_lock_irqsave(&conf->device_lock, flags);
1433 		if (plug) {
1434 			bio_list_add(&plug->pending, mbio);
1435 			plug->pending_cnt++;
1436 		} else {
1437 			bio_list_add(&conf->pending_bio_list, mbio);
1438 			conf->pending_count++;
1439 		}
1440 		spin_unlock_irqrestore(&conf->device_lock, flags);
1441 		if (!plug)
1442 			md_wakeup_thread(mddev->thread);
1443 	}
1444 	/* Mustn't call r1_bio_write_done before this next test,
1445 	 * as it could result in the bio being freed.
1446 	 */
1447 	if (sectors_handled < bio_sectors(bio)) {
1448 		r1_bio_write_done(r1_bio);
1449 		/* We need another r1_bio.  It has already been counted
1450 		 * in bio->bi_phys_segments
1451 		 */
1452 		r1_bio = mempool_alloc(conf->r1bio_pool, GFP_NOIO);
1453 		r1_bio->master_bio = bio;
1454 		r1_bio->sectors = bio_sectors(bio) - sectors_handled;
1455 		r1_bio->state = 0;
1456 		r1_bio->mddev = mddev;
1457 		r1_bio->sector = bio->bi_iter.bi_sector + sectors_handled;
1458 		goto retry_write;
1459 	}
1460 
1461 	r1_bio_write_done(r1_bio);
1462 
1463 	/* In case raid1d snuck in to freeze_array */
1464 	wake_up(&conf->wait_barrier);
1465 }
1466 
1467 static void raid1_status(struct seq_file *seq, struct mddev *mddev)
1468 {
1469 	struct r1conf *conf = mddev->private;
1470 	int i;
1471 
1472 	seq_printf(seq, " [%d/%d] [", conf->raid_disks,
1473 		   conf->raid_disks - mddev->degraded);
1474 	rcu_read_lock();
1475 	for (i = 0; i < conf->raid_disks; i++) {
1476 		struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev);
1477 		seq_printf(seq, "%s",
1478 			   rdev && test_bit(In_sync, &rdev->flags) ? "U" : "_");
1479 	}
1480 	rcu_read_unlock();
1481 	seq_printf(seq, "]");
1482 }
1483 
1484 static void raid1_error(struct mddev *mddev, struct md_rdev *rdev)
1485 {
1486 	char b[BDEVNAME_SIZE];
1487 	struct r1conf *conf = mddev->private;
1488 	unsigned long flags;
1489 
1490 	/*
1491 	 * If it is not operational, then we have already marked it as dead
1492 	 * else if it is the last working disks, ignore the error, let the
1493 	 * next level up know.
1494 	 * else mark the drive as failed
1495 	 */
1496 	spin_lock_irqsave(&conf->device_lock, flags);
1497 	if (test_bit(In_sync, &rdev->flags)
1498 	    && (conf->raid_disks - mddev->degraded) == 1) {
1499 		/*
1500 		 * Don't fail the drive, act as though we were just a
1501 		 * normal single drive.
1502 		 * However don't try a recovery from this drive as
1503 		 * it is very likely to fail.
1504 		 */
1505 		conf->recovery_disabled = mddev->recovery_disabled;
1506 		spin_unlock_irqrestore(&conf->device_lock, flags);
1507 		return;
1508 	}
1509 	set_bit(Blocked, &rdev->flags);
1510 	if (test_and_clear_bit(In_sync, &rdev->flags)) {
1511 		mddev->degraded++;
1512 		set_bit(Faulty, &rdev->flags);
1513 	} else
1514 		set_bit(Faulty, &rdev->flags);
1515 	spin_unlock_irqrestore(&conf->device_lock, flags);
1516 	/*
1517 	 * if recovery is running, make sure it aborts.
1518 	 */
1519 	set_bit(MD_RECOVERY_INTR, &mddev->recovery);
1520 	set_mask_bits(&mddev->sb_flags, 0,
1521 		      BIT(MD_SB_CHANGE_DEVS) | BIT(MD_SB_CHANGE_PENDING));
1522 	pr_crit("md/raid1:%s: Disk failure on %s, disabling device.\n"
1523 		"md/raid1:%s: Operation continuing on %d devices.\n",
1524 		mdname(mddev), bdevname(rdev->bdev, b),
1525 		mdname(mddev), conf->raid_disks - mddev->degraded);
1526 }
1527 
1528 static void print_conf(struct r1conf *conf)
1529 {
1530 	int i;
1531 
1532 	pr_debug("RAID1 conf printout:\n");
1533 	if (!conf) {
1534 		pr_debug("(!conf)\n");
1535 		return;
1536 	}
1537 	pr_debug(" --- wd:%d rd:%d\n", conf->raid_disks - conf->mddev->degraded,
1538 		 conf->raid_disks);
1539 
1540 	rcu_read_lock();
1541 	for (i = 0; i < conf->raid_disks; i++) {
1542 		char b[BDEVNAME_SIZE];
1543 		struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev);
1544 		if (rdev)
1545 			pr_debug(" disk %d, wo:%d, o:%d, dev:%s\n",
1546 				 i, !test_bit(In_sync, &rdev->flags),
1547 				 !test_bit(Faulty, &rdev->flags),
1548 				 bdevname(rdev->bdev,b));
1549 	}
1550 	rcu_read_unlock();
1551 }
1552 
1553 static void close_sync(struct r1conf *conf)
1554 {
1555 	wait_barrier(conf, NULL);
1556 	allow_barrier(conf, 0, 0);
1557 
1558 	mempool_destroy(conf->r1buf_pool);
1559 	conf->r1buf_pool = NULL;
1560 
1561 	spin_lock_irq(&conf->resync_lock);
1562 	conf->next_resync = MaxSector - 2 * NEXT_NORMALIO_DISTANCE;
1563 	conf->start_next_window = MaxSector;
1564 	conf->current_window_requests +=
1565 		conf->next_window_requests;
1566 	conf->next_window_requests = 0;
1567 	spin_unlock_irq(&conf->resync_lock);
1568 }
1569 
1570 static int raid1_spare_active(struct mddev *mddev)
1571 {
1572 	int i;
1573 	struct r1conf *conf = mddev->private;
1574 	int count = 0;
1575 	unsigned long flags;
1576 
1577 	/*
1578 	 * Find all failed disks within the RAID1 configuration
1579 	 * and mark them readable.
1580 	 * Called under mddev lock, so rcu protection not needed.
1581 	 * device_lock used to avoid races with raid1_end_read_request
1582 	 * which expects 'In_sync' flags and ->degraded to be consistent.
1583 	 */
1584 	spin_lock_irqsave(&conf->device_lock, flags);
1585 	for (i = 0; i < conf->raid_disks; i++) {
1586 		struct md_rdev *rdev = conf->mirrors[i].rdev;
1587 		struct md_rdev *repl = conf->mirrors[conf->raid_disks + i].rdev;
1588 		if (repl
1589 		    && !test_bit(Candidate, &repl->flags)
1590 		    && repl->recovery_offset == MaxSector
1591 		    && !test_bit(Faulty, &repl->flags)
1592 		    && !test_and_set_bit(In_sync, &repl->flags)) {
1593 			/* replacement has just become active */
1594 			if (!rdev ||
1595 			    !test_and_clear_bit(In_sync, &rdev->flags))
1596 				count++;
1597 			if (rdev) {
1598 				/* Replaced device not technically
1599 				 * faulty, but we need to be sure
1600 				 * it gets removed and never re-added
1601 				 */
1602 				set_bit(Faulty, &rdev->flags);
1603 				sysfs_notify_dirent_safe(
1604 					rdev->sysfs_state);
1605 			}
1606 		}
1607 		if (rdev
1608 		    && rdev->recovery_offset == MaxSector
1609 		    && !test_bit(Faulty, &rdev->flags)
1610 		    && !test_and_set_bit(In_sync, &rdev->flags)) {
1611 			count++;
1612 			sysfs_notify_dirent_safe(rdev->sysfs_state);
1613 		}
1614 	}
1615 	mddev->degraded -= count;
1616 	spin_unlock_irqrestore(&conf->device_lock, flags);
1617 
1618 	print_conf(conf);
1619 	return count;
1620 }
1621 
1622 static int raid1_add_disk(struct mddev *mddev, struct md_rdev *rdev)
1623 {
1624 	struct r1conf *conf = mddev->private;
1625 	int err = -EEXIST;
1626 	int mirror = 0;
1627 	struct raid1_info *p;
1628 	int first = 0;
1629 	int last = conf->raid_disks - 1;
1630 
1631 	if (mddev->recovery_disabled == conf->recovery_disabled)
1632 		return -EBUSY;
1633 
1634 	if (md_integrity_add_rdev(rdev, mddev))
1635 		return -ENXIO;
1636 
1637 	if (rdev->raid_disk >= 0)
1638 		first = last = rdev->raid_disk;
1639 
1640 	/*
1641 	 * find the disk ... but prefer rdev->saved_raid_disk
1642 	 * if possible.
1643 	 */
1644 	if (rdev->saved_raid_disk >= 0 &&
1645 	    rdev->saved_raid_disk >= first &&
1646 	    conf->mirrors[rdev->saved_raid_disk].rdev == NULL)
1647 		first = last = rdev->saved_raid_disk;
1648 
1649 	for (mirror = first; mirror <= last; mirror++) {
1650 		p = conf->mirrors+mirror;
1651 		if (!p->rdev) {
1652 
1653 			if (mddev->gendisk)
1654 				disk_stack_limits(mddev->gendisk, rdev->bdev,
1655 						  rdev->data_offset << 9);
1656 
1657 			p->head_position = 0;
1658 			rdev->raid_disk = mirror;
1659 			err = 0;
1660 			/* As all devices are equivalent, we don't need a full recovery
1661 			 * if this was recently any drive of the array
1662 			 */
1663 			if (rdev->saved_raid_disk < 0)
1664 				conf->fullsync = 1;
1665 			rcu_assign_pointer(p->rdev, rdev);
1666 			break;
1667 		}
1668 		if (test_bit(WantReplacement, &p->rdev->flags) &&
1669 		    p[conf->raid_disks].rdev == NULL) {
1670 			/* Add this device as a replacement */
1671 			clear_bit(In_sync, &rdev->flags);
1672 			set_bit(Replacement, &rdev->flags);
1673 			rdev->raid_disk = mirror;
1674 			err = 0;
1675 			conf->fullsync = 1;
1676 			rcu_assign_pointer(p[conf->raid_disks].rdev, rdev);
1677 			break;
1678 		}
1679 	}
1680 	if (mddev->queue && blk_queue_discard(bdev_get_queue(rdev->bdev)))
1681 		queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, mddev->queue);
1682 	print_conf(conf);
1683 	return err;
1684 }
1685 
1686 static int raid1_remove_disk(struct mddev *mddev, struct md_rdev *rdev)
1687 {
1688 	struct r1conf *conf = mddev->private;
1689 	int err = 0;
1690 	int number = rdev->raid_disk;
1691 	struct raid1_info *p = conf->mirrors + number;
1692 
1693 	if (rdev != p->rdev)
1694 		p = conf->mirrors + conf->raid_disks + number;
1695 
1696 	print_conf(conf);
1697 	if (rdev == p->rdev) {
1698 		if (test_bit(In_sync, &rdev->flags) ||
1699 		    atomic_read(&rdev->nr_pending)) {
1700 			err = -EBUSY;
1701 			goto abort;
1702 		}
1703 		/* Only remove non-faulty devices if recovery
1704 		 * is not possible.
1705 		 */
1706 		if (!test_bit(Faulty, &rdev->flags) &&
1707 		    mddev->recovery_disabled != conf->recovery_disabled &&
1708 		    mddev->degraded < conf->raid_disks) {
1709 			err = -EBUSY;
1710 			goto abort;
1711 		}
1712 		p->rdev = NULL;
1713 		if (!test_bit(RemoveSynchronized, &rdev->flags)) {
1714 			synchronize_rcu();
1715 			if (atomic_read(&rdev->nr_pending)) {
1716 				/* lost the race, try later */
1717 				err = -EBUSY;
1718 				p->rdev = rdev;
1719 				goto abort;
1720 			}
1721 		}
1722 		if (conf->mirrors[conf->raid_disks + number].rdev) {
1723 			/* We just removed a device that is being replaced.
1724 			 * Move down the replacement.  We drain all IO before
1725 			 * doing this to avoid confusion.
1726 			 */
1727 			struct md_rdev *repl =
1728 				conf->mirrors[conf->raid_disks + number].rdev;
1729 			freeze_array(conf, 0);
1730 			clear_bit(Replacement, &repl->flags);
1731 			p->rdev = repl;
1732 			conf->mirrors[conf->raid_disks + number].rdev = NULL;
1733 			unfreeze_array(conf);
1734 			clear_bit(WantReplacement, &rdev->flags);
1735 		} else
1736 			clear_bit(WantReplacement, &rdev->flags);
1737 		err = md_integrity_register(mddev);
1738 	}
1739 abort:
1740 
1741 	print_conf(conf);
1742 	return err;
1743 }
1744 
1745 static void end_sync_read(struct bio *bio)
1746 {
1747 	struct r1bio *r1_bio = bio->bi_private;
1748 
1749 	update_head_pos(r1_bio->read_disk, r1_bio);
1750 
1751 	/*
1752 	 * we have read a block, now it needs to be re-written,
1753 	 * or re-read if the read failed.
1754 	 * We don't do much here, just schedule handling by raid1d
1755 	 */
1756 	if (!bio->bi_error)
1757 		set_bit(R1BIO_Uptodate, &r1_bio->state);
1758 
1759 	if (atomic_dec_and_test(&r1_bio->remaining))
1760 		reschedule_retry(r1_bio);
1761 }
1762 
1763 static void end_sync_write(struct bio *bio)
1764 {
1765 	int uptodate = !bio->bi_error;
1766 	struct r1bio *r1_bio = bio->bi_private;
1767 	struct mddev *mddev = r1_bio->mddev;
1768 	struct r1conf *conf = mddev->private;
1769 	sector_t first_bad;
1770 	int bad_sectors;
1771 	struct md_rdev *rdev = conf->mirrors[find_bio_disk(r1_bio, bio)].rdev;
1772 
1773 	if (!uptodate) {
1774 		sector_t sync_blocks = 0;
1775 		sector_t s = r1_bio->sector;
1776 		long sectors_to_go = r1_bio->sectors;
1777 		/* make sure these bits doesn't get cleared. */
1778 		do {
1779 			bitmap_end_sync(mddev->bitmap, s,
1780 					&sync_blocks, 1);
1781 			s += sync_blocks;
1782 			sectors_to_go -= sync_blocks;
1783 		} while (sectors_to_go > 0);
1784 		set_bit(WriteErrorSeen, &rdev->flags);
1785 		if (!test_and_set_bit(WantReplacement, &rdev->flags))
1786 			set_bit(MD_RECOVERY_NEEDED, &
1787 				mddev->recovery);
1788 		set_bit(R1BIO_WriteError, &r1_bio->state);
1789 	} else if (is_badblock(rdev, r1_bio->sector, r1_bio->sectors,
1790 			       &first_bad, &bad_sectors) &&
1791 		   !is_badblock(conf->mirrors[r1_bio->read_disk].rdev,
1792 				r1_bio->sector,
1793 				r1_bio->sectors,
1794 				&first_bad, &bad_sectors)
1795 		)
1796 		set_bit(R1BIO_MadeGood, &r1_bio->state);
1797 
1798 	if (atomic_dec_and_test(&r1_bio->remaining)) {
1799 		int s = r1_bio->sectors;
1800 		if (test_bit(R1BIO_MadeGood, &r1_bio->state) ||
1801 		    test_bit(R1BIO_WriteError, &r1_bio->state))
1802 			reschedule_retry(r1_bio);
1803 		else {
1804 			put_buf(r1_bio);
1805 			md_done_sync(mddev, s, uptodate);
1806 		}
1807 	}
1808 }
1809 
1810 static int r1_sync_page_io(struct md_rdev *rdev, sector_t sector,
1811 			    int sectors, struct page *page, int rw)
1812 {
1813 	if (sync_page_io(rdev, sector, sectors << 9, page, rw, 0, false))
1814 		/* success */
1815 		return 1;
1816 	if (rw == WRITE) {
1817 		set_bit(WriteErrorSeen, &rdev->flags);
1818 		if (!test_and_set_bit(WantReplacement,
1819 				      &rdev->flags))
1820 			set_bit(MD_RECOVERY_NEEDED, &
1821 				rdev->mddev->recovery);
1822 	}
1823 	/* need to record an error - either for the block or the device */
1824 	if (!rdev_set_badblocks(rdev, sector, sectors, 0))
1825 		md_error(rdev->mddev, rdev);
1826 	return 0;
1827 }
1828 
1829 static int fix_sync_read_error(struct r1bio *r1_bio)
1830 {
1831 	/* Try some synchronous reads of other devices to get
1832 	 * good data, much like with normal read errors.  Only
1833 	 * read into the pages we already have so we don't
1834 	 * need to re-issue the read request.
1835 	 * We don't need to freeze the array, because being in an
1836 	 * active sync request, there is no normal IO, and
1837 	 * no overlapping syncs.
1838 	 * We don't need to check is_badblock() again as we
1839 	 * made sure that anything with a bad block in range
1840 	 * will have bi_end_io clear.
1841 	 */
1842 	struct mddev *mddev = r1_bio->mddev;
1843 	struct r1conf *conf = mddev->private;
1844 	struct bio *bio = r1_bio->bios[r1_bio->read_disk];
1845 	sector_t sect = r1_bio->sector;
1846 	int sectors = r1_bio->sectors;
1847 	int idx = 0;
1848 	struct md_rdev *rdev;
1849 
1850 	rdev = conf->mirrors[r1_bio->read_disk].rdev;
1851 	if (test_bit(FailFast, &rdev->flags)) {
1852 		/* Don't try recovering from here - just fail it
1853 		 * ... unless it is the last working device of course */
1854 		md_error(mddev, rdev);
1855 		if (test_bit(Faulty, &rdev->flags))
1856 			/* Don't try to read from here, but make sure
1857 			 * put_buf does it's thing
1858 			 */
1859 			bio->bi_end_io = end_sync_write;
1860 	}
1861 
1862 	while(sectors) {
1863 		int s = sectors;
1864 		int d = r1_bio->read_disk;
1865 		int success = 0;
1866 		int start;
1867 
1868 		if (s > (PAGE_SIZE>>9))
1869 			s = PAGE_SIZE >> 9;
1870 		do {
1871 			if (r1_bio->bios[d]->bi_end_io == end_sync_read) {
1872 				/* No rcu protection needed here devices
1873 				 * can only be removed when no resync is
1874 				 * active, and resync is currently active
1875 				 */
1876 				rdev = conf->mirrors[d].rdev;
1877 				if (sync_page_io(rdev, sect, s<<9,
1878 						 bio->bi_io_vec[idx].bv_page,
1879 						 REQ_OP_READ, 0, false)) {
1880 					success = 1;
1881 					break;
1882 				}
1883 			}
1884 			d++;
1885 			if (d == conf->raid_disks * 2)
1886 				d = 0;
1887 		} while (!success && d != r1_bio->read_disk);
1888 
1889 		if (!success) {
1890 			char b[BDEVNAME_SIZE];
1891 			int abort = 0;
1892 			/* Cannot read from anywhere, this block is lost.
1893 			 * Record a bad block on each device.  If that doesn't
1894 			 * work just disable and interrupt the recovery.
1895 			 * Don't fail devices as that won't really help.
1896 			 */
1897 			pr_crit_ratelimited("md/raid1:%s: %s: unrecoverable I/O read error for block %llu\n",
1898 					    mdname(mddev),
1899 					    bdevname(bio->bi_bdev, b),
1900 					    (unsigned long long)r1_bio->sector);
1901 			for (d = 0; d < conf->raid_disks * 2; d++) {
1902 				rdev = conf->mirrors[d].rdev;
1903 				if (!rdev || test_bit(Faulty, &rdev->flags))
1904 					continue;
1905 				if (!rdev_set_badblocks(rdev, sect, s, 0))
1906 					abort = 1;
1907 			}
1908 			if (abort) {
1909 				conf->recovery_disabled =
1910 					mddev->recovery_disabled;
1911 				set_bit(MD_RECOVERY_INTR, &mddev->recovery);
1912 				md_done_sync(mddev, r1_bio->sectors, 0);
1913 				put_buf(r1_bio);
1914 				return 0;
1915 			}
1916 			/* Try next page */
1917 			sectors -= s;
1918 			sect += s;
1919 			idx++;
1920 			continue;
1921 		}
1922 
1923 		start = d;
1924 		/* write it back and re-read */
1925 		while (d != r1_bio->read_disk) {
1926 			if (d == 0)
1927 				d = conf->raid_disks * 2;
1928 			d--;
1929 			if (r1_bio->bios[d]->bi_end_io != end_sync_read)
1930 				continue;
1931 			rdev = conf->mirrors[d].rdev;
1932 			if (r1_sync_page_io(rdev, sect, s,
1933 					    bio->bi_io_vec[idx].bv_page,
1934 					    WRITE) == 0) {
1935 				r1_bio->bios[d]->bi_end_io = NULL;
1936 				rdev_dec_pending(rdev, mddev);
1937 			}
1938 		}
1939 		d = start;
1940 		while (d != r1_bio->read_disk) {
1941 			if (d == 0)
1942 				d = conf->raid_disks * 2;
1943 			d--;
1944 			if (r1_bio->bios[d]->bi_end_io != end_sync_read)
1945 				continue;
1946 			rdev = conf->mirrors[d].rdev;
1947 			if (r1_sync_page_io(rdev, sect, s,
1948 					    bio->bi_io_vec[idx].bv_page,
1949 					    READ) != 0)
1950 				atomic_add(s, &rdev->corrected_errors);
1951 		}
1952 		sectors -= s;
1953 		sect += s;
1954 		idx ++;
1955 	}
1956 	set_bit(R1BIO_Uptodate, &r1_bio->state);
1957 	bio->bi_error = 0;
1958 	return 1;
1959 }
1960 
1961 static void process_checks(struct r1bio *r1_bio)
1962 {
1963 	/* We have read all readable devices.  If we haven't
1964 	 * got the block, then there is no hope left.
1965 	 * If we have, then we want to do a comparison
1966 	 * and skip the write if everything is the same.
1967 	 * If any blocks failed to read, then we need to
1968 	 * attempt an over-write
1969 	 */
1970 	struct mddev *mddev = r1_bio->mddev;
1971 	struct r1conf *conf = mddev->private;
1972 	int primary;
1973 	int i;
1974 	int vcnt;
1975 
1976 	/* Fix variable parts of all bios */
1977 	vcnt = (r1_bio->sectors + PAGE_SIZE / 512 - 1) >> (PAGE_SHIFT - 9);
1978 	for (i = 0; i < conf->raid_disks * 2; i++) {
1979 		int j;
1980 		int size;
1981 		int error;
1982 		struct bio *b = r1_bio->bios[i];
1983 		if (b->bi_end_io != end_sync_read)
1984 			continue;
1985 		/* fixup the bio for reuse, but preserve errno */
1986 		error = b->bi_error;
1987 		bio_reset(b);
1988 		b->bi_error = error;
1989 		b->bi_vcnt = vcnt;
1990 		b->bi_iter.bi_size = r1_bio->sectors << 9;
1991 		b->bi_iter.bi_sector = r1_bio->sector +
1992 			conf->mirrors[i].rdev->data_offset;
1993 		b->bi_bdev = conf->mirrors[i].rdev->bdev;
1994 		b->bi_end_io = end_sync_read;
1995 		b->bi_private = r1_bio;
1996 
1997 		size = b->bi_iter.bi_size;
1998 		for (j = 0; j < vcnt ; j++) {
1999 			struct bio_vec *bi;
2000 			bi = &b->bi_io_vec[j];
2001 			bi->bv_offset = 0;
2002 			if (size > PAGE_SIZE)
2003 				bi->bv_len = PAGE_SIZE;
2004 			else
2005 				bi->bv_len = size;
2006 			size -= PAGE_SIZE;
2007 		}
2008 	}
2009 	for (primary = 0; primary < conf->raid_disks * 2; primary++)
2010 		if (r1_bio->bios[primary]->bi_end_io == end_sync_read &&
2011 		    !r1_bio->bios[primary]->bi_error) {
2012 			r1_bio->bios[primary]->bi_end_io = NULL;
2013 			rdev_dec_pending(conf->mirrors[primary].rdev, mddev);
2014 			break;
2015 		}
2016 	r1_bio->read_disk = primary;
2017 	for (i = 0; i < conf->raid_disks * 2; i++) {
2018 		int j;
2019 		struct bio *pbio = r1_bio->bios[primary];
2020 		struct bio *sbio = r1_bio->bios[i];
2021 		int error = sbio->bi_error;
2022 
2023 		if (sbio->bi_end_io != end_sync_read)
2024 			continue;
2025 		/* Now we can 'fixup' the error value */
2026 		sbio->bi_error = 0;
2027 
2028 		if (!error) {
2029 			for (j = vcnt; j-- ; ) {
2030 				struct page *p, *s;
2031 				p = pbio->bi_io_vec[j].bv_page;
2032 				s = sbio->bi_io_vec[j].bv_page;
2033 				if (memcmp(page_address(p),
2034 					   page_address(s),
2035 					   sbio->bi_io_vec[j].bv_len))
2036 					break;
2037 			}
2038 		} else
2039 			j = 0;
2040 		if (j >= 0)
2041 			atomic64_add(r1_bio->sectors, &mddev->resync_mismatches);
2042 		if (j < 0 || (test_bit(MD_RECOVERY_CHECK, &mddev->recovery)
2043 			      && !error)) {
2044 			/* No need to write to this device. */
2045 			sbio->bi_end_io = NULL;
2046 			rdev_dec_pending(conf->mirrors[i].rdev, mddev);
2047 			continue;
2048 		}
2049 
2050 		bio_copy_data(sbio, pbio);
2051 	}
2052 }
2053 
2054 static void sync_request_write(struct mddev *mddev, struct r1bio *r1_bio)
2055 {
2056 	struct r1conf *conf = mddev->private;
2057 	int i;
2058 	int disks = conf->raid_disks * 2;
2059 	struct bio *bio, *wbio;
2060 
2061 	bio = r1_bio->bios[r1_bio->read_disk];
2062 
2063 	if (!test_bit(R1BIO_Uptodate, &r1_bio->state))
2064 		/* ouch - failed to read all of that. */
2065 		if (!fix_sync_read_error(r1_bio))
2066 			return;
2067 
2068 	if (test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery))
2069 		process_checks(r1_bio);
2070 
2071 	/*
2072 	 * schedule writes
2073 	 */
2074 	atomic_set(&r1_bio->remaining, 1);
2075 	for (i = 0; i < disks ; i++) {
2076 		wbio = r1_bio->bios[i];
2077 		if (wbio->bi_end_io == NULL ||
2078 		    (wbio->bi_end_io == end_sync_read &&
2079 		     (i == r1_bio->read_disk ||
2080 		      !test_bit(MD_RECOVERY_SYNC, &mddev->recovery))))
2081 			continue;
2082 
2083 		bio_set_op_attrs(wbio, REQ_OP_WRITE, 0);
2084 		if (test_bit(FailFast, &conf->mirrors[i].rdev->flags))
2085 			wbio->bi_opf |= MD_FAILFAST;
2086 
2087 		wbio->bi_end_io = end_sync_write;
2088 		atomic_inc(&r1_bio->remaining);
2089 		md_sync_acct(conf->mirrors[i].rdev->bdev, bio_sectors(wbio));
2090 
2091 		generic_make_request(wbio);
2092 	}
2093 
2094 	if (atomic_dec_and_test(&r1_bio->remaining)) {
2095 		/* if we're here, all write(s) have completed, so clean up */
2096 		int s = r1_bio->sectors;
2097 		if (test_bit(R1BIO_MadeGood, &r1_bio->state) ||
2098 		    test_bit(R1BIO_WriteError, &r1_bio->state))
2099 			reschedule_retry(r1_bio);
2100 		else {
2101 			put_buf(r1_bio);
2102 			md_done_sync(mddev, s, 1);
2103 		}
2104 	}
2105 }
2106 
2107 /*
2108  * This is a kernel thread which:
2109  *
2110  *	1.	Retries failed read operations on working mirrors.
2111  *	2.	Updates the raid superblock when problems encounter.
2112  *	3.	Performs writes following reads for array synchronising.
2113  */
2114 
2115 static void fix_read_error(struct r1conf *conf, int read_disk,
2116 			   sector_t sect, int sectors)
2117 {
2118 	struct mddev *mddev = conf->mddev;
2119 	while(sectors) {
2120 		int s = sectors;
2121 		int d = read_disk;
2122 		int success = 0;
2123 		int start;
2124 		struct md_rdev *rdev;
2125 
2126 		if (s > (PAGE_SIZE>>9))
2127 			s = PAGE_SIZE >> 9;
2128 
2129 		do {
2130 			sector_t first_bad;
2131 			int bad_sectors;
2132 
2133 			rcu_read_lock();
2134 			rdev = rcu_dereference(conf->mirrors[d].rdev);
2135 			if (rdev &&
2136 			    (test_bit(In_sync, &rdev->flags) ||
2137 			     (!test_bit(Faulty, &rdev->flags) &&
2138 			      rdev->recovery_offset >= sect + s)) &&
2139 			    is_badblock(rdev, sect, s,
2140 					&first_bad, &bad_sectors) == 0) {
2141 				atomic_inc(&rdev->nr_pending);
2142 				rcu_read_unlock();
2143 				if (sync_page_io(rdev, sect, s<<9,
2144 					 conf->tmppage, REQ_OP_READ, 0, false))
2145 					success = 1;
2146 				rdev_dec_pending(rdev, mddev);
2147 				if (success)
2148 					break;
2149 			} else
2150 				rcu_read_unlock();
2151 			d++;
2152 			if (d == conf->raid_disks * 2)
2153 				d = 0;
2154 		} while (!success && d != read_disk);
2155 
2156 		if (!success) {
2157 			/* Cannot read from anywhere - mark it bad */
2158 			struct md_rdev *rdev = conf->mirrors[read_disk].rdev;
2159 			if (!rdev_set_badblocks(rdev, sect, s, 0))
2160 				md_error(mddev, rdev);
2161 			break;
2162 		}
2163 		/* write it back and re-read */
2164 		start = d;
2165 		while (d != read_disk) {
2166 			if (d==0)
2167 				d = conf->raid_disks * 2;
2168 			d--;
2169 			rcu_read_lock();
2170 			rdev = rcu_dereference(conf->mirrors[d].rdev);
2171 			if (rdev &&
2172 			    !test_bit(Faulty, &rdev->flags)) {
2173 				atomic_inc(&rdev->nr_pending);
2174 				rcu_read_unlock();
2175 				r1_sync_page_io(rdev, sect, s,
2176 						conf->tmppage, WRITE);
2177 				rdev_dec_pending(rdev, mddev);
2178 			} else
2179 				rcu_read_unlock();
2180 		}
2181 		d = start;
2182 		while (d != read_disk) {
2183 			char b[BDEVNAME_SIZE];
2184 			if (d==0)
2185 				d = conf->raid_disks * 2;
2186 			d--;
2187 			rcu_read_lock();
2188 			rdev = rcu_dereference(conf->mirrors[d].rdev);
2189 			if (rdev &&
2190 			    !test_bit(Faulty, &rdev->flags)) {
2191 				atomic_inc(&rdev->nr_pending);
2192 				rcu_read_unlock();
2193 				if (r1_sync_page_io(rdev, sect, s,
2194 						    conf->tmppage, READ)) {
2195 					atomic_add(s, &rdev->corrected_errors);
2196 					pr_info("md/raid1:%s: read error corrected (%d sectors at %llu on %s)\n",
2197 						mdname(mddev), s,
2198 						(unsigned long long)(sect +
2199 								     rdev->data_offset),
2200 						bdevname(rdev->bdev, b));
2201 				}
2202 				rdev_dec_pending(rdev, mddev);
2203 			} else
2204 				rcu_read_unlock();
2205 		}
2206 		sectors -= s;
2207 		sect += s;
2208 	}
2209 }
2210 
2211 static int narrow_write_error(struct r1bio *r1_bio, int i)
2212 {
2213 	struct mddev *mddev = r1_bio->mddev;
2214 	struct r1conf *conf = mddev->private;
2215 	struct md_rdev *rdev = conf->mirrors[i].rdev;
2216 
2217 	/* bio has the data to be written to device 'i' where
2218 	 * we just recently had a write error.
2219 	 * We repeatedly clone the bio and trim down to one block,
2220 	 * then try the write.  Where the write fails we record
2221 	 * a bad block.
2222 	 * It is conceivable that the bio doesn't exactly align with
2223 	 * blocks.  We must handle this somehow.
2224 	 *
2225 	 * We currently own a reference on the rdev.
2226 	 */
2227 
2228 	int block_sectors;
2229 	sector_t sector;
2230 	int sectors;
2231 	int sect_to_write = r1_bio->sectors;
2232 	int ok = 1;
2233 
2234 	if (rdev->badblocks.shift < 0)
2235 		return 0;
2236 
2237 	block_sectors = roundup(1 << rdev->badblocks.shift,
2238 				bdev_logical_block_size(rdev->bdev) >> 9);
2239 	sector = r1_bio->sector;
2240 	sectors = ((sector + block_sectors)
2241 		   & ~(sector_t)(block_sectors - 1))
2242 		- sector;
2243 
2244 	while (sect_to_write) {
2245 		struct bio *wbio;
2246 		if (sectors > sect_to_write)
2247 			sectors = sect_to_write;
2248 		/* Write at 'sector' for 'sectors'*/
2249 
2250 		if (test_bit(R1BIO_BehindIO, &r1_bio->state)) {
2251 			unsigned vcnt = r1_bio->behind_page_count;
2252 			struct bio_vec *vec = r1_bio->behind_bvecs;
2253 
2254 			while (!vec->bv_page) {
2255 				vec++;
2256 				vcnt--;
2257 			}
2258 
2259 			wbio = bio_alloc_mddev(GFP_NOIO, vcnt, mddev);
2260 			memcpy(wbio->bi_io_vec, vec, vcnt * sizeof(struct bio_vec));
2261 
2262 			wbio->bi_vcnt = vcnt;
2263 		} else {
2264 			wbio = bio_clone_mddev(r1_bio->master_bio, GFP_NOIO, mddev);
2265 		}
2266 
2267 		bio_set_op_attrs(wbio, REQ_OP_WRITE, 0);
2268 		wbio->bi_iter.bi_sector = r1_bio->sector;
2269 		wbio->bi_iter.bi_size = r1_bio->sectors << 9;
2270 
2271 		bio_trim(wbio, sector - r1_bio->sector, sectors);
2272 		wbio->bi_iter.bi_sector += rdev->data_offset;
2273 		wbio->bi_bdev = rdev->bdev;
2274 
2275 		if (submit_bio_wait(wbio) < 0)
2276 			/* failure! */
2277 			ok = rdev_set_badblocks(rdev, sector,
2278 						sectors, 0)
2279 				&& ok;
2280 
2281 		bio_put(wbio);
2282 		sect_to_write -= sectors;
2283 		sector += sectors;
2284 		sectors = block_sectors;
2285 	}
2286 	return ok;
2287 }
2288 
2289 static void handle_sync_write_finished(struct r1conf *conf, struct r1bio *r1_bio)
2290 {
2291 	int m;
2292 	int s = r1_bio->sectors;
2293 	for (m = 0; m < conf->raid_disks * 2 ; m++) {
2294 		struct md_rdev *rdev = conf->mirrors[m].rdev;
2295 		struct bio *bio = r1_bio->bios[m];
2296 		if (bio->bi_end_io == NULL)
2297 			continue;
2298 		if (!bio->bi_error &&
2299 		    test_bit(R1BIO_MadeGood, &r1_bio->state)) {
2300 			rdev_clear_badblocks(rdev, r1_bio->sector, s, 0);
2301 		}
2302 		if (bio->bi_error &&
2303 		    test_bit(R1BIO_WriteError, &r1_bio->state)) {
2304 			if (!rdev_set_badblocks(rdev, r1_bio->sector, s, 0))
2305 				md_error(conf->mddev, rdev);
2306 		}
2307 	}
2308 	put_buf(r1_bio);
2309 	md_done_sync(conf->mddev, s, 1);
2310 }
2311 
2312 static void handle_write_finished(struct r1conf *conf, struct r1bio *r1_bio)
2313 {
2314 	int m;
2315 	bool fail = false;
2316 	for (m = 0; m < conf->raid_disks * 2 ; m++)
2317 		if (r1_bio->bios[m] == IO_MADE_GOOD) {
2318 			struct md_rdev *rdev = conf->mirrors[m].rdev;
2319 			rdev_clear_badblocks(rdev,
2320 					     r1_bio->sector,
2321 					     r1_bio->sectors, 0);
2322 			rdev_dec_pending(rdev, conf->mddev);
2323 		} else if (r1_bio->bios[m] != NULL) {
2324 			/* This drive got a write error.  We need to
2325 			 * narrow down and record precise write
2326 			 * errors.
2327 			 */
2328 			fail = true;
2329 			if (!narrow_write_error(r1_bio, m)) {
2330 				md_error(conf->mddev,
2331 					 conf->mirrors[m].rdev);
2332 				/* an I/O failed, we can't clear the bitmap */
2333 				set_bit(R1BIO_Degraded, &r1_bio->state);
2334 			}
2335 			rdev_dec_pending(conf->mirrors[m].rdev,
2336 					 conf->mddev);
2337 		}
2338 	if (fail) {
2339 		spin_lock_irq(&conf->device_lock);
2340 		list_add(&r1_bio->retry_list, &conf->bio_end_io_list);
2341 		conf->nr_queued++;
2342 		spin_unlock_irq(&conf->device_lock);
2343 		md_wakeup_thread(conf->mddev->thread);
2344 	} else {
2345 		if (test_bit(R1BIO_WriteError, &r1_bio->state))
2346 			close_write(r1_bio);
2347 		raid_end_bio_io(r1_bio);
2348 	}
2349 }
2350 
2351 static void handle_read_error(struct r1conf *conf, struct r1bio *r1_bio)
2352 {
2353 	int disk;
2354 	int max_sectors;
2355 	struct mddev *mddev = conf->mddev;
2356 	struct bio *bio;
2357 	char b[BDEVNAME_SIZE];
2358 	struct md_rdev *rdev;
2359 	dev_t bio_dev;
2360 	sector_t bio_sector;
2361 
2362 	clear_bit(R1BIO_ReadError, &r1_bio->state);
2363 	/* we got a read error. Maybe the drive is bad.  Maybe just
2364 	 * the block and we can fix it.
2365 	 * We freeze all other IO, and try reading the block from
2366 	 * other devices.  When we find one, we re-write
2367 	 * and check it that fixes the read error.
2368 	 * This is all done synchronously while the array is
2369 	 * frozen
2370 	 */
2371 
2372 	bio = r1_bio->bios[r1_bio->read_disk];
2373 	bdevname(bio->bi_bdev, b);
2374 	bio_dev = bio->bi_bdev->bd_dev;
2375 	bio_sector = conf->mirrors[r1_bio->read_disk].rdev->data_offset + r1_bio->sector;
2376 	bio_put(bio);
2377 	r1_bio->bios[r1_bio->read_disk] = NULL;
2378 
2379 	rdev = conf->mirrors[r1_bio->read_disk].rdev;
2380 	if (mddev->ro == 0
2381 	    && !test_bit(FailFast, &rdev->flags)) {
2382 		freeze_array(conf, 1);
2383 		fix_read_error(conf, r1_bio->read_disk,
2384 			       r1_bio->sector, r1_bio->sectors);
2385 		unfreeze_array(conf);
2386 	} else {
2387 		r1_bio->bios[r1_bio->read_disk] = IO_BLOCKED;
2388 	}
2389 
2390 	rdev_dec_pending(rdev, conf->mddev);
2391 
2392 read_more:
2393 	disk = read_balance(conf, r1_bio, &max_sectors);
2394 	if (disk == -1) {
2395 		pr_crit_ratelimited("md/raid1:%s: %s: unrecoverable I/O read error for block %llu\n",
2396 				    mdname(mddev), b, (unsigned long long)r1_bio->sector);
2397 		raid_end_bio_io(r1_bio);
2398 	} else {
2399 		const unsigned long do_sync
2400 			= r1_bio->master_bio->bi_opf & REQ_SYNC;
2401 		r1_bio->read_disk = disk;
2402 		bio = bio_clone_mddev(r1_bio->master_bio, GFP_NOIO, mddev);
2403 		bio_trim(bio, r1_bio->sector - bio->bi_iter.bi_sector,
2404 			 max_sectors);
2405 		r1_bio->bios[r1_bio->read_disk] = bio;
2406 		rdev = conf->mirrors[disk].rdev;
2407 		pr_info_ratelimited("md/raid1:%s: redirecting sector %llu to other mirror: %s\n",
2408 				    mdname(mddev),
2409 				    (unsigned long long)r1_bio->sector,
2410 				    bdevname(rdev->bdev, b));
2411 		bio->bi_iter.bi_sector = r1_bio->sector + rdev->data_offset;
2412 		bio->bi_bdev = rdev->bdev;
2413 		bio->bi_end_io = raid1_end_read_request;
2414 		bio_set_op_attrs(bio, REQ_OP_READ, do_sync);
2415 		if (test_bit(FailFast, &rdev->flags) &&
2416 		    test_bit(R1BIO_FailFast, &r1_bio->state))
2417 			bio->bi_opf |= MD_FAILFAST;
2418 		bio->bi_private = r1_bio;
2419 		if (max_sectors < r1_bio->sectors) {
2420 			/* Drat - have to split this up more */
2421 			struct bio *mbio = r1_bio->master_bio;
2422 			int sectors_handled = (r1_bio->sector + max_sectors
2423 					       - mbio->bi_iter.bi_sector);
2424 			r1_bio->sectors = max_sectors;
2425 			spin_lock_irq(&conf->device_lock);
2426 			if (mbio->bi_phys_segments == 0)
2427 				mbio->bi_phys_segments = 2;
2428 			else
2429 				mbio->bi_phys_segments++;
2430 			spin_unlock_irq(&conf->device_lock);
2431 			trace_block_bio_remap(bdev_get_queue(bio->bi_bdev),
2432 					      bio, bio_dev, bio_sector);
2433 			generic_make_request(bio);
2434 			bio = NULL;
2435 
2436 			r1_bio = mempool_alloc(conf->r1bio_pool, GFP_NOIO);
2437 
2438 			r1_bio->master_bio = mbio;
2439 			r1_bio->sectors = bio_sectors(mbio) - sectors_handled;
2440 			r1_bio->state = 0;
2441 			set_bit(R1BIO_ReadError, &r1_bio->state);
2442 			r1_bio->mddev = mddev;
2443 			r1_bio->sector = mbio->bi_iter.bi_sector +
2444 				sectors_handled;
2445 
2446 			goto read_more;
2447 		} else {
2448 			trace_block_bio_remap(bdev_get_queue(bio->bi_bdev),
2449 					      bio, bio_dev, bio_sector);
2450 			generic_make_request(bio);
2451 		}
2452 	}
2453 }
2454 
2455 static void raid1d(struct md_thread *thread)
2456 {
2457 	struct mddev *mddev = thread->mddev;
2458 	struct r1bio *r1_bio;
2459 	unsigned long flags;
2460 	struct r1conf *conf = mddev->private;
2461 	struct list_head *head = &conf->retry_list;
2462 	struct blk_plug plug;
2463 
2464 	md_check_recovery(mddev);
2465 
2466 	if (!list_empty_careful(&conf->bio_end_io_list) &&
2467 	    !test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)) {
2468 		LIST_HEAD(tmp);
2469 		spin_lock_irqsave(&conf->device_lock, flags);
2470 		if (!test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)) {
2471 			while (!list_empty(&conf->bio_end_io_list)) {
2472 				list_move(conf->bio_end_io_list.prev, &tmp);
2473 				conf->nr_queued--;
2474 			}
2475 		}
2476 		spin_unlock_irqrestore(&conf->device_lock, flags);
2477 		while (!list_empty(&tmp)) {
2478 			r1_bio = list_first_entry(&tmp, struct r1bio,
2479 						  retry_list);
2480 			list_del(&r1_bio->retry_list);
2481 			if (mddev->degraded)
2482 				set_bit(R1BIO_Degraded, &r1_bio->state);
2483 			if (test_bit(R1BIO_WriteError, &r1_bio->state))
2484 				close_write(r1_bio);
2485 			raid_end_bio_io(r1_bio);
2486 		}
2487 	}
2488 
2489 	blk_start_plug(&plug);
2490 	for (;;) {
2491 
2492 		flush_pending_writes(conf);
2493 
2494 		spin_lock_irqsave(&conf->device_lock, flags);
2495 		if (list_empty(head)) {
2496 			spin_unlock_irqrestore(&conf->device_lock, flags);
2497 			break;
2498 		}
2499 		r1_bio = list_entry(head->prev, struct r1bio, retry_list);
2500 		list_del(head->prev);
2501 		conf->nr_queued--;
2502 		spin_unlock_irqrestore(&conf->device_lock, flags);
2503 
2504 		mddev = r1_bio->mddev;
2505 		conf = mddev->private;
2506 		if (test_bit(R1BIO_IsSync, &r1_bio->state)) {
2507 			if (test_bit(R1BIO_MadeGood, &r1_bio->state) ||
2508 			    test_bit(R1BIO_WriteError, &r1_bio->state))
2509 				handle_sync_write_finished(conf, r1_bio);
2510 			else
2511 				sync_request_write(mddev, r1_bio);
2512 		} else if (test_bit(R1BIO_MadeGood, &r1_bio->state) ||
2513 			   test_bit(R1BIO_WriteError, &r1_bio->state))
2514 			handle_write_finished(conf, r1_bio);
2515 		else if (test_bit(R1BIO_ReadError, &r1_bio->state))
2516 			handle_read_error(conf, r1_bio);
2517 		else
2518 			/* just a partial read to be scheduled from separate
2519 			 * context
2520 			 */
2521 			generic_make_request(r1_bio->bios[r1_bio->read_disk]);
2522 
2523 		cond_resched();
2524 		if (mddev->sb_flags & ~(1<<MD_SB_CHANGE_PENDING))
2525 			md_check_recovery(mddev);
2526 	}
2527 	blk_finish_plug(&plug);
2528 }
2529 
2530 static int init_resync(struct r1conf *conf)
2531 {
2532 	int buffs;
2533 
2534 	buffs = RESYNC_WINDOW / RESYNC_BLOCK_SIZE;
2535 	BUG_ON(conf->r1buf_pool);
2536 	conf->r1buf_pool = mempool_create(buffs, r1buf_pool_alloc, r1buf_pool_free,
2537 					  conf->poolinfo);
2538 	if (!conf->r1buf_pool)
2539 		return -ENOMEM;
2540 	conf->next_resync = 0;
2541 	return 0;
2542 }
2543 
2544 /*
2545  * perform a "sync" on one "block"
2546  *
2547  * We need to make sure that no normal I/O request - particularly write
2548  * requests - conflict with active sync requests.
2549  *
2550  * This is achieved by tracking pending requests and a 'barrier' concept
2551  * that can be installed to exclude normal IO requests.
2552  */
2553 
2554 static sector_t raid1_sync_request(struct mddev *mddev, sector_t sector_nr,
2555 				   int *skipped)
2556 {
2557 	struct r1conf *conf = mddev->private;
2558 	struct r1bio *r1_bio;
2559 	struct bio *bio;
2560 	sector_t max_sector, nr_sectors;
2561 	int disk = -1;
2562 	int i;
2563 	int wonly = -1;
2564 	int write_targets = 0, read_targets = 0;
2565 	sector_t sync_blocks;
2566 	int still_degraded = 0;
2567 	int good_sectors = RESYNC_SECTORS;
2568 	int min_bad = 0; /* number of sectors that are bad in all devices */
2569 
2570 	if (!conf->r1buf_pool)
2571 		if (init_resync(conf))
2572 			return 0;
2573 
2574 	max_sector = mddev->dev_sectors;
2575 	if (sector_nr >= max_sector) {
2576 		/* If we aborted, we need to abort the
2577 		 * sync on the 'current' bitmap chunk (there will
2578 		 * only be one in raid1 resync.
2579 		 * We can find the current addess in mddev->curr_resync
2580 		 */
2581 		if (mddev->curr_resync < max_sector) /* aborted */
2582 			bitmap_end_sync(mddev->bitmap, mddev->curr_resync,
2583 						&sync_blocks, 1);
2584 		else /* completed sync */
2585 			conf->fullsync = 0;
2586 
2587 		bitmap_close_sync(mddev->bitmap);
2588 		close_sync(conf);
2589 
2590 		if (mddev_is_clustered(mddev)) {
2591 			conf->cluster_sync_low = 0;
2592 			conf->cluster_sync_high = 0;
2593 		}
2594 		return 0;
2595 	}
2596 
2597 	if (mddev->bitmap == NULL &&
2598 	    mddev->recovery_cp == MaxSector &&
2599 	    !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) &&
2600 	    conf->fullsync == 0) {
2601 		*skipped = 1;
2602 		return max_sector - sector_nr;
2603 	}
2604 	/* before building a request, check if we can skip these blocks..
2605 	 * This call the bitmap_start_sync doesn't actually record anything
2606 	 */
2607 	if (!bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, 1) &&
2608 	    !conf->fullsync && !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery)) {
2609 		/* We can skip this block, and probably several more */
2610 		*skipped = 1;
2611 		return sync_blocks;
2612 	}
2613 
2614 	/*
2615 	 * If there is non-resync activity waiting for a turn, then let it
2616 	 * though before starting on this new sync request.
2617 	 */
2618 	if (conf->nr_waiting)
2619 		schedule_timeout_uninterruptible(1);
2620 
2621 	/* we are incrementing sector_nr below. To be safe, we check against
2622 	 * sector_nr + two times RESYNC_SECTORS
2623 	 */
2624 
2625 	bitmap_cond_end_sync(mddev->bitmap, sector_nr,
2626 		mddev_is_clustered(mddev) && (sector_nr + 2 * RESYNC_SECTORS > conf->cluster_sync_high));
2627 	r1_bio = mempool_alloc(conf->r1buf_pool, GFP_NOIO);
2628 
2629 	raise_barrier(conf, sector_nr);
2630 
2631 	rcu_read_lock();
2632 	/*
2633 	 * If we get a correctably read error during resync or recovery,
2634 	 * we might want to read from a different device.  So we
2635 	 * flag all drives that could conceivably be read from for READ,
2636 	 * and any others (which will be non-In_sync devices) for WRITE.
2637 	 * If a read fails, we try reading from something else for which READ
2638 	 * is OK.
2639 	 */
2640 
2641 	r1_bio->mddev = mddev;
2642 	r1_bio->sector = sector_nr;
2643 	r1_bio->state = 0;
2644 	set_bit(R1BIO_IsSync, &r1_bio->state);
2645 
2646 	for (i = 0; i < conf->raid_disks * 2; i++) {
2647 		struct md_rdev *rdev;
2648 		bio = r1_bio->bios[i];
2649 		bio_reset(bio);
2650 
2651 		rdev = rcu_dereference(conf->mirrors[i].rdev);
2652 		if (rdev == NULL ||
2653 		    test_bit(Faulty, &rdev->flags)) {
2654 			if (i < conf->raid_disks)
2655 				still_degraded = 1;
2656 		} else if (!test_bit(In_sync, &rdev->flags)) {
2657 			bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
2658 			bio->bi_end_io = end_sync_write;
2659 			write_targets ++;
2660 		} else {
2661 			/* may need to read from here */
2662 			sector_t first_bad = MaxSector;
2663 			int bad_sectors;
2664 
2665 			if (is_badblock(rdev, sector_nr, good_sectors,
2666 					&first_bad, &bad_sectors)) {
2667 				if (first_bad > sector_nr)
2668 					good_sectors = first_bad - sector_nr;
2669 				else {
2670 					bad_sectors -= (sector_nr - first_bad);
2671 					if (min_bad == 0 ||
2672 					    min_bad > bad_sectors)
2673 						min_bad = bad_sectors;
2674 				}
2675 			}
2676 			if (sector_nr < first_bad) {
2677 				if (test_bit(WriteMostly, &rdev->flags)) {
2678 					if (wonly < 0)
2679 						wonly = i;
2680 				} else {
2681 					if (disk < 0)
2682 						disk = i;
2683 				}
2684 				bio_set_op_attrs(bio, REQ_OP_READ, 0);
2685 				bio->bi_end_io = end_sync_read;
2686 				read_targets++;
2687 			} else if (!test_bit(WriteErrorSeen, &rdev->flags) &&
2688 				test_bit(MD_RECOVERY_SYNC, &mddev->recovery) &&
2689 				!test_bit(MD_RECOVERY_CHECK, &mddev->recovery)) {
2690 				/*
2691 				 * The device is suitable for reading (InSync),
2692 				 * but has bad block(s) here. Let's try to correct them,
2693 				 * if we are doing resync or repair. Otherwise, leave
2694 				 * this device alone for this sync request.
2695 				 */
2696 				bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
2697 				bio->bi_end_io = end_sync_write;
2698 				write_targets++;
2699 			}
2700 		}
2701 		if (bio->bi_end_io) {
2702 			atomic_inc(&rdev->nr_pending);
2703 			bio->bi_iter.bi_sector = sector_nr + rdev->data_offset;
2704 			bio->bi_bdev = rdev->bdev;
2705 			bio->bi_private = r1_bio;
2706 			if (test_bit(FailFast, &rdev->flags))
2707 				bio->bi_opf |= MD_FAILFAST;
2708 		}
2709 	}
2710 	rcu_read_unlock();
2711 	if (disk < 0)
2712 		disk = wonly;
2713 	r1_bio->read_disk = disk;
2714 
2715 	if (read_targets == 0 && min_bad > 0) {
2716 		/* These sectors are bad on all InSync devices, so we
2717 		 * need to mark them bad on all write targets
2718 		 */
2719 		int ok = 1;
2720 		for (i = 0 ; i < conf->raid_disks * 2 ; i++)
2721 			if (r1_bio->bios[i]->bi_end_io == end_sync_write) {
2722 				struct md_rdev *rdev = conf->mirrors[i].rdev;
2723 				ok = rdev_set_badblocks(rdev, sector_nr,
2724 							min_bad, 0
2725 					) && ok;
2726 			}
2727 		set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
2728 		*skipped = 1;
2729 		put_buf(r1_bio);
2730 
2731 		if (!ok) {
2732 			/* Cannot record the badblocks, so need to
2733 			 * abort the resync.
2734 			 * If there are multiple read targets, could just
2735 			 * fail the really bad ones ???
2736 			 */
2737 			conf->recovery_disabled = mddev->recovery_disabled;
2738 			set_bit(MD_RECOVERY_INTR, &mddev->recovery);
2739 			return 0;
2740 		} else
2741 			return min_bad;
2742 
2743 	}
2744 	if (min_bad > 0 && min_bad < good_sectors) {
2745 		/* only resync enough to reach the next bad->good
2746 		 * transition */
2747 		good_sectors = min_bad;
2748 	}
2749 
2750 	if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery) && read_targets > 0)
2751 		/* extra read targets are also write targets */
2752 		write_targets += read_targets-1;
2753 
2754 	if (write_targets == 0 || read_targets == 0) {
2755 		/* There is nowhere to write, so all non-sync
2756 		 * drives must be failed - so we are finished
2757 		 */
2758 		sector_t rv;
2759 		if (min_bad > 0)
2760 			max_sector = sector_nr + min_bad;
2761 		rv = max_sector - sector_nr;
2762 		*skipped = 1;
2763 		put_buf(r1_bio);
2764 		return rv;
2765 	}
2766 
2767 	if (max_sector > mddev->resync_max)
2768 		max_sector = mddev->resync_max; /* Don't do IO beyond here */
2769 	if (max_sector > sector_nr + good_sectors)
2770 		max_sector = sector_nr + good_sectors;
2771 	nr_sectors = 0;
2772 	sync_blocks = 0;
2773 	do {
2774 		struct page *page;
2775 		int len = PAGE_SIZE;
2776 		if (sector_nr + (len>>9) > max_sector)
2777 			len = (max_sector - sector_nr) << 9;
2778 		if (len == 0)
2779 			break;
2780 		if (sync_blocks == 0) {
2781 			if (!bitmap_start_sync(mddev->bitmap, sector_nr,
2782 					       &sync_blocks, still_degraded) &&
2783 			    !conf->fullsync &&
2784 			    !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery))
2785 				break;
2786 			if ((len >> 9) > sync_blocks)
2787 				len = sync_blocks<<9;
2788 		}
2789 
2790 		for (i = 0 ; i < conf->raid_disks * 2; i++) {
2791 			bio = r1_bio->bios[i];
2792 			if (bio->bi_end_io) {
2793 				page = bio->bi_io_vec[bio->bi_vcnt].bv_page;
2794 				if (bio_add_page(bio, page, len, 0) == 0) {
2795 					/* stop here */
2796 					bio->bi_io_vec[bio->bi_vcnt].bv_page = page;
2797 					while (i > 0) {
2798 						i--;
2799 						bio = r1_bio->bios[i];
2800 						if (bio->bi_end_io==NULL)
2801 							continue;
2802 						/* remove last page from this bio */
2803 						bio->bi_vcnt--;
2804 						bio->bi_iter.bi_size -= len;
2805 						bio_clear_flag(bio, BIO_SEG_VALID);
2806 					}
2807 					goto bio_full;
2808 				}
2809 			}
2810 		}
2811 		nr_sectors += len>>9;
2812 		sector_nr += len>>9;
2813 		sync_blocks -= (len>>9);
2814 	} while (r1_bio->bios[disk]->bi_vcnt < RESYNC_PAGES);
2815  bio_full:
2816 	r1_bio->sectors = nr_sectors;
2817 
2818 	if (mddev_is_clustered(mddev) &&
2819 			conf->cluster_sync_high < sector_nr + nr_sectors) {
2820 		conf->cluster_sync_low = mddev->curr_resync_completed;
2821 		conf->cluster_sync_high = conf->cluster_sync_low + CLUSTER_RESYNC_WINDOW_SECTORS;
2822 		/* Send resync message */
2823 		md_cluster_ops->resync_info_update(mddev,
2824 				conf->cluster_sync_low,
2825 				conf->cluster_sync_high);
2826 	}
2827 
2828 	/* For a user-requested sync, we read all readable devices and do a
2829 	 * compare
2830 	 */
2831 	if (test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery)) {
2832 		atomic_set(&r1_bio->remaining, read_targets);
2833 		for (i = 0; i < conf->raid_disks * 2 && read_targets; i++) {
2834 			bio = r1_bio->bios[i];
2835 			if (bio->bi_end_io == end_sync_read) {
2836 				read_targets--;
2837 				md_sync_acct(bio->bi_bdev, nr_sectors);
2838 				if (read_targets == 1)
2839 					bio->bi_opf &= ~MD_FAILFAST;
2840 				generic_make_request(bio);
2841 			}
2842 		}
2843 	} else {
2844 		atomic_set(&r1_bio->remaining, 1);
2845 		bio = r1_bio->bios[r1_bio->read_disk];
2846 		md_sync_acct(bio->bi_bdev, nr_sectors);
2847 		if (read_targets == 1)
2848 			bio->bi_opf &= ~MD_FAILFAST;
2849 		generic_make_request(bio);
2850 
2851 	}
2852 	return nr_sectors;
2853 }
2854 
2855 static sector_t raid1_size(struct mddev *mddev, sector_t sectors, int raid_disks)
2856 {
2857 	if (sectors)
2858 		return sectors;
2859 
2860 	return mddev->dev_sectors;
2861 }
2862 
2863 static struct r1conf *setup_conf(struct mddev *mddev)
2864 {
2865 	struct r1conf *conf;
2866 	int i;
2867 	struct raid1_info *disk;
2868 	struct md_rdev *rdev;
2869 	int err = -ENOMEM;
2870 
2871 	conf = kzalloc(sizeof(struct r1conf), GFP_KERNEL);
2872 	if (!conf)
2873 		goto abort;
2874 
2875 	conf->mirrors = kzalloc(sizeof(struct raid1_info)
2876 				* mddev->raid_disks * 2,
2877 				 GFP_KERNEL);
2878 	if (!conf->mirrors)
2879 		goto abort;
2880 
2881 	conf->tmppage = alloc_page(GFP_KERNEL);
2882 	if (!conf->tmppage)
2883 		goto abort;
2884 
2885 	conf->poolinfo = kzalloc(sizeof(*conf->poolinfo), GFP_KERNEL);
2886 	if (!conf->poolinfo)
2887 		goto abort;
2888 	conf->poolinfo->raid_disks = mddev->raid_disks * 2;
2889 	conf->r1bio_pool = mempool_create(NR_RAID1_BIOS, r1bio_pool_alloc,
2890 					  r1bio_pool_free,
2891 					  conf->poolinfo);
2892 	if (!conf->r1bio_pool)
2893 		goto abort;
2894 
2895 	conf->poolinfo->mddev = mddev;
2896 
2897 	err = -EINVAL;
2898 	spin_lock_init(&conf->device_lock);
2899 	rdev_for_each(rdev, mddev) {
2900 		struct request_queue *q;
2901 		int disk_idx = rdev->raid_disk;
2902 		if (disk_idx >= mddev->raid_disks
2903 		    || disk_idx < 0)
2904 			continue;
2905 		if (test_bit(Replacement, &rdev->flags))
2906 			disk = conf->mirrors + mddev->raid_disks + disk_idx;
2907 		else
2908 			disk = conf->mirrors + disk_idx;
2909 
2910 		if (disk->rdev)
2911 			goto abort;
2912 		disk->rdev = rdev;
2913 		q = bdev_get_queue(rdev->bdev);
2914 
2915 		disk->head_position = 0;
2916 		disk->seq_start = MaxSector;
2917 	}
2918 	conf->raid_disks = mddev->raid_disks;
2919 	conf->mddev = mddev;
2920 	INIT_LIST_HEAD(&conf->retry_list);
2921 	INIT_LIST_HEAD(&conf->bio_end_io_list);
2922 
2923 	spin_lock_init(&conf->resync_lock);
2924 	init_waitqueue_head(&conf->wait_barrier);
2925 
2926 	bio_list_init(&conf->pending_bio_list);
2927 	conf->pending_count = 0;
2928 	conf->recovery_disabled = mddev->recovery_disabled - 1;
2929 
2930 	conf->start_next_window = MaxSector;
2931 	conf->current_window_requests = conf->next_window_requests = 0;
2932 
2933 	err = -EIO;
2934 	for (i = 0; i < conf->raid_disks * 2; i++) {
2935 
2936 		disk = conf->mirrors + i;
2937 
2938 		if (i < conf->raid_disks &&
2939 		    disk[conf->raid_disks].rdev) {
2940 			/* This slot has a replacement. */
2941 			if (!disk->rdev) {
2942 				/* No original, just make the replacement
2943 				 * a recovering spare
2944 				 */
2945 				disk->rdev =
2946 					disk[conf->raid_disks].rdev;
2947 				disk[conf->raid_disks].rdev = NULL;
2948 			} else if (!test_bit(In_sync, &disk->rdev->flags))
2949 				/* Original is not in_sync - bad */
2950 				goto abort;
2951 		}
2952 
2953 		if (!disk->rdev ||
2954 		    !test_bit(In_sync, &disk->rdev->flags)) {
2955 			disk->head_position = 0;
2956 			if (disk->rdev &&
2957 			    (disk->rdev->saved_raid_disk < 0))
2958 				conf->fullsync = 1;
2959 		}
2960 	}
2961 
2962 	err = -ENOMEM;
2963 	conf->thread = md_register_thread(raid1d, mddev, "raid1");
2964 	if (!conf->thread)
2965 		goto abort;
2966 
2967 	return conf;
2968 
2969  abort:
2970 	if (conf) {
2971 		mempool_destroy(conf->r1bio_pool);
2972 		kfree(conf->mirrors);
2973 		safe_put_page(conf->tmppage);
2974 		kfree(conf->poolinfo);
2975 		kfree(conf);
2976 	}
2977 	return ERR_PTR(err);
2978 }
2979 
2980 static void raid1_free(struct mddev *mddev, void *priv);
2981 static int raid1_run(struct mddev *mddev)
2982 {
2983 	struct r1conf *conf;
2984 	int i;
2985 	struct md_rdev *rdev;
2986 	int ret;
2987 	bool discard_supported = false;
2988 
2989 	if (mddev->level != 1) {
2990 		pr_warn("md/raid1:%s: raid level not set to mirroring (%d)\n",
2991 			mdname(mddev), mddev->level);
2992 		return -EIO;
2993 	}
2994 	if (mddev->reshape_position != MaxSector) {
2995 		pr_warn("md/raid1:%s: reshape_position set but not supported\n",
2996 			mdname(mddev));
2997 		return -EIO;
2998 	}
2999 	/*
3000 	 * copy the already verified devices into our private RAID1
3001 	 * bookkeeping area. [whatever we allocate in run(),
3002 	 * should be freed in raid1_free()]
3003 	 */
3004 	if (mddev->private == NULL)
3005 		conf = setup_conf(mddev);
3006 	else
3007 		conf = mddev->private;
3008 
3009 	if (IS_ERR(conf))
3010 		return PTR_ERR(conf);
3011 
3012 	if (mddev->queue)
3013 		blk_queue_max_write_same_sectors(mddev->queue, 0);
3014 
3015 	rdev_for_each(rdev, mddev) {
3016 		if (!mddev->gendisk)
3017 			continue;
3018 		disk_stack_limits(mddev->gendisk, rdev->bdev,
3019 				  rdev->data_offset << 9);
3020 		if (blk_queue_discard(bdev_get_queue(rdev->bdev)))
3021 			discard_supported = true;
3022 	}
3023 
3024 	mddev->degraded = 0;
3025 	for (i=0; i < conf->raid_disks; i++)
3026 		if (conf->mirrors[i].rdev == NULL ||
3027 		    !test_bit(In_sync, &conf->mirrors[i].rdev->flags) ||
3028 		    test_bit(Faulty, &conf->mirrors[i].rdev->flags))
3029 			mddev->degraded++;
3030 
3031 	if (conf->raid_disks - mddev->degraded == 1)
3032 		mddev->recovery_cp = MaxSector;
3033 
3034 	if (mddev->recovery_cp != MaxSector)
3035 		pr_info("md/raid1:%s: not clean -- starting background reconstruction\n",
3036 			mdname(mddev));
3037 	pr_info("md/raid1:%s: active with %d out of %d mirrors\n",
3038 		mdname(mddev), mddev->raid_disks - mddev->degraded,
3039 		mddev->raid_disks);
3040 
3041 	/*
3042 	 * Ok, everything is just fine now
3043 	 */
3044 	mddev->thread = conf->thread;
3045 	conf->thread = NULL;
3046 	mddev->private = conf;
3047 	set_bit(MD_FAILFAST_SUPPORTED, &mddev->flags);
3048 
3049 	md_set_array_sectors(mddev, raid1_size(mddev, 0, 0));
3050 
3051 	if (mddev->queue) {
3052 		if (discard_supported)
3053 			queue_flag_set_unlocked(QUEUE_FLAG_DISCARD,
3054 						mddev->queue);
3055 		else
3056 			queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD,
3057 						  mddev->queue);
3058 	}
3059 
3060 	ret =  md_integrity_register(mddev);
3061 	if (ret) {
3062 		md_unregister_thread(&mddev->thread);
3063 		raid1_free(mddev, conf);
3064 	}
3065 	return ret;
3066 }
3067 
3068 static void raid1_free(struct mddev *mddev, void *priv)
3069 {
3070 	struct r1conf *conf = priv;
3071 
3072 	mempool_destroy(conf->r1bio_pool);
3073 	kfree(conf->mirrors);
3074 	safe_put_page(conf->tmppage);
3075 	kfree(conf->poolinfo);
3076 	kfree(conf);
3077 }
3078 
3079 static int raid1_resize(struct mddev *mddev, sector_t sectors)
3080 {
3081 	/* no resync is happening, and there is enough space
3082 	 * on all devices, so we can resize.
3083 	 * We need to make sure resync covers any new space.
3084 	 * If the array is shrinking we should possibly wait until
3085 	 * any io in the removed space completes, but it hardly seems
3086 	 * worth it.
3087 	 */
3088 	sector_t newsize = raid1_size(mddev, sectors, 0);
3089 	if (mddev->external_size &&
3090 	    mddev->array_sectors > newsize)
3091 		return -EINVAL;
3092 	if (mddev->bitmap) {
3093 		int ret = bitmap_resize(mddev->bitmap, newsize, 0, 0);
3094 		if (ret)
3095 			return ret;
3096 	}
3097 	md_set_array_sectors(mddev, newsize);
3098 	set_capacity(mddev->gendisk, mddev->array_sectors);
3099 	revalidate_disk(mddev->gendisk);
3100 	if (sectors > mddev->dev_sectors &&
3101 	    mddev->recovery_cp > mddev->dev_sectors) {
3102 		mddev->recovery_cp = mddev->dev_sectors;
3103 		set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
3104 	}
3105 	mddev->dev_sectors = sectors;
3106 	mddev->resync_max_sectors = sectors;
3107 	return 0;
3108 }
3109 
3110 static int raid1_reshape(struct mddev *mddev)
3111 {
3112 	/* We need to:
3113 	 * 1/ resize the r1bio_pool
3114 	 * 2/ resize conf->mirrors
3115 	 *
3116 	 * We allocate a new r1bio_pool if we can.
3117 	 * Then raise a device barrier and wait until all IO stops.
3118 	 * Then resize conf->mirrors and swap in the new r1bio pool.
3119 	 *
3120 	 * At the same time, we "pack" the devices so that all the missing
3121 	 * devices have the higher raid_disk numbers.
3122 	 */
3123 	mempool_t *newpool, *oldpool;
3124 	struct pool_info *newpoolinfo;
3125 	struct raid1_info *newmirrors;
3126 	struct r1conf *conf = mddev->private;
3127 	int cnt, raid_disks;
3128 	unsigned long flags;
3129 	int d, d2, err;
3130 
3131 	/* Cannot change chunk_size, layout, or level */
3132 	if (mddev->chunk_sectors != mddev->new_chunk_sectors ||
3133 	    mddev->layout != mddev->new_layout ||
3134 	    mddev->level != mddev->new_level) {
3135 		mddev->new_chunk_sectors = mddev->chunk_sectors;
3136 		mddev->new_layout = mddev->layout;
3137 		mddev->new_level = mddev->level;
3138 		return -EINVAL;
3139 	}
3140 
3141 	if (!mddev_is_clustered(mddev)) {
3142 		err = md_allow_write(mddev);
3143 		if (err)
3144 			return err;
3145 	}
3146 
3147 	raid_disks = mddev->raid_disks + mddev->delta_disks;
3148 
3149 	if (raid_disks < conf->raid_disks) {
3150 		cnt=0;
3151 		for (d= 0; d < conf->raid_disks; d++)
3152 			if (conf->mirrors[d].rdev)
3153 				cnt++;
3154 		if (cnt > raid_disks)
3155 			return -EBUSY;
3156 	}
3157 
3158 	newpoolinfo = kmalloc(sizeof(*newpoolinfo), GFP_KERNEL);
3159 	if (!newpoolinfo)
3160 		return -ENOMEM;
3161 	newpoolinfo->mddev = mddev;
3162 	newpoolinfo->raid_disks = raid_disks * 2;
3163 
3164 	newpool = mempool_create(NR_RAID1_BIOS, r1bio_pool_alloc,
3165 				 r1bio_pool_free, newpoolinfo);
3166 	if (!newpool) {
3167 		kfree(newpoolinfo);
3168 		return -ENOMEM;
3169 	}
3170 	newmirrors = kzalloc(sizeof(struct raid1_info) * raid_disks * 2,
3171 			     GFP_KERNEL);
3172 	if (!newmirrors) {
3173 		kfree(newpoolinfo);
3174 		mempool_destroy(newpool);
3175 		return -ENOMEM;
3176 	}
3177 
3178 	freeze_array(conf, 0);
3179 
3180 	/* ok, everything is stopped */
3181 	oldpool = conf->r1bio_pool;
3182 	conf->r1bio_pool = newpool;
3183 
3184 	for (d = d2 = 0; d < conf->raid_disks; d++) {
3185 		struct md_rdev *rdev = conf->mirrors[d].rdev;
3186 		if (rdev && rdev->raid_disk != d2) {
3187 			sysfs_unlink_rdev(mddev, rdev);
3188 			rdev->raid_disk = d2;
3189 			sysfs_unlink_rdev(mddev, rdev);
3190 			if (sysfs_link_rdev(mddev, rdev))
3191 				pr_warn("md/raid1:%s: cannot register rd%d\n",
3192 					mdname(mddev), rdev->raid_disk);
3193 		}
3194 		if (rdev)
3195 			newmirrors[d2++].rdev = rdev;
3196 	}
3197 	kfree(conf->mirrors);
3198 	conf->mirrors = newmirrors;
3199 	kfree(conf->poolinfo);
3200 	conf->poolinfo = newpoolinfo;
3201 
3202 	spin_lock_irqsave(&conf->device_lock, flags);
3203 	mddev->degraded += (raid_disks - conf->raid_disks);
3204 	spin_unlock_irqrestore(&conf->device_lock, flags);
3205 	conf->raid_disks = mddev->raid_disks = raid_disks;
3206 	mddev->delta_disks = 0;
3207 
3208 	unfreeze_array(conf);
3209 
3210 	set_bit(MD_RECOVERY_RECOVER, &mddev->recovery);
3211 	set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
3212 	md_wakeup_thread(mddev->thread);
3213 
3214 	mempool_destroy(oldpool);
3215 	return 0;
3216 }
3217 
3218 static void raid1_quiesce(struct mddev *mddev, int state)
3219 {
3220 	struct r1conf *conf = mddev->private;
3221 
3222 	switch(state) {
3223 	case 2: /* wake for suspend */
3224 		wake_up(&conf->wait_barrier);
3225 		break;
3226 	case 1:
3227 		freeze_array(conf, 0);
3228 		break;
3229 	case 0:
3230 		unfreeze_array(conf);
3231 		break;
3232 	}
3233 }
3234 
3235 static void *raid1_takeover(struct mddev *mddev)
3236 {
3237 	/* raid1 can take over:
3238 	 *  raid5 with 2 devices, any layout or chunk size
3239 	 */
3240 	if (mddev->level == 5 && mddev->raid_disks == 2) {
3241 		struct r1conf *conf;
3242 		mddev->new_level = 1;
3243 		mddev->new_layout = 0;
3244 		mddev->new_chunk_sectors = 0;
3245 		conf = setup_conf(mddev);
3246 		if (!IS_ERR(conf)) {
3247 			/* Array must appear to be quiesced */
3248 			conf->array_frozen = 1;
3249 			clear_bit(MD_HAS_JOURNAL, &mddev->flags);
3250 			clear_bit(MD_JOURNAL_CLEAN, &mddev->flags);
3251 		}
3252 		return conf;
3253 	}
3254 	return ERR_PTR(-EINVAL);
3255 }
3256 
3257 static struct md_personality raid1_personality =
3258 {
3259 	.name		= "raid1",
3260 	.level		= 1,
3261 	.owner		= THIS_MODULE,
3262 	.make_request	= raid1_make_request,
3263 	.run		= raid1_run,
3264 	.free		= raid1_free,
3265 	.status		= raid1_status,
3266 	.error_handler	= raid1_error,
3267 	.hot_add_disk	= raid1_add_disk,
3268 	.hot_remove_disk= raid1_remove_disk,
3269 	.spare_active	= raid1_spare_active,
3270 	.sync_request	= raid1_sync_request,
3271 	.resize		= raid1_resize,
3272 	.size		= raid1_size,
3273 	.check_reshape	= raid1_reshape,
3274 	.quiesce	= raid1_quiesce,
3275 	.takeover	= raid1_takeover,
3276 	.congested	= raid1_congested,
3277 };
3278 
3279 static int __init raid_init(void)
3280 {
3281 	return register_md_personality(&raid1_personality);
3282 }
3283 
3284 static void raid_exit(void)
3285 {
3286 	unregister_md_personality(&raid1_personality);
3287 }
3288 
3289 module_init(raid_init);
3290 module_exit(raid_exit);
3291 MODULE_LICENSE("GPL");
3292 MODULE_DESCRIPTION("RAID1 (mirroring) personality for MD");
3293 MODULE_ALIAS("md-personality-3"); /* RAID1 */
3294 MODULE_ALIAS("md-raid1");
3295 MODULE_ALIAS("md-level-1");
3296 
3297 module_param(max_queued_requests, int, S_IRUGO|S_IWUSR);
3298