1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause
20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
58 	struct gv_raid5_packet *wp;  in gv_plex_start()  local
62 	bcount = bp->bio_length;  in gv_plex_start()
63 	addr = bp->bio_data;  in gv_plex_start()
64 	boff = bp->bio_offset;  in gv_plex_start()
68 		wp = NULL;  in gv_plex_start()
72 		 * might involve several read/write sub-requests.  in gv_plex_start()
74 		if (p->org == GV_PLEX_RAID5) {  in gv_plex_start()
75 			wp = gv_raid5_start(p, bp, addr, boff, bcount);  in gv_plex_start()
76  			if (wp == NULL)  in gv_plex_start()
79 			len = wp->length;  in gv_plex_start()
81 			if (TAILQ_EMPTY(&wp->bits))  in gv_plex_start()
82 				g_free(wp);  in gv_plex_start()
83 			else if (wp->lockbase != -1)  in gv_plex_start()
84 				TAILQ_INSERT_TAIL(&p->packets, wp, list);  in gv_plex_start()
96 		bcount -= len;  in gv_plex_start()
102 	 * Fire off all sub-requests.  We get the correct consumer (== drive)  in gv_plex_start()
104 	 * cbp->bio_caller1.  in gv_plex_start()
106 	cbp = bioq_takefirst(p->bqueue);  in gv_plex_start()
109 		 * RAID5 sub-requests need to come in correct order, otherwise  in gv_plex_start()
111 		 * another sub-request.  We abuse cbp->bio_caller2 to mark  in gv_plex_start()
114 		if (cbp->bio_caller2 != NULL && gv_stripe_active(p, cbp)) {  in gv_plex_start()
116 			cbp->bio_pflags |= GV_BIO_ONHOLD;  in gv_plex_start()
117 			bioq_disksort(p->wqueue, cbp);  in gv_plex_start()
119 			s = cbp->bio_caller1;  in gv_plex_start()
120 			g_io_request(cbp, s->drive_sc->consumer);  in gv_plex_start()
122 		cbp = bioq_takefirst(p->bqueue);  in gv_plex_start()
134 	switch (p->org) {  in gv_plex_offset()
141 		LIST_FOREACH(s, &p->subdisks, in_plex) {  in gv_plex_offset()
142 			if (s->plex_offset <= boff &&  in gv_plex_offset()
143 			    s->plex_offset + s->size > boff) {  in gv_plex_offset()
149 		if (s == NULL || s->drive_sc == NULL)  in gv_plex_offset()
153 		*real_off = boff - s->plex_offset;  in gv_plex_offset()
154 		len_left = s->size - (*real_off);  in gv_plex_offset()
161 		stripeno = boff / p->stripesize;  in gv_plex_offset()
165 		sdcount = gv_sdcount(p, (boff >= p->synced));  in gv_plex_offset()
167 		if (!(boff + bcount <= p->synced) &&  in gv_plex_offset()
168 		    (p->flags & GV_PLEX_GROWING) &&  in gv_plex_offset()
175 		    p->stripesize;  in gv_plex_offset()
177 		stripeend = stripestart + p->stripesize;  in gv_plex_offset()
178 		*real_off = boff - (stripeno * p->stripesize) +  in gv_plex_offset()
180 		len_left = stripeend - *real_off;  in gv_plex_offset()
205 	sdno = -1;  in gv_plex_normal_request()
210 	if (p == NULL || LIST_EMPTY(&p->subdisks))   in gv_plex_normal_request()
214 	    &real_len, &sdno, (bp->bio_pflags & GV_BIO_GROW));  in gv_plex_normal_request()
217 		bioq_disksort(p->rqueue, bp);  in gv_plex_normal_request()
218 		return (-1); /* "Fail", and delay request. */  in gv_plex_normal_request()
228 	LIST_FOREACH(s, &p->subdisks, in_plex) {  in gv_plex_normal_request()
235 	if (s == NULL || s->drive_sc == NULL)  in gv_plex_normal_request()
239 	switch (s->state) {  in gv_plex_normal_request()
244 		if (bp->bio_pflags & GV_BIO_INTERNAL)  in gv_plex_normal_request()
249 		if (!(bp->bio_pflags & GV_BIO_SYNCREQ)) {  in gv_plex_normal_request()
255 		G_VINUM_DEBUG(1, "sd %s is initializing", s->name);  in gv_plex_normal_request()
259 		if (bp->bio_cmd == BIO_READ)  in gv_plex_normal_request()
273 	cbp->bio_offset = real_off + s->drive_offset;  in gv_plex_normal_request()
274 	cbp->bio_length = real_len;  in gv_plex_normal_request()
275 	cbp->bio_data = addr;  in gv_plex_normal_request()
276 	cbp->bio_done = gv_done;  in gv_plex_normal_request()
277 	cbp->bio_caller1 = s;  in gv_plex_normal_request()
278 	s->drive_sc->active++;  in gv_plex_normal_request()
280 	/* Store the sub-requests now and let others issue them. */  in gv_plex_normal_request()
281 	bioq_insert_tail(p->bqueue, cbp);   in gv_plex_normal_request()
285 	/* Building the sub-request failed. If internal BIO, do not deliver. */  in gv_plex_normal_request()
286 	if (bp->bio_pflags & GV_BIO_INTERNAL) {  in gv_plex_normal_request()
287 		if (bp->bio_pflags & GV_BIO_MALLOC)  in gv_plex_normal_request()
288 			g_free(bp->bio_data);  in gv_plex_normal_request()
290 		p->flags &= ~(GV_PLEX_SYNCING | GV_PLEX_REBUILDING |  in gv_plex_normal_request()
292 		return (-1);  in gv_plex_normal_request()
295 	return (-1);  in gv_plex_normal_request()
306 	pbp = bp->bio_parent;  in gv_plex_normal_done()
307 	if (pbp->bio_error == 0)  in gv_plex_normal_done()
308 		pbp->bio_error = bp->bio_error;  in gv_plex_normal_done()
310 	pbp->bio_inbed++;  in gv_plex_normal_done()
311 	if (pbp->bio_children == pbp->bio_inbed) {  in gv_plex_normal_done()
314 		pbp->bio_completed = pbp->bio_length;  in gv_plex_normal_done()
315 		if (pbp->bio_pflags & GV_BIO_SYNCREQ)  in gv_plex_normal_done()
317 		else if (pbp->bio_pflags & GV_BIO_GROW)  in gv_plex_normal_done()
320 			g_io_deliver(pbp, pbp->bio_error);  in gv_plex_normal_done()
325  * Handle a completed request to a RAID-5 plex.
333 	struct gv_raid5_packet *wp;  in gv_plex_raid5_done()  local
338 	sc = p->vinumconf;  in gv_plex_raid5_done()
339 	wp = bp->bio_caller2;  in gv_plex_raid5_done()
341 	switch (bp->bio_parent->bio_cmd) {  in gv_plex_raid5_done()
343 		if (wp == NULL) {  in gv_plex_raid5_done()
344 			completed = bp->bio_completed;  in gv_plex_raid5_done()
348 		TAILQ_FOREACH_SAFE(bq, &wp->bits, queue, bq2) {  in gv_plex_raid5_done()
349 			if (bq->bp != bp)  in gv_plex_raid5_done()
351 			TAILQ_REMOVE(&wp->bits, bq, queue);  in gv_plex_raid5_done()
353 			for (i = 0; i < wp->length; i++)  in gv_plex_raid5_done()
354 				wp->data[i] ^= bp->bio_data[i];  in gv_plex_raid5_done()
357 		if (TAILQ_EMPTY(&wp->bits)) {  in gv_plex_raid5_done()
358 			completed = wp->length;  in gv_plex_raid5_done()
359 			if (wp->lockbase != -1) {  in gv_plex_raid5_done()
360 				TAILQ_REMOVE(&p->packets, wp, list);  in gv_plex_raid5_done()
362 				pbp = bioq_takefirst(p->wqueue);  in gv_plex_raid5_done()
365 					pbp = bioq_takefirst(p->wqueue);  in gv_plex_raid5_done()
368 			g_free(wp);  in gv_plex_raid5_done()
375 		if (wp == NULL) {  in gv_plex_raid5_done()
376 			completed = bp->bio_completed;  in gv_plex_raid5_done()
381 		TAILQ_FOREACH_SAFE(bq, &wp->bits, queue, bq2) {  in gv_plex_raid5_done()
382 			if (bq->bp != bp)  in gv_plex_raid5_done()
384 			TAILQ_REMOVE(&wp->bits, bq, queue);  in gv_plex_raid5_done()
386 			cbp = wp->parity;  in gv_plex_raid5_done()
388 				for (i = 0; i < wp->length; i++)  in gv_plex_raid5_done()
389 					cbp->bio_data[i] ^= bp->bio_data[i];  in gv_plex_raid5_done()
395 		if (TAILQ_EMPTY(&wp->bits)) {  in gv_plex_raid5_done()
396 			if (bp->bio_parent->bio_pflags & GV_BIO_CHECK)  in gv_plex_raid5_done()
397 				i = gv_check_parity(p, bp, wp);  in gv_plex_raid5_done()
399 				i = gv_normal_parity(p, bp, wp);  in gv_plex_raid5_done()
401 			/* All of our sub-requests have finished. */  in gv_plex_raid5_done()
403 				completed = wp->length;  in gv_plex_raid5_done()
404 				TAILQ_REMOVE(&p->packets, wp, list);  in gv_plex_raid5_done()
406 				pbp = bioq_takefirst(p->wqueue);  in gv_plex_raid5_done()
409 					pbp = bioq_takefirst(p->wqueue);  in gv_plex_raid5_done()
411 				g_free(wp);  in gv_plex_raid5_done()
418 	pbp = bp->bio_parent;  in gv_plex_raid5_done()
419 	if (pbp->bio_error == 0)  in gv_plex_raid5_done()
420 		pbp->bio_error = bp->bio_error;  in gv_plex_raid5_done()
421 	pbp->bio_completed += completed;  in gv_plex_raid5_done()
424 	pbp->bio_inbed++;  in gv_plex_raid5_done()
425 	if (pbp->bio_inbed == pbp->bio_children) {  in gv_plex_raid5_done()
427 		if (pbp->bio_cmd == BIO_WRITE &&  in gv_plex_raid5_done()
428 		    (pbp->bio_pflags & GV_BIO_CHECK)) {  in gv_plex_raid5_done()
430 		} else if (pbp->bio_cmd == BIO_WRITE &&  in gv_plex_raid5_done()
431 		    (pbp->bio_pflags & GV_BIO_REBUILD)) {  in gv_plex_raid5_done()
433 		} else if (pbp->bio_pflags & GV_BIO_INIT) {  in gv_plex_raid5_done()
435 		} else if (pbp->bio_pflags & GV_BIO_SYNCREQ) {  in gv_plex_raid5_done()
437 		} else if (pbp->bio_pflags & GV_BIO_GROW) {  in gv_plex_raid5_done()
440 			g_io_deliver(pbp, pbp->bio_error);  in gv_plex_raid5_done()
445 	if (bp->bio_cflags & GV_BIO_MALLOC)  in gv_plex_raid5_done()
446 		g_free(bp->bio_data);  in gv_plex_raid5_done()
451 gv_check_parity(struct gv_plex *p, struct bio *bp, struct gv_raid5_packet *wp)  in gv_check_parity()  argument
460 	if (wp->waiting != NULL) {  in gv_check_parity()
461 		pbp = wp->waiting;  in gv_check_parity()
462 		wp->waiting = NULL;  in gv_check_parity()
463 		s = pbp->bio_caller1;  in gv_check_parity()
464 		g_io_request(pbp, s->drive_sc->consumer);  in gv_check_parity()
467 	} else if (wp->parity != NULL) {  in gv_check_parity()
468 		pbp = wp->parity;  in gv_check_parity()
469 		wp->parity = NULL;  in gv_check_parity()
472 		for (i = 0; i < wp->length; i++) {  in gv_check_parity()
473 			if (bp->bio_data[i] != pbp->bio_data[i]) {  in gv_check_parity()
481 			bp->bio_parent->bio_error = EAGAIN;  in gv_check_parity()
484 			if (bp->bio_parent->bio_pflags & GV_BIO_PARITY) {  in gv_check_parity()
485 				s = pbp->bio_caller1;  in gv_check_parity()
486 				g_io_request(pbp, s->drive_sc->consumer);  in gv_check_parity()
496 			bp->bio_parent->bio_inbed++;  in gv_check_parity()
505 gv_normal_parity(struct gv_plex *p, struct bio *bp, struct gv_raid5_packet *wp)  in gv_normal_parity()  argument
513 	if (wp->waiting != NULL) {  in gv_normal_parity()
514 		pbp = wp->waiting;  in gv_normal_parity()
515 		wp->waiting = NULL;  in gv_normal_parity()
516 		cbp = wp->parity;  in gv_normal_parity()
517 		for (i = 0; i < wp->length; i++)  in gv_normal_parity()
518 			cbp->bio_data[i] ^= pbp->bio_data[i];  in gv_normal_parity()
519 		s = pbp->bio_caller1;  in gv_normal_parity()
520 		g_io_request(pbp, s->drive_sc->consumer);  in gv_normal_parity()
523 	} else if (wp->parity != NULL) {  in gv_normal_parity()
524 		cbp = wp->parity;  in gv_normal_parity()
525 		wp->parity = NULL;  in gv_normal_parity()
526 		s = cbp->bio_caller1;  in gv_normal_parity()
527 		g_io_request(cbp, s->drive_sc->consumer);  in gv_normal_parity()
540 	bp = bioq_takefirst(p->rqueue);  in gv_plex_flush()
543 		bp = bioq_takefirst(p->rqueue);  in gv_plex_flush()
553 	mtx_lock(&sc->bqueue_mtx);  in gv_post_bio()
554 	bioq_disksort(sc->bqueue_down, bp);  in gv_post_bio()
556 	mtx_unlock(&sc->bqueue_mtx);  in gv_post_bio()
568 	sc = from->vinumconf;  in gv_sync_request()
574 		    " %jd; out of memory", from->name, offset);  in gv_sync_request()
577 	bp->bio_length = length;  in gv_sync_request()
578 	bp->bio_done = NULL;  in gv_sync_request()
579 	bp->bio_pflags |= GV_BIO_SYNCREQ;  in gv_sync_request()
580 	bp->bio_offset = offset;  in gv_sync_request()
581 	bp->bio_caller1 = from;  in gv_sync_request()
582 	bp->bio_caller2 = to;  in gv_sync_request()
583 	bp->bio_cmd = type;  in gv_sync_request()
586 	bp->bio_pflags |= GV_BIO_MALLOC; /* Free on the next run. */  in gv_sync_request()
587 	bp->bio_data = data;  in gv_sync_request()
613 	from = bp->bio_caller2;  in gv_sync_complete()
615 	v = to->vol_sc;  in gv_sync_complete()
617 	sc = v->vinumconf;  in gv_sync_complete()
621 	if (bp->bio_cmd == BIO_READ) {  in gv_sync_complete()
622 		err = gv_sync_request(from, to, bp->bio_offset, bp->bio_length,  in gv_sync_complete()
623 	    	    BIO_WRITE, bp->bio_data);  in gv_sync_complete()
625 	} else if (bp->bio_cmd == BIO_WRITE) {  in gv_sync_complete()
626 		if (bp->bio_pflags & GV_BIO_MALLOC)  in gv_sync_complete()
627 			g_free(bp->bio_data);  in gv_sync_complete()
628 		to->synced += bp->bio_length;  in gv_sync_complete()
630 		if (bp->bio_offset + bp->bio_length >= from->size) {  in gv_sync_complete()
632 			    to->name, from->name);  in gv_sync_complete()
634 			LIST_FOREACH(s, &to->subdisks, in_plex)  in gv_sync_complete()
637 			to->flags &= ~GV_PLEX_SYNCING;  in gv_sync_complete()
638 			to->synced = 0;  in gv_sync_complete()
641 			offset = bp->bio_offset + bp->bio_length;  in gv_sync_complete()
643 			    MIN(bp->bio_length, from->size - offset),  in gv_sync_complete()
650 		to->flags &= ~GV_PLEX_SYNCING;  in gv_sync_complete()
656 	LIST_FOREACH(p, &v->plexes, in_volume) {  in gv_sync_complete()
657 		if (p->flags & GV_PLEX_SYNCING) {  in gv_sync_complete()
659 			return (-1);  in gv_sync_complete()
663 	gv_access(v->provider, -1, -1, 0);  in gv_sync_complete()
681 	sc = p->vinumconf;  in gv_grow_request()
687 		    "out of memory", p->name);  in gv_grow_request()
691 	bp->bio_cmd = type;  in gv_grow_request()
692 	bp->bio_done = NULL;  in gv_grow_request()
693 	bp->bio_error = 0;  in gv_grow_request()
694 	bp->bio_caller1 = p;  in gv_grow_request()
695 	bp->bio_offset = offset;  in gv_grow_request()
696 	bp->bio_length = length;  in gv_grow_request()
697 	bp->bio_pflags |= GV_BIO_GROW;  in gv_grow_request()
700 	bp->bio_pflags |= GV_BIO_MALLOC;  in gv_grow_request()
701 	bp->bio_data = data;  in gv_grow_request()
720 	v = p->vol_sc;  in gv_grow_complete()
722 	sc = v->vinumconf;  in gv_grow_complete()
727 	if (bp->bio_cmd == BIO_READ) {  in gv_grow_complete()
728 		p->synced += bp->bio_length;  in gv_grow_complete()
729 		err = gv_grow_request(p, bp->bio_offset, bp->bio_length,  in gv_grow_complete()
730 		    BIO_WRITE, bp->bio_data);  in gv_grow_complete()
732 	} else if (bp->bio_cmd == BIO_WRITE) {  in gv_grow_complete()
733 		if (bp->bio_pflags & GV_BIO_MALLOC)  in gv_grow_complete()
734 			g_free(bp->bio_data);  in gv_grow_complete()
738 		s = LIST_FIRST(&p->subdisks);  in gv_grow_complete()
740 		origsize = (s->size * (sdcount - 1));  in gv_grow_complete()
741 		if (bp->bio_offset + bp->bio_length >= origsize) {  in gv_grow_complete()
742 			G_VINUM_DEBUG(1, "growing of %s completed", p->name);  in gv_grow_complete()
743 			p->flags &= ~GV_PLEX_GROWING;  in gv_grow_complete()
744 			LIST_FOREACH(s, &p->subdisks, in_plex) {  in gv_grow_complete()
745 				s->flags &= ~GV_SD_GROW;  in gv_grow_complete()
748 			p->size = gv_plex_size(p);  in gv_grow_complete()
752 			gv_access(v->provider, -1, -1, 0);  in gv_grow_complete()
754 			p->synced = 0;  in gv_grow_complete()
759 			offset = bp->bio_offset + bp->bio_length;  in gv_grow_complete()
761 			   MIN(bp->bio_length, origsize - offset),  in gv_grow_complete()
768 		p->flags &= ~GV_PLEX_GROWING;  in gv_grow_complete()
785 	d = s->drive_sc;  in gv_init_request()
787 	cp = d->consumer;  in gv_init_request()
793 		    " (drive offset %jd); out of memory", s->name,  in gv_init_request()
794 		    (intmax_t)s->initialized, (intmax_t)start);  in gv_init_request()
797 	bp->bio_cmd = BIO_WRITE;  in gv_init_request()
798 	bp->bio_data = data;  in gv_init_request()
799 	bp->bio_done = NULL;  in gv_init_request()
800 	bp->bio_error = 0;  in gv_init_request()
801 	bp->bio_length = length;  in gv_init_request()
802 	bp->bio_pflags |= GV_BIO_INIT;  in gv_init_request()
803 	bp->bio_offset = start;  in gv_init_request()
804 	bp->bio_caller1 = s;  in gv_init_request()
810 		    " (drive offset %jd); out of memory", s->name,  in gv_init_request()
811 		    (intmax_t)s->initialized, (intmax_t)start);  in gv_init_request()
814 	cbp->bio_done = gv_done;  in gv_init_request()
815 	cbp->bio_caller1 = s;  in gv_init_request()
816 	d->active++;  in gv_init_request()
835 	s = bp->bio_caller1;  in gv_init_complete()
836 	start = bp->bio_offset;  in gv_init_complete()
837 	length = bp->bio_length;  in gv_init_complete()
838 	error = bp->bio_error;  in gv_init_complete()
839 	data = bp->bio_data;  in gv_init_complete()
842 	d = s->drive_sc;  in gv_init_complete()
844 	cp = d->consumer;  in gv_init_complete()
846 	sc = p->vinumconf;  in gv_init_complete()
856 	if (start >= s->drive_offset + s->size) {  in gv_init_complete()
861 		g_access(cp, 0, -1, 0);  in gv_init_complete()
868 			s->initialized = 0;  in gv_init_complete()
871 			    "successfully", s->name);  in gv_init_complete()
875 	s->initialized += length;  in gv_init_complete()
891 	sc = p->vinumconf;  in gv_parity_request()
897 		    "out of memory", p->name);  in gv_parity_request()
901 	bp->bio_cmd = BIO_WRITE;  in gv_parity_request()
902 	bp->bio_done = NULL;  in gv_parity_request()
903 	bp->bio_error = 0;  in gv_parity_request()
904 	bp->bio_length = p->stripesize;  in gv_parity_request()
905 	bp->bio_caller1 = p;  in gv_parity_request()
912 		bp->bio_data = g_malloc(GV_DFLT_SYNCSIZE, M_WAITOK);  in gv_parity_request()
914 		bp->bio_data = g_malloc(p->stripesize, M_WAITOK | M_ZERO);  in gv_parity_request()
920 	bp->bio_pflags = flags;  in gv_parity_request()
921 	bp->bio_pflags |= GV_BIO_MALLOC;  in gv_parity_request()
924 	bp->bio_offset = offset;  in gv_parity_request()
938 	error = bp->bio_error;  in gv_parity_complete()
939 	flags = bp->bio_pflags;  in gv_parity_complete()
942 	sc = p->vinumconf;  in gv_parity_complete()
946 	if (bp->bio_pflags & GV_BIO_MALLOC)  in gv_parity_complete()
947 		g_free(bp->bio_data);  in gv_parity_complete()
952 		    (intmax_t)p->synced);  in gv_parity_complete()
960 		gv_access(p->vol_sc->provider, -1, -1, 0);  in gv_parity_complete()
963 		    "errno %d", p->name, (intmax_t)p->synced, error);  in gv_parity_complete()
966 		p->synced += p->stripesize;  in gv_parity_complete()
969 	if (p->synced >= p->size) {  in gv_parity_complete()
973 		gv_access(p->vol_sc->provider, -1, -1, 0);  in gv_parity_complete()
976 		G_VINUM_DEBUG(1, "parity operation on %s finished", p->name);  in gv_parity_complete()
977 		p->synced = 0;  in gv_parity_complete()
983 	gv_parity_request(p, flags, p->synced);  in gv_parity_complete()
997 	error = bp->bio_error;  in gv_rebuild_complete()
998 	flags = bp->bio_pflags;  in gv_rebuild_complete()
999 	offset = bp->bio_offset;  in gv_rebuild_complete()
1001 	sc = p->vinumconf;  in gv_rebuild_complete()
1005 	if (bp->bio_pflags & GV_BIO_MALLOC)  in gv_rebuild_complete()
1006 		g_free(bp->bio_data);  in gv_rebuild_complete()
1012 		gv_access(p->vol_sc->provider, -1, -1, 0);  in gv_rebuild_complete()
1016 		    p->name, (intmax_t)offset, error);  in gv_rebuild_complete()
1017 		p->flags &= ~GV_PLEX_REBUILDING;  in gv_rebuild_complete()
1018 		p->synced = 0;  in gv_rebuild_complete()
1023 	offset += (p->stripesize * (gv_sdcount(p, 1) - 1));  in gv_rebuild_complete()
1024 	if (offset >= p->size) {  in gv_rebuild_complete()
1028 		gv_access(p->vol_sc->provider, -1, -1, 0);  in gv_rebuild_complete()
1031 		G_VINUM_DEBUG(1, "rebuild of %s finished", p->name);  in gv_rebuild_complete()
1032 		gv_save_config(p->vinumconf);  in gv_rebuild_complete()
1033 		p->flags &= ~GV_PLEX_REBUILDING;  in gv_rebuild_complete()
1034 		p->synced = 0;  in gv_rebuild_complete()
1036 		LIST_FOREACH(s, &p->subdisks, in_plex)  in gv_rebuild_complete()