534 vdev_raidz_generate_parity_p(raidz_map_t *rm)  in vdev_raidz_generate_parity_p()  argument
540 	pcount = rm->rm_col[VDEV_RAIDZ_P].rc_size / sizeof (src[0]);  in vdev_raidz_generate_parity_p()
542 	for (c = rm->rm_firstdatacol; c < rm->rm_cols; c++) {  in vdev_raidz_generate_parity_p()
543 		src = rm->rm_col[c].rc_data;  in vdev_raidz_generate_parity_p()
544 		p = rm->rm_col[VDEV_RAIDZ_P].rc_data;  in vdev_raidz_generate_parity_p()
545 		ccount = rm->rm_col[c].rc_size / sizeof (src[0]);  in vdev_raidz_generate_parity_p()
547 		if (c == rm->rm_firstdatacol) {  in vdev_raidz_generate_parity_p()
562 vdev_raidz_generate_parity_pq(raidz_map_t *rm)  in vdev_raidz_generate_parity_pq()  argument
567 	pcnt = rm->rm_col[VDEV_RAIDZ_P].rc_size / sizeof (src[0]);  in vdev_raidz_generate_parity_pq()
568 	ASSERT(rm->rm_col[VDEV_RAIDZ_P].rc_size ==  in vdev_raidz_generate_parity_pq()
569 	    rm->rm_col[VDEV_RAIDZ_Q].rc_size);  in vdev_raidz_generate_parity_pq()
571 	for (c = rm->rm_firstdatacol; c < rm->rm_cols; c++) {  in vdev_raidz_generate_parity_pq()
572 		src = rm->rm_col[c].rc_data;  in vdev_raidz_generate_parity_pq()
573 		p = rm->rm_col[VDEV_RAIDZ_P].rc_data;  in vdev_raidz_generate_parity_pq()
574 		q = rm->rm_col[VDEV_RAIDZ_Q].rc_data;  in vdev_raidz_generate_parity_pq()
576 		ccnt = rm->rm_col[c].rc_size / sizeof (src[0]);  in vdev_raidz_generate_parity_pq()
578 		if (c == rm->rm_firstdatacol) {  in vdev_raidz_generate_parity_pq()
614 vdev_raidz_generate_parity_pqr(raidz_map_t *rm)  in vdev_raidz_generate_parity_pqr()  argument
619 	pcnt = rm->rm_col[VDEV_RAIDZ_P].rc_size / sizeof (src[0]);  in vdev_raidz_generate_parity_pqr()
620 	ASSERT(rm->rm_col[VDEV_RAIDZ_P].rc_size ==  in vdev_raidz_generate_parity_pqr()
621 	    rm->rm_col[VDEV_RAIDZ_Q].rc_size);  in vdev_raidz_generate_parity_pqr()
622 	ASSERT(rm->rm_col[VDEV_RAIDZ_P].rc_size ==  in vdev_raidz_generate_parity_pqr()
623 	    rm->rm_col[VDEV_RAIDZ_R].rc_size);  in vdev_raidz_generate_parity_pqr()
625 	for (c = rm->rm_firstdatacol; c < rm->rm_cols; c++) {  in vdev_raidz_generate_parity_pqr()
626 		src = rm->rm_col[c].rc_data;  in vdev_raidz_generate_parity_pqr()
627 		p = rm->rm_col[VDEV_RAIDZ_P].rc_data;  in vdev_raidz_generate_parity_pqr()
628 		q = rm->rm_col[VDEV_RAIDZ_Q].rc_data;  in vdev_raidz_generate_parity_pqr()
629 		r = rm->rm_col[VDEV_RAIDZ_R].rc_data;  in vdev_raidz_generate_parity_pqr()
631 		ccnt = rm->rm_col[c].rc_size / sizeof (src[0]);  in vdev_raidz_generate_parity_pqr()
633 		if (c == rm->rm_firstdatacol) {  in vdev_raidz_generate_parity_pqr()
679 vdev_raidz_generate_parity(raidz_map_t *rm)  in vdev_raidz_generate_parity()  argument
681 	switch (rm->rm_firstdatacol) {  in vdev_raidz_generate_parity()
683 		vdev_raidz_generate_parity_p(rm);  in vdev_raidz_generate_parity()
686 		vdev_raidz_generate_parity_pq(rm);  in vdev_raidz_generate_parity()
689 		vdev_raidz_generate_parity_pqr(rm);  in vdev_raidz_generate_parity()
851 vdev_raidz_matrix_init(raidz_map_t *rm, int n, int nmap, int *map,  in vdev_raidz_matrix_init()  argument
857 	ASSERT(n == rm->rm_cols - rm->rm_firstdatacol);  in vdev_raidz_matrix_init()
881 vdev_raidz_matrix_invert(raidz_map_t *rm, int n, int nmissing, int *missing,  in vdev_raidz_matrix_invert()  argument
893 		ASSERT3S(used[i], <, rm->rm_firstdatacol);  in vdev_raidz_matrix_invert()
896 		ASSERT3S(used[i], >=, rm->rm_firstdatacol);  in vdev_raidz_matrix_invert()
913 			ASSERT3U(used[j], >=, rm->rm_firstdatacol);  in vdev_raidz_matrix_invert()
914 			jj = used[j] - rm->rm_firstdatacol;  in vdev_raidz_matrix_invert()
975 vdev_raidz_matrix_reconstruct(raidz_map_t *rm, int n, int nmissing,  in vdev_raidz_matrix_reconstruct()  argument
1011 		ASSERT3U(c, <, rm->rm_cols);  in vdev_raidz_matrix_reconstruct()
1013 		src = rm->rm_col[c].rc_data;  in vdev_raidz_matrix_reconstruct()
1014 		ccount = rm->rm_col[c].rc_size;  in vdev_raidz_matrix_reconstruct()
1016 			cc = missing[j] + rm->rm_firstdatacol;  in vdev_raidz_matrix_reconstruct()
1017 			ASSERT3U(cc, >=, rm->rm_firstdatacol);  in vdev_raidz_matrix_reconstruct()
1018 			ASSERT3U(cc, <, rm->rm_cols);  in vdev_raidz_matrix_reconstruct()
1021 			dst[j] = rm->rm_col[cc].rc_data;  in vdev_raidz_matrix_reconstruct()
1022 			dcount[j] = rm->rm_col[cc].rc_size;  in vdev_raidz_matrix_reconstruct()
1025 		ASSERT(ccount >= rm->rm_col[missing[0]].rc_size || i > 0);  in vdev_raidz_matrix_reconstruct()
1055 vdev_raidz_reconstruct_general(raidz_map_t *rm, int *tgts, int ntgts)  in vdev_raidz_reconstruct_general()  argument
1072 	n = rm->rm_cols - rm->rm_firstdatacol;  in vdev_raidz_reconstruct_general()
1079 		if (tgts[t] >= rm->rm_firstdatacol) {  in vdev_raidz_reconstruct_general()
1081 			    tgts[t] - rm->rm_firstdatacol;  in vdev_raidz_reconstruct_general()
1091 		ASSERT(c < rm->rm_firstdatacol);  in vdev_raidz_reconstruct_general()
1130 	for (tt = 0, c = rm->rm_firstdatacol; c < rm->rm_cols; c++) {  in vdev_raidz_reconstruct_general()
1132 		    c == missing_rows[tt] + rm->rm_firstdatacol) {  in vdev_raidz_reconstruct_general()
1145 	vdev_raidz_matrix_init(rm, n, nmissing_rows, parity_map, rows);  in vdev_raidz_reconstruct_general()
1150 	vdev_raidz_matrix_invert(rm, n, nmissing_rows, missing_rows, rows,  in vdev_raidz_reconstruct_general()
1156 	vdev_raidz_matrix_reconstruct(rm, n, nmissing_rows, missing_rows,  in vdev_raidz_reconstruct_general()
1165 vdev_raidz_reconstruct(raidz_map_t *rm, int *t, int nt)  in vdev_raidz_reconstruct()  argument
1180 	nbadparity = rm->rm_firstdatacol;  in vdev_raidz_reconstruct()
1181 	nbaddata = rm->rm_cols - nbadparity;  in vdev_raidz_reconstruct()
1183 	for (i = 0, c = 0; c < rm->rm_cols; c++) {  in vdev_raidz_reconstruct()
1187 		} else if (rm->rm_col[c].rc_error != 0) {  in vdev_raidz_reconstruct()
1189 		} else if (c >= rm->rm_firstdatacol) {  in vdev_raidz_reconstruct()
1200 	code = vdev_raidz_reconstruct_general(rm, tgts, ntgts);  in vdev_raidz_reconstruct()
1210 	raidz_map_t *rm;  in vdev_raidz_map_alloc()  local
1232 	rm = malloc(offsetof(raidz_map_t, rm_col[scols]));  in vdev_raidz_map_alloc()
1233 	if (rm == NULL)  in vdev_raidz_map_alloc()
1234 		return (rm);  in vdev_raidz_map_alloc()
1236 	rm->rm_cols = acols;  in vdev_raidz_map_alloc()
1237 	rm->rm_scols = scols;  in vdev_raidz_map_alloc()
1238 	rm->rm_bigcols = bc;  in vdev_raidz_map_alloc()
1239 	rm->rm_skipstart = bc;  in vdev_raidz_map_alloc()
1240 	rm->rm_missingdata = 0;  in vdev_raidz_map_alloc()
1241 	rm->rm_missingparity = 0;  in vdev_raidz_map_alloc()
1242 	rm->rm_firstdatacol = nparity;  in vdev_raidz_map_alloc()
1243 	rm->rm_reports = 0;  in vdev_raidz_map_alloc()
1244 	rm->rm_freed = 0;  in vdev_raidz_map_alloc()
1245 	rm->rm_ecksuminjected = 0;  in vdev_raidz_map_alloc()
1256 		rm->rm_col[c].rc_devidx = col;  in vdev_raidz_map_alloc()
1257 		rm->rm_col[c].rc_offset = coff;  in vdev_raidz_map_alloc()
1258 		rm->rm_col[c].rc_data = NULL;  in vdev_raidz_map_alloc()
1259 		rm->rm_col[c].rc_error = 0;  in vdev_raidz_map_alloc()
1260 		rm->rm_col[c].rc_tried = 0;  in vdev_raidz_map_alloc()
1261 		rm->rm_col[c].rc_skipped = 0;  in vdev_raidz_map_alloc()
1264 			rm->rm_col[c].rc_size = 0;  in vdev_raidz_map_alloc()
1266 			rm->rm_col[c].rc_size = (q + 1) << unit_shift;  in vdev_raidz_map_alloc()
1268 			rm->rm_col[c].rc_size = q << unit_shift;  in vdev_raidz_map_alloc()
1270 		asize += rm->rm_col[c].rc_size;  in vdev_raidz_map_alloc()
1274 	rm->rm_asize = roundup(asize, (nparity + 1) << unit_shift);  in vdev_raidz_map_alloc()
1275 	rm->rm_nskip = roundup(tot, nparity + 1) - tot;  in vdev_raidz_map_alloc()
1276 	ASSERT3U(rm->rm_asize - asize, ==, rm->rm_nskip << unit_shift);  in vdev_raidz_map_alloc()
1277 	ASSERT3U(rm->rm_nskip, <=, nparity);  in vdev_raidz_map_alloc()
1279 	for (c = 0; c < rm->rm_firstdatacol; c++) {  in vdev_raidz_map_alloc()
1280 		rm->rm_col[c].rc_data = malloc(rm->rm_col[c].rc_size);  in vdev_raidz_map_alloc()
1281 		if (rm->rm_col[c].rc_data == NULL) {  in vdev_raidz_map_alloc()
1284 				free(rm->rm_col[--c].rc_data);  in vdev_raidz_map_alloc()
1285 			free(rm);  in vdev_raidz_map_alloc()
1290 	rm->rm_col[c].rc_data = data;  in vdev_raidz_map_alloc()
1293 		rm->rm_col[c].rc_data = (char *)rm->rm_col[c - 1].rc_data +  in vdev_raidz_map_alloc()
1294 		    rm->rm_col[c - 1].rc_size;  in vdev_raidz_map_alloc()
1316 	ASSERT(rm->rm_cols >= 2);  in vdev_raidz_map_alloc()
1317 	ASSERT(rm->rm_col[0].rc_size == rm->rm_col[1].rc_size);  in vdev_raidz_map_alloc()
1319 	if (rm->rm_firstdatacol == 1 && (offset & (1ULL << 20))) {  in vdev_raidz_map_alloc()
1320 		devidx = rm->rm_col[0].rc_devidx;  in vdev_raidz_map_alloc()
1321 		o = rm->rm_col[0].rc_offset;  in vdev_raidz_map_alloc()
1322 		rm->rm_col[0].rc_devidx = rm->rm_col[1].rc_devidx;  in vdev_raidz_map_alloc()
1323 		rm->rm_col[0].rc_offset = rm->rm_col[1].rc_offset;  in vdev_raidz_map_alloc()
1324 		rm->rm_col[1].rc_devidx = devidx;  in vdev_raidz_map_alloc()
1325 		rm->rm_col[1].rc_offset = o;  in vdev_raidz_map_alloc()
1327 		if (rm->rm_skipstart == 0)  in vdev_raidz_map_alloc()
1328 			rm->rm_skipstart = 1;  in vdev_raidz_map_alloc()
1331 	return (rm);  in vdev_raidz_map_alloc()
1335 vdev_raidz_map_free(raidz_map_t *rm)  in vdev_raidz_map_free()  argument
1339 	for (c = rm->rm_firstdatacol - 1; c >= 0; c--)  in vdev_raidz_map_free()
1340 		free(rm->rm_col[c].rc_data);  in vdev_raidz_map_free()
1342 	free(rm);  in vdev_raidz_map_free()
1376 raidz_parity_verify(raidz_map_t *rm)  in raidz_parity_verify()  argument
1382 	for (c = 0; c < rm->rm_firstdatacol; c++) {  in raidz_parity_verify()
1383 		rc = &rm->rm_col[c];  in raidz_parity_verify()
1394 	vdev_raidz_generate_parity(rm);  in raidz_parity_verify()
1396 	for (c = rm->rm_firstdatacol - 1; c >= 0; c--) {  in raidz_parity_verify()
1397 		rc = &rm->rm_col[c];  in raidz_parity_verify()
1420 vdev_raidz_combrec(const spa_t *spa, raidz_map_t *rm, const blkptr_t *bp,  in vdev_raidz_combrec()  argument
1430 	ASSERT(total_errors < rm->rm_firstdatacol);  in vdev_raidz_combrec()
1437 	for (n = 1; n <= rm->rm_firstdatacol - total_errors; n++) {  in vdev_raidz_combrec()
1449 			    c < rm->rm_firstdatacol) {  in vdev_raidz_combrec()
1450 				c = rm->rm_firstdatacol;  in vdev_raidz_combrec()
1453 			while (rm->rm_col[c].rc_error != 0) {  in vdev_raidz_combrec()
1455 				ASSERT3S(c, <, rm->rm_cols);  in vdev_raidz_combrec()
1464 		tgts[n] = rm->rm_cols;  in vdev_raidz_combrec()
1473 		orig[n - 1] = malloc(rm->rm_col[0].rc_size);  in vdev_raidz_combrec()
1494 				ASSERT3S(c, <, rm->rm_cols);  in vdev_raidz_combrec()
1495 				rc = &rm->rm_col[c];  in vdev_raidz_combrec()
1503 			code = vdev_raidz_reconstruct(rm, tgts, n);  in vdev_raidz_combrec()
1507 					rc = &rm->rm_col[c];  in vdev_raidz_combrec()
1521 				rc = &rm->rm_col[c];  in vdev_raidz_combrec()
1531 				    next < rm->rm_cols &&  in vdev_raidz_combrec()
1532 				    rm->rm_col[next].rc_error != 0; next++)  in vdev_raidz_combrec()
1548 				    rm->rm_col[c].rc_error != 0; c++)  in vdev_raidz_combrec()
1572 	raidz_map_t *rm;  in vdev_raidz_read()  local
1587 	rm = vdev_raidz_map_alloc(data, offset, bytes, tvd->v_ashift,  in vdev_raidz_read()
1589 	if (rm == NULL)  in vdev_raidz_read()
1596 	for (c = rm->rm_cols - 1; c >= 0; c--) {  in vdev_raidz_read()
1597 		rc = &rm->rm_col[c];  in vdev_raidz_read()
1600 			if (c >= rm->rm_firstdatacol)  in vdev_raidz_read()
1601 				rm->rm_missingdata++;  in vdev_raidz_read()
1603 				rm->rm_missingparity++;  in vdev_raidz_read()
1611 			if (c >= rm->rm_firstdatacol)  in vdev_raidz_read()
1612 				rm->rm_missingdata++;  in vdev_raidz_read()
1614 				rm->rm_missingparity++;  in vdev_raidz_read()
1620 		if (c >= rm->rm_firstdatacol || rm->rm_missingdata > 0) {  in vdev_raidz_read()
1635 	ASSERT(rm->rm_missingparity <= rm->rm_firstdatacol);  in vdev_raidz_read()
1636 	ASSERT(rm->rm_missingdata <= rm->rm_cols - rm->rm_firstdatacol);  in vdev_raidz_read()
1638 	for (c = 0; c < rm->rm_cols; c++) {  in vdev_raidz_read()
1639 		rc = &rm->rm_col[c];  in vdev_raidz_read()
1644 			if (c < rm->rm_firstdatacol)  in vdev_raidz_read()
1653 		} else if (c < rm->rm_firstdatacol && !rc->rc_tried) {  in vdev_raidz_read()
1675 	if (total_errors <= rm->rm_firstdatacol - parity_untried) {  in vdev_raidz_read()
1690 				    rm->rm_firstdatacol) {  in vdev_raidz_read()
1691 					n = raidz_parity_verify(rm);  in vdev_raidz_read()
1694 					    rm->rm_firstdatacol);  in vdev_raidz_read()
1707 			ASSERT(parity_errors < rm->rm_firstdatacol);  in vdev_raidz_read()
1713 			for (c = rm->rm_firstdatacol; c < rm->rm_cols; c++) {  in vdev_raidz_read()
1714 				rc = &rm->rm_col[c];  in vdev_raidz_read()
1721 			ASSERT(rm->rm_firstdatacol >= n);  in vdev_raidz_read()
1723 			code = vdev_raidz_reconstruct(rm, tgts, n);  in vdev_raidz_read()
1740 				if (parity_errors < rm->rm_firstdatacol - n) {  in vdev_raidz_read()
1741 					n = raidz_parity_verify(rm);  in vdev_raidz_read()
1744 					    rm->rm_firstdatacol);  in vdev_raidz_read()
1761 	rm->rm_missingdata = 0;  in vdev_raidz_read()
1762 	rm->rm_missingparity = 0;  in vdev_raidz_read()
1765 	for (c = 0; c < rm->rm_cols; c++) {  in vdev_raidz_read()
1766 		rc = &rm->rm_col[c];  in vdev_raidz_read()
1797 	if (total_errors > rm->rm_firstdatacol) {  in vdev_raidz_read()
1799 	} else if (total_errors < rm->rm_firstdatacol &&  in vdev_raidz_read()
1800 	    (code = vdev_raidz_combrec(vd->v_spa, rm, bp, data, offset, bytes,  in vdev_raidz_read()
1807 		if (code != (1 << rm->rm_firstdatacol) - 1)  in vdev_raidz_read()
1808 			(void) raidz_parity_verify(rm);  in vdev_raidz_read()
1826 	vdev_raidz_map_free(rm);  in vdev_raidz_read()