Lines Matching full:rr
382 vdev_raidz_row_free(raidz_row_t *rr) in vdev_raidz_row_free() argument
384 for (int c = 0; c < rr->rr_cols; c++) { in vdev_raidz_row_free()
385 raidz_col_t *rc = &rr->rr_col[c]; in vdev_raidz_row_free()
393 if (rr->rr_abd_empty != NULL) in vdev_raidz_row_free()
394 abd_free(rr->rr_abd_empty); in vdev_raidz_row_free()
396 kmem_free(rr, offsetof(raidz_row_t, rr_col[rr->rr_scols])); in vdev_raidz_row_free()
443 raidz_row_t *rr = in vdev_raidz_row_alloc() local
446 rr->rr_cols = cols; in vdev_raidz_row_alloc()
447 rr->rr_scols = cols; in vdev_raidz_row_alloc()
450 raidz_col_t *rc = &rr->rr_col[c]; in vdev_raidz_row_alloc()
465 return (rr); in vdev_raidz_row_alloc()
474 raidz_row_t *rr = rm->rm_row[0]; in vdev_raidz_map_alloc_write() local
483 if (rm->rm_skipstart < rr->rr_firstdatacol) { in vdev_raidz_map_alloc_write()
486 } else if (rr->rr_scols < (rm->rm_skipstart + rm->rm_nskip)) { in vdev_raidz_map_alloc_write()
488 (rm->rm_skipstart + rm->rm_nskip) % rr->rr_scols; in vdev_raidz_map_alloc_write()
495 int skipped = rr->rr_scols - rr->rr_cols; in vdev_raidz_map_alloc_write()
498 for (c = 0; c < rr->rr_firstdatacol; c++) { in vdev_raidz_map_alloc_write()
499 raidz_col_t *rc = &rr->rr_col[c]; in vdev_raidz_map_alloc_write()
520 for (off = 0; c < rr->rr_cols; c++) { in vdev_raidz_map_alloc_write()
521 raidz_col_t *rc = &rr->rr_col[c]; in vdev_raidz_map_alloc_write()
559 raidz_row_t *rr = rm->rm_row[0]; in vdev_raidz_map_alloc_read() local
564 for (c = 0; c < rr->rr_firstdatacol; c++) in vdev_raidz_map_alloc_read()
565 rr->rr_col[c].rc_abd = in vdev_raidz_map_alloc_read()
566 abd_alloc_linear(rr->rr_col[c].rc_size, B_FALSE); in vdev_raidz_map_alloc_read()
568 for (uint64_t off = 0; c < rr->rr_cols; c++) { in vdev_raidz_map_alloc_read()
569 raidz_col_t *rc = &rr->rr_col[c]; in vdev_raidz_map_alloc_read()
587 raidz_row_t *rr; in vdev_raidz_map_alloc() local
637 rr = vdev_raidz_row_alloc(scols, zio); in vdev_raidz_map_alloc()
638 rm->rm_row[0] = rr; in vdev_raidz_map_alloc()
639 rr->rr_cols = acols; in vdev_raidz_map_alloc()
640 rr->rr_bigcols = bc; in vdev_raidz_map_alloc()
641 rr->rr_firstdatacol = nparity; in vdev_raidz_map_alloc()
643 rr->rr_offset = zio->io_offset; in vdev_raidz_map_alloc()
644 rr->rr_size = zio->io_size; in vdev_raidz_map_alloc()
650 raidz_col_t *rc = &rr->rr_col[c]; in vdev_raidz_map_alloc()
694 ASSERT(rr->rr_cols >= 2); in vdev_raidz_map_alloc()
695 ASSERT(rr->rr_col[0].rc_size == rr->rr_col[1].rc_size); in vdev_raidz_map_alloc()
697 if (rr->rr_firstdatacol == 1 && (zio->io_offset & (1ULL << 20))) { in vdev_raidz_map_alloc()
698 uint64_t devidx = rr->rr_col[0].rc_devidx; in vdev_raidz_map_alloc()
699 o = rr->rr_col[0].rc_offset; in vdev_raidz_map_alloc()
700 rr->rr_col[0].rc_devidx = rr->rr_col[1].rc_devidx; in vdev_raidz_map_alloc()
701 rr->rr_col[0].rc_offset = rr->rr_col[1].rc_offset; in vdev_raidz_map_alloc()
702 rr->rr_col[1].rc_devidx = devidx; in vdev_raidz_map_alloc()
703 rr->rr_col[1].rc_offset = o; in vdev_raidz_map_alloc()
783 raidz_row_t *rr = vdev_raidz_row_alloc(cols, zio); in vdev_raidz_map_alloc_expanded() local
784 rm->rm_row[row] = rr; in vdev_raidz_map_alloc_expanded()
827 rr->rr_firstdatacol = nparity; in vdev_raidz_map_alloc_expanded()
832 rr->rr_offset = b << ashift; in vdev_raidz_map_alloc_expanded()
833 rr->rr_size = (rr->rr_cols - rr->rr_firstdatacol) << ashift; in vdev_raidz_map_alloc_expanded()
836 for (int c = 0; c < rr->rr_cols; c++, child_id++) { in vdev_raidz_map_alloc_expanded()
841 raidz_col_t *rc = &rr->rr_col[c]; in vdev_raidz_map_alloc_expanded()
856 uint64_t dc = c - rr->rr_firstdatacol; in vdev_raidz_map_alloc_expanded()
857 if (c < rr->rr_firstdatacol) { in vdev_raidz_map_alloc_expanded()
923 if (rr->rr_firstdatacol == 1 && rr->rr_cols > 1 && in vdev_raidz_map_alloc_expanded()
925 ASSERT(rr->rr_cols >= 2); in vdev_raidz_map_alloc_expanded()
926 ASSERT(rr->rr_col[0].rc_size == rr->rr_col[1].rc_size); in vdev_raidz_map_alloc_expanded()
928 int devidx0 = rr->rr_col[0].rc_devidx; in vdev_raidz_map_alloc_expanded()
929 uint64_t offset0 = rr->rr_col[0].rc_offset; in vdev_raidz_map_alloc_expanded()
930 int shadow_devidx0 = rr->rr_col[0].rc_shadow_devidx; in vdev_raidz_map_alloc_expanded()
932 rr->rr_col[0].rc_shadow_offset; in vdev_raidz_map_alloc_expanded()
934 rr->rr_col[0].rc_devidx = rr->rr_col[1].rc_devidx; in vdev_raidz_map_alloc_expanded()
935 rr->rr_col[0].rc_offset = rr->rr_col[1].rc_offset; in vdev_raidz_map_alloc_expanded()
936 rr->rr_col[0].rc_shadow_devidx = in vdev_raidz_map_alloc_expanded()
937 rr->rr_col[1].rc_shadow_devidx; in vdev_raidz_map_alloc_expanded()
938 rr->rr_col[0].rc_shadow_offset = in vdev_raidz_map_alloc_expanded()
939 rr->rr_col[1].rc_shadow_offset; in vdev_raidz_map_alloc_expanded()
941 rr->rr_col[1].rc_devidx = devidx0; in vdev_raidz_map_alloc_expanded()
942 rr->rr_col[1].rc_offset = offset0; in vdev_raidz_map_alloc_expanded()
943 rr->rr_col[1].rc_shadow_devidx = shadow_devidx0; in vdev_raidz_map_alloc_expanded()
944 rr->rr_col[1].rc_shadow_offset = shadow_offset0; in vdev_raidz_map_alloc_expanded()
965 raidz_row_t *rr = rm->rm_row[i]; in vdev_raidz_map_alloc_expanded() local
966 for (int c = 0; c < rr->rr_cols; c++) { in vdev_raidz_map_alloc_expanded()
967 raidz_col_t *rc = &rr->rr_col[c]; in vdev_raidz_map_alloc_expanded()
1019 raidz_row_t *rr = rm->rm_row[i]; in vdev_raidz_map_alloc_expanded() local
1020 for (int c = 0; c < rr->rr_firstdatacol; c++) { in vdev_raidz_map_alloc_expanded()
1021 raidz_col_t *rc = &rr->rr_col[c]; in vdev_raidz_map_alloc_expanded()
1036 raidz_row_t *rr = rm->rm_row[i]; in vdev_raidz_map_alloc_expanded() local
1037 for (int c = 0; c < rr->rr_firstdatacol; c++) { in vdev_raidz_map_alloc_expanded()
1038 raidz_col_t *rc = &rr->rr_col[c]; in vdev_raidz_map_alloc_expanded()
1113 vdev_raidz_generate_parity_p(raidz_row_t *rr) in vdev_raidz_generate_parity_p() argument
1115 uint64_t *p = abd_to_buf(rr->rr_col[VDEV_RAIDZ_P].rc_abd); in vdev_raidz_generate_parity_p()
1117 for (int c = rr->rr_firstdatacol; c < rr->rr_cols; c++) { in vdev_raidz_generate_parity_p()
1118 abd_t *src = rr->rr_col[c].rc_abd; in vdev_raidz_generate_parity_p()
1120 if (c == rr->rr_firstdatacol) { in vdev_raidz_generate_parity_p()
1121 abd_copy_to_buf(p, src, rr->rr_col[c].rc_size); in vdev_raidz_generate_parity_p()
1124 (void) abd_iterate_func(src, 0, rr->rr_col[c].rc_size, in vdev_raidz_generate_parity_p()
1131 vdev_raidz_generate_parity_pq(raidz_row_t *rr) in vdev_raidz_generate_parity_pq() argument
1133 uint64_t *p = abd_to_buf(rr->rr_col[VDEV_RAIDZ_P].rc_abd); in vdev_raidz_generate_parity_pq()
1134 uint64_t *q = abd_to_buf(rr->rr_col[VDEV_RAIDZ_Q].rc_abd); in vdev_raidz_generate_parity_pq()
1135 uint64_t pcnt = rr->rr_col[VDEV_RAIDZ_P].rc_size / sizeof (p[0]); in vdev_raidz_generate_parity_pq()
1136 ASSERT(rr->rr_col[VDEV_RAIDZ_P].rc_size == in vdev_raidz_generate_parity_pq()
1137 rr->rr_col[VDEV_RAIDZ_Q].rc_size); in vdev_raidz_generate_parity_pq()
1139 for (int c = rr->rr_firstdatacol; c < rr->rr_cols; c++) { in vdev_raidz_generate_parity_pq()
1140 abd_t *src = rr->rr_col[c].rc_abd; in vdev_raidz_generate_parity_pq()
1142 uint64_t ccnt = rr->rr_col[c].rc_size / sizeof (p[0]); in vdev_raidz_generate_parity_pq()
1144 if (c == rr->rr_firstdatacol) { in vdev_raidz_generate_parity_pq()
1146 abd_copy_to_buf(p, src, rr->rr_col[c].rc_size); in vdev_raidz_generate_parity_pq()
1147 (void) memcpy(q, p, rr->rr_col[c].rc_size); in vdev_raidz_generate_parity_pq()
1157 (void) abd_iterate_func(src, 0, rr->rr_col[c].rc_size, in vdev_raidz_generate_parity_pq()
1173 vdev_raidz_generate_parity_pqr(raidz_row_t *rr) in vdev_raidz_generate_parity_pqr() argument
1175 uint64_t *p = abd_to_buf(rr->rr_col[VDEV_RAIDZ_P].rc_abd); in vdev_raidz_generate_parity_pqr()
1176 uint64_t *q = abd_to_buf(rr->rr_col[VDEV_RAIDZ_Q].rc_abd); in vdev_raidz_generate_parity_pqr()
1177 uint64_t *r = abd_to_buf(rr->rr_col[VDEV_RAIDZ_R].rc_abd); in vdev_raidz_generate_parity_pqr()
1178 uint64_t pcnt = rr->rr_col[VDEV_RAIDZ_P].rc_size / sizeof (p[0]); in vdev_raidz_generate_parity_pqr()
1179 ASSERT(rr->rr_col[VDEV_RAIDZ_P].rc_size == in vdev_raidz_generate_parity_pqr()
1180 rr->rr_col[VDEV_RAIDZ_Q].rc_size); in vdev_raidz_generate_parity_pqr()
1181 ASSERT(rr->rr_col[VDEV_RAIDZ_P].rc_size == in vdev_raidz_generate_parity_pqr()
1182 rr->rr_col[VDEV_RAIDZ_R].rc_size); in vdev_raidz_generate_parity_pqr()
1184 for (int c = rr->rr_firstdatacol; c < rr->rr_cols; c++) { in vdev_raidz_generate_parity_pqr()
1185 abd_t *src = rr->rr_col[c].rc_abd; in vdev_raidz_generate_parity_pqr()
1187 uint64_t ccnt = rr->rr_col[c].rc_size / sizeof (p[0]); in vdev_raidz_generate_parity_pqr()
1189 if (c == rr->rr_firstdatacol) { in vdev_raidz_generate_parity_pqr()
1191 abd_copy_to_buf(p, src, rr->rr_col[c].rc_size); in vdev_raidz_generate_parity_pqr()
1192 (void) memcpy(q, p, rr->rr_col[c].rc_size); in vdev_raidz_generate_parity_pqr()
1193 (void) memcpy(r, p, rr->rr_col[c].rc_size); in vdev_raidz_generate_parity_pqr()
1204 (void) abd_iterate_func(src, 0, rr->rr_col[c].rc_size, in vdev_raidz_generate_parity_pqr()
1225 vdev_raidz_generate_parity_row(raidz_map_t *rm, raidz_row_t *rr) in vdev_raidz_generate_parity_row() argument
1227 if (rr->rr_cols == 0) { in vdev_raidz_generate_parity_row()
1238 if (vdev_raidz_math_generate(rm, rr) != RAIDZ_ORIGINAL_IMPL) in vdev_raidz_generate_parity_row()
1241 switch (rr->rr_firstdatacol) { in vdev_raidz_generate_parity_row()
1243 vdev_raidz_generate_parity_p(rr); in vdev_raidz_generate_parity_row()
1246 vdev_raidz_generate_parity_pq(rr); in vdev_raidz_generate_parity_row()
1249 vdev_raidz_generate_parity_pqr(rr); in vdev_raidz_generate_parity_row()
1260 raidz_row_t *rr = rm->rm_row[i]; in vdev_raidz_generate_parity() local
1261 vdev_raidz_generate_parity_row(rm, rr); in vdev_raidz_generate_parity()
1382 vdev_raidz_reconstruct_p(raidz_row_t *rr, int *tgts, int ntgts) in vdev_raidz_reconstruct_p() argument
1388 zfs_dbgmsg("reconstruct_p(rm=%px x=%u)", rr, x); in vdev_raidz_reconstruct_p()
1391 ASSERT3U(x, >=, rr->rr_firstdatacol); in vdev_raidz_reconstruct_p()
1392 ASSERT3U(x, <, rr->rr_cols); in vdev_raidz_reconstruct_p()
1394 ASSERT3U(rr->rr_col[x].rc_size, <=, rr->rr_col[VDEV_RAIDZ_P].rc_size); in vdev_raidz_reconstruct_p()
1396 src = rr->rr_col[VDEV_RAIDZ_P].rc_abd; in vdev_raidz_reconstruct_p()
1397 dst = rr->rr_col[x].rc_abd; in vdev_raidz_reconstruct_p()
1399 abd_copy_from_buf(dst, abd_to_buf(src), rr->rr_col[x].rc_size); in vdev_raidz_reconstruct_p()
1401 for (int c = rr->rr_firstdatacol; c < rr->rr_cols; c++) { in vdev_raidz_reconstruct_p()
1402 uint64_t size = MIN(rr->rr_col[x].rc_size, in vdev_raidz_reconstruct_p()
1403 rr->rr_col[c].rc_size); in vdev_raidz_reconstruct_p()
1405 src = rr->rr_col[c].rc_abd; in vdev_raidz_reconstruct_p()
1416 vdev_raidz_reconstruct_q(raidz_row_t *rr, int *tgts, int ntgts) in vdev_raidz_reconstruct_q() argument
1423 zfs_dbgmsg("reconstruct_q(rm=%px x=%u)", rr, x); in vdev_raidz_reconstruct_q()
1427 ASSERT(rr->rr_col[x].rc_size <= rr->rr_col[VDEV_RAIDZ_Q].rc_size); in vdev_raidz_reconstruct_q()
1429 for (c = rr->rr_firstdatacol; c < rr->rr_cols; c++) { in vdev_raidz_reconstruct_q()
1430 uint64_t size = (c == x) ? 0 : MIN(rr->rr_col[x].rc_size, in vdev_raidz_reconstruct_q()
1431 rr->rr_col[c].rc_size); in vdev_raidz_reconstruct_q()
1433 src = rr->rr_col[c].rc_abd; in vdev_raidz_reconstruct_q()
1434 dst = rr->rr_col[x].rc_abd; in vdev_raidz_reconstruct_q()
1436 if (c == rr->rr_firstdatacol) { in vdev_raidz_reconstruct_q()
1438 if (rr->rr_col[x].rc_size > size) { in vdev_raidz_reconstruct_q()
1440 rr->rr_col[x].rc_size - size); in vdev_raidz_reconstruct_q()
1443 ASSERT3U(size, <=, rr->rr_col[x].rc_size); in vdev_raidz_reconstruct_q()
1447 size, rr->rr_col[x].rc_size - size, in vdev_raidz_reconstruct_q()
1452 src = rr->rr_col[VDEV_RAIDZ_Q].rc_abd; in vdev_raidz_reconstruct_q()
1453 dst = rr->rr_col[x].rc_abd; in vdev_raidz_reconstruct_q()
1454 exp = 255 - (rr->rr_cols - 1 - x); in vdev_raidz_reconstruct_q()
1457 (void) abd_iterate_func(dst, 0, rr->rr_col[x].rc_size, in vdev_raidz_reconstruct_q()
1462 vdev_raidz_reconstruct_pq(raidz_row_t *rr, int *tgts, int ntgts) in vdev_raidz_reconstruct_pq() argument
1472 zfs_dbgmsg("reconstruct_pq(rm=%px x=%u y=%u)", rr, x, y); in vdev_raidz_reconstruct_pq()
1476 ASSERT(x >= rr->rr_firstdatacol); in vdev_raidz_reconstruct_pq()
1477 ASSERT(y < rr->rr_cols); in vdev_raidz_reconstruct_pq()
1479 ASSERT(rr->rr_col[x].rc_size >= rr->rr_col[y].rc_size); in vdev_raidz_reconstruct_pq()
1488 pdata = rr->rr_col[VDEV_RAIDZ_P].rc_abd; in vdev_raidz_reconstruct_pq()
1489 qdata = rr->rr_col[VDEV_RAIDZ_Q].rc_abd; in vdev_raidz_reconstruct_pq()
1490 xsize = rr->rr_col[x].rc_size; in vdev_raidz_reconstruct_pq()
1491 ysize = rr->rr_col[y].rc_size; in vdev_raidz_reconstruct_pq()
1493 rr->rr_col[VDEV_RAIDZ_P].rc_abd = in vdev_raidz_reconstruct_pq()
1494 abd_alloc_linear(rr->rr_col[VDEV_RAIDZ_P].rc_size, B_TRUE); in vdev_raidz_reconstruct_pq()
1495 rr->rr_col[VDEV_RAIDZ_Q].rc_abd = in vdev_raidz_reconstruct_pq()
1496 abd_alloc_linear(rr->rr_col[VDEV_RAIDZ_Q].rc_size, B_TRUE); in vdev_raidz_reconstruct_pq()
1497 rr->rr_col[x].rc_size = 0; in vdev_raidz_reconstruct_pq()
1498 rr->rr_col[y].rc_size = 0; in vdev_raidz_reconstruct_pq()
1500 vdev_raidz_generate_parity_pq(rr); in vdev_raidz_reconstruct_pq()
1502 rr->rr_col[x].rc_size = xsize; in vdev_raidz_reconstruct_pq()
1503 rr->rr_col[y].rc_size = ysize; in vdev_raidz_reconstruct_pq()
1507 pxy = abd_to_buf(rr->rr_col[VDEV_RAIDZ_P].rc_abd); in vdev_raidz_reconstruct_pq()
1508 qxy = abd_to_buf(rr->rr_col[VDEV_RAIDZ_Q].rc_abd); in vdev_raidz_reconstruct_pq()
1509 xd = rr->rr_col[x].rc_abd; in vdev_raidz_reconstruct_pq()
1510 yd = rr->rr_col[y].rc_abd; in vdev_raidz_reconstruct_pq()
1528 b = vdev_raidz_pow2[255 - (rr->rr_cols - 1 - x)]; in vdev_raidz_reconstruct_pq()
1542 abd_free(rr->rr_col[VDEV_RAIDZ_P].rc_abd); in vdev_raidz_reconstruct_pq()
1543 abd_free(rr->rr_col[VDEV_RAIDZ_Q].rc_abd); in vdev_raidz_reconstruct_pq()
1548 rr->rr_col[VDEV_RAIDZ_P].rc_abd = pdata; in vdev_raidz_reconstruct_pq()
1549 rr->rr_col[VDEV_RAIDZ_Q].rc_abd = qdata; in vdev_raidz_reconstruct_pq()
1705 vdev_raidz_matrix_init(raidz_row_t *rr, int n, int nmap, int *map, in vdev_raidz_matrix_init() argument
1711 ASSERT(n == rr->rr_cols - rr->rr_firstdatacol); in vdev_raidz_matrix_init()
1735 vdev_raidz_matrix_invert(raidz_row_t *rr, int n, int nmissing, int *missing, in vdev_raidz_matrix_invert() argument
1747 ASSERT3S(used[i], <, rr->rr_firstdatacol); in vdev_raidz_matrix_invert()
1750 ASSERT3S(used[i], >=, rr->rr_firstdatacol); in vdev_raidz_matrix_invert()
1767 ASSERT3U(used[j], >=, rr->rr_firstdatacol); in vdev_raidz_matrix_invert()
1768 jj = used[j] - rr->rr_firstdatacol; in vdev_raidz_matrix_invert()
1829 vdev_raidz_matrix_reconstruct(raidz_row_t *rr, int n, int nmissing, in vdev_raidz_matrix_reconstruct() argument
1861 ASSERT3U(c, <, rr->rr_cols); in vdev_raidz_matrix_reconstruct()
1863 ccount = rr->rr_col[c].rc_size; in vdev_raidz_matrix_reconstruct()
1864 ASSERT(ccount >= rr->rr_col[missing[0]].rc_size || i > 0); in vdev_raidz_matrix_reconstruct()
1867 src = abd_to_buf(rr->rr_col[c].rc_abd); in vdev_raidz_matrix_reconstruct()
1869 cc = missing[j] + rr->rr_firstdatacol; in vdev_raidz_matrix_reconstruct()
1870 ASSERT3U(cc, >=, rr->rr_firstdatacol); in vdev_raidz_matrix_reconstruct()
1871 ASSERT3U(cc, <, rr->rr_cols); in vdev_raidz_matrix_reconstruct()
1874 dcount[j] = rr->rr_col[cc].rc_size; in vdev_raidz_matrix_reconstruct()
1876 dst[j] = abd_to_buf(rr->rr_col[cc].rc_abd); in vdev_raidz_matrix_reconstruct()
1907 vdev_raidz_reconstruct_general(raidz_row_t *rr, int *tgts, int ntgts) in vdev_raidz_reconstruct_general() argument
1923 zfs_dbgmsg("reconstruct_general(rm=%px ntgts=%u)", rr, ntgts); in vdev_raidz_reconstruct_general()
1928 for (i = rr->rr_firstdatacol; i < rr->rr_cols; i++) { in vdev_raidz_reconstruct_general()
1929 ASSERT(rr->rr_col[i].rc_abd != NULL); in vdev_raidz_reconstruct_general()
1930 if (!abd_is_linear(rr->rr_col[i].rc_abd)) { in vdev_raidz_reconstruct_general()
1931 bufs = kmem_alloc(rr->rr_cols * sizeof (abd_t *), in vdev_raidz_reconstruct_general()
1934 for (c = rr->rr_firstdatacol; c < rr->rr_cols; c++) { in vdev_raidz_reconstruct_general()
1935 raidz_col_t *col = &rr->rr_col[c]; in vdev_raidz_reconstruct_general()
1950 n = rr->rr_cols - rr->rr_firstdatacol; in vdev_raidz_reconstruct_general()
1957 if (tgts[t] >= rr->rr_firstdatacol) { in vdev_raidz_reconstruct_general()
1959 tgts[t] - rr->rr_firstdatacol; in vdev_raidz_reconstruct_general()
1969 ASSERT(c < rr->rr_firstdatacol); in vdev_raidz_reconstruct_general()
1999 for (tt = 0, c = rr->rr_firstdatacol; c < rr->rr_cols; c++) { in vdev_raidz_reconstruct_general()
2001 c == missing_rows[tt] + rr->rr_firstdatacol) { in vdev_raidz_reconstruct_general()
2014 vdev_raidz_matrix_init(rr, n, nmissing_rows, parity_map, rows); in vdev_raidz_reconstruct_general()
2019 vdev_raidz_matrix_invert(rr, n, nmissing_rows, missing_rows, rows, in vdev_raidz_reconstruct_general()
2025 vdev_raidz_matrix_reconstruct(rr, n, nmissing_rows, missing_rows, in vdev_raidz_reconstruct_general()
2034 for (c = rr->rr_firstdatacol; c < rr->rr_cols; c++) { in vdev_raidz_reconstruct_general()
2035 raidz_col_t *col = &rr->rr_col[c]; in vdev_raidz_reconstruct_general()
2043 kmem_free(bufs, rr->rr_cols * sizeof (abd_t *)); in vdev_raidz_reconstruct_general()
2048 vdev_raidz_reconstruct_row(raidz_map_t *rm, raidz_row_t *rr, in vdev_raidz_reconstruct_row() argument
2059 rr, nt, (int)rr->rr_cols, (int)rr->rr_missingdata, in vdev_raidz_reconstruct_row()
2060 (int)rr->rr_missingparity); in vdev_raidz_reconstruct_row()
2063 nbadparity = rr->rr_firstdatacol; in vdev_raidz_reconstruct_row()
2064 nbaddata = rr->rr_cols - nbadparity; in vdev_raidz_reconstruct_row()
2066 for (i = 0, c = 0; c < rr->rr_cols; c++) { in vdev_raidz_reconstruct_row()
2070 rr, c, (int)rr->rr_col[c].rc_devidx, in vdev_raidz_reconstruct_row()
2071 (long long)rr->rr_col[c].rc_offset, in vdev_raidz_reconstruct_row()
2072 (int)rr->rr_col[c].rc_error); in vdev_raidz_reconstruct_row()
2074 if (c < rr->rr_firstdatacol) in vdev_raidz_reconstruct_row()
2080 } else if (rr->rr_col[c].rc_error != 0) { in vdev_raidz_reconstruct_row()
2082 } else if (c >= rr->rr_firstdatacol) { in vdev_raidz_reconstruct_row()
2097 ret = vdev_raidz_math_reconstruct(rm, rr, parity_valid, dt, nbaddata); in vdev_raidz_reconstruct_row()
2107 vdev_raidz_reconstruct_p(rr, dt, 1); in vdev_raidz_reconstruct_row()
2111 ASSERT(rr->rr_firstdatacol > 1); in vdev_raidz_reconstruct_row()
2114 vdev_raidz_reconstruct_q(rr, dt, 1); in vdev_raidz_reconstruct_row()
2118 ASSERT(rr->rr_firstdatacol > 2); in vdev_raidz_reconstruct_row()
2122 ASSERT(rr->rr_firstdatacol > 1); in vdev_raidz_reconstruct_row()
2126 vdev_raidz_reconstruct_pq(rr, dt, 2); in vdev_raidz_reconstruct_row()
2130 ASSERT(rr->rr_firstdatacol > 2); in vdev_raidz_reconstruct_row()
2135 vdev_raidz_reconstruct_general(rr, tgts, ntgts); in vdev_raidz_reconstruct_row()
2332 vdev_raidz_io_verify(zio_t *zio, raidz_map_t *rm, raidz_row_t *rr, int col) in vdev_raidz_io_verify() argument
2337 logical_rs.rs_start = rr->rr_offset; in vdev_raidz_io_verify()
2339 vdev_raidz_psize_to_asize(zio->io_vd, rr->rr_size, in vdev_raidz_io_verify()
2342 raidz_col_t *rc = &rr->rr_col[col]; in vdev_raidz_io_verify()
2373 vdev_raidz_io_start_write(zio_t *zio, raidz_row_t *rr) in vdev_raidz_io_start_write() argument
2378 vdev_raidz_generate_parity_row(rm, rr); in vdev_raidz_io_start_write()
2380 for (int c = 0; c < rr->rr_scols; c++) { in vdev_raidz_io_start_write()
2381 raidz_col_t *rc = &rr->rr_col[c]; in vdev_raidz_io_start_write()
2385 vdev_raidz_io_verify(zio, rm, rr, c); in vdev_raidz_io_start_write()
2426 raidz_row_t *rr = rm->rm_row[0]; in raidz_start_skip_writes() local
2427 for (int c = 0; c < rr->rr_scols; c++) { in raidz_start_skip_writes()
2428 raidz_col_t *rc = &rr->rr_col[c]; in raidz_start_skip_writes()
2444 vdev_raidz_io_start_read_row(zio_t *zio, raidz_row_t *rr, boolean_t forceparity) in vdev_raidz_io_start_read_row() argument
2452 for (int c = rr->rr_cols - 1; c >= 0; c--) { in vdev_raidz_io_start_read_row()
2453 raidz_col_t *rc = &rr->rr_col[c]; in vdev_raidz_io_start_read_row()
2458 if (c >= rr->rr_firstdatacol) in vdev_raidz_io_start_read_row()
2459 rr->rr_missingdata++; in vdev_raidz_io_start_read_row()
2461 rr->rr_missingparity++; in vdev_raidz_io_start_read_row()
2468 if (c >= rr->rr_firstdatacol) in vdev_raidz_io_start_read_row()
2469 rr->rr_missingdata++; in vdev_raidz_io_start_read_row()
2471 rr->rr_missingparity++; in vdev_raidz_io_start_read_row()
2477 c >= rr->rr_firstdatacol || rr->rr_missingdata > 0 || in vdev_raidz_io_start_read_row()
2531 raidz_row_t *rr = rm->rm_row[i]; in vdev_raidz_io_start_read() local
2532 vdev_raidz_io_start_read_row(zio, rr, forceparity); in vdev_raidz_io_start_read()
2705 raidz_parity_verify(zio_t *zio, raidz_row_t *rr) in raidz_parity_verify() argument
2719 for (c = 0; c < rr->rr_firstdatacol; c++) { in raidz_parity_verify()
2720 rc = &rr->rr_col[c]; in raidz_parity_verify()
2733 if (rr->rr_nempty && rr->rr_abd_empty != NULL) in raidz_parity_verify()
2734 ret += vdev_draid_map_verify_empty(zio, rr); in raidz_parity_verify()
2741 vdev_raidz_generate_parity_row(rm, rr); in raidz_parity_verify()
2743 for (c = 0; c < rr->rr_firstdatacol; c++) { in raidz_parity_verify()
2744 rc = &rr->rr_col[c]; in raidz_parity_verify()
2763 vdev_raidz_worst_error(raidz_row_t *rr) in vdev_raidz_worst_error() argument
2767 for (int c = 0; c < rr->rr_cols; c++) { in vdev_raidz_worst_error()
2768 error = zio_worst_error(error, rr->rr_col[c].rc_error); in vdev_raidz_worst_error()
2769 error = zio_worst_error(error, rr->rr_col[c].rc_shadow_error); in vdev_raidz_worst_error()
2776 vdev_raidz_io_done_verified(zio_t *zio, raidz_row_t *rr) in vdev_raidz_io_done_verified() argument
2785 for (int c = 0; c < rr->rr_cols; c++) { in vdev_raidz_io_done_verified()
2786 raidz_col_t *rc = &rr->rr_col[c]; in vdev_raidz_io_done_verified()
2789 if (c < rr->rr_firstdatacol) in vdev_raidz_io_done_verified()
2796 } else if (c < rr->rr_firstdatacol && !rc->rc_tried) { in vdev_raidz_io_done_verified()
2813 rr->rr_firstdatacol - data_errors || in vdev_raidz_io_done_verified()
2815 int n = raidz_parity_verify(zio, rr); in vdev_raidz_io_done_verified()
2824 for (int c = 0; c < rr->rr_cols; c++) { in vdev_raidz_io_done_verified()
2825 raidz_col_t *rc = &rr->rr_col[c]; in vdev_raidz_io_done_verified()
2864 for (int c = 0; c < rr->rr_cols; c++) { in vdev_raidz_io_done_verified()
2865 raidz_col_t *rc = &rr->rr_col[c]; in vdev_raidz_io_done_verified()
2896 raidz_row_t *rr = rm->rm_row[i]; in raidz_restore_orig_data() local
2897 for (int c = 0; c < rr->rr_cols; c++) { in raidz_restore_orig_data()
2898 raidz_col_t *rc = &rr->rr_col[c]; in raidz_restore_orig_data()
2966 raidz_row_t *rr = rm->rm_row[r]; in raidz_reconstruct() local
2975 for (int c = 0; c < rr->rr_cols; c++) { in raidz_reconstruct()
2976 raidz_col_t *rc = &rr->rr_col[c]; in raidz_reconstruct()
3038 vdev_raidz_reconstruct_row(rm, rr, my_tgts, t); in raidz_reconstruct()
3048 raidz_row_t *rr = rm->rm_row[i]; in raidz_reconstruct() local
3050 for (int c = 0; c < rr->rr_cols; c++) { in raidz_reconstruct()
3051 raidz_col_t *rc = &rr->rr_col[c]; in raidz_reconstruct()
3064 c >= rr->rr_firstdatacol) { in raidz_reconstruct()
3074 vdev_raidz_io_done_verified(zio, rr); in raidz_reconstruct()
3154 raidz_row_t *rr = rm->rm_row[i]; in vdev_raidz_combrec() local
3157 for (int c = 0; c < rr->rr_cols; c++) { in vdev_raidz_combrec()
3158 if (rr->rr_col[c].rc_error) in vdev_raidz_combrec()
3163 return (vdev_raidz_worst_error(rr)); in vdev_raidz_combrec()
3253 raidz_row_t *rr = rm->rm_row[row]; in vdev_raidz_reconstruct() local
3254 vdev_raidz_reconstruct_row(rm, rr, t, nt); in vdev_raidz_reconstruct()
3268 vdev_raidz_io_done_write_impl(zio_t *zio, raidz_row_t *rr) in vdev_raidz_io_done_write_impl() argument
3273 ASSERT3U(rr->rr_missingparity, <=, rr->rr_firstdatacol); in vdev_raidz_io_done_write_impl()
3274 ASSERT3U(rr->rr_missingdata, <=, rr->rr_cols - rr->rr_firstdatacol); in vdev_raidz_io_done_write_impl()
3277 for (int c = 0; c < rr->rr_cols; c++) { in vdev_raidz_io_done_write_impl()
3278 raidz_col_t *rc = &rr->rr_col[c]; in vdev_raidz_io_done_write_impl()
3303 if (normal_errors > rr->rr_firstdatacol || in vdev_raidz_io_done_write_impl()
3304 shadow_errors > rr->rr_firstdatacol) { in vdev_raidz_io_done_write_impl()
3306 vdev_raidz_worst_error(rr)); in vdev_raidz_io_done_write_impl()
3312 raidz_row_t *rr) in vdev_raidz_io_done_reconstruct_known_missing() argument
3319 ASSERT3U(rr->rr_missingparity, <=, rr->rr_firstdatacol); in vdev_raidz_io_done_reconstruct_known_missing()
3320 ASSERT3U(rr->rr_missingdata, <=, rr->rr_cols - rr->rr_firstdatacol); in vdev_raidz_io_done_reconstruct_known_missing()
3322 for (int c = 0; c < rr->rr_cols; c++) { in vdev_raidz_io_done_reconstruct_known_missing()
3323 raidz_col_t *rc = &rr->rr_col[c]; in vdev_raidz_io_done_reconstruct_known_missing()
3339 if (c < rr->rr_firstdatacol) in vdev_raidz_io_done_reconstruct_known_missing()
3345 } else if (c < rr->rr_firstdatacol && !rc->rc_tried) { in vdev_raidz_io_done_reconstruct_known_missing()
3356 total_errors <= rr->rr_firstdatacol - parity_untried) { in vdev_raidz_io_done_reconstruct_known_missing()
3365 ASSERT(parity_errors < rr->rr_firstdatacol); in vdev_raidz_io_done_reconstruct_known_missing()
3372 for (int c = rr->rr_firstdatacol; c < rr->rr_cols; c++) { in vdev_raidz_io_done_reconstruct_known_missing()
3373 raidz_col_t *rc = &rr->rr_col[c]; in vdev_raidz_io_done_reconstruct_known_missing()
3380 ASSERT(rr->rr_firstdatacol >= n); in vdev_raidz_io_done_reconstruct_known_missing()
3382 vdev_raidz_reconstruct_row(rm, rr, tgts, n); in vdev_raidz_io_done_reconstruct_known_missing()
3390 vdev_raidz_read_all(zio_t *zio, raidz_row_t *rr) in vdev_raidz_read_all() argument
3395 rr->rr_missingdata = 0; in vdev_raidz_read_all()
3396 rr->rr_missingparity = 0; in vdev_raidz_read_all()
3403 if (rr->rr_nempty != 0 && rr->rr_abd_empty == NULL) in vdev_raidz_read_all()
3404 vdev_draid_map_alloc_empty(zio, rr); in vdev_raidz_read_all()
3406 for (int c = 0; c < rr->rr_cols; c++) { in vdev_raidz_read_all()
3407 raidz_col_t *rc = &rr->rr_col[c]; in vdev_raidz_read_all()
3433 raidz_row_t *rr = rm->rm_row[i]; in vdev_raidz_io_done_unrecoverable() local
3435 for (int c = 0; c < rr->rr_cols; c++) { in vdev_raidz_io_done_unrecoverable()
3436 raidz_col_t *rc = &rr->rr_col[c]; in vdev_raidz_io_done_unrecoverable()
3472 raidz_row_t *rr = rm->rm_row[i]; in vdev_raidz_io_done() local
3474 for (int c = 0; c < rr->rr_cols; c++) { in vdev_raidz_io_done()
3475 raidz_col_t *rc = &rr->rr_col[c]; in vdev_raidz_io_done()
3484 if (c >= rr->rr_firstdatacol) { in vdev_raidz_io_done()
3503 raidz_row_t *rr = rm->rm_row[i]; in vdev_raidz_io_done() local
3505 rm, rr); in vdev_raidz_io_done()
3513 raidz_row_t *rr = rm->rm_row[i]; in vdev_raidz_io_done() local
3514 vdev_raidz_io_done_verified(zio, rr); in vdev_raidz_io_done()