Lines Matching refs:rm
141 vdev_raidz_map_free(raidz_map_t *rm) in vdev_raidz_map_free() argument
145 for (c = 0; c < rm->rm_firstdatacol; c++) { in vdev_raidz_map_free()
146 abd_free(rm->rm_col[c].rc_abd); in vdev_raidz_map_free()
148 if (rm->rm_col[c].rc_gdata != NULL) in vdev_raidz_map_free()
149 abd_free(rm->rm_col[c].rc_gdata); in vdev_raidz_map_free()
152 for (c = rm->rm_firstdatacol; c < rm->rm_cols; c++) in vdev_raidz_map_free()
153 abd_put(rm->rm_col[c].rc_abd); in vdev_raidz_map_free()
155 if (rm->rm_abd_copy != NULL) in vdev_raidz_map_free()
156 abd_free(rm->rm_abd_copy); in vdev_raidz_map_free()
158 kmem_free(rm, offsetof(raidz_map_t, rm_col[rm->rm_scols])); in vdev_raidz_map_free()
164 raidz_map_t *rm = zio->io_vsd; in vdev_raidz_map_free_vsd() local
166 ASSERT0(rm->rm_freed); in vdev_raidz_map_free_vsd()
167 rm->rm_freed = 1; in vdev_raidz_map_free_vsd()
169 if (rm->rm_reports == 0) in vdev_raidz_map_free_vsd()
170 vdev_raidz_map_free(rm); in vdev_raidz_map_free_vsd()
177 raidz_map_t *rm = arg; in vdev_raidz_cksum_free() local
179 ASSERT3U(rm->rm_reports, >, 0); in vdev_raidz_cksum_free()
181 if (--rm->rm_reports == 0 && rm->rm_freed != 0) in vdev_raidz_cksum_free()
182 vdev_raidz_map_free(rm); in vdev_raidz_cksum_free()
188 raidz_map_t *rm = zcr->zcr_cbdata; in vdev_raidz_cksum_finish() local
193 const abd_t *bad = rm->rm_col[c].rc_abd; in vdev_raidz_cksum_finish()
200 if (c < rm->rm_firstdatacol) { in vdev_raidz_cksum_finish()
206 if (rm->rm_col[0].rc_gdata == NULL) { in vdev_raidz_cksum_finish()
214 for (x = 0; x < rm->rm_firstdatacol; x++) { in vdev_raidz_cksum_finish()
215 bad_parity[x] = rm->rm_col[x].rc_abd; in vdev_raidz_cksum_finish()
216 rm->rm_col[x].rc_abd = in vdev_raidz_cksum_finish()
217 rm->rm_col[x].rc_gdata = in vdev_raidz_cksum_finish()
218 abd_alloc_sametype(rm->rm_col[x].rc_abd, in vdev_raidz_cksum_finish()
219 rm->rm_col[x].rc_size); in vdev_raidz_cksum_finish()
224 for (; x < rm->rm_cols; x++) { in vdev_raidz_cksum_finish()
225 abd_put(rm->rm_col[x].rc_abd); in vdev_raidz_cksum_finish()
227 rm->rm_col[x].rc_abd = in vdev_raidz_cksum_finish()
229 offset, rm->rm_col[x].rc_size); in vdev_raidz_cksum_finish()
230 offset += rm->rm_col[x].rc_size; in vdev_raidz_cksum_finish()
236 vdev_raidz_generate_parity(rm); in vdev_raidz_cksum_finish()
239 for (x = 0; x < rm->rm_firstdatacol; x++) in vdev_raidz_cksum_finish()
240 rm->rm_col[x].rc_abd = bad_parity[x]; in vdev_raidz_cksum_finish()
243 for (x = rm->rm_firstdatacol; x < rm->rm_cols; x++) { in vdev_raidz_cksum_finish()
244 abd_put(rm->rm_col[x].rc_abd); in vdev_raidz_cksum_finish()
245 rm->rm_col[x].rc_abd = abd_get_offset_size( in vdev_raidz_cksum_finish()
246 rm->rm_abd_copy, offset, in vdev_raidz_cksum_finish()
247 rm->rm_col[x].rc_size); in vdev_raidz_cksum_finish()
248 offset += rm->rm_col[x].rc_size; in vdev_raidz_cksum_finish()
252 ASSERT3P(rm->rm_col[c].rc_gdata, !=, NULL); in vdev_raidz_cksum_finish()
253 good = abd_get_offset_size(rm->rm_col[c].rc_gdata, 0, in vdev_raidz_cksum_finish()
254 rm->rm_col[c].rc_size); in vdev_raidz_cksum_finish()
258 for (x = rm->rm_firstdatacol; x < c; x++) in vdev_raidz_cksum_finish()
259 offset += rm->rm_col[x].rc_size; in vdev_raidz_cksum_finish()
262 rm->rm_col[c].rc_size); in vdev_raidz_cksum_finish()
282 raidz_map_t *rm = zio->io_vsd; in vdev_raidz_cksum_report() local
286 zcr->zcr_cbdata = rm; in vdev_raidz_cksum_report()
291 rm->rm_reports++; in vdev_raidz_cksum_report()
292 ASSERT3U(rm->rm_reports, >, 0); in vdev_raidz_cksum_report()
294 if (rm->rm_abd_copy != NULL) in vdev_raidz_cksum_report()
307 for (c = rm->rm_firstdatacol; c < rm->rm_cols; c++) in vdev_raidz_cksum_report()
308 size += rm->rm_col[c].rc_size; in vdev_raidz_cksum_report()
310 rm->rm_abd_copy = abd_alloc_for_io(size, B_FALSE); in vdev_raidz_cksum_report()
312 for (offset = 0, c = rm->rm_firstdatacol; c < rm->rm_cols; c++) { in vdev_raidz_cksum_report()
313 raidz_col_t *col = &rm->rm_col[c]; in vdev_raidz_cksum_report()
314 abd_t *tmp = abd_get_offset_size(rm->rm_abd_copy, offset, in vdev_raidz_cksum_report()
341 raidz_map_t *rm; in vdev_raidz_map_alloc() local
387 rm = kmem_alloc(offsetof(raidz_map_t, rm_col[scols]), KM_SLEEP); in vdev_raidz_map_alloc()
389 rm->rm_cols = acols; in vdev_raidz_map_alloc()
390 rm->rm_scols = scols; in vdev_raidz_map_alloc()
391 rm->rm_bigcols = bc; in vdev_raidz_map_alloc()
392 rm->rm_skipstart = bc; in vdev_raidz_map_alloc()
393 rm->rm_missingdata = 0; in vdev_raidz_map_alloc()
394 rm->rm_missingparity = 0; in vdev_raidz_map_alloc()
395 rm->rm_firstdatacol = nparity; in vdev_raidz_map_alloc()
396 rm->rm_abd_copy = NULL; in vdev_raidz_map_alloc()
397 rm->rm_reports = 0; in vdev_raidz_map_alloc()
398 rm->rm_freed = 0; in vdev_raidz_map_alloc()
399 rm->rm_ecksuminjected = 0; in vdev_raidz_map_alloc()
410 rm->rm_col[c].rc_devidx = col; in vdev_raidz_map_alloc()
411 rm->rm_col[c].rc_offset = coff; in vdev_raidz_map_alloc()
412 rm->rm_col[c].rc_abd = NULL; in vdev_raidz_map_alloc()
413 rm->rm_col[c].rc_gdata = NULL; in vdev_raidz_map_alloc()
414 rm->rm_col[c].rc_error = 0; in vdev_raidz_map_alloc()
415 rm->rm_col[c].rc_tried = 0; in vdev_raidz_map_alloc()
416 rm->rm_col[c].rc_skipped = 0; in vdev_raidz_map_alloc()
419 rm->rm_col[c].rc_size = 0; in vdev_raidz_map_alloc()
421 rm->rm_col[c].rc_size = (q + 1) << ashift; in vdev_raidz_map_alloc()
423 rm->rm_col[c].rc_size = q << ashift; in vdev_raidz_map_alloc()
425 asize += rm->rm_col[c].rc_size; in vdev_raidz_map_alloc()
429 rm->rm_asize = roundup(asize, (nparity + 1) << ashift); in vdev_raidz_map_alloc()
430 rm->rm_nskip = roundup(tot, nparity + 1) - tot; in vdev_raidz_map_alloc()
431 ASSERT3U(rm->rm_asize - asize, ==, rm->rm_nskip << ashift); in vdev_raidz_map_alloc()
432 ASSERT3U(rm->rm_nskip, <=, nparity); in vdev_raidz_map_alloc()
434 for (c = 0; c < rm->rm_firstdatacol; c++) in vdev_raidz_map_alloc()
435 rm->rm_col[c].rc_abd = in vdev_raidz_map_alloc()
436 abd_alloc_linear(rm->rm_col[c].rc_size, B_FALSE); in vdev_raidz_map_alloc()
438 rm->rm_col[c].rc_abd = abd_get_offset_size(zio->io_abd, 0, in vdev_raidz_map_alloc()
439 rm->rm_col[c].rc_size); in vdev_raidz_map_alloc()
440 off = rm->rm_col[c].rc_size; in vdev_raidz_map_alloc()
443 rm->rm_col[c].rc_abd = abd_get_offset_size(zio->io_abd, off, in vdev_raidz_map_alloc()
444 rm->rm_col[c].rc_size); in vdev_raidz_map_alloc()
445 off += rm->rm_col[c].rc_size; in vdev_raidz_map_alloc()
468 ASSERT(rm->rm_cols >= 2); in vdev_raidz_map_alloc()
469 ASSERT(rm->rm_col[0].rc_size == rm->rm_col[1].rc_size); in vdev_raidz_map_alloc()
471 if (rm->rm_firstdatacol == 1 && (zio->io_offset & (1ULL << 20))) { in vdev_raidz_map_alloc()
472 devidx = rm->rm_col[0].rc_devidx; in vdev_raidz_map_alloc()
473 o = rm->rm_col[0].rc_offset; in vdev_raidz_map_alloc()
474 rm->rm_col[0].rc_devidx = rm->rm_col[1].rc_devidx; in vdev_raidz_map_alloc()
475 rm->rm_col[0].rc_offset = rm->rm_col[1].rc_offset; in vdev_raidz_map_alloc()
476 rm->rm_col[1].rc_devidx = devidx; in vdev_raidz_map_alloc()
477 rm->rm_col[1].rc_offset = o; in vdev_raidz_map_alloc()
479 if (rm->rm_skipstart == 0) in vdev_raidz_map_alloc()
480 rm->rm_skipstart = 1; in vdev_raidz_map_alloc()
484 rm->rm_ops = vdev_raidz_math_get_ops(); in vdev_raidz_map_alloc()
486 return (rm); in vdev_raidz_map_alloc()
551 vdev_raidz_generate_parity_p(raidz_map_t *rm) in vdev_raidz_generate_parity_p() argument
557 for (c = rm->rm_firstdatacol; c < rm->rm_cols; c++) { in vdev_raidz_generate_parity_p()
558 src = rm->rm_col[c].rc_abd; in vdev_raidz_generate_parity_p()
559 p = abd_to_buf(rm->rm_col[VDEV_RAIDZ_P].rc_abd); in vdev_raidz_generate_parity_p()
561 if (c == rm->rm_firstdatacol) { in vdev_raidz_generate_parity_p()
562 abd_copy_to_buf_off(p, src, 0, rm->rm_col[c].rc_size); in vdev_raidz_generate_parity_p()
565 (void) abd_iterate_func(src, 0, rm->rm_col[c].rc_size, in vdev_raidz_generate_parity_p()
572 vdev_raidz_generate_parity_pq(raidz_map_t *rm) in vdev_raidz_generate_parity_pq() argument
578 pcnt = rm->rm_col[VDEV_RAIDZ_P].rc_size / sizeof (p[0]); in vdev_raidz_generate_parity_pq()
579 ASSERT(rm->rm_col[VDEV_RAIDZ_P].rc_size == in vdev_raidz_generate_parity_pq()
580 rm->rm_col[VDEV_RAIDZ_Q].rc_size); in vdev_raidz_generate_parity_pq()
582 for (c = rm->rm_firstdatacol; c < rm->rm_cols; c++) { in vdev_raidz_generate_parity_pq()
583 src = rm->rm_col[c].rc_abd; in vdev_raidz_generate_parity_pq()
584 p = abd_to_buf(rm->rm_col[VDEV_RAIDZ_P].rc_abd); in vdev_raidz_generate_parity_pq()
585 q = abd_to_buf(rm->rm_col[VDEV_RAIDZ_Q].rc_abd); in vdev_raidz_generate_parity_pq()
587 ccnt = rm->rm_col[c].rc_size / sizeof (p[0]); in vdev_raidz_generate_parity_pq()
589 if (c == rm->rm_firstdatacol) { in vdev_raidz_generate_parity_pq()
592 abd_copy_to_buf_off(p, src, 0, rm->rm_col[c].rc_size); in vdev_raidz_generate_parity_pq()
593 (void) memcpy(q, p, rm->rm_col[c].rc_size); in vdev_raidz_generate_parity_pq()
603 (void) abd_iterate_func(src, 0, rm->rm_col[c].rc_size, in vdev_raidz_generate_parity_pq()
618 vdev_raidz_generate_parity_pqr(raidz_map_t *rm) in vdev_raidz_generate_parity_pqr() argument
624 pcnt = rm->rm_col[VDEV_RAIDZ_P].rc_size / sizeof (p[0]); in vdev_raidz_generate_parity_pqr()
625 ASSERT(rm->rm_col[VDEV_RAIDZ_P].rc_size == in vdev_raidz_generate_parity_pqr()
626 rm->rm_col[VDEV_RAIDZ_Q].rc_size); in vdev_raidz_generate_parity_pqr()
627 ASSERT(rm->rm_col[VDEV_RAIDZ_P].rc_size == in vdev_raidz_generate_parity_pqr()
628 rm->rm_col[VDEV_RAIDZ_R].rc_size); in vdev_raidz_generate_parity_pqr()
630 for (c = rm->rm_firstdatacol; c < rm->rm_cols; c++) { in vdev_raidz_generate_parity_pqr()
631 src = rm->rm_col[c].rc_abd; in vdev_raidz_generate_parity_pqr()
632 p = abd_to_buf(rm->rm_col[VDEV_RAIDZ_P].rc_abd); in vdev_raidz_generate_parity_pqr()
633 q = abd_to_buf(rm->rm_col[VDEV_RAIDZ_Q].rc_abd); in vdev_raidz_generate_parity_pqr()
634 r = abd_to_buf(rm->rm_col[VDEV_RAIDZ_R].rc_abd); in vdev_raidz_generate_parity_pqr()
636 ccnt = rm->rm_col[c].rc_size / sizeof (p[0]); in vdev_raidz_generate_parity_pqr()
638 if (c == rm->rm_firstdatacol) { in vdev_raidz_generate_parity_pqr()
639 ASSERT3S(src->abd_size, >=, rm->rm_col[c].rc_size); in vdev_raidz_generate_parity_pqr()
641 abd_copy_to_buf_off(p, src, 0, rm->rm_col[c].rc_size); in vdev_raidz_generate_parity_pqr()
642 (void) memcpy(q, p, rm->rm_col[c].rc_size); in vdev_raidz_generate_parity_pqr()
643 (void) memcpy(r, p, rm->rm_col[c].rc_size); in vdev_raidz_generate_parity_pqr()
654 (void) abd_iterate_func(src, 0, rm->rm_col[c].rc_size, in vdev_raidz_generate_parity_pqr()
674 vdev_raidz_generate_parity(raidz_map_t *rm) in vdev_raidz_generate_parity() argument
677 if (vdev_raidz_math_generate(rm) != RAIDZ_ORIGINAL_IMPL) in vdev_raidz_generate_parity()
680 switch (rm->rm_firstdatacol) { in vdev_raidz_generate_parity()
682 vdev_raidz_generate_parity_p(rm); in vdev_raidz_generate_parity()
685 vdev_raidz_generate_parity_pq(rm); in vdev_raidz_generate_parity()
688 vdev_raidz_generate_parity_pqr(rm); in vdev_raidz_generate_parity()
812 vdev_raidz_reconstruct_p(raidz_map_t *rm, int *tgts, int ntgts) in vdev_raidz_reconstruct_p() argument
819 ASSERT(x >= rm->rm_firstdatacol); in vdev_raidz_reconstruct_p()
820 ASSERT(x < rm->rm_cols); in vdev_raidz_reconstruct_p()
822 ASSERT(rm->rm_col[x].rc_size <= rm->rm_col[VDEV_RAIDZ_P].rc_size); in vdev_raidz_reconstruct_p()
823 ASSERT(rm->rm_col[x].rc_size > 0); in vdev_raidz_reconstruct_p()
825 src = rm->rm_col[VDEV_RAIDZ_P].rc_abd; in vdev_raidz_reconstruct_p()
826 dst = rm->rm_col[x].rc_abd; in vdev_raidz_reconstruct_p()
828 ASSERT3S(dst->abd_size, >=, rm->rm_col[x].rc_size); in vdev_raidz_reconstruct_p()
829 ASSERT3S(src->abd_size, >=, rm->rm_col[x].rc_size); in vdev_raidz_reconstruct_p()
830 abd_copy_off(dst, src, 0, 0, rm->rm_col[x].rc_size); in vdev_raidz_reconstruct_p()
832 for (c = rm->rm_firstdatacol; c < rm->rm_cols; c++) { in vdev_raidz_reconstruct_p()
833 uint64_t size = MIN(rm->rm_col[x].rc_size, in vdev_raidz_reconstruct_p()
834 rm->rm_col[c].rc_size); in vdev_raidz_reconstruct_p()
836 src = rm->rm_col[c].rc_abd; in vdev_raidz_reconstruct_p()
837 dst = rm->rm_col[x].rc_abd; in vdev_raidz_reconstruct_p()
850 vdev_raidz_reconstruct_q(raidz_map_t *rm, int *tgts, int ntgts) in vdev_raidz_reconstruct_q() argument
858 ASSERT(rm->rm_col[x].rc_size <= rm->rm_col[VDEV_RAIDZ_Q].rc_size); in vdev_raidz_reconstruct_q()
860 for (c = rm->rm_firstdatacol; c < rm->rm_cols; c++) { in vdev_raidz_reconstruct_q()
861 uint64_t size = (c == x) ? 0 : MIN(rm->rm_col[x].rc_size, in vdev_raidz_reconstruct_q()
862 rm->rm_col[c].rc_size); in vdev_raidz_reconstruct_q()
864 src = rm->rm_col[c].rc_abd; in vdev_raidz_reconstruct_q()
865 dst = rm->rm_col[x].rc_abd; in vdev_raidz_reconstruct_q()
867 if (c == rm->rm_firstdatacol) { in vdev_raidz_reconstruct_q()
873 if (rm->rm_col[x].rc_size > size) in vdev_raidz_reconstruct_q()
875 rm->rm_col[x].rc_size - size); in vdev_raidz_reconstruct_q()
877 ASSERT3U(size, <=, rm->rm_col[x].rc_size); in vdev_raidz_reconstruct_q()
882 size, rm->rm_col[x].rc_size - size, in vdev_raidz_reconstruct_q()
887 src = rm->rm_col[VDEV_RAIDZ_Q].rc_abd; in vdev_raidz_reconstruct_q()
888 dst = rm->rm_col[x].rc_abd; in vdev_raidz_reconstruct_q()
889 exp = 255 - (rm->rm_cols - 1 - x); in vdev_raidz_reconstruct_q()
892 (void) abd_iterate_func(dst, 0, rm->rm_col[x].rc_size, in vdev_raidz_reconstruct_q()
899 vdev_raidz_reconstruct_pq(raidz_map_t *rm, int *tgts, int ntgts) in vdev_raidz_reconstruct_pq() argument
910 ASSERT(x >= rm->rm_firstdatacol); in vdev_raidz_reconstruct_pq()
911 ASSERT(y < rm->rm_cols); in vdev_raidz_reconstruct_pq()
913 ASSERT(rm->rm_col[x].rc_size >= rm->rm_col[y].rc_size); in vdev_raidz_reconstruct_pq()
922 pdata = rm->rm_col[VDEV_RAIDZ_P].rc_abd; in vdev_raidz_reconstruct_pq()
923 qdata = rm->rm_col[VDEV_RAIDZ_Q].rc_abd; in vdev_raidz_reconstruct_pq()
924 xsize = rm->rm_col[x].rc_size; in vdev_raidz_reconstruct_pq()
925 ysize = rm->rm_col[y].rc_size; in vdev_raidz_reconstruct_pq()
927 rm->rm_col[VDEV_RAIDZ_P].rc_abd = in vdev_raidz_reconstruct_pq()
928 abd_alloc_linear(rm->rm_col[VDEV_RAIDZ_P].rc_size, B_TRUE); in vdev_raidz_reconstruct_pq()
929 rm->rm_col[VDEV_RAIDZ_Q].rc_abd = in vdev_raidz_reconstruct_pq()
930 abd_alloc_linear(rm->rm_col[VDEV_RAIDZ_Q].rc_size, B_TRUE); in vdev_raidz_reconstruct_pq()
931 rm->rm_col[x].rc_size = 0; in vdev_raidz_reconstruct_pq()
932 rm->rm_col[y].rc_size = 0; in vdev_raidz_reconstruct_pq()
934 vdev_raidz_generate_parity_pq(rm); in vdev_raidz_reconstruct_pq()
936 rm->rm_col[x].rc_size = xsize; in vdev_raidz_reconstruct_pq()
937 rm->rm_col[y].rc_size = ysize; in vdev_raidz_reconstruct_pq()
941 pxy = abd_to_buf(rm->rm_col[VDEV_RAIDZ_P].rc_abd); in vdev_raidz_reconstruct_pq()
942 qxy = abd_to_buf(rm->rm_col[VDEV_RAIDZ_Q].rc_abd); in vdev_raidz_reconstruct_pq()
943 xd = rm->rm_col[x].rc_abd; in vdev_raidz_reconstruct_pq()
944 yd = rm->rm_col[y].rc_abd; in vdev_raidz_reconstruct_pq()
962 b = vdev_raidz_pow2[255 - (rm->rm_cols - 1 - x)]; in vdev_raidz_reconstruct_pq()
975 abd_free(rm->rm_col[VDEV_RAIDZ_P].rc_abd); in vdev_raidz_reconstruct_pq()
976 abd_free(rm->rm_col[VDEV_RAIDZ_Q].rc_abd); in vdev_raidz_reconstruct_pq()
981 rm->rm_col[VDEV_RAIDZ_P].rc_abd = pdata; in vdev_raidz_reconstruct_pq()
982 rm->rm_col[VDEV_RAIDZ_Q].rc_abd = qdata; in vdev_raidz_reconstruct_pq()
1142 vdev_raidz_matrix_init(raidz_map_t *rm, int n, int nmap, int *map, in vdev_raidz_matrix_init() argument
1148 ASSERT(n == rm->rm_cols - rm->rm_firstdatacol); in vdev_raidz_matrix_init()
1172 vdev_raidz_matrix_invert(raidz_map_t *rm, int n, int nmissing, int *missing, in vdev_raidz_matrix_invert() argument
1184 ASSERT3S(used[i], <, rm->rm_firstdatacol); in vdev_raidz_matrix_invert()
1187 ASSERT3S(used[i], >=, rm->rm_firstdatacol); in vdev_raidz_matrix_invert()
1204 ASSERT3U(used[j], >=, rm->rm_firstdatacol); in vdev_raidz_matrix_invert()
1205 jj = used[j] - rm->rm_firstdatacol; in vdev_raidz_matrix_invert()
1266 vdev_raidz_matrix_reconstruct(raidz_map_t *rm, int n, int nmissing, in vdev_raidz_matrix_reconstruct() argument
1298 ASSERT3U(c, <, rm->rm_cols); in vdev_raidz_matrix_reconstruct()
1300 src = abd_to_buf(rm->rm_col[c].rc_abd); in vdev_raidz_matrix_reconstruct()
1301 ccount = rm->rm_col[c].rc_size; in vdev_raidz_matrix_reconstruct()
1303 cc = missing[j] + rm->rm_firstdatacol; in vdev_raidz_matrix_reconstruct()
1304 ASSERT3U(cc, >=, rm->rm_firstdatacol); in vdev_raidz_matrix_reconstruct()
1305 ASSERT3U(cc, <, rm->rm_cols); in vdev_raidz_matrix_reconstruct()
1308 dst[j] = abd_to_buf(rm->rm_col[cc].rc_abd); in vdev_raidz_matrix_reconstruct()
1309 dcount[j] = rm->rm_col[cc].rc_size; in vdev_raidz_matrix_reconstruct()
1312 ASSERT(ccount >= rm->rm_col[missing[0]].rc_size || i > 0); in vdev_raidz_matrix_reconstruct()
1342 vdev_raidz_reconstruct_general(raidz_map_t *rm, int *tgts, int ntgts) in vdev_raidz_reconstruct_general() argument
1364 if (!abd_is_linear(rm->rm_col[rm->rm_firstdatacol].rc_abd)) { in vdev_raidz_reconstruct_general()
1365 bufs = kmem_alloc(rm->rm_cols * sizeof (abd_t *), KM_PUSHPAGE); in vdev_raidz_reconstruct_general()
1367 for (c = rm->rm_firstdatacol; c < rm->rm_cols; c++) { in vdev_raidz_reconstruct_general()
1368 raidz_col_t *col = &rm->rm_col[c]; in vdev_raidz_reconstruct_general()
1378 n = rm->rm_cols - rm->rm_firstdatacol; in vdev_raidz_reconstruct_general()
1385 if (tgts[t] >= rm->rm_firstdatacol) { in vdev_raidz_reconstruct_general()
1387 tgts[t] - rm->rm_firstdatacol; in vdev_raidz_reconstruct_general()
1397 ASSERT(c < rm->rm_firstdatacol); in vdev_raidz_reconstruct_general()
1432 for (tt = 0, c = rm->rm_firstdatacol; c < rm->rm_cols; c++) { in vdev_raidz_reconstruct_general()
1434 c == missing_rows[tt] + rm->rm_firstdatacol) { in vdev_raidz_reconstruct_general()
1447 vdev_raidz_matrix_init(rm, n, nmissing_rows, parity_map, rows); in vdev_raidz_reconstruct_general()
1452 vdev_raidz_matrix_invert(rm, n, nmissing_rows, missing_rows, rows, in vdev_raidz_reconstruct_general()
1458 vdev_raidz_matrix_reconstruct(rm, n, nmissing_rows, missing_rows, in vdev_raidz_reconstruct_general()
1467 for (c = rm->rm_firstdatacol; c < rm->rm_cols; c++) { in vdev_raidz_reconstruct_general()
1468 raidz_col_t *col = &rm->rm_col[c]; in vdev_raidz_reconstruct_general()
1476 kmem_free(bufs, rm->rm_cols * sizeof (abd_t *)); in vdev_raidz_reconstruct_general()
1483 vdev_raidz_reconstruct(raidz_map_t *rm, const int *t, int nt) in vdev_raidz_reconstruct() argument
1499 nbadparity = rm->rm_firstdatacol; in vdev_raidz_reconstruct()
1500 nbaddata = rm->rm_cols - nbadparity; in vdev_raidz_reconstruct()
1502 for (i = 0, c = 0; c < rm->rm_cols; c++) { in vdev_raidz_reconstruct()
1503 if (c < rm->rm_firstdatacol) in vdev_raidz_reconstruct()
1509 } else if (rm->rm_col[c].rc_error != 0) { in vdev_raidz_reconstruct()
1511 } else if (c >= rm->rm_firstdatacol) { in vdev_raidz_reconstruct()
1526 ret = vdev_raidz_math_reconstruct(rm, parity_valid, dt, nbaddata); in vdev_raidz_reconstruct()
1536 return (vdev_raidz_reconstruct_p(rm, dt, 1)); in vdev_raidz_reconstruct()
1538 ASSERT(rm->rm_firstdatacol > 1); in vdev_raidz_reconstruct()
1541 return (vdev_raidz_reconstruct_q(rm, dt, 1)); in vdev_raidz_reconstruct()
1543 ASSERT(rm->rm_firstdatacol > 2); in vdev_raidz_reconstruct()
1547 ASSERT(rm->rm_firstdatacol > 1); in vdev_raidz_reconstruct()
1551 return (vdev_raidz_reconstruct_pq(rm, dt, 2)); in vdev_raidz_reconstruct()
1553 ASSERT(rm->rm_firstdatacol > 2); in vdev_raidz_reconstruct()
1558 code = vdev_raidz_reconstruct_general(rm, tgts, ntgts); in vdev_raidz_reconstruct()
1685 raidz_map_t *rm; in vdev_raidz_dumpio() local
1719 rm = vdev_raidz_map_alloc(zio, tvd->vdev_ashift, vd->vdev_children, in vdev_raidz_dumpio()
1724 for (c = rm->rm_firstdatacol; c < rm->rm_cols; in vdev_raidz_dumpio()
1726 rc = &rm->rm_col[c]; in vdev_raidz_dumpio()
1762 vdev_raidz_map_free(rm); in vdev_raidz_dumpio()
1797 vdev_raidz_io_verify(zio_t *zio, raidz_map_t *rm, int col) in vdev_raidz_io_verify() argument
1808 raidz_col_t *rc = &rm->rm_col[col]; in vdev_raidz_io_verify()
1852 raidz_map_t *rm; in vdev_raidz_io_start() local
1856 rm = vdev_raidz_map_alloc(zio, tvd->vdev_ashift, vd->vdev_children, in vdev_raidz_io_start()
1859 zio->io_vsd = rm; in vdev_raidz_io_start()
1862 ASSERT3U(rm->rm_asize, ==, vdev_psize_to_asize(vd, zio->io_size)); in vdev_raidz_io_start()
1865 vdev_raidz_generate_parity(rm); in vdev_raidz_io_start()
1867 for (c = 0; c < rm->rm_cols; c++) { in vdev_raidz_io_start()
1868 rc = &rm->rm_col[c]; in vdev_raidz_io_start()
1874 vdev_raidz_io_verify(zio, rm, c); in vdev_raidz_io_start()
1886 for (c = rm->rm_skipstart, i = 0; i < rm->rm_nskip; c++, i++) { in vdev_raidz_io_start()
1887 ASSERT(c <= rm->rm_scols); in vdev_raidz_io_start()
1888 if (c == rm->rm_scols) in vdev_raidz_io_start()
1890 rc = &rm->rm_col[c]; in vdev_raidz_io_start()
1909 for (c = rm->rm_cols - 1; c >= 0; c--) { in vdev_raidz_io_start()
1910 rc = &rm->rm_col[c]; in vdev_raidz_io_start()
1913 if (c >= rm->rm_firstdatacol) in vdev_raidz_io_start()
1914 rm->rm_missingdata++; in vdev_raidz_io_start()
1916 rm->rm_missingparity++; in vdev_raidz_io_start()
1923 if (c >= rm->rm_firstdatacol) in vdev_raidz_io_start()
1924 rm->rm_missingdata++; in vdev_raidz_io_start()
1926 rm->rm_missingparity++; in vdev_raidz_io_start()
1931 if (c >= rm->rm_firstdatacol || rm->rm_missingdata > 0 || in vdev_raidz_io_start()
1954 raidz_map_t *rm = zio->io_vsd; in raidz_checksum_error() local
1961 zbc.zbc_injected = rm->rm_ecksuminjected; in raidz_checksum_error()
1977 raidz_map_t *rm = zio->io_vsd; in raidz_checksum_verify() local
1981 rm->rm_ecksuminjected = 1; in raidz_checksum_verify()
1993 raidz_parity_verify(zio_t *zio, raidz_map_t *rm) in raidz_parity_verify() argument
2006 for (c = 0; c < rm->rm_firstdatacol; c++) { in raidz_parity_verify()
2007 rc = &rm->rm_col[c]; in raidz_parity_verify()
2014 vdev_raidz_generate_parity(rm); in raidz_parity_verify()
2016 for (c = 0; c < rm->rm_firstdatacol; c++) { in raidz_parity_verify()
2017 rc = &rm->rm_col[c]; in raidz_parity_verify()
2032 vdev_raidz_worst_error(raidz_map_t *rm) in vdev_raidz_worst_error() argument
2036 for (int c = 0; c < rm->rm_cols; c++) in vdev_raidz_worst_error()
2037 error = zio_worst_error(error, rm->rm_col[c].rc_error); in vdev_raidz_worst_error()
2053 raidz_map_t *rm = zio->io_vsd; in vdev_raidz_combrec() local
2061 ASSERT(total_errors < rm->rm_firstdatacol); in vdev_raidz_combrec()
2068 for (n = 1; n <= rm->rm_firstdatacol - total_errors; n++) { in vdev_raidz_combrec()
2080 c < rm->rm_firstdatacol) { in vdev_raidz_combrec()
2081 c = rm->rm_firstdatacol; in vdev_raidz_combrec()
2084 while (rm->rm_col[c].rc_error != 0) { in vdev_raidz_combrec()
2086 ASSERT3S(c, <, rm->rm_cols); in vdev_raidz_combrec()
2095 tgts[n] = rm->rm_cols; in vdev_raidz_combrec()
2104 orig[n - 1] = abd_alloc_sametype(rm->rm_col[0].rc_abd, in vdev_raidz_combrec()
2105 rm->rm_col[0].rc_size); in vdev_raidz_combrec()
2122 ASSERT3S(c, <, rm->rm_cols); in vdev_raidz_combrec()
2123 rc = &rm->rm_col[c]; in vdev_raidz_combrec()
2134 code = vdev_raidz_reconstruct(rm, tgts, n); in vdev_raidz_combrec()
2139 rc = &rm->rm_col[c]; in vdev_raidz_combrec()
2156 rc = &rm->rm_col[c]; in vdev_raidz_combrec()
2169 next < rm->rm_cols && in vdev_raidz_combrec()
2170 rm->rm_col[next].rc_error != 0; next++) in vdev_raidz_combrec()
2186 rm->rm_col[c].rc_error != 0; c++) in vdev_raidz_combrec()
2229 raidz_map_t *rm = zio->io_vsd; in vdev_raidz_io_done() local
2242 ASSERT(rm->rm_missingparity <= rm->rm_firstdatacol); in vdev_raidz_io_done()
2243 ASSERT(rm->rm_missingdata <= rm->rm_cols - rm->rm_firstdatacol); in vdev_raidz_io_done()
2245 for (c = 0; c < rm->rm_cols; c++) { in vdev_raidz_io_done()
2246 rc = &rm->rm_col[c]; in vdev_raidz_io_done()
2251 if (c < rm->rm_firstdatacol) in vdev_raidz_io_done()
2260 } else if (c < rm->rm_firstdatacol && !rc->rc_tried) { in vdev_raidz_io_done()
2277 if (total_errors > rm->rm_firstdatacol) in vdev_raidz_io_done()
2278 zio->io_error = vdev_raidz_worst_error(rm); in vdev_raidz_io_done()
2301 if (total_errors <= rm->rm_firstdatacol - parity_untried) { in vdev_raidz_io_done()
2313 rm->rm_firstdatacol || in vdev_raidz_io_done()
2315 n = raidz_parity_verify(zio, rm); in vdev_raidz_io_done()
2318 rm->rm_firstdatacol); in vdev_raidz_io_done()
2331 ASSERT(parity_errors < rm->rm_firstdatacol); in vdev_raidz_io_done()
2337 for (c = rm->rm_firstdatacol; c < rm->rm_cols; c++) { in vdev_raidz_io_done()
2338 rc = &rm->rm_col[c]; in vdev_raidz_io_done()
2345 ASSERT(rm->rm_firstdatacol >= n); in vdev_raidz_io_done()
2347 code = vdev_raidz_reconstruct(rm, tgts, n); in vdev_raidz_io_done()
2363 if (parity_errors < rm->rm_firstdatacol - n || in vdev_raidz_io_done()
2365 n = raidz_parity_verify(zio, rm); in vdev_raidz_io_done()
2368 rm->rm_firstdatacol); in vdev_raidz_io_done()
2384 rm->rm_missingdata = 0; in vdev_raidz_io_done()
2385 rm->rm_missingparity = 0; in vdev_raidz_io_done()
2387 for (c = 0; c < rm->rm_cols; c++) { in vdev_raidz_io_done()
2388 if (rm->rm_col[c].rc_tried) in vdev_raidz_io_done()
2393 rc = &rm->rm_col[c]; in vdev_raidz_io_done()
2401 } while (++c < rm->rm_cols); in vdev_raidz_io_done()
2416 if (total_errors > rm->rm_firstdatacol) { in vdev_raidz_io_done()
2417 zio->io_error = vdev_raidz_worst_error(rm); in vdev_raidz_io_done()
2419 } else if (total_errors < rm->rm_firstdatacol && in vdev_raidz_io_done()
2426 if (code != (1 << rm->rm_firstdatacol) - 1) in vdev_raidz_io_done()
2427 (void) raidz_parity_verify(zio, rm); in vdev_raidz_io_done()
2444 for (c = 0; c < rm->rm_cols; c++) { in vdev_raidz_io_done()
2445 rc = &rm->rm_col[c]; in vdev_raidz_io_done()
2450 rm->rm_ecksuminjected; in vdev_raidz_io_done()
2471 for (c = 0; c < rm->rm_cols; c++) { in vdev_raidz_io_done()
2472 rc = &rm->rm_col[c]; in vdev_raidz_io_done()