8 #include "dm-verity-fec.h"
11 #define DM_MSG_PREFIX	"verity-fec"
18 	return v->fec && v->fec->dev;  in verity_fec_is_enabled()
38 	mod = do_div(offset, v->fec->rsn);  in fec_interleave()
39 	return offset + mod * (v->fec->rounds << v->data_dev_block_bits);  in fec_interleave()
46 			  u8 *data, u8 *fec, int neras)  in fec_decode_rs8()  argument
51 	for (i = 0; i < v->fec->roots; i++)  in fec_decode_rs8()
52 		par[i] = fec[i];  in fec_decode_rs8()
54 	return decode_rs8(fio->rs, data, par, v->fec->rsn, NULL, neras,  in fec_decode_rs8()
69 	position = (index + rsb) * v->fec->roots;  in fec_read_parity()
70 	block = div64_u64_rem(position, v->fec->io_size, &rem);  in fec_read_parity()
73 	res = dm_bufio_read_with_ioprio(v->fec->bufio, block, buf, ioprio);  in fec_read_parity()
75 		DMERR("%s: FEC %llu: parity read failed (block %llu): %ld",  in fec_read_parity()
109 	return &fio->bufs[i][j * v->fec->rsn];  in fec_buffer_rs_block()
141 	 * one corrected target byte and consumes fec->roots parity bytes.  in fec_decode_bufs()
159 		offset += v->fec->roots;  in fec_decode_bufs()
160 		if (offset >= v->fec->io_size) {  in fec_decode_bufs()
174 		DMERR_LIMIT("%s: FEC %llu: failed to correct: %d",  in fec_decode_bufs()
177 		DMWARN_LIMIT("%s: FEC %llu: corrected %d errors",  in fec_decode_bufs()
226 	for (i = 0; i < v->fec->rsn; i++) {  in fec_read_bufs()
227 		ileaved = fec_interleave(v, rsb * v->fec->rsn + i);  in fec_read_bufs()
237 		bufio = v->fec->data_bufio;  in fec_read_bufs()
246 			if (unlikely(block >= v->fec->hash_blocks))  in fec_read_bufs()
255 			DMWARN_LIMIT("%s: FEC %llu: read failed (%llu): %ld",  in fec_read_bufs()
261 			if (neras && *neras <= v->fec->roots)  in fec_read_bufs()
268 		if (bufio == v->fec->data_bufio &&  in fec_read_bufs()
277 			 * maximum number (i.e. fec->roots) of erasures  in fec_read_bufs()
279 			if (neras && *neras <= v->fec->roots &&  in fec_read_bufs()
305  * Allocate RS control structure and FEC buffers from preallocated mempools,
313 		fio->rs = mempool_alloc(&v->fec->rs_pool, GFP_NOIO);  in fec_alloc_bufs()
319 		fio->bufs[n] = mempool_alloc(&v->fec->prealloc_pool, GFP_NOWAIT);  in fec_alloc_bufs()
321 			DMERR("failed to allocate FEC buffer");  in fec_alloc_bufs()
331 		fio->bufs[n] = mempool_alloc(&v->fec->extra_pool, GFP_NOWAIT);  in fec_alloc_bufs()
339 		fio->output = mempool_alloc(&v->fec->output_pool, GFP_NOIO);  in fec_alloc_bufs()
353 		memset(fio->bufs[n], 0, v->fec->rsn << DM_VERITY_FEC_BUF_RS_BITS);  in fec_init_bufs()
397 		DMERR_LIMIT("%s: FEC %llu: failed to correct (%d erasures)",  in fec_decode_rsb()
417 		DMWARN_LIMIT("%s: FEC: recursion too deep", v->data_dev->name);  in verity_fec_decode()
427 	 * For RS(M, N), the continuous FEC data is divided into blocks of N  in verity_fec_decode()
437 	res = div64_u64(offset, v->fec->rounds << v->data_dev_block_bits);  in verity_fec_decode()
443 	rsb = offset - res * (v->fec->rounds << v->data_dev_block_bits);  in verity_fec_decode()
470 	struct dm_verity_fec *f = io->v->fec;  in verity_fec_finish_io()
517 	       v->fec->dev->name,  in verity_fec_status_table()
518 	       (unsigned long long)v->fec->blocks,  in verity_fec_status_table()
519 	       (unsigned long long)v->fec->start,  in verity_fec_status_table()
520 	       v->fec->roots);  in verity_fec_status_table()
527 	struct dm_verity_fec *f = v->fec;  in verity_fec_dtr()
547 	v->fec = NULL;  in verity_fec_dtr()
554 	return init_rs_gfp(8, 0x11d, 0, 1, v->fec->roots, gfp_mask);  in fec_rs_alloc()
584 		ti->error = "FEC feature arguments require a value";  in verity_fec_parse_opt_args()
592 		r = dm_get_device(ti, arg_value, BLK_OPEN_READ, &v->fec->dev);  in verity_fec_parse_opt_args()
594 			ti->error = "FEC device lookup failed";  in verity_fec_parse_opt_args()
605 		v->fec->blocks = num_ll;  in verity_fec_parse_opt_args()
614 		v->fec->start = num_ll;  in verity_fec_parse_opt_args()
623 		v->fec->roots = num_c;  in verity_fec_parse_opt_args()
626 		ti->error = "Unrecognized verity FEC feature request";  in verity_fec_parse_opt_args()
634  * Allocate dm_verity_fec for v->fec. Must be called before verity_fec_ctr.
642 		v->ti->error = "Cannot allocate FEC structure";  in verity_fec_ctr_alloc()
645 	v->fec = f;  in verity_fec_ctr_alloc()
656 	struct dm_verity_fec *f = v->fec;  in verity_fec_ctr()
667 	 * FEC is computed over data blocks, possible metadata, and  in verity_fec_ctr()
668 	 * hash blocks. In other words, FEC covers total of fec_blocks  in verity_fec_ctr()
674 	 * where all data is stored on the same device and FEC covers  in verity_fec_ctr()
688 		ti->error = "Block sizes must match to use FEC";  in verity_fec_ctr()
730 		f->io_size = v->fec->roots << SECTOR_SHIFT;  in verity_fec_ctr()
736 		ti->error = "Cannot initialize FEC bufio client";  in verity_fec_ctr()
742 	fec_blocks = div64_u64(f->rounds * f->roots, v->fec->roots << SECTOR_SHIFT);  in verity_fec_ctr()
744 		ti->error = "FEC device is too small";  in verity_fec_ctr()
752 		ti->error = "Cannot initialize FEC data bufio client";  in verity_fec_ctr()
773 		ti->error = "Cannot create FEC buffer cache";  in verity_fec_ctr()
782 		ti->error = "Cannot allocate FEC buffer prealloc pool";  in verity_fec_ctr()
788 		ti->error = "Cannot allocate FEC buffer extra pool";  in verity_fec_ctr()
796 		ti->error = "Cannot allocate FEC output pool";  in verity_fec_ctr()