1 /* 2 * Copyright (C) 2015 Google, Inc. 3 * 4 * Author: Sami Tolvanen <samitolvanen@google.com> 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License as published by the Free 8 * Software Foundation; either version 2 of the License, or (at your option) 9 * any later version. 10 */ 11 12 #include "dm-verity-fec.h" 13 #include <linux/math64.h> 14 15 #define DM_MSG_PREFIX "verity-fec" 16 17 /* 18 * If error correction has been configured, returns true. 19 */ 20 bool verity_fec_is_enabled(struct dm_verity *v) 21 { 22 return v->fec && v->fec->dev; 23 } 24 25 /* 26 * Return a pointer to dm_verity_fec_io after dm_verity_io and its variable 27 * length fields. 28 */ 29 static inline struct dm_verity_fec_io *fec_io(struct dm_verity_io *io) 30 { 31 return (struct dm_verity_fec_io *) verity_io_digest_end(io->v, io); 32 } 33 34 /* 35 * Return an interleaved offset for a byte in RS block. 36 */ 37 static inline u64 fec_interleave(struct dm_verity *v, u64 offset) 38 { 39 u32 mod; 40 41 mod = do_div(offset, v->fec->rsn); 42 return offset + mod * (v->fec->rounds << v->data_dev_block_bits); 43 } 44 45 /* 46 * Decode an RS block using Reed-Solomon. 47 */ 48 static int fec_decode_rs8(struct dm_verity *v, struct dm_verity_fec_io *fio, 49 u8 *data, u8 *fec, int neras) 50 { 51 int i; 52 uint16_t par[DM_VERITY_FEC_RSM - DM_VERITY_FEC_MIN_RSN]; 53 54 for (i = 0; i < v->fec->roots; i++) 55 par[i] = fec[i]; 56 57 return decode_rs8(fio->rs, data, par, v->fec->rsn, NULL, neras, 58 fio->erasures, 0, NULL); 59 } 60 61 /* 62 * Read error-correcting codes for the requested RS block. Returns a pointer 63 * to the data block. Caller is responsible for releasing buf. 64 */ 65 static u8 *fec_read_parity(struct dm_verity *v, u64 rsb, int index, 66 unsigned *offset, struct dm_buffer **buf) 67 { 68 u64 position, block; 69 u8 *res; 70 71 position = (index + rsb) * v->fec->roots; 72 block = position >> v->data_dev_block_bits; 73 *offset = (unsigned)(position - (block << v->data_dev_block_bits)); 74 75 res = dm_bufio_read(v->fec->bufio, v->fec->start + block, buf); 76 if (unlikely(IS_ERR(res))) { 77 DMERR("%s: FEC %llu: parity read failed (block %llu): %ld", 78 v->data_dev->name, (unsigned long long)rsb, 79 (unsigned long long)(v->fec->start + block), 80 PTR_ERR(res)); 81 *buf = NULL; 82 } 83 84 return res; 85 } 86 87 /* Loop over each preallocated buffer slot. */ 88 #define fec_for_each_prealloc_buffer(__i) \ 89 for (__i = 0; __i < DM_VERITY_FEC_BUF_PREALLOC; __i++) 90 91 /* Loop over each extra buffer slot. */ 92 #define fec_for_each_extra_buffer(io, __i) \ 93 for (__i = DM_VERITY_FEC_BUF_PREALLOC; __i < DM_VERITY_FEC_BUF_MAX; __i++) 94 95 /* Loop over each allocated buffer. */ 96 #define fec_for_each_buffer(io, __i) \ 97 for (__i = 0; __i < (io)->nbufs; __i++) 98 99 /* Loop over each RS block in each allocated buffer. */ 100 #define fec_for_each_buffer_rs_block(io, __i, __j) \ 101 fec_for_each_buffer(io, __i) \ 102 for (__j = 0; __j < 1 << DM_VERITY_FEC_BUF_RS_BITS; __j++) 103 104 /* 105 * Return a pointer to the current RS block when called inside 106 * fec_for_each_buffer_rs_block. 107 */ 108 static inline u8 *fec_buffer_rs_block(struct dm_verity *v, 109 struct dm_verity_fec_io *fio, 110 unsigned i, unsigned j) 111 { 112 return &fio->bufs[i][j * v->fec->rsn]; 113 } 114 115 /* 116 * Return an index to the current RS block when called inside 117 * fec_for_each_buffer_rs_block. 118 */ 119 static inline unsigned fec_buffer_rs_index(unsigned i, unsigned j) 120 { 121 return (i << DM_VERITY_FEC_BUF_RS_BITS) + j; 122 } 123 124 /* 125 * Decode all RS blocks from buffers and copy corrected bytes into fio->output 126 * starting from block_offset. 127 */ 128 static int fec_decode_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio, 129 u64 rsb, int byte_index, unsigned block_offset, 130 int neras) 131 { 132 int r, corrected = 0, res; 133 struct dm_buffer *buf; 134 unsigned n, i, offset; 135 u8 *par, *block; 136 137 par = fec_read_parity(v, rsb, block_offset, &offset, &buf); 138 if (IS_ERR(par)) 139 return PTR_ERR(par); 140 141 /* 142 * Decode the RS blocks we have in bufs. Each RS block results in 143 * one corrected target byte and consumes fec->roots parity bytes. 144 */ 145 fec_for_each_buffer_rs_block(fio, n, i) { 146 block = fec_buffer_rs_block(v, fio, n, i); 147 res = fec_decode_rs8(v, fio, block, &par[offset], neras); 148 if (res < 0) { 149 r = res; 150 goto error; 151 } 152 153 corrected += res; 154 fio->output[block_offset] = block[byte_index]; 155 156 block_offset++; 157 if (block_offset >= 1 << v->data_dev_block_bits) 158 goto done; 159 160 /* read the next block when we run out of parity bytes */ 161 offset += v->fec->roots; 162 if (offset >= 1 << v->data_dev_block_bits) { 163 dm_bufio_release(buf); 164 165 par = fec_read_parity(v, rsb, block_offset, &offset, &buf); 166 if (unlikely(IS_ERR(par))) 167 return PTR_ERR(par); 168 } 169 } 170 done: 171 r = corrected; 172 error: 173 dm_bufio_release(buf); 174 175 if (r < 0 && neras) 176 DMERR_LIMIT("%s: FEC %llu: failed to correct: %d", 177 v->data_dev->name, (unsigned long long)rsb, r); 178 else if (r > 0) 179 DMWARN_LIMIT("%s: FEC %llu: corrected %d errors", 180 v->data_dev->name, (unsigned long long)rsb, r); 181 182 return r; 183 } 184 185 /* 186 * Locate data block erasures using verity hashes. 187 */ 188 static int fec_is_erasure(struct dm_verity *v, struct dm_verity_io *io, 189 u8 *want_digest, u8 *data) 190 { 191 if (unlikely(verity_hash(v, verity_io_hash_req(v, io), 192 data, 1 << v->data_dev_block_bits, 193 verity_io_real_digest(v, io)))) 194 return 0; 195 196 return memcmp(verity_io_real_digest(v, io), want_digest, 197 v->digest_size) != 0; 198 } 199 200 /* 201 * Read data blocks that are part of the RS block and deinterleave as much as 202 * fits into buffers. Check for erasure locations if @neras is non-NULL. 203 */ 204 static int fec_read_bufs(struct dm_verity *v, struct dm_verity_io *io, 205 u64 rsb, u64 target, unsigned block_offset, 206 int *neras) 207 { 208 bool is_zero; 209 int i, j, target_index = -1; 210 struct dm_buffer *buf; 211 struct dm_bufio_client *bufio; 212 struct dm_verity_fec_io *fio = fec_io(io); 213 u64 block, ileaved; 214 u8 *bbuf, *rs_block; 215 u8 want_digest[v->digest_size]; 216 unsigned n, k; 217 218 if (neras) 219 *neras = 0; 220 221 /* 222 * read each of the rsn data blocks that are part of the RS block, and 223 * interleave contents to available bufs 224 */ 225 for (i = 0; i < v->fec->rsn; i++) { 226 ileaved = fec_interleave(v, rsb * v->fec->rsn + i); 227 228 /* 229 * target is the data block we want to correct, target_index is 230 * the index of this block within the rsn RS blocks 231 */ 232 if (ileaved == target) 233 target_index = i; 234 235 block = ileaved >> v->data_dev_block_bits; 236 bufio = v->fec->data_bufio; 237 238 if (block >= v->data_blocks) { 239 block -= v->data_blocks; 240 241 /* 242 * blocks outside the area were assumed to contain 243 * zeros when encoding data was generated 244 */ 245 if (unlikely(block >= v->fec->hash_blocks)) 246 continue; 247 248 block += v->hash_start; 249 bufio = v->bufio; 250 } 251 252 bbuf = dm_bufio_read(bufio, block, &buf); 253 if (unlikely(IS_ERR(bbuf))) { 254 DMWARN_LIMIT("%s: FEC %llu: read failed (%llu): %ld", 255 v->data_dev->name, 256 (unsigned long long)rsb, 257 (unsigned long long)block, PTR_ERR(bbuf)); 258 259 /* assume the block is corrupted */ 260 if (neras && *neras <= v->fec->roots) 261 fio->erasures[(*neras)++] = i; 262 263 continue; 264 } 265 266 /* locate erasures if the block is on the data device */ 267 if (bufio == v->fec->data_bufio && 268 verity_hash_for_block(v, io, block, want_digest, 269 &is_zero) == 0) { 270 /* skip known zero blocks entirely */ 271 if (is_zero) 272 goto done; 273 274 /* 275 * skip if we have already found the theoretical 276 * maximum number (i.e. fec->roots) of erasures 277 */ 278 if (neras && *neras <= v->fec->roots && 279 fec_is_erasure(v, io, want_digest, bbuf)) 280 fio->erasures[(*neras)++] = i; 281 } 282 283 /* 284 * deinterleave and copy the bytes that fit into bufs, 285 * starting from block_offset 286 */ 287 fec_for_each_buffer_rs_block(fio, n, j) { 288 k = fec_buffer_rs_index(n, j) + block_offset; 289 290 if (k >= 1 << v->data_dev_block_bits) 291 goto done; 292 293 rs_block = fec_buffer_rs_block(v, fio, n, j); 294 rs_block[i] = bbuf[k]; 295 } 296 done: 297 dm_bufio_release(buf); 298 } 299 300 return target_index; 301 } 302 303 /* 304 * Allocate RS control structure and FEC buffers from preallocated mempools, 305 * and attempt to allocate as many extra buffers as available. 306 */ 307 static int fec_alloc_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio) 308 { 309 unsigned n; 310 311 if (!fio->rs) { 312 fio->rs = mempool_alloc(v->fec->rs_pool, 0); 313 if (unlikely(!fio->rs)) { 314 DMERR("failed to allocate RS"); 315 return -ENOMEM; 316 } 317 } 318 319 fec_for_each_prealloc_buffer(n) { 320 if (fio->bufs[n]) 321 continue; 322 323 fio->bufs[n] = mempool_alloc(v->fec->prealloc_pool, GFP_NOIO); 324 if (unlikely(!fio->bufs[n])) { 325 DMERR("failed to allocate FEC buffer"); 326 return -ENOMEM; 327 } 328 } 329 330 /* try to allocate the maximum number of buffers */ 331 fec_for_each_extra_buffer(fio, n) { 332 if (fio->bufs[n]) 333 continue; 334 335 fio->bufs[n] = mempool_alloc(v->fec->extra_pool, GFP_NOIO); 336 /* we can manage with even one buffer if necessary */ 337 if (unlikely(!fio->bufs[n])) 338 break; 339 } 340 fio->nbufs = n; 341 342 if (!fio->output) { 343 fio->output = mempool_alloc(v->fec->output_pool, GFP_NOIO); 344 345 if (!fio->output) { 346 DMERR("failed to allocate FEC page"); 347 return -ENOMEM; 348 } 349 } 350 351 return 0; 352 } 353 354 /* 355 * Initialize buffers and clear erasures. fec_read_bufs() assumes buffers are 356 * zeroed before deinterleaving. 357 */ 358 static void fec_init_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio) 359 { 360 unsigned n; 361 362 fec_for_each_buffer(fio, n) 363 memset(fio->bufs[n], 0, v->fec->rsn << DM_VERITY_FEC_BUF_RS_BITS); 364 365 memset(fio->erasures, 0, sizeof(fio->erasures)); 366 } 367 368 /* 369 * Decode all RS blocks in a single data block and return the target block 370 * (indicated by @offset) in fio->output. If @use_erasures is non-zero, uses 371 * hashes to locate erasures. 372 */ 373 static int fec_decode_rsb(struct dm_verity *v, struct dm_verity_io *io, 374 struct dm_verity_fec_io *fio, u64 rsb, u64 offset, 375 bool use_erasures) 376 { 377 int r, neras = 0; 378 unsigned pos; 379 380 r = fec_alloc_bufs(v, fio); 381 if (unlikely(r < 0)) 382 return r; 383 384 for (pos = 0; pos < 1 << v->data_dev_block_bits; ) { 385 fec_init_bufs(v, fio); 386 387 r = fec_read_bufs(v, io, rsb, offset, pos, 388 use_erasures ? &neras : NULL); 389 if (unlikely(r < 0)) 390 return r; 391 392 r = fec_decode_bufs(v, fio, rsb, r, pos, neras); 393 if (r < 0) 394 return r; 395 396 pos += fio->nbufs << DM_VERITY_FEC_BUF_RS_BITS; 397 } 398 399 /* Always re-validate the corrected block against the expected hash */ 400 r = verity_hash(v, verity_io_hash_req(v, io), fio->output, 401 1 << v->data_dev_block_bits, 402 verity_io_real_digest(v, io)); 403 if (unlikely(r < 0)) 404 return r; 405 406 if (memcmp(verity_io_real_digest(v, io), verity_io_want_digest(v, io), 407 v->digest_size)) { 408 DMERR_LIMIT("%s: FEC %llu: failed to correct (%d erasures)", 409 v->data_dev->name, (unsigned long long)rsb, neras); 410 return -EILSEQ; 411 } 412 413 return 0; 414 } 415 416 static int fec_bv_copy(struct dm_verity *v, struct dm_verity_io *io, u8 *data, 417 size_t len) 418 { 419 struct dm_verity_fec_io *fio = fec_io(io); 420 421 memcpy(data, &fio->output[fio->output_pos], len); 422 fio->output_pos += len; 423 424 return 0; 425 } 426 427 /* 428 * Correct errors in a block. Copies corrected block to dest if non-NULL, 429 * otherwise to a bio_vec starting from iter. 430 */ 431 int verity_fec_decode(struct dm_verity *v, struct dm_verity_io *io, 432 enum verity_block_type type, sector_t block, u8 *dest, 433 struct bvec_iter *iter) 434 { 435 int r; 436 struct dm_verity_fec_io *fio = fec_io(io); 437 u64 offset, res, rsb; 438 439 if (!verity_fec_is_enabled(v)) 440 return -EOPNOTSUPP; 441 442 if (fio->level >= DM_VERITY_FEC_MAX_RECURSION) { 443 DMWARN_LIMIT("%s: FEC: recursion too deep", v->data_dev->name); 444 return -EIO; 445 } 446 447 fio->level++; 448 449 if (type == DM_VERITY_BLOCK_TYPE_METADATA) 450 block += v->data_blocks; 451 452 /* 453 * For RS(M, N), the continuous FEC data is divided into blocks of N 454 * bytes. Since block size may not be divisible by N, the last block 455 * is zero padded when decoding. 456 * 457 * Each byte of the block is covered by a different RS(M, N) code, 458 * and each code is interleaved over N blocks to make it less likely 459 * that bursty corruption will leave us in unrecoverable state. 460 */ 461 462 offset = block << v->data_dev_block_bits; 463 res = div64_u64(offset, v->fec->rounds << v->data_dev_block_bits); 464 465 /* 466 * The base RS block we can feed to the interleaver to find out all 467 * blocks required for decoding. 468 */ 469 rsb = offset - res * (v->fec->rounds << v->data_dev_block_bits); 470 471 /* 472 * Locating erasures is slow, so attempt to recover the block without 473 * them first. Do a second attempt with erasures if the corruption is 474 * bad enough. 475 */ 476 r = fec_decode_rsb(v, io, fio, rsb, offset, false); 477 if (r < 0) { 478 r = fec_decode_rsb(v, io, fio, rsb, offset, true); 479 if (r < 0) 480 goto done; 481 } 482 483 if (dest) 484 memcpy(dest, fio->output, 1 << v->data_dev_block_bits); 485 else if (iter) { 486 fio->output_pos = 0; 487 r = verity_for_bv_block(v, io, iter, fec_bv_copy); 488 } 489 490 done: 491 fio->level--; 492 return r; 493 } 494 495 /* 496 * Clean up per-bio data. 497 */ 498 void verity_fec_finish_io(struct dm_verity_io *io) 499 { 500 unsigned n; 501 struct dm_verity_fec *f = io->v->fec; 502 struct dm_verity_fec_io *fio = fec_io(io); 503 504 if (!verity_fec_is_enabled(io->v)) 505 return; 506 507 mempool_free(fio->rs, f->rs_pool); 508 509 fec_for_each_prealloc_buffer(n) 510 mempool_free(fio->bufs[n], f->prealloc_pool); 511 512 fec_for_each_extra_buffer(fio, n) 513 mempool_free(fio->bufs[n], f->extra_pool); 514 515 mempool_free(fio->output, f->output_pool); 516 } 517 518 /* 519 * Initialize per-bio data. 520 */ 521 void verity_fec_init_io(struct dm_verity_io *io) 522 { 523 struct dm_verity_fec_io *fio = fec_io(io); 524 525 if (!verity_fec_is_enabled(io->v)) 526 return; 527 528 fio->rs = NULL; 529 memset(fio->bufs, 0, sizeof(fio->bufs)); 530 fio->nbufs = 0; 531 fio->output = NULL; 532 fio->level = 0; 533 } 534 535 /* 536 * Append feature arguments and values to the status table. 537 */ 538 unsigned verity_fec_status_table(struct dm_verity *v, unsigned sz, 539 char *result, unsigned maxlen) 540 { 541 if (!verity_fec_is_enabled(v)) 542 return sz; 543 544 DMEMIT(" " DM_VERITY_OPT_FEC_DEV " %s " 545 DM_VERITY_OPT_FEC_BLOCKS " %llu " 546 DM_VERITY_OPT_FEC_START " %llu " 547 DM_VERITY_OPT_FEC_ROOTS " %d", 548 v->fec->dev->name, 549 (unsigned long long)v->fec->blocks, 550 (unsigned long long)v->fec->start, 551 v->fec->roots); 552 553 return sz; 554 } 555 556 void verity_fec_dtr(struct dm_verity *v) 557 { 558 struct dm_verity_fec *f = v->fec; 559 560 if (!verity_fec_is_enabled(v)) 561 goto out; 562 563 mempool_destroy(f->rs_pool); 564 mempool_destroy(f->prealloc_pool); 565 mempool_destroy(f->extra_pool); 566 kmem_cache_destroy(f->cache); 567 568 if (f->data_bufio) 569 dm_bufio_client_destroy(f->data_bufio); 570 if (f->bufio) 571 dm_bufio_client_destroy(f->bufio); 572 573 if (f->dev) 574 dm_put_device(v->ti, f->dev); 575 out: 576 kfree(f); 577 v->fec = NULL; 578 } 579 580 static void *fec_rs_alloc(gfp_t gfp_mask, void *pool_data) 581 { 582 struct dm_verity *v = (struct dm_verity *)pool_data; 583 584 return init_rs(8, 0x11d, 0, 1, v->fec->roots); 585 } 586 587 static void fec_rs_free(void *element, void *pool_data) 588 { 589 struct rs_control *rs = (struct rs_control *)element; 590 591 if (rs) 592 free_rs(rs); 593 } 594 595 bool verity_is_fec_opt_arg(const char *arg_name) 596 { 597 return (!strcasecmp(arg_name, DM_VERITY_OPT_FEC_DEV) || 598 !strcasecmp(arg_name, DM_VERITY_OPT_FEC_BLOCKS) || 599 !strcasecmp(arg_name, DM_VERITY_OPT_FEC_START) || 600 !strcasecmp(arg_name, DM_VERITY_OPT_FEC_ROOTS)); 601 } 602 603 int verity_fec_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v, 604 unsigned *argc, const char *arg_name) 605 { 606 int r; 607 struct dm_target *ti = v->ti; 608 const char *arg_value; 609 unsigned long long num_ll; 610 unsigned char num_c; 611 char dummy; 612 613 if (!*argc) { 614 ti->error = "FEC feature arguments require a value"; 615 return -EINVAL; 616 } 617 618 arg_value = dm_shift_arg(as); 619 (*argc)--; 620 621 if (!strcasecmp(arg_name, DM_VERITY_OPT_FEC_DEV)) { 622 r = dm_get_device(ti, arg_value, FMODE_READ, &v->fec->dev); 623 if (r) { 624 ti->error = "FEC device lookup failed"; 625 return r; 626 } 627 628 } else if (!strcasecmp(arg_name, DM_VERITY_OPT_FEC_BLOCKS)) { 629 if (sscanf(arg_value, "%llu%c", &num_ll, &dummy) != 1 || 630 ((sector_t)(num_ll << (v->data_dev_block_bits - SECTOR_SHIFT)) 631 >> (v->data_dev_block_bits - SECTOR_SHIFT) != num_ll)) { 632 ti->error = "Invalid " DM_VERITY_OPT_FEC_BLOCKS; 633 return -EINVAL; 634 } 635 v->fec->blocks = num_ll; 636 637 } else if (!strcasecmp(arg_name, DM_VERITY_OPT_FEC_START)) { 638 if (sscanf(arg_value, "%llu%c", &num_ll, &dummy) != 1 || 639 ((sector_t)(num_ll << (v->data_dev_block_bits - SECTOR_SHIFT)) >> 640 (v->data_dev_block_bits - SECTOR_SHIFT) != num_ll)) { 641 ti->error = "Invalid " DM_VERITY_OPT_FEC_START; 642 return -EINVAL; 643 } 644 v->fec->start = num_ll; 645 646 } else if (!strcasecmp(arg_name, DM_VERITY_OPT_FEC_ROOTS)) { 647 if (sscanf(arg_value, "%hhu%c", &num_c, &dummy) != 1 || !num_c || 648 num_c < (DM_VERITY_FEC_RSM - DM_VERITY_FEC_MAX_RSN) || 649 num_c > (DM_VERITY_FEC_RSM - DM_VERITY_FEC_MIN_RSN)) { 650 ti->error = "Invalid " DM_VERITY_OPT_FEC_ROOTS; 651 return -EINVAL; 652 } 653 v->fec->roots = num_c; 654 655 } else { 656 ti->error = "Unrecognized verity FEC feature request"; 657 return -EINVAL; 658 } 659 660 return 0; 661 } 662 663 /* 664 * Allocate dm_verity_fec for v->fec. Must be called before verity_fec_ctr. 665 */ 666 int verity_fec_ctr_alloc(struct dm_verity *v) 667 { 668 struct dm_verity_fec *f; 669 670 f = kzalloc(sizeof(struct dm_verity_fec), GFP_KERNEL); 671 if (!f) { 672 v->ti->error = "Cannot allocate FEC structure"; 673 return -ENOMEM; 674 } 675 v->fec = f; 676 677 return 0; 678 } 679 680 /* 681 * Validate arguments and preallocate memory. Must be called after arguments 682 * have been parsed using verity_fec_parse_opt_args. 683 */ 684 int verity_fec_ctr(struct dm_verity *v) 685 { 686 struct dm_verity_fec *f = v->fec; 687 struct dm_target *ti = v->ti; 688 u64 hash_blocks; 689 690 if (!verity_fec_is_enabled(v)) { 691 verity_fec_dtr(v); 692 return 0; 693 } 694 695 /* 696 * FEC is computed over data blocks, possible metadata, and 697 * hash blocks. In other words, FEC covers total of fec_blocks 698 * blocks consisting of the following: 699 * 700 * data blocks | hash blocks | metadata (optional) 701 * 702 * We allow metadata after hash blocks to support a use case 703 * where all data is stored on the same device and FEC covers 704 * the entire area. 705 * 706 * If metadata is included, we require it to be available on the 707 * hash device after the hash blocks. 708 */ 709 710 hash_blocks = v->hash_blocks - v->hash_start; 711 712 /* 713 * Require matching block sizes for data and hash devices for 714 * simplicity. 715 */ 716 if (v->data_dev_block_bits != v->hash_dev_block_bits) { 717 ti->error = "Block sizes must match to use FEC"; 718 return -EINVAL; 719 } 720 721 if (!f->roots) { 722 ti->error = "Missing " DM_VERITY_OPT_FEC_ROOTS; 723 return -EINVAL; 724 } 725 f->rsn = DM_VERITY_FEC_RSM - f->roots; 726 727 if (!f->blocks) { 728 ti->error = "Missing " DM_VERITY_OPT_FEC_BLOCKS; 729 return -EINVAL; 730 } 731 732 f->rounds = f->blocks; 733 if (sector_div(f->rounds, f->rsn)) 734 f->rounds++; 735 736 /* 737 * Due to optional metadata, f->blocks can be larger than 738 * data_blocks and hash_blocks combined. 739 */ 740 if (f->blocks < v->data_blocks + hash_blocks || !f->rounds) { 741 ti->error = "Invalid " DM_VERITY_OPT_FEC_BLOCKS; 742 return -EINVAL; 743 } 744 745 /* 746 * Metadata is accessed through the hash device, so we require 747 * it to be large enough. 748 */ 749 f->hash_blocks = f->blocks - v->data_blocks; 750 if (dm_bufio_get_device_size(v->bufio) < f->hash_blocks) { 751 ti->error = "Hash device is too small for " 752 DM_VERITY_OPT_FEC_BLOCKS; 753 return -E2BIG; 754 } 755 756 f->bufio = dm_bufio_client_create(f->dev->bdev, 757 1 << v->data_dev_block_bits, 758 1, 0, NULL, NULL); 759 if (IS_ERR(f->bufio)) { 760 ti->error = "Cannot initialize FEC bufio client"; 761 return PTR_ERR(f->bufio); 762 } 763 764 if (dm_bufio_get_device_size(f->bufio) < 765 ((f->start + f->rounds * f->roots) >> v->data_dev_block_bits)) { 766 ti->error = "FEC device is too small"; 767 return -E2BIG; 768 } 769 770 f->data_bufio = dm_bufio_client_create(v->data_dev->bdev, 771 1 << v->data_dev_block_bits, 772 1, 0, NULL, NULL); 773 if (IS_ERR(f->data_bufio)) { 774 ti->error = "Cannot initialize FEC data bufio client"; 775 return PTR_ERR(f->data_bufio); 776 } 777 778 if (dm_bufio_get_device_size(f->data_bufio) < v->data_blocks) { 779 ti->error = "Data device is too small"; 780 return -E2BIG; 781 } 782 783 /* Preallocate an rs_control structure for each worker thread */ 784 f->rs_pool = mempool_create(num_online_cpus(), fec_rs_alloc, 785 fec_rs_free, (void *) v); 786 if (!f->rs_pool) { 787 ti->error = "Cannot allocate RS pool"; 788 return -ENOMEM; 789 } 790 791 f->cache = kmem_cache_create("dm_verity_fec_buffers", 792 f->rsn << DM_VERITY_FEC_BUF_RS_BITS, 793 0, 0, NULL); 794 if (!f->cache) { 795 ti->error = "Cannot create FEC buffer cache"; 796 return -ENOMEM; 797 } 798 799 /* Preallocate DM_VERITY_FEC_BUF_PREALLOC buffers for each thread */ 800 f->prealloc_pool = mempool_create_slab_pool(num_online_cpus() * 801 DM_VERITY_FEC_BUF_PREALLOC, 802 f->cache); 803 if (!f->prealloc_pool) { 804 ti->error = "Cannot allocate FEC buffer prealloc pool"; 805 return -ENOMEM; 806 } 807 808 f->extra_pool = mempool_create_slab_pool(0, f->cache); 809 if (!f->extra_pool) { 810 ti->error = "Cannot allocate FEC buffer extra pool"; 811 return -ENOMEM; 812 } 813 814 /* Preallocate an output buffer for each thread */ 815 f->output_pool = mempool_create_kmalloc_pool(num_online_cpus(), 816 1 << v->data_dev_block_bits); 817 if (!f->output_pool) { 818 ti->error = "Cannot allocate FEC output pool"; 819 return -ENOMEM; 820 } 821 822 /* Reserve space for our per-bio data */ 823 ti->per_io_data_size += sizeof(struct dm_verity_fec_io); 824 825 return 0; 826 } 827