1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Some low level IO code, and hacks for various block layer limitations
4 *
5 * Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com>
6 * Copyright 2012 Google, Inc.
7 */
8
9 #include "bcachefs.h"
10 #include "alloc_background.h"
11 #include "alloc_foreground.h"
12 #include "btree_update.h"
13 #include "buckets.h"
14 #include "checksum.h"
15 #include "clock.h"
16 #include "compress.h"
17 #include "data_update.h"
18 #include "disk_groups.h"
19 #include "ec.h"
20 #include "error.h"
21 #include "io_read.h"
22 #include "io_misc.h"
23 #include "io_write.h"
24 #include "reflink.h"
25 #include "subvolume.h"
26 #include "trace.h"
27
28 #include <linux/sched/mm.h>
29
30 #ifndef CONFIG_BCACHEFS_NO_LATENCY_ACCT
31
bch2_target_congested(struct bch_fs * c,u16 target)32 static bool bch2_target_congested(struct bch_fs *c, u16 target)
33 {
34 const struct bch_devs_mask *devs;
35 unsigned d, nr = 0, total = 0;
36 u64 now = local_clock(), last;
37 s64 congested;
38 struct bch_dev *ca;
39
40 if (!target)
41 return false;
42
43 rcu_read_lock();
44 devs = bch2_target_to_mask(c, target) ?:
45 &c->rw_devs[BCH_DATA_user];
46
47 for_each_set_bit(d, devs->d, BCH_SB_MEMBERS_MAX) {
48 ca = rcu_dereference(c->devs[d]);
49 if (!ca)
50 continue;
51
52 congested = atomic_read(&ca->congested);
53 last = READ_ONCE(ca->congested_last);
54 if (time_after64(now, last))
55 congested -= (now - last) >> 12;
56
57 total += max(congested, 0LL);
58 nr++;
59 }
60 rcu_read_unlock();
61
62 return bch2_rand_range(nr * CONGESTED_MAX) < total;
63 }
64
65 #else
66
bch2_target_congested(struct bch_fs * c,u16 target)67 static bool bch2_target_congested(struct bch_fs *c, u16 target)
68 {
69 return false;
70 }
71
72 #endif
73
74 /* Cache promotion on read */
75
76 struct promote_op {
77 struct rcu_head rcu;
78 u64 start_time;
79
80 struct rhash_head hash;
81 struct bpos pos;
82
83 struct data_update write;
84 struct bio_vec bi_inline_vecs[]; /* must be last */
85 };
86
87 static const struct rhashtable_params bch_promote_params = {
88 .head_offset = offsetof(struct promote_op, hash),
89 .key_offset = offsetof(struct promote_op, pos),
90 .key_len = sizeof(struct bpos),
91 .automatic_shrinking = true,
92 };
93
have_io_error(struct bch_io_failures * failed)94 static inline bool have_io_error(struct bch_io_failures *failed)
95 {
96 return failed && failed->nr;
97 }
98
should_promote(struct bch_fs * c,struct bkey_s_c k,struct bpos pos,struct bch_io_opts opts,unsigned flags,struct bch_io_failures * failed)99 static inline int should_promote(struct bch_fs *c, struct bkey_s_c k,
100 struct bpos pos,
101 struct bch_io_opts opts,
102 unsigned flags,
103 struct bch_io_failures *failed)
104 {
105 if (!have_io_error(failed)) {
106 BUG_ON(!opts.promote_target);
107
108 if (!(flags & BCH_READ_MAY_PROMOTE))
109 return -BCH_ERR_nopromote_may_not;
110
111 if (bch2_bkey_has_target(c, k, opts.promote_target))
112 return -BCH_ERR_nopromote_already_promoted;
113
114 if (bkey_extent_is_unwritten(k))
115 return -BCH_ERR_nopromote_unwritten;
116
117 if (bch2_target_congested(c, opts.promote_target))
118 return -BCH_ERR_nopromote_congested;
119 }
120
121 if (rhashtable_lookup_fast(&c->promote_table, &pos,
122 bch_promote_params))
123 return -BCH_ERR_nopromote_in_flight;
124
125 return 0;
126 }
127
promote_free(struct bch_fs * c,struct promote_op * op)128 static void promote_free(struct bch_fs *c, struct promote_op *op)
129 {
130 int ret;
131
132 bch2_data_update_exit(&op->write);
133
134 ret = rhashtable_remove_fast(&c->promote_table, &op->hash,
135 bch_promote_params);
136 BUG_ON(ret);
137 bch2_write_ref_put(c, BCH_WRITE_REF_promote);
138 kfree_rcu(op, rcu);
139 }
140
promote_done(struct bch_write_op * wop)141 static void promote_done(struct bch_write_op *wop)
142 {
143 struct promote_op *op =
144 container_of(wop, struct promote_op, write.op);
145 struct bch_fs *c = op->write.op.c;
146
147 bch2_time_stats_update(&c->times[BCH_TIME_data_promote],
148 op->start_time);
149 promote_free(c, op);
150 }
151
promote_start(struct promote_op * op,struct bch_read_bio * rbio)152 static void promote_start(struct promote_op *op, struct bch_read_bio *rbio)
153 {
154 struct bio *bio = &op->write.op.wbio.bio;
155
156 trace_and_count(op->write.op.c, read_promote, &rbio->bio);
157
158 /* we now own pages: */
159 BUG_ON(!rbio->bounce);
160 BUG_ON(rbio->bio.bi_vcnt > bio->bi_max_vecs);
161
162 memcpy(bio->bi_io_vec, rbio->bio.bi_io_vec,
163 sizeof(struct bio_vec) * rbio->bio.bi_vcnt);
164 swap(bio->bi_vcnt, rbio->bio.bi_vcnt);
165
166 bch2_data_update_read_done(&op->write, rbio->pick.crc);
167 }
168
__promote_alloc(struct btree_trans * trans,enum btree_id btree_id,struct bkey_s_c k,struct bpos pos,struct extent_ptr_decoded * pick,struct bch_io_opts opts,unsigned sectors,struct bch_read_bio ** rbio,struct bch_io_failures * failed)169 static struct promote_op *__promote_alloc(struct btree_trans *trans,
170 enum btree_id btree_id,
171 struct bkey_s_c k,
172 struct bpos pos,
173 struct extent_ptr_decoded *pick,
174 struct bch_io_opts opts,
175 unsigned sectors,
176 struct bch_read_bio **rbio,
177 struct bch_io_failures *failed)
178 {
179 struct bch_fs *c = trans->c;
180 struct promote_op *op = NULL;
181 struct bio *bio;
182 unsigned pages = DIV_ROUND_UP(sectors, PAGE_SECTORS);
183 int ret;
184
185 if (!bch2_write_ref_tryget(c, BCH_WRITE_REF_promote))
186 return ERR_PTR(-BCH_ERR_nopromote_no_writes);
187
188 op = kzalloc(struct_size(op, bi_inline_vecs, pages), GFP_KERNEL);
189 if (!op) {
190 ret = -BCH_ERR_nopromote_enomem;
191 goto err;
192 }
193
194 op->start_time = local_clock();
195 op->pos = pos;
196
197 /*
198 * We don't use the mempool here because extents that aren't
199 * checksummed or compressed can be too big for the mempool:
200 */
201 *rbio = kzalloc(sizeof(struct bch_read_bio) +
202 sizeof(struct bio_vec) * pages,
203 GFP_KERNEL);
204 if (!*rbio) {
205 ret = -BCH_ERR_nopromote_enomem;
206 goto err;
207 }
208
209 rbio_init(&(*rbio)->bio, opts);
210 bio_init(&(*rbio)->bio, NULL, (*rbio)->bio.bi_inline_vecs, pages, 0);
211
212 if (bch2_bio_alloc_pages(&(*rbio)->bio, sectors << 9, GFP_KERNEL)) {
213 ret = -BCH_ERR_nopromote_enomem;
214 goto err;
215 }
216
217 (*rbio)->bounce = true;
218 (*rbio)->split = true;
219 (*rbio)->kmalloc = true;
220
221 if (rhashtable_lookup_insert_fast(&c->promote_table, &op->hash,
222 bch_promote_params)) {
223 ret = -BCH_ERR_nopromote_in_flight;
224 goto err;
225 }
226
227 bio = &op->write.op.wbio.bio;
228 bio_init(bio, NULL, bio->bi_inline_vecs, pages, 0);
229
230 struct data_update_opts update_opts = {};
231
232 if (!have_io_error(failed)) {
233 update_opts.target = opts.promote_target;
234 update_opts.extra_replicas = 1;
235 update_opts.write_flags = BCH_WRITE_ALLOC_NOWAIT|BCH_WRITE_CACHED;
236 } else {
237 update_opts.target = opts.foreground_target;
238
239 struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k);
240 unsigned ptr_bit = 1;
241 bkey_for_each_ptr(ptrs, ptr) {
242 if (bch2_dev_io_failures(failed, ptr->dev))
243 update_opts.rewrite_ptrs |= ptr_bit;
244 ptr_bit <<= 1;
245 }
246 }
247
248 ret = bch2_data_update_init(trans, NULL, NULL, &op->write,
249 writepoint_hashed((unsigned long) current),
250 opts,
251 update_opts,
252 btree_id, k);
253 /*
254 * possible errors: -BCH_ERR_nocow_lock_blocked,
255 * -BCH_ERR_ENOSPC_disk_reservation:
256 */
257 if (ret) {
258 BUG_ON(rhashtable_remove_fast(&c->promote_table, &op->hash,
259 bch_promote_params));
260 goto err;
261 }
262
263 op->write.op.end_io = promote_done;
264
265 return op;
266 err:
267 if (*rbio)
268 bio_free_pages(&(*rbio)->bio);
269 kfree(*rbio);
270 *rbio = NULL;
271 /* We may have added to the rhashtable and thus need rcu freeing: */
272 kfree_rcu(op, rcu);
273 bch2_write_ref_put(c, BCH_WRITE_REF_promote);
274 return ERR_PTR(ret);
275 }
276
277 noinline
promote_alloc(struct btree_trans * trans,struct bvec_iter iter,struct bkey_s_c k,struct extent_ptr_decoded * pick,struct bch_io_opts opts,unsigned flags,struct bch_read_bio ** rbio,bool * bounce,bool * read_full,struct bch_io_failures * failed)278 static struct promote_op *promote_alloc(struct btree_trans *trans,
279 struct bvec_iter iter,
280 struct bkey_s_c k,
281 struct extent_ptr_decoded *pick,
282 struct bch_io_opts opts,
283 unsigned flags,
284 struct bch_read_bio **rbio,
285 bool *bounce,
286 bool *read_full,
287 struct bch_io_failures *failed)
288 {
289 struct bch_fs *c = trans->c;
290 /*
291 * if failed != NULL we're not actually doing a promote, we're
292 * recovering from an io/checksum error
293 */
294 bool promote_full = (have_io_error(failed) ||
295 *read_full ||
296 READ_ONCE(c->opts.promote_whole_extents));
297 /* data might have to be decompressed in the write path: */
298 unsigned sectors = promote_full
299 ? max(pick->crc.compressed_size, pick->crc.live_size)
300 : bvec_iter_sectors(iter);
301 struct bpos pos = promote_full
302 ? bkey_start_pos(k.k)
303 : POS(k.k->p.inode, iter.bi_sector);
304 struct promote_op *promote;
305 int ret;
306
307 ret = should_promote(c, k, pos, opts, flags, failed);
308 if (ret)
309 goto nopromote;
310
311 promote = __promote_alloc(trans,
312 k.k->type == KEY_TYPE_reflink_v
313 ? BTREE_ID_reflink
314 : BTREE_ID_extents,
315 k, pos, pick, opts, sectors, rbio, failed);
316 ret = PTR_ERR_OR_ZERO(promote);
317 if (ret)
318 goto nopromote;
319
320 *bounce = true;
321 *read_full = promote_full;
322 return promote;
323 nopromote:
324 trace_read_nopromote(c, ret);
325 return NULL;
326 }
327
328 /* Read */
329
bch2_read_err_msg_trans(struct btree_trans * trans,struct printbuf * out,struct bch_read_bio * rbio,struct bpos read_pos)330 static int bch2_read_err_msg_trans(struct btree_trans *trans, struct printbuf *out,
331 struct bch_read_bio *rbio, struct bpos read_pos)
332 {
333 return bch2_inum_offset_err_msg_trans(trans, out,
334 (subvol_inum) { rbio->subvol, read_pos.inode },
335 read_pos.offset << 9);
336 }
337
bch2_read_err_msg(struct bch_fs * c,struct printbuf * out,struct bch_read_bio * rbio,struct bpos read_pos)338 static void bch2_read_err_msg(struct bch_fs *c, struct printbuf *out,
339 struct bch_read_bio *rbio, struct bpos read_pos)
340 {
341 bch2_trans_run(c, bch2_read_err_msg_trans(trans, out, rbio, read_pos));
342 }
343
344 #define READ_RETRY_AVOID 1
345 #define READ_RETRY 2
346 #define READ_ERR 3
347
348 enum rbio_context {
349 RBIO_CONTEXT_NULL,
350 RBIO_CONTEXT_HIGHPRI,
351 RBIO_CONTEXT_UNBOUND,
352 };
353
354 static inline struct bch_read_bio *
bch2_rbio_parent(struct bch_read_bio * rbio)355 bch2_rbio_parent(struct bch_read_bio *rbio)
356 {
357 return rbio->split ? rbio->parent : rbio;
358 }
359
360 __always_inline
bch2_rbio_punt(struct bch_read_bio * rbio,work_func_t fn,enum rbio_context context,struct workqueue_struct * wq)361 static void bch2_rbio_punt(struct bch_read_bio *rbio, work_func_t fn,
362 enum rbio_context context,
363 struct workqueue_struct *wq)
364 {
365 if (context <= rbio->context) {
366 fn(&rbio->work);
367 } else {
368 rbio->work.func = fn;
369 rbio->context = context;
370 queue_work(wq, &rbio->work);
371 }
372 }
373
bch2_rbio_free(struct bch_read_bio * rbio)374 static inline struct bch_read_bio *bch2_rbio_free(struct bch_read_bio *rbio)
375 {
376 BUG_ON(rbio->bounce && !rbio->split);
377
378 if (rbio->promote)
379 promote_free(rbio->c, rbio->promote);
380 rbio->promote = NULL;
381
382 if (rbio->bounce)
383 bch2_bio_free_pages_pool(rbio->c, &rbio->bio);
384
385 if (rbio->split) {
386 struct bch_read_bio *parent = rbio->parent;
387
388 if (rbio->kmalloc)
389 kfree(rbio);
390 else
391 bio_put(&rbio->bio);
392
393 rbio = parent;
394 }
395
396 return rbio;
397 }
398
399 /*
400 * Only called on a top level bch_read_bio to complete an entire read request,
401 * not a split:
402 */
bch2_rbio_done(struct bch_read_bio * rbio)403 static void bch2_rbio_done(struct bch_read_bio *rbio)
404 {
405 if (rbio->start_time)
406 bch2_time_stats_update(&rbio->c->times[BCH_TIME_data_read],
407 rbio->start_time);
408 bio_endio(&rbio->bio);
409 }
410
bch2_read_retry_nodecode(struct bch_fs * c,struct bch_read_bio * rbio,struct bvec_iter bvec_iter,struct bch_io_failures * failed,unsigned flags)411 static void bch2_read_retry_nodecode(struct bch_fs *c, struct bch_read_bio *rbio,
412 struct bvec_iter bvec_iter,
413 struct bch_io_failures *failed,
414 unsigned flags)
415 {
416 struct btree_trans *trans = bch2_trans_get(c);
417 struct btree_iter iter;
418 struct bkey_buf sk;
419 struct bkey_s_c k;
420 int ret;
421
422 flags &= ~BCH_READ_LAST_FRAGMENT;
423 flags |= BCH_READ_MUST_CLONE;
424
425 bch2_bkey_buf_init(&sk);
426
427 bch2_trans_iter_init(trans, &iter, rbio->data_btree,
428 rbio->read_pos, BTREE_ITER_slots);
429 retry:
430 bch2_trans_begin(trans);
431 rbio->bio.bi_status = 0;
432
433 ret = lockrestart_do(trans, bkey_err(k = bch2_btree_iter_peek_slot(&iter)));
434 if (ret)
435 goto err;
436
437 bch2_bkey_buf_reassemble(&sk, c, k);
438 k = bkey_i_to_s_c(sk.k);
439
440 if (!bch2_bkey_matches_ptr(c, k,
441 rbio->pick.ptr,
442 rbio->data_pos.offset -
443 rbio->pick.crc.offset)) {
444 /* extent we wanted to read no longer exists: */
445 rbio->hole = true;
446 goto out;
447 }
448
449 ret = __bch2_read_extent(trans, rbio, bvec_iter,
450 rbio->read_pos,
451 rbio->data_btree,
452 k, 0, failed, flags);
453 if (ret == READ_RETRY)
454 goto retry;
455 if (ret)
456 goto err;
457 out:
458 bch2_rbio_done(rbio);
459 bch2_trans_iter_exit(trans, &iter);
460 bch2_trans_put(trans);
461 bch2_bkey_buf_exit(&sk, c);
462 return;
463 err:
464 rbio->bio.bi_status = BLK_STS_IOERR;
465 goto out;
466 }
467
bch2_rbio_retry(struct work_struct * work)468 static void bch2_rbio_retry(struct work_struct *work)
469 {
470 struct bch_read_bio *rbio =
471 container_of(work, struct bch_read_bio, work);
472 struct bch_fs *c = rbio->c;
473 struct bvec_iter iter = rbio->bvec_iter;
474 unsigned flags = rbio->flags;
475 subvol_inum inum = {
476 .subvol = rbio->subvol,
477 .inum = rbio->read_pos.inode,
478 };
479 struct bch_io_failures failed = { .nr = 0 };
480
481 trace_and_count(c, read_retry, &rbio->bio);
482
483 if (rbio->retry == READ_RETRY_AVOID)
484 bch2_mark_io_failure(&failed, &rbio->pick);
485
486 rbio->bio.bi_status = 0;
487
488 rbio = bch2_rbio_free(rbio);
489
490 flags |= BCH_READ_IN_RETRY;
491 flags &= ~BCH_READ_MAY_PROMOTE;
492
493 if (flags & BCH_READ_NODECODE) {
494 bch2_read_retry_nodecode(c, rbio, iter, &failed, flags);
495 } else {
496 flags &= ~BCH_READ_LAST_FRAGMENT;
497 flags |= BCH_READ_MUST_CLONE;
498
499 __bch2_read(c, rbio, iter, inum, &failed, flags);
500 }
501 }
502
bch2_rbio_error(struct bch_read_bio * rbio,int retry,blk_status_t error)503 static void bch2_rbio_error(struct bch_read_bio *rbio, int retry,
504 blk_status_t error)
505 {
506 rbio->retry = retry;
507
508 if (rbio->flags & BCH_READ_IN_RETRY)
509 return;
510
511 if (retry == READ_ERR) {
512 rbio = bch2_rbio_free(rbio);
513
514 rbio->bio.bi_status = error;
515 bch2_rbio_done(rbio);
516 } else {
517 bch2_rbio_punt(rbio, bch2_rbio_retry,
518 RBIO_CONTEXT_UNBOUND, system_unbound_wq);
519 }
520 }
521
bch2_read_io_err(struct work_struct * work)522 static void bch2_read_io_err(struct work_struct *work)
523 {
524 struct bch_read_bio *rbio =
525 container_of(work, struct bch_read_bio, work);
526 struct bio *bio = &rbio->bio;
527 struct bch_fs *c = rbio->c;
528 struct bch_dev *ca = rbio->have_ioref ? bch2_dev_have_ref(c, rbio->pick.ptr.dev) : NULL;
529 struct printbuf buf = PRINTBUF;
530
531 bch2_read_err_msg(c, &buf, rbio, rbio->read_pos);
532 prt_printf(&buf, "data read error: %s", bch2_blk_status_to_str(bio->bi_status));
533
534 if (ca) {
535 bch2_io_error(ca, BCH_MEMBER_ERROR_read);
536 bch_err_ratelimited(ca, "%s", buf.buf);
537 } else {
538 bch_err_ratelimited(c, "%s", buf.buf);
539 }
540
541 printbuf_exit(&buf);
542 bch2_rbio_error(rbio, READ_RETRY_AVOID, bio->bi_status);
543 }
544
__bch2_rbio_narrow_crcs(struct btree_trans * trans,struct bch_read_bio * rbio)545 static int __bch2_rbio_narrow_crcs(struct btree_trans *trans,
546 struct bch_read_bio *rbio)
547 {
548 struct bch_fs *c = rbio->c;
549 u64 data_offset = rbio->data_pos.offset - rbio->pick.crc.offset;
550 struct bch_extent_crc_unpacked new_crc;
551 struct btree_iter iter;
552 struct bkey_i *new;
553 struct bkey_s_c k;
554 int ret = 0;
555
556 if (crc_is_compressed(rbio->pick.crc))
557 return 0;
558
559 k = bch2_bkey_get_iter(trans, &iter, rbio->data_btree, rbio->data_pos,
560 BTREE_ITER_slots|BTREE_ITER_intent);
561 if ((ret = bkey_err(k)))
562 goto out;
563
564 if (bversion_cmp(k.k->bversion, rbio->version) ||
565 !bch2_bkey_matches_ptr(c, k, rbio->pick.ptr, data_offset))
566 goto out;
567
568 /* Extent was merged? */
569 if (bkey_start_offset(k.k) < data_offset ||
570 k.k->p.offset > data_offset + rbio->pick.crc.uncompressed_size)
571 goto out;
572
573 if (bch2_rechecksum_bio(c, &rbio->bio, rbio->version,
574 rbio->pick.crc, NULL, &new_crc,
575 bkey_start_offset(k.k) - data_offset, k.k->size,
576 rbio->pick.crc.csum_type)) {
577 bch_err(c, "error verifying existing checksum while narrowing checksum (memory corruption?)");
578 ret = 0;
579 goto out;
580 }
581
582 /*
583 * going to be temporarily appending another checksum entry:
584 */
585 new = bch2_trans_kmalloc(trans, bkey_bytes(k.k) +
586 sizeof(struct bch_extent_crc128));
587 if ((ret = PTR_ERR_OR_ZERO(new)))
588 goto out;
589
590 bkey_reassemble(new, k);
591
592 if (!bch2_bkey_narrow_crcs(new, new_crc))
593 goto out;
594
595 ret = bch2_trans_update(trans, &iter, new,
596 BTREE_UPDATE_internal_snapshot_node);
597 out:
598 bch2_trans_iter_exit(trans, &iter);
599 return ret;
600 }
601
bch2_rbio_narrow_crcs(struct bch_read_bio * rbio)602 static noinline void bch2_rbio_narrow_crcs(struct bch_read_bio *rbio)
603 {
604 bch2_trans_commit_do(rbio->c, NULL, NULL, BCH_TRANS_COMMIT_no_enospc,
605 __bch2_rbio_narrow_crcs(trans, rbio));
606 }
607
bch2_read_csum_err(struct work_struct * work)608 static void bch2_read_csum_err(struct work_struct *work)
609 {
610 struct bch_read_bio *rbio =
611 container_of(work, struct bch_read_bio, work);
612 struct bch_fs *c = rbio->c;
613 struct bio *src = &rbio->bio;
614 struct bch_extent_crc_unpacked crc = rbio->pick.crc;
615 struct nonce nonce = extent_nonce(rbio->version, crc);
616 struct bch_csum csum = bch2_checksum_bio(c, crc.csum_type, nonce, src);
617 struct printbuf buf = PRINTBUF;
618
619 bch2_read_err_msg(c, &buf, rbio, rbio->read_pos);
620 prt_str(&buf, "data ");
621 bch2_csum_err_msg(&buf, crc.csum_type, rbio->pick.crc.csum, csum);
622
623 struct bch_dev *ca = rbio->have_ioref ? bch2_dev_have_ref(c, rbio->pick.ptr.dev) : NULL;
624 if (ca) {
625 bch2_io_error(ca, BCH_MEMBER_ERROR_checksum);
626 bch_err_ratelimited(ca, "%s", buf.buf);
627 } else {
628 bch_err_ratelimited(c, "%s", buf.buf);
629 }
630
631 bch2_rbio_error(rbio, READ_RETRY_AVOID, BLK_STS_IOERR);
632 printbuf_exit(&buf);
633 }
634
bch2_read_decompress_err(struct work_struct * work)635 static void bch2_read_decompress_err(struct work_struct *work)
636 {
637 struct bch_read_bio *rbio =
638 container_of(work, struct bch_read_bio, work);
639 struct bch_fs *c = rbio->c;
640 struct printbuf buf = PRINTBUF;
641
642 bch2_read_err_msg(c, &buf, rbio, rbio->read_pos);
643 prt_str(&buf, "decompression error");
644
645 struct bch_dev *ca = rbio->have_ioref ? bch2_dev_have_ref(c, rbio->pick.ptr.dev) : NULL;
646 if (ca)
647 bch_err_ratelimited(ca, "%s", buf.buf);
648 else
649 bch_err_ratelimited(c, "%s", buf.buf);
650
651 bch2_rbio_error(rbio, READ_ERR, BLK_STS_IOERR);
652 printbuf_exit(&buf);
653 }
654
bch2_read_decrypt_err(struct work_struct * work)655 static void bch2_read_decrypt_err(struct work_struct *work)
656 {
657 struct bch_read_bio *rbio =
658 container_of(work, struct bch_read_bio, work);
659 struct bch_fs *c = rbio->c;
660 struct printbuf buf = PRINTBUF;
661
662 bch2_read_err_msg(c, &buf, rbio, rbio->read_pos);
663 prt_str(&buf, "decrypt error");
664
665 struct bch_dev *ca = rbio->have_ioref ? bch2_dev_have_ref(c, rbio->pick.ptr.dev) : NULL;
666 if (ca)
667 bch_err_ratelimited(ca, "%s", buf.buf);
668 else
669 bch_err_ratelimited(c, "%s", buf.buf);
670
671 bch2_rbio_error(rbio, READ_ERR, BLK_STS_IOERR);
672 printbuf_exit(&buf);
673 }
674
675 /* Inner part that may run in process context */
__bch2_read_endio(struct work_struct * work)676 static void __bch2_read_endio(struct work_struct *work)
677 {
678 struct bch_read_bio *rbio =
679 container_of(work, struct bch_read_bio, work);
680 struct bch_fs *c = rbio->c;
681 struct bio *src = &rbio->bio;
682 struct bio *dst = &bch2_rbio_parent(rbio)->bio;
683 struct bvec_iter dst_iter = rbio->bvec_iter;
684 struct bch_extent_crc_unpacked crc = rbio->pick.crc;
685 struct nonce nonce = extent_nonce(rbio->version, crc);
686 unsigned nofs_flags;
687 struct bch_csum csum;
688 int ret;
689
690 nofs_flags = memalloc_nofs_save();
691
692 /* Reset iterator for checksumming and copying bounced data: */
693 if (rbio->bounce) {
694 src->bi_iter.bi_size = crc.compressed_size << 9;
695 src->bi_iter.bi_idx = 0;
696 src->bi_iter.bi_bvec_done = 0;
697 } else {
698 src->bi_iter = rbio->bvec_iter;
699 }
700
701 csum = bch2_checksum_bio(c, crc.csum_type, nonce, src);
702 if (bch2_crc_cmp(csum, rbio->pick.crc.csum) && !c->opts.no_data_io)
703 goto csum_err;
704
705 /*
706 * XXX
707 * We need to rework the narrow_crcs path to deliver the read completion
708 * first, and then punt to a different workqueue, otherwise we're
709 * holding up reads while doing btree updates which is bad for memory
710 * reclaim.
711 */
712 if (unlikely(rbio->narrow_crcs))
713 bch2_rbio_narrow_crcs(rbio);
714
715 if (rbio->flags & BCH_READ_NODECODE)
716 goto nodecode;
717
718 /* Adjust crc to point to subset of data we want: */
719 crc.offset += rbio->offset_into_extent;
720 crc.live_size = bvec_iter_sectors(rbio->bvec_iter);
721
722 if (crc_is_compressed(crc)) {
723 ret = bch2_encrypt_bio(c, crc.csum_type, nonce, src);
724 if (ret)
725 goto decrypt_err;
726
727 if (bch2_bio_uncompress(c, src, dst, dst_iter, crc) &&
728 !c->opts.no_data_io)
729 goto decompression_err;
730 } else {
731 /* don't need to decrypt the entire bio: */
732 nonce = nonce_add(nonce, crc.offset << 9);
733 bio_advance(src, crc.offset << 9);
734
735 BUG_ON(src->bi_iter.bi_size < dst_iter.bi_size);
736 src->bi_iter.bi_size = dst_iter.bi_size;
737
738 ret = bch2_encrypt_bio(c, crc.csum_type, nonce, src);
739 if (ret)
740 goto decrypt_err;
741
742 if (rbio->bounce) {
743 struct bvec_iter src_iter = src->bi_iter;
744
745 bio_copy_data_iter(dst, &dst_iter, src, &src_iter);
746 }
747 }
748
749 if (rbio->promote) {
750 /*
751 * Re encrypt data we decrypted, so it's consistent with
752 * rbio->crc:
753 */
754 ret = bch2_encrypt_bio(c, crc.csum_type, nonce, src);
755 if (ret)
756 goto decrypt_err;
757
758 promote_start(rbio->promote, rbio);
759 rbio->promote = NULL;
760 }
761 nodecode:
762 if (likely(!(rbio->flags & BCH_READ_IN_RETRY))) {
763 rbio = bch2_rbio_free(rbio);
764 bch2_rbio_done(rbio);
765 }
766 out:
767 memalloc_nofs_restore(nofs_flags);
768 return;
769 csum_err:
770 /*
771 * Checksum error: if the bio wasn't bounced, we may have been
772 * reading into buffers owned by userspace (that userspace can
773 * scribble over) - retry the read, bouncing it this time:
774 */
775 if (!rbio->bounce && (rbio->flags & BCH_READ_USER_MAPPED)) {
776 rbio->flags |= BCH_READ_MUST_BOUNCE;
777 bch2_rbio_error(rbio, READ_RETRY, BLK_STS_IOERR);
778 goto out;
779 }
780
781 bch2_rbio_punt(rbio, bch2_read_csum_err, RBIO_CONTEXT_UNBOUND, system_unbound_wq);
782 goto out;
783 decompression_err:
784 bch2_rbio_punt(rbio, bch2_read_decompress_err, RBIO_CONTEXT_UNBOUND, system_unbound_wq);
785 goto out;
786 decrypt_err:
787 bch2_rbio_punt(rbio, bch2_read_decrypt_err, RBIO_CONTEXT_UNBOUND, system_unbound_wq);
788 goto out;
789 }
790
bch2_read_endio(struct bio * bio)791 static void bch2_read_endio(struct bio *bio)
792 {
793 struct bch_read_bio *rbio =
794 container_of(bio, struct bch_read_bio, bio);
795 struct bch_fs *c = rbio->c;
796 struct bch_dev *ca = rbio->have_ioref ? bch2_dev_have_ref(c, rbio->pick.ptr.dev) : NULL;
797 struct workqueue_struct *wq = NULL;
798 enum rbio_context context = RBIO_CONTEXT_NULL;
799
800 if (rbio->have_ioref) {
801 bch2_latency_acct(ca, rbio->submit_time, READ);
802 percpu_ref_put(&ca->io_ref);
803 }
804
805 if (!rbio->split)
806 rbio->bio.bi_end_io = rbio->end_io;
807
808 if (unlikely(bio->bi_status)) {
809 bch2_rbio_punt(rbio, bch2_read_io_err, RBIO_CONTEXT_UNBOUND, system_unbound_wq);
810 return;
811 }
812
813 if (((rbio->flags & BCH_READ_RETRY_IF_STALE) && race_fault()) ||
814 (ca && dev_ptr_stale(ca, &rbio->pick.ptr))) {
815 trace_and_count(c, read_reuse_race, &rbio->bio);
816
817 if (rbio->flags & BCH_READ_RETRY_IF_STALE)
818 bch2_rbio_error(rbio, READ_RETRY, BLK_STS_AGAIN);
819 else
820 bch2_rbio_error(rbio, READ_ERR, BLK_STS_AGAIN);
821 return;
822 }
823
824 if (rbio->narrow_crcs ||
825 rbio->promote ||
826 crc_is_compressed(rbio->pick.crc) ||
827 bch2_csum_type_is_encryption(rbio->pick.crc.csum_type))
828 context = RBIO_CONTEXT_UNBOUND, wq = system_unbound_wq;
829 else if (rbio->pick.crc.csum_type)
830 context = RBIO_CONTEXT_HIGHPRI, wq = system_highpri_wq;
831
832 bch2_rbio_punt(rbio, __bch2_read_endio, context, wq);
833 }
834
read_from_stale_dirty_pointer(struct btree_trans * trans,struct bch_dev * ca,struct bkey_s_c k,struct bch_extent_ptr ptr)835 static noinline void read_from_stale_dirty_pointer(struct btree_trans *trans,
836 struct bch_dev *ca,
837 struct bkey_s_c k,
838 struct bch_extent_ptr ptr)
839 {
840 struct bch_fs *c = trans->c;
841 struct btree_iter iter;
842 struct printbuf buf = PRINTBUF;
843 int ret;
844
845 bch2_trans_iter_init(trans, &iter, BTREE_ID_alloc,
846 PTR_BUCKET_POS(ca, &ptr),
847 BTREE_ITER_cached);
848
849 int gen = bucket_gen_get(ca, iter.pos.offset);
850 if (gen >= 0) {
851 prt_printf(&buf, "Attempting to read from stale dirty pointer:\n");
852 printbuf_indent_add(&buf, 2);
853
854 bch2_bkey_val_to_text(&buf, c, k);
855 prt_newline(&buf);
856
857 prt_printf(&buf, "memory gen: %u", gen);
858
859 ret = lockrestart_do(trans, bkey_err(k = bch2_btree_iter_peek_slot(&iter)));
860 if (!ret) {
861 prt_newline(&buf);
862 bch2_bkey_val_to_text(&buf, c, k);
863 }
864 } else {
865 prt_printf(&buf, "Attempting to read from invalid bucket %llu:%llu:\n",
866 iter.pos.inode, iter.pos.offset);
867 printbuf_indent_add(&buf, 2);
868
869 prt_printf(&buf, "first bucket %u nbuckets %llu\n",
870 ca->mi.first_bucket, ca->mi.nbuckets);
871
872 bch2_bkey_val_to_text(&buf, c, k);
873 prt_newline(&buf);
874 }
875
876 bch2_fs_inconsistent(c, "%s", buf.buf);
877
878 bch2_trans_iter_exit(trans, &iter);
879 printbuf_exit(&buf);
880 }
881
__bch2_read_extent(struct btree_trans * trans,struct bch_read_bio * orig,struct bvec_iter iter,struct bpos read_pos,enum btree_id data_btree,struct bkey_s_c k,unsigned offset_into_extent,struct bch_io_failures * failed,unsigned flags)882 int __bch2_read_extent(struct btree_trans *trans, struct bch_read_bio *orig,
883 struct bvec_iter iter, struct bpos read_pos,
884 enum btree_id data_btree, struct bkey_s_c k,
885 unsigned offset_into_extent,
886 struct bch_io_failures *failed, unsigned flags)
887 {
888 struct bch_fs *c = trans->c;
889 struct extent_ptr_decoded pick;
890 struct bch_read_bio *rbio = NULL;
891 struct promote_op *promote = NULL;
892 bool bounce = false, read_full = false, narrow_crcs = false;
893 struct bpos data_pos = bkey_start_pos(k.k);
894 int pick_ret;
895
896 if (bkey_extent_is_inline_data(k.k)) {
897 unsigned bytes = min_t(unsigned, iter.bi_size,
898 bkey_inline_data_bytes(k.k));
899
900 swap(iter.bi_size, bytes);
901 memcpy_to_bio(&orig->bio, iter, bkey_inline_data_p(k));
902 swap(iter.bi_size, bytes);
903 bio_advance_iter(&orig->bio, &iter, bytes);
904 zero_fill_bio_iter(&orig->bio, iter);
905 goto out_read_done;
906 }
907 retry_pick:
908 pick_ret = bch2_bkey_pick_read_device(c, k, failed, &pick);
909
910 /* hole or reservation - just zero fill: */
911 if (!pick_ret)
912 goto hole;
913
914 if (unlikely(pick_ret < 0)) {
915 struct printbuf buf = PRINTBUF;
916 bch2_read_err_msg_trans(trans, &buf, orig, read_pos);
917 prt_printf(&buf, "no device to read from: %s\n ", bch2_err_str(pick_ret));
918 bch2_bkey_val_to_text(&buf, c, k);
919
920 bch_err_ratelimited(c, "%s", buf.buf);
921 printbuf_exit(&buf);
922 goto err;
923 }
924
925 if (unlikely(bch2_csum_type_is_encryption(pick.crc.csum_type)) && !c->chacha20) {
926 struct printbuf buf = PRINTBUF;
927 bch2_read_err_msg_trans(trans, &buf, orig, read_pos);
928 prt_printf(&buf, "attempting to read encrypted data without encryption key\n ");
929 bch2_bkey_val_to_text(&buf, c, k);
930
931 bch_err_ratelimited(c, "%s", buf.buf);
932 printbuf_exit(&buf);
933 goto err;
934 }
935
936 struct bch_dev *ca = bch2_dev_get_ioref(c, pick.ptr.dev, READ);
937
938 /*
939 * Stale dirty pointers are treated as IO errors, but @failed isn't
940 * allocated unless we're in the retry path - so if we're not in the
941 * retry path, don't check here, it'll be caught in bch2_read_endio()
942 * and we'll end up in the retry path:
943 */
944 if ((flags & BCH_READ_IN_RETRY) &&
945 !pick.ptr.cached &&
946 ca &&
947 unlikely(dev_ptr_stale(ca, &pick.ptr))) {
948 read_from_stale_dirty_pointer(trans, ca, k, pick.ptr);
949 bch2_mark_io_failure(failed, &pick);
950 percpu_ref_put(&ca->io_ref);
951 goto retry_pick;
952 }
953
954 /*
955 * Unlock the iterator while the btree node's lock is still in
956 * cache, before doing the IO:
957 */
958 bch2_trans_unlock(trans);
959
960 if (flags & BCH_READ_NODECODE) {
961 /*
962 * can happen if we retry, and the extent we were going to read
963 * has been merged in the meantime:
964 */
965 if (pick.crc.compressed_size > orig->bio.bi_vcnt * PAGE_SECTORS) {
966 if (ca)
967 percpu_ref_put(&ca->io_ref);
968 goto hole;
969 }
970
971 iter.bi_size = pick.crc.compressed_size << 9;
972 goto get_bio;
973 }
974
975 if (!(flags & BCH_READ_LAST_FRAGMENT) ||
976 bio_flagged(&orig->bio, BIO_CHAIN))
977 flags |= BCH_READ_MUST_CLONE;
978
979 narrow_crcs = !(flags & BCH_READ_IN_RETRY) &&
980 bch2_can_narrow_extent_crcs(k, pick.crc);
981
982 if (narrow_crcs && (flags & BCH_READ_USER_MAPPED))
983 flags |= BCH_READ_MUST_BOUNCE;
984
985 EBUG_ON(offset_into_extent + bvec_iter_sectors(iter) > k.k->size);
986
987 if (crc_is_compressed(pick.crc) ||
988 (pick.crc.csum_type != BCH_CSUM_none &&
989 (bvec_iter_sectors(iter) != pick.crc.uncompressed_size ||
990 (bch2_csum_type_is_encryption(pick.crc.csum_type) &&
991 (flags & BCH_READ_USER_MAPPED)) ||
992 (flags & BCH_READ_MUST_BOUNCE)))) {
993 read_full = true;
994 bounce = true;
995 }
996
997 if (orig->opts.promote_target || have_io_error(failed))
998 promote = promote_alloc(trans, iter, k, &pick, orig->opts, flags,
999 &rbio, &bounce, &read_full, failed);
1000
1001 if (!read_full) {
1002 EBUG_ON(crc_is_compressed(pick.crc));
1003 EBUG_ON(pick.crc.csum_type &&
1004 (bvec_iter_sectors(iter) != pick.crc.uncompressed_size ||
1005 bvec_iter_sectors(iter) != pick.crc.live_size ||
1006 pick.crc.offset ||
1007 offset_into_extent));
1008
1009 data_pos.offset += offset_into_extent;
1010 pick.ptr.offset += pick.crc.offset +
1011 offset_into_extent;
1012 offset_into_extent = 0;
1013 pick.crc.compressed_size = bvec_iter_sectors(iter);
1014 pick.crc.uncompressed_size = bvec_iter_sectors(iter);
1015 pick.crc.offset = 0;
1016 pick.crc.live_size = bvec_iter_sectors(iter);
1017 }
1018 get_bio:
1019 if (rbio) {
1020 /*
1021 * promote already allocated bounce rbio:
1022 * promote needs to allocate a bio big enough for uncompressing
1023 * data in the write path, but we're not going to use it all
1024 * here:
1025 */
1026 EBUG_ON(rbio->bio.bi_iter.bi_size <
1027 pick.crc.compressed_size << 9);
1028 rbio->bio.bi_iter.bi_size =
1029 pick.crc.compressed_size << 9;
1030 } else if (bounce) {
1031 unsigned sectors = pick.crc.compressed_size;
1032
1033 rbio = rbio_init(bio_alloc_bioset(NULL,
1034 DIV_ROUND_UP(sectors, PAGE_SECTORS),
1035 0,
1036 GFP_NOFS,
1037 &c->bio_read_split),
1038 orig->opts);
1039
1040 bch2_bio_alloc_pages_pool(c, &rbio->bio, sectors << 9);
1041 rbio->bounce = true;
1042 rbio->split = true;
1043 } else if (flags & BCH_READ_MUST_CLONE) {
1044 /*
1045 * Have to clone if there were any splits, due to error
1046 * reporting issues (if a split errored, and retrying didn't
1047 * work, when it reports the error to its parent (us) we don't
1048 * know if the error was from our bio, and we should retry, or
1049 * from the whole bio, in which case we don't want to retry and
1050 * lose the error)
1051 */
1052 rbio = rbio_init(bio_alloc_clone(NULL, &orig->bio, GFP_NOFS,
1053 &c->bio_read_split),
1054 orig->opts);
1055 rbio->bio.bi_iter = iter;
1056 rbio->split = true;
1057 } else {
1058 rbio = orig;
1059 rbio->bio.bi_iter = iter;
1060 EBUG_ON(bio_flagged(&rbio->bio, BIO_CHAIN));
1061 }
1062
1063 EBUG_ON(bio_sectors(&rbio->bio) != pick.crc.compressed_size);
1064
1065 rbio->c = c;
1066 rbio->submit_time = local_clock();
1067 if (rbio->split)
1068 rbio->parent = orig;
1069 else
1070 rbio->end_io = orig->bio.bi_end_io;
1071 rbio->bvec_iter = iter;
1072 rbio->offset_into_extent= offset_into_extent;
1073 rbio->flags = flags;
1074 rbio->have_ioref = ca != NULL;
1075 rbio->narrow_crcs = narrow_crcs;
1076 rbio->hole = 0;
1077 rbio->retry = 0;
1078 rbio->context = 0;
1079 /* XXX: only initialize this if needed */
1080 rbio->devs_have = bch2_bkey_devs(k);
1081 rbio->pick = pick;
1082 rbio->subvol = orig->subvol;
1083 rbio->read_pos = read_pos;
1084 rbio->data_btree = data_btree;
1085 rbio->data_pos = data_pos;
1086 rbio->version = k.k->bversion;
1087 rbio->promote = promote;
1088 INIT_WORK(&rbio->work, NULL);
1089
1090 if (flags & BCH_READ_NODECODE)
1091 orig->pick = pick;
1092
1093 rbio->bio.bi_opf = orig->bio.bi_opf;
1094 rbio->bio.bi_iter.bi_sector = pick.ptr.offset;
1095 rbio->bio.bi_end_io = bch2_read_endio;
1096
1097 if (rbio->bounce)
1098 trace_and_count(c, read_bounce, &rbio->bio);
1099
1100 this_cpu_add(c->counters[BCH_COUNTER_io_read], bio_sectors(&rbio->bio));
1101 bch2_increment_clock(c, bio_sectors(&rbio->bio), READ);
1102
1103 /*
1104 * If it's being moved internally, we don't want to flag it as a cache
1105 * hit:
1106 */
1107 if (ca && pick.ptr.cached && !(flags & BCH_READ_NODECODE))
1108 bch2_bucket_io_time_reset(trans, pick.ptr.dev,
1109 PTR_BUCKET_NR(ca, &pick.ptr), READ);
1110
1111 if (!(flags & (BCH_READ_IN_RETRY|BCH_READ_LAST_FRAGMENT))) {
1112 bio_inc_remaining(&orig->bio);
1113 trace_and_count(c, read_split, &orig->bio);
1114 }
1115
1116 if (!rbio->pick.idx) {
1117 if (unlikely(!rbio->have_ioref)) {
1118 struct printbuf buf = PRINTBUF;
1119 bch2_read_err_msg_trans(trans, &buf, rbio, read_pos);
1120 prt_printf(&buf, "no device to read from:\n ");
1121 bch2_bkey_val_to_text(&buf, c, k);
1122
1123 bch_err_ratelimited(c, "%s", buf.buf);
1124 printbuf_exit(&buf);
1125
1126 bch2_rbio_error(rbio, READ_RETRY_AVOID, BLK_STS_IOERR);
1127 goto out;
1128 }
1129
1130 this_cpu_add(ca->io_done->sectors[READ][BCH_DATA_user],
1131 bio_sectors(&rbio->bio));
1132 bio_set_dev(&rbio->bio, ca->disk_sb.bdev);
1133
1134 if (unlikely(c->opts.no_data_io)) {
1135 if (likely(!(flags & BCH_READ_IN_RETRY)))
1136 bio_endio(&rbio->bio);
1137 } else {
1138 if (likely(!(flags & BCH_READ_IN_RETRY)))
1139 submit_bio(&rbio->bio);
1140 else
1141 submit_bio_wait(&rbio->bio);
1142 }
1143
1144 /*
1145 * We just submitted IO which may block, we expect relock fail
1146 * events and shouldn't count them:
1147 */
1148 trans->notrace_relock_fail = true;
1149 } else {
1150 /* Attempting reconstruct read: */
1151 if (bch2_ec_read_extent(trans, rbio, k)) {
1152 bch2_rbio_error(rbio, READ_RETRY_AVOID, BLK_STS_IOERR);
1153 goto out;
1154 }
1155
1156 if (likely(!(flags & BCH_READ_IN_RETRY)))
1157 bio_endio(&rbio->bio);
1158 }
1159 out:
1160 if (likely(!(flags & BCH_READ_IN_RETRY))) {
1161 return 0;
1162 } else {
1163 int ret;
1164
1165 rbio->context = RBIO_CONTEXT_UNBOUND;
1166 bch2_read_endio(&rbio->bio);
1167
1168 ret = rbio->retry;
1169 rbio = bch2_rbio_free(rbio);
1170
1171 if (ret == READ_RETRY_AVOID) {
1172 bch2_mark_io_failure(failed, &pick);
1173 ret = READ_RETRY;
1174 }
1175
1176 if (!ret)
1177 goto out_read_done;
1178
1179 return ret;
1180 }
1181
1182 err:
1183 if (flags & BCH_READ_IN_RETRY)
1184 return READ_ERR;
1185
1186 orig->bio.bi_status = BLK_STS_IOERR;
1187 goto out_read_done;
1188
1189 hole:
1190 /*
1191 * won't normally happen in the BCH_READ_NODECODE
1192 * (bch2_move_extent()) path, but if we retry and the extent we wanted
1193 * to read no longer exists we have to signal that:
1194 */
1195 if (flags & BCH_READ_NODECODE)
1196 orig->hole = true;
1197
1198 zero_fill_bio_iter(&orig->bio, iter);
1199 out_read_done:
1200 if (flags & BCH_READ_LAST_FRAGMENT)
1201 bch2_rbio_done(orig);
1202 return 0;
1203 }
1204
__bch2_read(struct bch_fs * c,struct bch_read_bio * rbio,struct bvec_iter bvec_iter,subvol_inum inum,struct bch_io_failures * failed,unsigned flags)1205 void __bch2_read(struct bch_fs *c, struct bch_read_bio *rbio,
1206 struct bvec_iter bvec_iter, subvol_inum inum,
1207 struct bch_io_failures *failed, unsigned flags)
1208 {
1209 struct btree_trans *trans = bch2_trans_get(c);
1210 struct btree_iter iter;
1211 struct bkey_buf sk;
1212 struct bkey_s_c k;
1213 int ret;
1214
1215 BUG_ON(flags & BCH_READ_NODECODE);
1216
1217 bch2_bkey_buf_init(&sk);
1218 bch2_trans_iter_init(trans, &iter, BTREE_ID_extents,
1219 POS(inum.inum, bvec_iter.bi_sector),
1220 BTREE_ITER_slots);
1221
1222 while (1) {
1223 enum btree_id data_btree = BTREE_ID_extents;
1224
1225 bch2_trans_begin(trans);
1226
1227 u32 snapshot;
1228 ret = bch2_subvolume_get_snapshot(trans, inum.subvol, &snapshot);
1229 if (ret)
1230 goto err;
1231
1232 bch2_btree_iter_set_snapshot(&iter, snapshot);
1233
1234 bch2_btree_iter_set_pos(&iter,
1235 POS(inum.inum, bvec_iter.bi_sector));
1236
1237 k = bch2_btree_iter_peek_slot(&iter);
1238 ret = bkey_err(k);
1239 if (ret)
1240 goto err;
1241
1242 s64 offset_into_extent = iter.pos.offset -
1243 bkey_start_offset(k.k);
1244 unsigned sectors = k.k->size - offset_into_extent;
1245
1246 bch2_bkey_buf_reassemble(&sk, c, k);
1247
1248 ret = bch2_read_indirect_extent(trans, &data_btree,
1249 &offset_into_extent, &sk);
1250 if (ret)
1251 goto err;
1252
1253 k = bkey_i_to_s_c(sk.k);
1254
1255 /*
1256 * With indirect extents, the amount of data to read is the min
1257 * of the original extent and the indirect extent:
1258 */
1259 sectors = min_t(unsigned, sectors, k.k->size - offset_into_extent);
1260
1261 unsigned bytes = min(sectors, bvec_iter_sectors(bvec_iter)) << 9;
1262 swap(bvec_iter.bi_size, bytes);
1263
1264 if (bvec_iter.bi_size == bytes)
1265 flags |= BCH_READ_LAST_FRAGMENT;
1266
1267 ret = __bch2_read_extent(trans, rbio, bvec_iter, iter.pos,
1268 data_btree, k,
1269 offset_into_extent, failed, flags);
1270 if (ret)
1271 goto err;
1272
1273 if (flags & BCH_READ_LAST_FRAGMENT)
1274 break;
1275
1276 swap(bvec_iter.bi_size, bytes);
1277 bio_advance_iter(&rbio->bio, &bvec_iter, bytes);
1278 err:
1279 if (ret &&
1280 !bch2_err_matches(ret, BCH_ERR_transaction_restart) &&
1281 ret != READ_RETRY &&
1282 ret != READ_RETRY_AVOID)
1283 break;
1284 }
1285
1286 bch2_trans_iter_exit(trans, &iter);
1287
1288 if (ret) {
1289 struct printbuf buf = PRINTBUF;
1290 bch2_inum_offset_err_msg_trans(trans, &buf, inum, bvec_iter.bi_sector << 9);
1291 prt_printf(&buf, "read error %i from btree lookup", ret);
1292 bch_err_ratelimited(c, "%s", buf.buf);
1293 printbuf_exit(&buf);
1294
1295 rbio->bio.bi_status = BLK_STS_IOERR;
1296 bch2_rbio_done(rbio);
1297 }
1298
1299 bch2_trans_put(trans);
1300 bch2_bkey_buf_exit(&sk, c);
1301 }
1302
bch2_fs_io_read_exit(struct bch_fs * c)1303 void bch2_fs_io_read_exit(struct bch_fs *c)
1304 {
1305 if (c->promote_table.tbl)
1306 rhashtable_destroy(&c->promote_table);
1307 bioset_exit(&c->bio_read_split);
1308 bioset_exit(&c->bio_read);
1309 }
1310
bch2_fs_io_read_init(struct bch_fs * c)1311 int bch2_fs_io_read_init(struct bch_fs *c)
1312 {
1313 if (bioset_init(&c->bio_read, 1, offsetof(struct bch_read_bio, bio),
1314 BIOSET_NEED_BVECS))
1315 return -BCH_ERR_ENOMEM_bio_read_init;
1316
1317 if (bioset_init(&c->bio_read_split, 1, offsetof(struct bch_read_bio, bio),
1318 BIOSET_NEED_BVECS))
1319 return -BCH_ERR_ENOMEM_bio_read_split_init;
1320
1321 if (rhashtable_init(&c->promote_table, &bch_promote_params))
1322 return -BCH_ERR_ENOMEM_promote_table_init;
1323
1324 return 0;
1325 }
1326