1 // SPDX-License-Identifier: GPL-2.0
2
3 #include "bcachefs.h"
4 #include "alloc_foreground.h"
5 #include "btree_gc.h"
6 #include "btree_io.h"
7 #include "btree_iter.h"
8 #include "btree_journal_iter.h"
9 #include "btree_key_cache.h"
10 #include "btree_update_interior.h"
11 #include "btree_write_buffer.h"
12 #include "buckets.h"
13 #include "disk_accounting.h"
14 #include "errcode.h"
15 #include "error.h"
16 #include "journal.h"
17 #include "journal_io.h"
18 #include "journal_reclaim.h"
19 #include "replicas.h"
20 #include "snapshot.h"
21
22 #include <linux/prefetch.h>
23
24 static const char * const trans_commit_flags_strs[] = {
25 #define x(n, ...) #n,
26 BCH_TRANS_COMMIT_FLAGS()
27 #undef x
28 NULL
29 };
30
bch2_trans_commit_flags_to_text(struct printbuf * out,enum bch_trans_commit_flags flags)31 void bch2_trans_commit_flags_to_text(struct printbuf *out, enum bch_trans_commit_flags flags)
32 {
33 enum bch_watermark watermark = flags & BCH_WATERMARK_MASK;
34
35 prt_printf(out, "watermark=%s", bch2_watermarks[watermark]);
36
37 flags >>= BCH_WATERMARK_BITS;
38 if (flags) {
39 prt_char(out, ' ');
40 bch2_prt_bitflags(out, trans_commit_flags_strs, flags);
41 }
42 }
43
verify_update_old_key(struct btree_trans * trans,struct btree_insert_entry * i)44 static void verify_update_old_key(struct btree_trans *trans, struct btree_insert_entry *i)
45 {
46 #ifdef CONFIG_BCACHEFS_DEBUG
47 struct bch_fs *c = trans->c;
48 struct bkey u;
49 struct bkey_s_c k = bch2_btree_path_peek_slot_exact(trans->paths + i->path, &u);
50
51 if (unlikely(trans->journal_replay_not_finished)) {
52 struct bkey_i *j_k =
53 bch2_journal_keys_peek_slot(c, i->btree_id, i->level, i->k->k.p);
54
55 if (j_k)
56 k = bkey_i_to_s_c(j_k);
57 }
58
59 u = *k.k;
60 u.needs_whiteout = i->old_k.needs_whiteout;
61
62 BUG_ON(memcmp(&i->old_k, &u, sizeof(struct bkey)));
63 BUG_ON(i->old_v != k.v);
64 #endif
65 }
66
insert_l(struct btree_trans * trans,struct btree_insert_entry * i)67 static inline struct btree_path_level *insert_l(struct btree_trans *trans, struct btree_insert_entry *i)
68 {
69 return (trans->paths + i->path)->l + i->level;
70 }
71
same_leaf_as_prev(struct btree_trans * trans,struct btree_insert_entry * i)72 static inline bool same_leaf_as_prev(struct btree_trans *trans,
73 struct btree_insert_entry *i)
74 {
75 return i != trans->updates &&
76 insert_l(trans, &i[0])->b == insert_l(trans, &i[-1])->b;
77 }
78
same_leaf_as_next(struct btree_trans * trans,struct btree_insert_entry * i)79 static inline bool same_leaf_as_next(struct btree_trans *trans,
80 struct btree_insert_entry *i)
81 {
82 return i + 1 < trans->updates + trans->nr_updates &&
83 insert_l(trans, &i[0])->b == insert_l(trans, &i[1])->b;
84 }
85
bch2_btree_node_prep_for_write(struct btree_trans * trans,struct btree_path * path,struct btree * b)86 inline void bch2_btree_node_prep_for_write(struct btree_trans *trans,
87 struct btree_path *path,
88 struct btree *b)
89 {
90 struct bch_fs *c = trans->c;
91
92 if (unlikely(btree_node_just_written(b)) &&
93 bch2_btree_post_write_cleanup(c, b))
94 bch2_trans_node_reinit_iter(trans, b);
95
96 /*
97 * If the last bset has been written, or if it's gotten too big - start
98 * a new bset to insert into:
99 */
100 if (want_new_bset(c, b))
101 bch2_btree_init_next(trans, b);
102 }
103
trans_lock_write_fail(struct btree_trans * trans,struct btree_insert_entry * i)104 static noinline int trans_lock_write_fail(struct btree_trans *trans, struct btree_insert_entry *i)
105 {
106 while (--i >= trans->updates) {
107 if (same_leaf_as_prev(trans, i))
108 continue;
109
110 bch2_btree_node_unlock_write(trans, trans->paths + i->path, insert_l(trans, i)->b);
111 }
112
113 trace_and_count(trans->c, trans_restart_would_deadlock_write, trans);
114 return btree_trans_restart(trans, BCH_ERR_transaction_restart_would_deadlock_write);
115 }
116
bch2_trans_lock_write(struct btree_trans * trans)117 static inline int bch2_trans_lock_write(struct btree_trans *trans)
118 {
119 EBUG_ON(trans->write_locked);
120
121 trans_for_each_update(trans, i) {
122 if (same_leaf_as_prev(trans, i))
123 continue;
124
125 if (bch2_btree_node_lock_write(trans, trans->paths + i->path, &insert_l(trans, i)->b->c))
126 return trans_lock_write_fail(trans, i);
127
128 if (!i->cached)
129 bch2_btree_node_prep_for_write(trans, trans->paths + i->path, insert_l(trans, i)->b);
130 }
131
132 trans->write_locked = true;
133 return 0;
134 }
135
bch2_trans_unlock_updates_write(struct btree_trans * trans)136 static inline void bch2_trans_unlock_updates_write(struct btree_trans *trans)
137 {
138 if (likely(trans->write_locked)) {
139 trans_for_each_update(trans, i)
140 if (btree_node_locked_type(trans->paths + i->path, i->level) ==
141 BTREE_NODE_WRITE_LOCKED)
142 bch2_btree_node_unlock_write_inlined(trans,
143 trans->paths + i->path, insert_l(trans, i)->b);
144 trans->write_locked = false;
145 }
146 }
147
148 /* Inserting into a given leaf node (last stage of insert): */
149
150 /* Handle overwrites and do insert, for non extents: */
bch2_btree_bset_insert_key(struct btree_trans * trans,struct btree_path * path,struct btree * b,struct btree_node_iter * node_iter,struct bkey_i * insert)151 bool bch2_btree_bset_insert_key(struct btree_trans *trans,
152 struct btree_path *path,
153 struct btree *b,
154 struct btree_node_iter *node_iter,
155 struct bkey_i *insert)
156 {
157 struct bkey_packed *k;
158 unsigned clobber_u64s = 0, new_u64s = 0;
159
160 EBUG_ON(btree_node_just_written(b));
161 EBUG_ON(bset_written(b, btree_bset_last(b)));
162 EBUG_ON(bkey_deleted(&insert->k) && bkey_val_u64s(&insert->k));
163 EBUG_ON(bpos_lt(insert->k.p, b->data->min_key));
164 EBUG_ON(bpos_gt(insert->k.p, b->data->max_key));
165 EBUG_ON(insert->k.u64s > bch2_btree_keys_u64s_remaining(b));
166 EBUG_ON(!b->c.level && !bpos_eq(insert->k.p, path->pos));
167
168 k = bch2_btree_node_iter_peek_all(node_iter, b);
169 if (k && bkey_cmp_left_packed(b, k, &insert->k.p))
170 k = NULL;
171
172 /* @k is the key being overwritten/deleted, if any: */
173 EBUG_ON(k && bkey_deleted(k));
174
175 /* Deleting, but not found? nothing to do: */
176 if (bkey_deleted(&insert->k) && !k)
177 return false;
178
179 if (bkey_deleted(&insert->k)) {
180 /* Deleting: */
181 btree_account_key_drop(b, k);
182 k->type = KEY_TYPE_deleted;
183
184 if (k->needs_whiteout)
185 push_whiteout(b, insert->k.p);
186 k->needs_whiteout = false;
187
188 if (k >= btree_bset_last(b)->start) {
189 clobber_u64s = k->u64s;
190 bch2_bset_delete(b, k, clobber_u64s);
191 goto fix_iter;
192 } else {
193 bch2_btree_path_fix_key_modified(trans, b, k);
194 }
195
196 return true;
197 }
198
199 if (k) {
200 /* Overwriting: */
201 btree_account_key_drop(b, k);
202 k->type = KEY_TYPE_deleted;
203
204 insert->k.needs_whiteout = k->needs_whiteout;
205 k->needs_whiteout = false;
206
207 if (k >= btree_bset_last(b)->start) {
208 clobber_u64s = k->u64s;
209 goto overwrite;
210 } else {
211 bch2_btree_path_fix_key_modified(trans, b, k);
212 }
213 }
214
215 k = bch2_btree_node_iter_bset_pos(node_iter, b, bset_tree_last(b));
216 overwrite:
217 bch2_bset_insert(b, k, insert, clobber_u64s);
218 new_u64s = k->u64s;
219 fix_iter:
220 if (clobber_u64s != new_u64s)
221 bch2_btree_node_iter_fix(trans, path, b, node_iter, k,
222 clobber_u64s, new_u64s);
223 return true;
224 }
225
__btree_node_flush(struct journal * j,struct journal_entry_pin * pin,unsigned i,u64 seq)226 static int __btree_node_flush(struct journal *j, struct journal_entry_pin *pin,
227 unsigned i, u64 seq)
228 {
229 struct bch_fs *c = container_of(j, struct bch_fs, journal);
230 struct btree_write *w = container_of(pin, struct btree_write, journal);
231 struct btree *b = container_of(w, struct btree, writes[i]);
232 struct btree_trans *trans = bch2_trans_get(c);
233 unsigned long old, new;
234 unsigned idx = w - b->writes;
235
236 btree_node_lock_nopath_nofail(trans, &b->c, SIX_LOCK_read);
237
238 old = READ_ONCE(b->flags);
239 do {
240 new = old;
241
242 if (!(old & (1 << BTREE_NODE_dirty)) ||
243 !!(old & (1 << BTREE_NODE_write_idx)) != idx ||
244 w->journal.seq != seq)
245 break;
246
247 new &= ~BTREE_WRITE_TYPE_MASK;
248 new |= BTREE_WRITE_journal_reclaim;
249 new |= 1 << BTREE_NODE_need_write;
250 } while (!try_cmpxchg(&b->flags, &old, new));
251
252 btree_node_write_if_need(trans, b, SIX_LOCK_read);
253 six_unlock_read(&b->c.lock);
254
255 bch2_trans_put(trans);
256 return 0;
257 }
258
bch2_btree_node_flush0(struct journal * j,struct journal_entry_pin * pin,u64 seq)259 int bch2_btree_node_flush0(struct journal *j, struct journal_entry_pin *pin, u64 seq)
260 {
261 return __btree_node_flush(j, pin, 0, seq);
262 }
263
bch2_btree_node_flush1(struct journal * j,struct journal_entry_pin * pin,u64 seq)264 int bch2_btree_node_flush1(struct journal *j, struct journal_entry_pin *pin, u64 seq)
265 {
266 return __btree_node_flush(j, pin, 1, seq);
267 }
268
bch2_btree_add_journal_pin(struct bch_fs * c,struct btree * b,u64 seq)269 inline void bch2_btree_add_journal_pin(struct bch_fs *c,
270 struct btree *b, u64 seq)
271 {
272 struct btree_write *w = btree_current_write(b);
273
274 bch2_journal_pin_add(&c->journal, seq, &w->journal,
275 btree_node_write_idx(b) == 0
276 ? bch2_btree_node_flush0
277 : bch2_btree_node_flush1);
278 }
279
280 /**
281 * bch2_btree_insert_key_leaf() - insert a key one key into a leaf node
282 * @trans: btree transaction object
283 * @path: path pointing to @insert's pos
284 * @insert: key to insert
285 * @journal_seq: sequence number of journal reservation
286 */
bch2_btree_insert_key_leaf(struct btree_trans * trans,struct btree_path * path,struct bkey_i * insert,u64 journal_seq)287 inline void bch2_btree_insert_key_leaf(struct btree_trans *trans,
288 struct btree_path *path,
289 struct bkey_i *insert,
290 u64 journal_seq)
291 {
292 struct bch_fs *c = trans->c;
293 struct btree *b = path_l(path)->b;
294 struct bset_tree *t = bset_tree_last(b);
295 struct bset *i = bset(b, t);
296 int old_u64s = bset_u64s(t);
297 int old_live_u64s = b->nr.live_u64s;
298 int live_u64s_added, u64s_added;
299
300 if (unlikely(!bch2_btree_bset_insert_key(trans, path, b,
301 &path_l(path)->iter, insert)))
302 return;
303
304 i->journal_seq = cpu_to_le64(max(journal_seq, le64_to_cpu(i->journal_seq)));
305
306 bch2_btree_add_journal_pin(c, b, journal_seq);
307
308 if (unlikely(!btree_node_dirty(b))) {
309 EBUG_ON(test_bit(BCH_FS_clean_shutdown, &c->flags));
310 set_btree_node_dirty_acct(c, b);
311 }
312
313 live_u64s_added = (int) b->nr.live_u64s - old_live_u64s;
314 u64s_added = (int) bset_u64s(t) - old_u64s;
315
316 if (b->sib_u64s[0] != U16_MAX && live_u64s_added < 0)
317 b->sib_u64s[0] = max(0, (int) b->sib_u64s[0] + live_u64s_added);
318 if (b->sib_u64s[1] != U16_MAX && live_u64s_added < 0)
319 b->sib_u64s[1] = max(0, (int) b->sib_u64s[1] + live_u64s_added);
320
321 if (u64s_added > live_u64s_added &&
322 bch2_maybe_compact_whiteouts(c, b))
323 bch2_trans_node_reinit_iter(trans, b);
324 }
325
326 /* Cached btree updates: */
327
328 /* Normal update interface: */
329
btree_insert_entry_checks(struct btree_trans * trans,struct btree_insert_entry * i)330 static inline void btree_insert_entry_checks(struct btree_trans *trans,
331 struct btree_insert_entry *i)
332 {
333 struct btree_path *path = trans->paths + i->path;
334
335 BUG_ON(!bpos_eq(i->k->k.p, path->pos));
336 BUG_ON(i->cached != path->cached);
337 BUG_ON(i->level != path->level);
338 BUG_ON(i->btree_id != path->btree_id);
339 EBUG_ON(!i->level &&
340 btree_type_has_snapshots(i->btree_id) &&
341 !(i->flags & BTREE_UPDATE_internal_snapshot_node) &&
342 test_bit(JOURNAL_replay_done, &trans->c->journal.flags) &&
343 i->k->k.p.snapshot &&
344 bch2_snapshot_is_internal_node(trans->c, i->k->k.p.snapshot) > 0);
345 }
346
bch2_trans_journal_res_get(struct btree_trans * trans,unsigned flags)347 static __always_inline int bch2_trans_journal_res_get(struct btree_trans *trans,
348 unsigned flags)
349 {
350 return bch2_journal_res_get(&trans->c->journal, &trans->journal_res,
351 trans->journal_u64s, flags, trans);
352 }
353
354 #define JSET_ENTRY_LOG_U64s 4
355
journal_transaction_name(struct btree_trans * trans)356 static noinline void journal_transaction_name(struct btree_trans *trans)
357 {
358 struct bch_fs *c = trans->c;
359 struct journal *j = &c->journal;
360 struct jset_entry *entry =
361 bch2_journal_add_entry(j, &trans->journal_res,
362 BCH_JSET_ENTRY_log, 0, 0,
363 JSET_ENTRY_LOG_U64s);
364 struct jset_entry_log *l =
365 container_of(entry, struct jset_entry_log, entry);
366
367 strncpy(l->d, trans->fn, JSET_ENTRY_LOG_U64s * sizeof(u64));
368 }
369
btree_key_can_insert(struct btree_trans * trans,struct btree * b,unsigned u64s)370 static inline int btree_key_can_insert(struct btree_trans *trans,
371 struct btree *b, unsigned u64s)
372 {
373 if (!bch2_btree_node_insert_fits(b, u64s))
374 return -BCH_ERR_btree_insert_btree_node_full;
375
376 return 0;
377 }
378
379 noinline static int
btree_key_can_insert_cached_slowpath(struct btree_trans * trans,unsigned flags,struct btree_path * path,unsigned new_u64s)380 btree_key_can_insert_cached_slowpath(struct btree_trans *trans, unsigned flags,
381 struct btree_path *path, unsigned new_u64s)
382 {
383 struct bkey_cached *ck = (void *) path->l[0].b;
384 struct bkey_i *new_k;
385 int ret;
386
387 bch2_trans_unlock_updates_write(trans);
388 bch2_trans_unlock(trans);
389
390 new_k = kmalloc(new_u64s * sizeof(u64), GFP_KERNEL);
391 if (!new_k) {
392 bch_err(trans->c, "error allocating memory for key cache key, btree %s u64s %u",
393 bch2_btree_id_str(path->btree_id), new_u64s);
394 return -BCH_ERR_ENOMEM_btree_key_cache_insert;
395 }
396
397 ret = bch2_trans_relock(trans) ?:
398 bch2_trans_lock_write(trans);
399 if (unlikely(ret)) {
400 kfree(new_k);
401 return ret;
402 }
403
404 memcpy(new_k, ck->k, ck->u64s * sizeof(u64));
405
406 trans_for_each_update(trans, i)
407 if (i->old_v == &ck->k->v)
408 i->old_v = &new_k->v;
409
410 kfree(ck->k);
411 ck->u64s = new_u64s;
412 ck->k = new_k;
413 return 0;
414 }
415
btree_key_can_insert_cached(struct btree_trans * trans,unsigned flags,struct btree_path * path,unsigned u64s)416 static int btree_key_can_insert_cached(struct btree_trans *trans, unsigned flags,
417 struct btree_path *path, unsigned u64s)
418 {
419 struct bch_fs *c = trans->c;
420 struct bkey_cached *ck = (void *) path->l[0].b;
421 unsigned new_u64s;
422 struct bkey_i *new_k;
423 unsigned watermark = flags & BCH_WATERMARK_MASK;
424
425 EBUG_ON(path->level);
426
427 if (watermark < BCH_WATERMARK_reclaim &&
428 !test_bit(BKEY_CACHED_DIRTY, &ck->flags) &&
429 bch2_btree_key_cache_must_wait(c))
430 return -BCH_ERR_btree_insert_need_journal_reclaim;
431
432 /*
433 * bch2_varint_decode can read past the end of the buffer by at most 7
434 * bytes (it won't be used):
435 */
436 u64s += 1;
437
438 if (u64s <= ck->u64s)
439 return 0;
440
441 new_u64s = roundup_pow_of_two(u64s);
442 new_k = krealloc(ck->k, new_u64s * sizeof(u64), GFP_NOWAIT|__GFP_NOWARN);
443 if (unlikely(!new_k))
444 return btree_key_can_insert_cached_slowpath(trans, flags, path, new_u64s);
445
446 trans_for_each_update(trans, i)
447 if (i->old_v == &ck->k->v)
448 i->old_v = &new_k->v;
449
450 ck->u64s = new_u64s;
451 ck->k = new_k;
452 return 0;
453 }
454
455 /* Triggers: */
456
run_one_mem_trigger(struct btree_trans * trans,struct btree_insert_entry * i,unsigned flags)457 static int run_one_mem_trigger(struct btree_trans *trans,
458 struct btree_insert_entry *i,
459 unsigned flags)
460 {
461 verify_update_old_key(trans, i);
462
463 if (unlikely(flags & BTREE_TRIGGER_norun))
464 return 0;
465
466 struct bkey_s_c old = { &i->old_k, i->old_v };
467 struct bkey_i *new = i->k;
468 const struct bkey_ops *old_ops = bch2_bkey_type_ops(old.k->type);
469 const struct bkey_ops *new_ops = bch2_bkey_type_ops(i->k->k.type);
470
471 if (old_ops->trigger == new_ops->trigger)
472 return bch2_key_trigger(trans, i->btree_id, i->level,
473 old, bkey_i_to_s(new),
474 BTREE_TRIGGER_insert|BTREE_TRIGGER_overwrite|flags);
475 else
476 return bch2_key_trigger_new(trans, i->btree_id, i->level,
477 bkey_i_to_s(new), flags) ?:
478 bch2_key_trigger_old(trans, i->btree_id, i->level,
479 old, flags);
480 }
481
run_one_trans_trigger(struct btree_trans * trans,struct btree_insert_entry * i)482 static int run_one_trans_trigger(struct btree_trans *trans, struct btree_insert_entry *i)
483 {
484 verify_update_old_key(trans, i);
485
486 if ((i->flags & BTREE_TRIGGER_norun) ||
487 !btree_node_type_has_trans_triggers(i->bkey_type))
488 return 0;
489
490 /*
491 * Transactional triggers create new btree_insert_entries, so we can't
492 * pass them a pointer to a btree_insert_entry, that memory is going to
493 * move:
494 */
495 struct bkey old_k = i->old_k;
496 struct bkey_s_c old = { &old_k, i->old_v };
497 const struct bkey_ops *old_ops = bch2_bkey_type_ops(old.k->type);
498 const struct bkey_ops *new_ops = bch2_bkey_type_ops(i->k->k.type);
499 unsigned flags = i->flags|BTREE_TRIGGER_transactional;
500
501 if (!i->insert_trigger_run &&
502 !i->overwrite_trigger_run &&
503 old_ops->trigger == new_ops->trigger) {
504 i->overwrite_trigger_run = true;
505 i->insert_trigger_run = true;
506 return bch2_key_trigger(trans, i->btree_id, i->level, old, bkey_i_to_s(i->k),
507 BTREE_TRIGGER_insert|
508 BTREE_TRIGGER_overwrite|flags) ?: 1;
509 } else if (!i->overwrite_trigger_run) {
510 i->overwrite_trigger_run = true;
511 return bch2_key_trigger_old(trans, i->btree_id, i->level, old, flags) ?: 1;
512 } else if (!i->insert_trigger_run) {
513 i->insert_trigger_run = true;
514 return bch2_key_trigger_new(trans, i->btree_id, i->level, bkey_i_to_s(i->k), flags) ?: 1;
515 } else {
516 return 0;
517 }
518 }
519
run_btree_triggers(struct btree_trans * trans,enum btree_id btree_id,unsigned * btree_id_updates_start)520 static int run_btree_triggers(struct btree_trans *trans, enum btree_id btree_id,
521 unsigned *btree_id_updates_start)
522 {
523 bool trans_trigger_run;
524
525 /*
526 * Running triggers will append more updates to the list of updates as
527 * we're walking it:
528 */
529 do {
530 trans_trigger_run = false;
531
532 for (unsigned i = *btree_id_updates_start;
533 i < trans->nr_updates && trans->updates[i].btree_id <= btree_id;
534 i++) {
535 if (trans->updates[i].btree_id < btree_id) {
536 *btree_id_updates_start = i;
537 continue;
538 }
539
540 int ret = run_one_trans_trigger(trans, trans->updates + i);
541 if (ret < 0)
542 return ret;
543 if (ret)
544 trans_trigger_run = true;
545 }
546 } while (trans_trigger_run);
547
548 trans_for_each_update(trans, i)
549 BUG_ON(!(i->flags & BTREE_TRIGGER_norun) &&
550 i->btree_id == btree_id &&
551 btree_node_type_has_trans_triggers(i->bkey_type) &&
552 (!i->insert_trigger_run || !i->overwrite_trigger_run));
553
554 return 0;
555 }
556
bch2_trans_commit_run_triggers(struct btree_trans * trans)557 static int bch2_trans_commit_run_triggers(struct btree_trans *trans)
558 {
559 unsigned btree_id = 0, btree_id_updates_start = 0;
560 int ret = 0;
561
562 /*
563 *
564 * For a given btree, this algorithm runs insert triggers before
565 * overwrite triggers: this is so that when extents are being moved
566 * (e.g. by FALLOCATE_FL_INSERT_RANGE), we don't drop references before
567 * they are re-added.
568 */
569 for (btree_id = 0; btree_id < BTREE_ID_NR; btree_id++) {
570 if (btree_id == BTREE_ID_alloc)
571 continue;
572
573 ret = run_btree_triggers(trans, btree_id, &btree_id_updates_start);
574 if (ret)
575 return ret;
576 }
577
578 btree_id_updates_start = 0;
579 ret = run_btree_triggers(trans, BTREE_ID_alloc, &btree_id_updates_start);
580 if (ret)
581 return ret;
582
583 #ifdef CONFIG_BCACHEFS_DEBUG
584 trans_for_each_update(trans, i)
585 BUG_ON(!(i->flags & BTREE_TRIGGER_norun) &&
586 btree_node_type_has_trans_triggers(i->bkey_type) &&
587 (!i->insert_trigger_run || !i->overwrite_trigger_run));
588 #endif
589 return 0;
590 }
591
bch2_trans_commit_run_gc_triggers(struct btree_trans * trans)592 static noinline int bch2_trans_commit_run_gc_triggers(struct btree_trans *trans)
593 {
594 trans_for_each_update(trans, i)
595 if (btree_node_type_has_triggers(i->bkey_type) &&
596 gc_visited(trans->c, gc_pos_btree(i->btree_id, i->level, i->k->k.p))) {
597 int ret = run_one_mem_trigger(trans, i, i->flags|BTREE_TRIGGER_gc);
598 if (ret)
599 return ret;
600 }
601
602 return 0;
603 }
604
605 static inline int
bch2_trans_commit_write_locked(struct btree_trans * trans,unsigned flags,struct btree_insert_entry ** stopped_at,unsigned long trace_ip)606 bch2_trans_commit_write_locked(struct btree_trans *trans, unsigned flags,
607 struct btree_insert_entry **stopped_at,
608 unsigned long trace_ip)
609 {
610 struct bch_fs *c = trans->c;
611 struct btree_trans_commit_hook *h;
612 unsigned u64s = 0;
613 int ret = 0;
614
615 bch2_trans_verify_not_unlocked_or_in_restart(trans);
616
617 if (race_fault()) {
618 trace_and_count(c, trans_restart_fault_inject, trans, trace_ip);
619 return btree_trans_restart(trans, BCH_ERR_transaction_restart_fault_inject);
620 }
621
622 /*
623 * Check if the insert will fit in the leaf node with the write lock
624 * held, otherwise another thread could write the node changing the
625 * amount of space available:
626 */
627
628 prefetch(&trans->c->journal.flags);
629
630 trans_for_each_update(trans, i) {
631 /* Multiple inserts might go to same leaf: */
632 if (!same_leaf_as_prev(trans, i))
633 u64s = 0;
634
635 u64s += i->k->k.u64s;
636 ret = !i->cached
637 ? btree_key_can_insert(trans, insert_l(trans, i)->b, u64s)
638 : btree_key_can_insert_cached(trans, flags, trans->paths + i->path, u64s);
639 if (ret) {
640 *stopped_at = i;
641 return ret;
642 }
643
644 i->k->k.needs_whiteout = false;
645 }
646
647 /*
648 * Don't get journal reservation until after we know insert will
649 * succeed:
650 */
651 if (likely(!(flags & BCH_TRANS_COMMIT_no_journal_res))) {
652 ret = bch2_trans_journal_res_get(trans,
653 (flags & BCH_WATERMARK_MASK)|
654 JOURNAL_RES_GET_NONBLOCK);
655 if (ret)
656 return ret;
657
658 if (unlikely(trans->journal_transaction_names))
659 journal_transaction_name(trans);
660 }
661
662 /*
663 * Not allowed to fail after we've gotten our journal reservation - we
664 * have to use it:
665 */
666
667 if (IS_ENABLED(CONFIG_BCACHEFS_DEBUG) &&
668 !(flags & BCH_TRANS_COMMIT_no_journal_res)) {
669 if (bch2_journal_seq_verify)
670 trans_for_each_update(trans, i)
671 i->k->k.bversion.lo = trans->journal_res.seq;
672 else if (bch2_inject_invalid_keys)
673 trans_for_each_update(trans, i)
674 i->k->k.bversion = MAX_VERSION;
675 }
676
677 h = trans->hooks;
678 while (h) {
679 ret = h->fn(trans, h);
680 if (ret)
681 return ret;
682 h = h->next;
683 }
684
685 struct jset_entry *entry = trans->journal_entries;
686
687 percpu_down_read(&c->mark_lock);
688 for (entry = trans->journal_entries;
689 entry != (void *) ((u64 *) trans->journal_entries + trans->journal_entries_u64s);
690 entry = vstruct_next(entry))
691 if (entry->type == BCH_JSET_ENTRY_write_buffer_keys &&
692 entry->start->k.type == KEY_TYPE_accounting) {
693 ret = bch2_accounting_trans_commit_hook(trans, bkey_i_to_accounting(entry->start), flags);
694 if (ret)
695 goto revert_fs_usage;
696 }
697 percpu_up_read(&c->mark_lock);
698
699 /* XXX: we only want to run this if deltas are nonzero */
700 bch2_trans_account_disk_usage_change(trans);
701
702 trans_for_each_update(trans, i)
703 if (btree_node_type_has_atomic_triggers(i->bkey_type)) {
704 ret = run_one_mem_trigger(trans, i, BTREE_TRIGGER_atomic|i->flags);
705 if (ret)
706 goto fatal_err;
707 }
708
709 if (unlikely(c->gc_pos.phase)) {
710 ret = bch2_trans_commit_run_gc_triggers(trans);
711 if (ret)
712 goto fatal_err;
713 }
714
715 struct bkey_validate_context validate_context = { .from = BKEY_VALIDATE_commit };
716
717 if (!(flags & BCH_TRANS_COMMIT_no_journal_res))
718 validate_context.flags = BCH_VALIDATE_write|BCH_VALIDATE_commit;
719
720 for (struct jset_entry *i = trans->journal_entries;
721 i != (void *) ((u64 *) trans->journal_entries + trans->journal_entries_u64s);
722 i = vstruct_next(i)) {
723 ret = bch2_journal_entry_validate(c, NULL, i,
724 bcachefs_metadata_version_current,
725 CPU_BIG_ENDIAN, validate_context);
726 if (unlikely(ret)) {
727 bch2_trans_inconsistent(trans, "invalid journal entry on insert from %s\n",
728 trans->fn);
729 goto fatal_err;
730 }
731 }
732
733 trans_for_each_update(trans, i) {
734 validate_context.level = i->level;
735 validate_context.btree = i->btree_id;
736
737 ret = bch2_bkey_validate(c, bkey_i_to_s_c(i->k), validate_context);
738 if (unlikely(ret)){
739 bch2_trans_inconsistent(trans, "invalid bkey on insert from %s -> %ps\n",
740 trans->fn, (void *) i->ip_allocated);
741 goto fatal_err;
742 }
743 btree_insert_entry_checks(trans, i);
744 }
745
746 if (likely(!(flags & BCH_TRANS_COMMIT_no_journal_res))) {
747 struct journal *j = &c->journal;
748 struct jset_entry *entry;
749
750 trans_for_each_update(trans, i) {
751 if (i->key_cache_already_flushed)
752 continue;
753
754 if (i->flags & BTREE_UPDATE_nojournal)
755 continue;
756
757 verify_update_old_key(trans, i);
758
759 if (trans->journal_transaction_names) {
760 entry = bch2_journal_add_entry(j, &trans->journal_res,
761 BCH_JSET_ENTRY_overwrite,
762 i->btree_id, i->level,
763 i->old_k.u64s);
764 bkey_reassemble((struct bkey_i *) entry->start,
765 (struct bkey_s_c) { &i->old_k, i->old_v });
766 }
767
768 entry = bch2_journal_add_entry(j, &trans->journal_res,
769 BCH_JSET_ENTRY_btree_keys,
770 i->btree_id, i->level,
771 i->k->k.u64s);
772 bkey_copy((struct bkey_i *) entry->start, i->k);
773 }
774
775 memcpy_u64s_small(journal_res_entry(&c->journal, &trans->journal_res),
776 trans->journal_entries,
777 trans->journal_entries_u64s);
778
779 trans->journal_res.offset += trans->journal_entries_u64s;
780 trans->journal_res.u64s -= trans->journal_entries_u64s;
781
782 if (trans->journal_seq)
783 *trans->journal_seq = trans->journal_res.seq;
784 }
785
786 trans_for_each_update(trans, i) {
787 struct btree_path *path = trans->paths + i->path;
788
789 if (!i->cached)
790 bch2_btree_insert_key_leaf(trans, path, i->k, trans->journal_res.seq);
791 else if (!i->key_cache_already_flushed)
792 bch2_btree_insert_key_cached(trans, flags, i);
793 else
794 bch2_btree_key_cache_drop(trans, path);
795 }
796
797 return 0;
798 fatal_err:
799 bch2_fs_fatal_error(c, "fatal error in transaction commit: %s", bch2_err_str(ret));
800 percpu_down_read(&c->mark_lock);
801 revert_fs_usage:
802 for (struct jset_entry *entry2 = trans->journal_entries;
803 entry2 != entry;
804 entry2 = vstruct_next(entry2))
805 if (entry2->type == BCH_JSET_ENTRY_write_buffer_keys &&
806 entry2->start->k.type == KEY_TYPE_accounting)
807 bch2_accounting_trans_commit_revert(trans,
808 bkey_i_to_accounting(entry2->start), flags);
809 percpu_up_read(&c->mark_lock);
810 return ret;
811 }
812
bch2_drop_overwrites_from_journal(struct btree_trans * trans)813 static noinline void bch2_drop_overwrites_from_journal(struct btree_trans *trans)
814 {
815 /*
816 * Accounting keys aren't deduped in the journal: we have to compare
817 * each individual update against what's in the btree to see if it has
818 * been applied yet, and accounting updates also don't overwrite,
819 * they're deltas that accumulate.
820 */
821 trans_for_each_update(trans, i)
822 if (i->k->k.type != KEY_TYPE_accounting)
823 bch2_journal_key_overwritten(trans->c, i->btree_id, i->level, i->k->k.p);
824 }
825
bch2_trans_commit_journal_pin_flush(struct journal * j,struct journal_entry_pin * _pin,u64 seq)826 static int bch2_trans_commit_journal_pin_flush(struct journal *j,
827 struct journal_entry_pin *_pin, u64 seq)
828 {
829 return 0;
830 }
831
832 /*
833 * Get journal reservation, take write locks, and attempt to do btree update(s):
834 */
do_bch2_trans_commit(struct btree_trans * trans,unsigned flags,struct btree_insert_entry ** stopped_at,unsigned long trace_ip)835 static inline int do_bch2_trans_commit(struct btree_trans *trans, unsigned flags,
836 struct btree_insert_entry **stopped_at,
837 unsigned long trace_ip)
838 {
839 struct bch_fs *c = trans->c;
840 int ret = 0, u64s_delta = 0;
841
842 for (unsigned idx = 0; idx < trans->nr_updates; idx++) {
843 struct btree_insert_entry *i = trans->updates + idx;
844 if (i->cached)
845 continue;
846
847 u64s_delta += !bkey_deleted(&i->k->k) ? i->k->k.u64s : 0;
848 u64s_delta -= i->old_btree_u64s;
849
850 if (!same_leaf_as_next(trans, i)) {
851 if (u64s_delta <= 0) {
852 ret = bch2_foreground_maybe_merge(trans, i->path,
853 i->level, flags);
854 if (unlikely(ret))
855 return ret;
856 }
857
858 u64s_delta = 0;
859 }
860 }
861
862 ret = bch2_trans_lock_write(trans);
863 if (unlikely(ret))
864 return ret;
865
866 ret = bch2_trans_commit_write_locked(trans, flags, stopped_at, trace_ip);
867
868 if (!ret && unlikely(trans->journal_replay_not_finished))
869 bch2_drop_overwrites_from_journal(trans);
870
871 bch2_trans_unlock_updates_write(trans);
872
873 if (!ret && trans->journal_pin)
874 bch2_journal_pin_add(&c->journal, trans->journal_res.seq,
875 trans->journal_pin,
876 bch2_trans_commit_journal_pin_flush);
877
878 /*
879 * Drop journal reservation after dropping write locks, since dropping
880 * the journal reservation may kick off a journal write:
881 */
882 if (likely(!(flags & BCH_TRANS_COMMIT_no_journal_res)))
883 bch2_journal_res_put(&c->journal, &trans->journal_res);
884
885 return ret;
886 }
887
journal_reclaim_wait_done(struct bch_fs * c)888 static int journal_reclaim_wait_done(struct bch_fs *c)
889 {
890 int ret = bch2_journal_error(&c->journal) ?:
891 bch2_btree_key_cache_wait_done(c);
892
893 if (!ret)
894 journal_reclaim_kick(&c->journal);
895 return ret;
896 }
897
898 static noinline
bch2_trans_commit_error(struct btree_trans * trans,unsigned flags,struct btree_insert_entry * i,int ret,unsigned long trace_ip)899 int bch2_trans_commit_error(struct btree_trans *trans, unsigned flags,
900 struct btree_insert_entry *i,
901 int ret, unsigned long trace_ip)
902 {
903 struct bch_fs *c = trans->c;
904 enum bch_watermark watermark = flags & BCH_WATERMARK_MASK;
905
906 switch (ret) {
907 case -BCH_ERR_btree_insert_btree_node_full:
908 ret = bch2_btree_split_leaf(trans, i->path, flags);
909 if (bch2_err_matches(ret, BCH_ERR_transaction_restart))
910 trace_and_count(c, trans_restart_btree_node_split, trans,
911 trace_ip, trans->paths + i->path);
912 break;
913 case -BCH_ERR_btree_insert_need_mark_replicas:
914 ret = drop_locks_do(trans,
915 bch2_accounting_update_sb(trans));
916 break;
917 case -BCH_ERR_journal_res_get_blocked:
918 /*
919 * XXX: this should probably be a separate BTREE_INSERT_NONBLOCK
920 * flag
921 */
922 if ((flags & BCH_TRANS_COMMIT_journal_reclaim) &&
923 watermark < BCH_WATERMARK_reclaim) {
924 ret = -BCH_ERR_journal_reclaim_would_deadlock;
925 break;
926 }
927
928 ret = drop_locks_do(trans,
929 bch2_trans_journal_res_get(trans,
930 (flags & BCH_WATERMARK_MASK)|
931 JOURNAL_RES_GET_CHECK));
932 break;
933 case -BCH_ERR_btree_insert_need_journal_reclaim:
934 bch2_trans_unlock(trans);
935
936 trace_and_count(c, trans_blocked_journal_reclaim, trans, trace_ip);
937 track_event_change(&c->times[BCH_TIME_blocked_key_cache_flush], true);
938
939 wait_event_freezable(c->journal.reclaim_wait,
940 (ret = journal_reclaim_wait_done(c)));
941
942 track_event_change(&c->times[BCH_TIME_blocked_key_cache_flush], false);
943
944 if (ret < 0)
945 break;
946
947 ret = bch2_trans_relock(trans);
948 break;
949 default:
950 BUG_ON(ret >= 0);
951 break;
952 }
953
954 BUG_ON(bch2_err_matches(ret, BCH_ERR_transaction_restart) != !!trans->restarted);
955
956 bch2_fs_inconsistent_on(bch2_err_matches(ret, ENOSPC) &&
957 (flags & BCH_TRANS_COMMIT_no_enospc), c,
958 "%s: incorrectly got %s\n", __func__, bch2_err_str(ret));
959
960 return ret;
961 }
962
963 /*
964 * This is for updates done in the early part of fsck - btree_gc - before we've
965 * gone RW. we only add the new key to the list of keys for journal replay to
966 * do.
967 */
968 static noinline int
do_bch2_trans_commit_to_journal_replay(struct btree_trans * trans)969 do_bch2_trans_commit_to_journal_replay(struct btree_trans *trans)
970 {
971 struct bch_fs *c = trans->c;
972
973 BUG_ON(current != c->recovery_task);
974
975 trans_for_each_update(trans, i) {
976 int ret = bch2_journal_key_insert(c, i->btree_id, i->level, i->k);
977 if (ret)
978 return ret;
979 }
980
981 for (struct jset_entry *i = trans->journal_entries;
982 i != (void *) ((u64 *) trans->journal_entries + trans->journal_entries_u64s);
983 i = vstruct_next(i))
984 if (i->type == BCH_JSET_ENTRY_btree_keys ||
985 i->type == BCH_JSET_ENTRY_write_buffer_keys) {
986 int ret = bch2_journal_key_insert(c, i->btree_id, i->level, i->start);
987 if (ret)
988 return ret;
989 }
990
991 return 0;
992 }
993
__bch2_trans_commit(struct btree_trans * trans,unsigned flags)994 int __bch2_trans_commit(struct btree_trans *trans, unsigned flags)
995 {
996 struct btree_insert_entry *errored_at = NULL;
997 struct bch_fs *c = trans->c;
998 int ret = 0;
999
1000 bch2_trans_verify_not_unlocked_or_in_restart(trans);
1001
1002 ret = trans_maybe_inject_restart(trans, _RET_IP_);
1003 if (unlikely(ret))
1004 goto out_reset;
1005
1006 if (!trans->nr_updates &&
1007 !trans->journal_entries_u64s)
1008 goto out_reset;
1009
1010 ret = bch2_trans_commit_run_triggers(trans);
1011 if (ret)
1012 goto out_reset;
1013
1014 if (!(flags & BCH_TRANS_COMMIT_no_check_rw) &&
1015 unlikely(!bch2_write_ref_tryget(c, BCH_WRITE_REF_trans))) {
1016 if (unlikely(!test_bit(BCH_FS_may_go_rw, &c->flags)))
1017 ret = do_bch2_trans_commit_to_journal_replay(trans);
1018 else
1019 ret = -BCH_ERR_erofs_trans_commit;
1020 goto out_reset;
1021 }
1022
1023 EBUG_ON(test_bit(BCH_FS_clean_shutdown, &c->flags));
1024
1025 trans->journal_u64s = trans->journal_entries_u64s;
1026 trans->journal_transaction_names = READ_ONCE(c->opts.journal_transaction_names);
1027 if (trans->journal_transaction_names)
1028 trans->journal_u64s += jset_u64s(JSET_ENTRY_LOG_U64s);
1029
1030 trans_for_each_update(trans, i) {
1031 struct btree_path *path = trans->paths + i->path;
1032
1033 EBUG_ON(!path->should_be_locked);
1034
1035 ret = bch2_btree_path_upgrade(trans, path, i->level + 1);
1036 if (unlikely(ret))
1037 goto out;
1038
1039 EBUG_ON(!btree_node_intent_locked(path, i->level));
1040
1041 if (i->key_cache_already_flushed)
1042 continue;
1043
1044 if (i->flags & BTREE_UPDATE_nojournal)
1045 continue;
1046
1047 /* we're going to journal the key being updated: */
1048 trans->journal_u64s += jset_u64s(i->k->k.u64s);
1049
1050 /* and we're also going to log the overwrite: */
1051 if (trans->journal_transaction_names)
1052 trans->journal_u64s += jset_u64s(i->old_k.u64s);
1053 }
1054
1055 if (trans->extra_disk_res) {
1056 ret = bch2_disk_reservation_add(c, trans->disk_res,
1057 trans->extra_disk_res,
1058 (flags & BCH_TRANS_COMMIT_no_enospc)
1059 ? BCH_DISK_RESERVATION_NOFAIL : 0);
1060 if (ret)
1061 goto err;
1062 }
1063 retry:
1064 errored_at = NULL;
1065 bch2_trans_verify_not_unlocked_or_in_restart(trans);
1066 if (likely(!(flags & BCH_TRANS_COMMIT_no_journal_res)))
1067 memset(&trans->journal_res, 0, sizeof(trans->journal_res));
1068 memset(&trans->fs_usage_delta, 0, sizeof(trans->fs_usage_delta));
1069
1070 ret = do_bch2_trans_commit(trans, flags, &errored_at, _RET_IP_);
1071
1072 /* make sure we didn't drop or screw up locks: */
1073 bch2_trans_verify_locks(trans);
1074
1075 if (ret)
1076 goto err;
1077
1078 trace_and_count(c, transaction_commit, trans, _RET_IP_);
1079 out:
1080 if (likely(!(flags & BCH_TRANS_COMMIT_no_check_rw)))
1081 bch2_write_ref_put(c, BCH_WRITE_REF_trans);
1082 out_reset:
1083 if (!ret)
1084 bch2_trans_downgrade(trans);
1085 bch2_trans_reset_updates(trans);
1086
1087 return ret;
1088 err:
1089 ret = bch2_trans_commit_error(trans, flags, errored_at, ret, _RET_IP_);
1090 if (ret)
1091 goto out;
1092
1093 /*
1094 * We might have done another transaction commit in the error path -
1095 * i.e. btree write buffer flush - which will have made use of
1096 * trans->journal_res, but with BCH_TRANS_COMMIT_no_journal_res that is
1097 * how the journal sequence number to pin is passed in - so we must
1098 * restart:
1099 */
1100 if (flags & BCH_TRANS_COMMIT_no_journal_res) {
1101 ret = -BCH_ERR_transaction_restart_nested;
1102 goto out;
1103 }
1104
1105 goto retry;
1106 }
1107