xref: /linux/fs/bcachefs/btree_trans_commit.c (revision 1854c7f79dcaaba9f1c0b131445ace03f9fd532d)
1 // SPDX-License-Identifier: GPL-2.0
2 
3 #include "bcachefs.h"
4 #include "alloc_foreground.h"
5 #include "btree_gc.h"
6 #include "btree_io.h"
7 #include "btree_iter.h"
8 #include "btree_journal_iter.h"
9 #include "btree_key_cache.h"
10 #include "btree_update_interior.h"
11 #include "btree_write_buffer.h"
12 #include "buckets.h"
13 #include "disk_accounting.h"
14 #include "errcode.h"
15 #include "error.h"
16 #include "journal.h"
17 #include "journal_io.h"
18 #include "journal_reclaim.h"
19 #include "replicas.h"
20 #include "snapshot.h"
21 
22 #include <linux/prefetch.h>
23 
24 static const char * const trans_commit_flags_strs[] = {
25 #define x(n, ...) #n,
26 	BCH_TRANS_COMMIT_FLAGS()
27 #undef x
28 	NULL
29 };
30 
bch2_trans_commit_flags_to_text(struct printbuf * out,enum bch_trans_commit_flags flags)31 void bch2_trans_commit_flags_to_text(struct printbuf *out, enum bch_trans_commit_flags flags)
32 {
33 	enum bch_watermark watermark = flags & BCH_WATERMARK_MASK;
34 
35 	prt_printf(out, "watermark=%s", bch2_watermarks[watermark]);
36 
37 	flags >>= BCH_WATERMARK_BITS;
38 	if (flags) {
39 		prt_char(out, ' ');
40 		bch2_prt_bitflags(out, trans_commit_flags_strs, flags);
41 	}
42 }
43 
verify_update_old_key(struct btree_trans * trans,struct btree_insert_entry * i)44 static void verify_update_old_key(struct btree_trans *trans, struct btree_insert_entry *i)
45 {
46 #ifdef CONFIG_BCACHEFS_DEBUG
47 	struct bch_fs *c = trans->c;
48 	struct bkey u;
49 	struct bkey_s_c k = bch2_btree_path_peek_slot_exact(trans->paths + i->path, &u);
50 
51 	if (unlikely(trans->journal_replay_not_finished)) {
52 		struct bkey_i *j_k =
53 			bch2_journal_keys_peek_slot(c, i->btree_id, i->level, i->k->k.p);
54 
55 		if (j_k)
56 			k = bkey_i_to_s_c(j_k);
57 	}
58 
59 	u = *k.k;
60 	u.needs_whiteout = i->old_k.needs_whiteout;
61 
62 	BUG_ON(memcmp(&i->old_k, &u, sizeof(struct bkey)));
63 	BUG_ON(i->old_v != k.v);
64 #endif
65 }
66 
insert_l(struct btree_trans * trans,struct btree_insert_entry * i)67 static inline struct btree_path_level *insert_l(struct btree_trans *trans, struct btree_insert_entry *i)
68 {
69 	return (trans->paths + i->path)->l + i->level;
70 }
71 
same_leaf_as_prev(struct btree_trans * trans,struct btree_insert_entry * i)72 static inline bool same_leaf_as_prev(struct btree_trans *trans,
73 				     struct btree_insert_entry *i)
74 {
75 	return i != trans->updates &&
76 		insert_l(trans, &i[0])->b == insert_l(trans, &i[-1])->b;
77 }
78 
same_leaf_as_next(struct btree_trans * trans,struct btree_insert_entry * i)79 static inline bool same_leaf_as_next(struct btree_trans *trans,
80 				     struct btree_insert_entry *i)
81 {
82 	return i + 1 < trans->updates + trans->nr_updates &&
83 		insert_l(trans, &i[0])->b == insert_l(trans, &i[1])->b;
84 }
85 
bch2_btree_node_prep_for_write(struct btree_trans * trans,struct btree_path * path,struct btree * b)86 inline void bch2_btree_node_prep_for_write(struct btree_trans *trans,
87 					   struct btree_path *path,
88 					   struct btree *b)
89 {
90 	struct bch_fs *c = trans->c;
91 
92 	if (unlikely(btree_node_just_written(b)) &&
93 	    bch2_btree_post_write_cleanup(c, b))
94 		bch2_trans_node_reinit_iter(trans, b);
95 
96 	/*
97 	 * If the last bset has been written, or if it's gotten too big - start
98 	 * a new bset to insert into:
99 	 */
100 	if (want_new_bset(c, b))
101 		bch2_btree_init_next(trans, b);
102 }
103 
trans_lock_write_fail(struct btree_trans * trans,struct btree_insert_entry * i)104 static noinline int trans_lock_write_fail(struct btree_trans *trans, struct btree_insert_entry *i)
105 {
106 	while (--i >= trans->updates) {
107 		if (same_leaf_as_prev(trans, i))
108 			continue;
109 
110 		bch2_btree_node_unlock_write(trans, trans->paths + i->path, insert_l(trans, i)->b);
111 	}
112 
113 	trace_and_count(trans->c, trans_restart_would_deadlock_write, trans);
114 	return btree_trans_restart(trans, BCH_ERR_transaction_restart_would_deadlock_write);
115 }
116 
bch2_trans_lock_write(struct btree_trans * trans)117 static inline int bch2_trans_lock_write(struct btree_trans *trans)
118 {
119 	EBUG_ON(trans->write_locked);
120 
121 	trans_for_each_update(trans, i) {
122 		if (same_leaf_as_prev(trans, i))
123 			continue;
124 
125 		if (bch2_btree_node_lock_write(trans, trans->paths + i->path, &insert_l(trans, i)->b->c))
126 			return trans_lock_write_fail(trans, i);
127 
128 		if (!i->cached)
129 			bch2_btree_node_prep_for_write(trans, trans->paths + i->path, insert_l(trans, i)->b);
130 	}
131 
132 	trans->write_locked = true;
133 	return 0;
134 }
135 
bch2_trans_unlock_updates_write(struct btree_trans * trans)136 static inline void bch2_trans_unlock_updates_write(struct btree_trans *trans)
137 {
138 	if (likely(trans->write_locked)) {
139 		trans_for_each_update(trans, i)
140 			if (btree_node_locked_type(trans->paths + i->path, i->level) ==
141 			    BTREE_NODE_WRITE_LOCKED)
142 				bch2_btree_node_unlock_write_inlined(trans,
143 						trans->paths + i->path, insert_l(trans, i)->b);
144 		trans->write_locked = false;
145 	}
146 }
147 
148 /* Inserting into a given leaf node (last stage of insert): */
149 
150 /* Handle overwrites and do insert, for non extents: */
bch2_btree_bset_insert_key(struct btree_trans * trans,struct btree_path * path,struct btree * b,struct btree_node_iter * node_iter,struct bkey_i * insert)151 bool bch2_btree_bset_insert_key(struct btree_trans *trans,
152 				struct btree_path *path,
153 				struct btree *b,
154 				struct btree_node_iter *node_iter,
155 				struct bkey_i *insert)
156 {
157 	struct bkey_packed *k;
158 	unsigned clobber_u64s = 0, new_u64s = 0;
159 
160 	EBUG_ON(btree_node_just_written(b));
161 	EBUG_ON(bset_written(b, btree_bset_last(b)));
162 	EBUG_ON(bkey_deleted(&insert->k) && bkey_val_u64s(&insert->k));
163 	EBUG_ON(bpos_lt(insert->k.p, b->data->min_key));
164 	EBUG_ON(bpos_gt(insert->k.p, b->data->max_key));
165 	EBUG_ON(insert->k.u64s > bch2_btree_keys_u64s_remaining(b));
166 	EBUG_ON(!b->c.level && !bpos_eq(insert->k.p, path->pos));
167 
168 	k = bch2_btree_node_iter_peek_all(node_iter, b);
169 	if (k && bkey_cmp_left_packed(b, k, &insert->k.p))
170 		k = NULL;
171 
172 	/* @k is the key being overwritten/deleted, if any: */
173 	EBUG_ON(k && bkey_deleted(k));
174 
175 	/* Deleting, but not found? nothing to do: */
176 	if (bkey_deleted(&insert->k) && !k)
177 		return false;
178 
179 	if (bkey_deleted(&insert->k)) {
180 		/* Deleting: */
181 		btree_account_key_drop(b, k);
182 		k->type = KEY_TYPE_deleted;
183 
184 		if (k->needs_whiteout)
185 			push_whiteout(b, insert->k.p);
186 		k->needs_whiteout = false;
187 
188 		if (k >= btree_bset_last(b)->start) {
189 			clobber_u64s = k->u64s;
190 			bch2_bset_delete(b, k, clobber_u64s);
191 			goto fix_iter;
192 		} else {
193 			bch2_btree_path_fix_key_modified(trans, b, k);
194 		}
195 
196 		return true;
197 	}
198 
199 	if (k) {
200 		/* Overwriting: */
201 		btree_account_key_drop(b, k);
202 		k->type = KEY_TYPE_deleted;
203 
204 		insert->k.needs_whiteout = k->needs_whiteout;
205 		k->needs_whiteout = false;
206 
207 		if (k >= btree_bset_last(b)->start) {
208 			clobber_u64s = k->u64s;
209 			goto overwrite;
210 		} else {
211 			bch2_btree_path_fix_key_modified(trans, b, k);
212 		}
213 	}
214 
215 	k = bch2_btree_node_iter_bset_pos(node_iter, b, bset_tree_last(b));
216 overwrite:
217 	bch2_bset_insert(b, k, insert, clobber_u64s);
218 	new_u64s = k->u64s;
219 fix_iter:
220 	if (clobber_u64s != new_u64s)
221 		bch2_btree_node_iter_fix(trans, path, b, node_iter, k,
222 					 clobber_u64s, new_u64s);
223 	return true;
224 }
225 
__btree_node_flush(struct journal * j,struct journal_entry_pin * pin,unsigned i,u64 seq)226 static int __btree_node_flush(struct journal *j, struct journal_entry_pin *pin,
227 			       unsigned i, u64 seq)
228 {
229 	struct bch_fs *c = container_of(j, struct bch_fs, journal);
230 	struct btree_write *w = container_of(pin, struct btree_write, journal);
231 	struct btree *b = container_of(w, struct btree, writes[i]);
232 	struct btree_trans *trans = bch2_trans_get(c);
233 	unsigned long old, new;
234 	unsigned idx = w - b->writes;
235 
236 	btree_node_lock_nopath_nofail(trans, &b->c, SIX_LOCK_read);
237 
238 	old = READ_ONCE(b->flags);
239 	do {
240 		new = old;
241 
242 		if (!(old & (1 << BTREE_NODE_dirty)) ||
243 		    !!(old & (1 << BTREE_NODE_write_idx)) != idx ||
244 		    w->journal.seq != seq)
245 			break;
246 
247 		new &= ~BTREE_WRITE_TYPE_MASK;
248 		new |= BTREE_WRITE_journal_reclaim;
249 		new |= 1 << BTREE_NODE_need_write;
250 	} while (!try_cmpxchg(&b->flags, &old, new));
251 
252 	btree_node_write_if_need(trans, b, SIX_LOCK_read);
253 	six_unlock_read(&b->c.lock);
254 
255 	bch2_trans_put(trans);
256 	return 0;
257 }
258 
bch2_btree_node_flush0(struct journal * j,struct journal_entry_pin * pin,u64 seq)259 int bch2_btree_node_flush0(struct journal *j, struct journal_entry_pin *pin, u64 seq)
260 {
261 	return __btree_node_flush(j, pin, 0, seq);
262 }
263 
bch2_btree_node_flush1(struct journal * j,struct journal_entry_pin * pin,u64 seq)264 int bch2_btree_node_flush1(struct journal *j, struct journal_entry_pin *pin, u64 seq)
265 {
266 	return __btree_node_flush(j, pin, 1, seq);
267 }
268 
bch2_btree_add_journal_pin(struct bch_fs * c,struct btree * b,u64 seq)269 inline void bch2_btree_add_journal_pin(struct bch_fs *c,
270 				       struct btree *b, u64 seq)
271 {
272 	struct btree_write *w = btree_current_write(b);
273 
274 	bch2_journal_pin_add(&c->journal, seq, &w->journal,
275 			     btree_node_write_idx(b) == 0
276 			     ? bch2_btree_node_flush0
277 			     : bch2_btree_node_flush1);
278 }
279 
280 /**
281  * bch2_btree_insert_key_leaf() - insert a key one key into a leaf node
282  * @trans:		btree transaction object
283  * @path:		path pointing to @insert's pos
284  * @insert:		key to insert
285  * @journal_seq:	sequence number of journal reservation
286  */
bch2_btree_insert_key_leaf(struct btree_trans * trans,struct btree_path * path,struct bkey_i * insert,u64 journal_seq)287 inline void bch2_btree_insert_key_leaf(struct btree_trans *trans,
288 				       struct btree_path *path,
289 				       struct bkey_i *insert,
290 				       u64 journal_seq)
291 {
292 	struct bch_fs *c = trans->c;
293 	struct btree *b = path_l(path)->b;
294 	struct bset_tree *t = bset_tree_last(b);
295 	struct bset *i = bset(b, t);
296 	int old_u64s = bset_u64s(t);
297 	int old_live_u64s = b->nr.live_u64s;
298 	int live_u64s_added, u64s_added;
299 
300 	if (unlikely(!bch2_btree_bset_insert_key(trans, path, b,
301 					&path_l(path)->iter, insert)))
302 		return;
303 
304 	i->journal_seq = cpu_to_le64(max(journal_seq, le64_to_cpu(i->journal_seq)));
305 
306 	bch2_btree_add_journal_pin(c, b, journal_seq);
307 
308 	if (unlikely(!btree_node_dirty(b))) {
309 		EBUG_ON(test_bit(BCH_FS_clean_shutdown, &c->flags));
310 		set_btree_node_dirty_acct(c, b);
311 	}
312 
313 	live_u64s_added = (int) b->nr.live_u64s - old_live_u64s;
314 	u64s_added = (int) bset_u64s(t) - old_u64s;
315 
316 	if (b->sib_u64s[0] != U16_MAX && live_u64s_added < 0)
317 		b->sib_u64s[0] = max(0, (int) b->sib_u64s[0] + live_u64s_added);
318 	if (b->sib_u64s[1] != U16_MAX && live_u64s_added < 0)
319 		b->sib_u64s[1] = max(0, (int) b->sib_u64s[1] + live_u64s_added);
320 
321 	if (u64s_added > live_u64s_added &&
322 	    bch2_maybe_compact_whiteouts(c, b))
323 		bch2_trans_node_reinit_iter(trans, b);
324 }
325 
326 /* Cached btree updates: */
327 
328 /* Normal update interface: */
329 
btree_insert_entry_checks(struct btree_trans * trans,struct btree_insert_entry * i)330 static inline void btree_insert_entry_checks(struct btree_trans *trans,
331 					     struct btree_insert_entry *i)
332 {
333 	struct btree_path *path = trans->paths + i->path;
334 
335 	BUG_ON(!bpos_eq(i->k->k.p, path->pos));
336 	BUG_ON(i->cached	!= path->cached);
337 	BUG_ON(i->level		!= path->level);
338 	BUG_ON(i->btree_id	!= path->btree_id);
339 	EBUG_ON(!i->level &&
340 		btree_type_has_snapshots(i->btree_id) &&
341 		!(i->flags & BTREE_UPDATE_internal_snapshot_node) &&
342 		test_bit(JOURNAL_replay_done, &trans->c->journal.flags) &&
343 		i->k->k.p.snapshot &&
344 		bch2_snapshot_is_internal_node(trans->c, i->k->k.p.snapshot) > 0);
345 }
346 
bch2_trans_journal_res_get(struct btree_trans * trans,unsigned flags)347 static __always_inline int bch2_trans_journal_res_get(struct btree_trans *trans,
348 						      unsigned flags)
349 {
350 	return bch2_journal_res_get(&trans->c->journal, &trans->journal_res,
351 				    trans->journal_u64s, flags, trans);
352 }
353 
354 #define JSET_ENTRY_LOG_U64s		4
355 
journal_transaction_name(struct btree_trans * trans)356 static noinline void journal_transaction_name(struct btree_trans *trans)
357 {
358 	struct bch_fs *c = trans->c;
359 	struct journal *j = &c->journal;
360 	struct jset_entry *entry =
361 		bch2_journal_add_entry(j, &trans->journal_res,
362 				       BCH_JSET_ENTRY_log, 0, 0,
363 				       JSET_ENTRY_LOG_U64s);
364 	struct jset_entry_log *l =
365 		container_of(entry, struct jset_entry_log, entry);
366 
367 	strncpy(l->d, trans->fn, JSET_ENTRY_LOG_U64s * sizeof(u64));
368 }
369 
btree_key_can_insert(struct btree_trans * trans,struct btree * b,unsigned u64s)370 static inline int btree_key_can_insert(struct btree_trans *trans,
371 				       struct btree *b, unsigned u64s)
372 {
373 	if (!bch2_btree_node_insert_fits(b, u64s))
374 		return -BCH_ERR_btree_insert_btree_node_full;
375 
376 	return 0;
377 }
378 
379 noinline static int
btree_key_can_insert_cached_slowpath(struct btree_trans * trans,unsigned flags,struct btree_path * path,unsigned new_u64s)380 btree_key_can_insert_cached_slowpath(struct btree_trans *trans, unsigned flags,
381 				     struct btree_path *path, unsigned new_u64s)
382 {
383 	struct bkey_cached *ck = (void *) path->l[0].b;
384 	struct bkey_i *new_k;
385 	int ret;
386 
387 	bch2_trans_unlock_updates_write(trans);
388 	bch2_trans_unlock(trans);
389 
390 	new_k = kmalloc(new_u64s * sizeof(u64), GFP_KERNEL);
391 	if (!new_k) {
392 		bch_err(trans->c, "error allocating memory for key cache key, btree %s u64s %u",
393 			bch2_btree_id_str(path->btree_id), new_u64s);
394 		return -BCH_ERR_ENOMEM_btree_key_cache_insert;
395 	}
396 
397 	ret =   bch2_trans_relock(trans) ?:
398 		bch2_trans_lock_write(trans);
399 	if (unlikely(ret)) {
400 		kfree(new_k);
401 		return ret;
402 	}
403 
404 	memcpy(new_k, ck->k, ck->u64s * sizeof(u64));
405 
406 	trans_for_each_update(trans, i)
407 		if (i->old_v == &ck->k->v)
408 			i->old_v = &new_k->v;
409 
410 	kfree(ck->k);
411 	ck->u64s	= new_u64s;
412 	ck->k		= new_k;
413 	return 0;
414 }
415 
btree_key_can_insert_cached(struct btree_trans * trans,unsigned flags,struct btree_path * path,unsigned u64s)416 static int btree_key_can_insert_cached(struct btree_trans *trans, unsigned flags,
417 				       struct btree_path *path, unsigned u64s)
418 {
419 	struct bch_fs *c = trans->c;
420 	struct bkey_cached *ck = (void *) path->l[0].b;
421 	unsigned new_u64s;
422 	struct bkey_i *new_k;
423 	unsigned watermark = flags & BCH_WATERMARK_MASK;
424 
425 	EBUG_ON(path->level);
426 
427 	if (watermark < BCH_WATERMARK_reclaim &&
428 	    !test_bit(BKEY_CACHED_DIRTY, &ck->flags) &&
429 	    bch2_btree_key_cache_must_wait(c))
430 		return -BCH_ERR_btree_insert_need_journal_reclaim;
431 
432 	/*
433 	 * bch2_varint_decode can read past the end of the buffer by at most 7
434 	 * bytes (it won't be used):
435 	 */
436 	u64s += 1;
437 
438 	if (u64s <= ck->u64s)
439 		return 0;
440 
441 	new_u64s	= roundup_pow_of_two(u64s);
442 	new_k		= krealloc(ck->k, new_u64s * sizeof(u64), GFP_NOWAIT|__GFP_NOWARN);
443 	if (unlikely(!new_k))
444 		return btree_key_can_insert_cached_slowpath(trans, flags, path, new_u64s);
445 
446 	trans_for_each_update(trans, i)
447 		if (i->old_v == &ck->k->v)
448 			i->old_v = &new_k->v;
449 
450 	ck->u64s	= new_u64s;
451 	ck->k		= new_k;
452 	return 0;
453 }
454 
455 /* Triggers: */
456 
run_one_mem_trigger(struct btree_trans * trans,struct btree_insert_entry * i,unsigned flags)457 static int run_one_mem_trigger(struct btree_trans *trans,
458 			       struct btree_insert_entry *i,
459 			       unsigned flags)
460 {
461 	verify_update_old_key(trans, i);
462 
463 	if (unlikely(flags & BTREE_TRIGGER_norun))
464 		return 0;
465 
466 	struct bkey_s_c old = { &i->old_k, i->old_v };
467 	struct bkey_i *new = i->k;
468 	const struct bkey_ops *old_ops = bch2_bkey_type_ops(old.k->type);
469 	const struct bkey_ops *new_ops = bch2_bkey_type_ops(i->k->k.type);
470 
471 	if (old_ops->trigger == new_ops->trigger)
472 		return bch2_key_trigger(trans, i->btree_id, i->level,
473 				old, bkey_i_to_s(new),
474 				BTREE_TRIGGER_insert|BTREE_TRIGGER_overwrite|flags);
475 	else
476 		return bch2_key_trigger_new(trans, i->btree_id, i->level,
477 				bkey_i_to_s(new), flags) ?:
478 		       bch2_key_trigger_old(trans, i->btree_id, i->level,
479 				old, flags);
480 }
481 
run_one_trans_trigger(struct btree_trans * trans,struct btree_insert_entry * i)482 static int run_one_trans_trigger(struct btree_trans *trans, struct btree_insert_entry *i)
483 {
484 	verify_update_old_key(trans, i);
485 
486 	if ((i->flags & BTREE_TRIGGER_norun) ||
487 	    !btree_node_type_has_trans_triggers(i->bkey_type))
488 		return 0;
489 
490 	/*
491 	 * Transactional triggers create new btree_insert_entries, so we can't
492 	 * pass them a pointer to a btree_insert_entry, that memory is going to
493 	 * move:
494 	 */
495 	struct bkey old_k = i->old_k;
496 	struct bkey_s_c old = { &old_k, i->old_v };
497 	const struct bkey_ops *old_ops = bch2_bkey_type_ops(old.k->type);
498 	const struct bkey_ops *new_ops = bch2_bkey_type_ops(i->k->k.type);
499 	unsigned flags = i->flags|BTREE_TRIGGER_transactional;
500 
501 	if (!i->insert_trigger_run &&
502 	    !i->overwrite_trigger_run &&
503 	    old_ops->trigger == new_ops->trigger) {
504 		i->overwrite_trigger_run = true;
505 		i->insert_trigger_run = true;
506 		return bch2_key_trigger(trans, i->btree_id, i->level, old, bkey_i_to_s(i->k),
507 					BTREE_TRIGGER_insert|
508 					BTREE_TRIGGER_overwrite|flags) ?: 1;
509 	} else if (!i->overwrite_trigger_run) {
510 		i->overwrite_trigger_run = true;
511 		return bch2_key_trigger_old(trans, i->btree_id, i->level, old, flags) ?: 1;
512 	} else if (!i->insert_trigger_run) {
513 		i->insert_trigger_run = true;
514 		return bch2_key_trigger_new(trans, i->btree_id, i->level, bkey_i_to_s(i->k), flags) ?: 1;
515 	} else {
516 		return 0;
517 	}
518 }
519 
run_btree_triggers(struct btree_trans * trans,enum btree_id btree_id,unsigned * btree_id_updates_start)520 static int run_btree_triggers(struct btree_trans *trans, enum btree_id btree_id,
521 			      unsigned *btree_id_updates_start)
522 {
523 	bool trans_trigger_run;
524 
525 	/*
526 	 * Running triggers will append more updates to the list of updates as
527 	 * we're walking it:
528 	 */
529 	do {
530 		trans_trigger_run = false;
531 
532 		for (unsigned i = *btree_id_updates_start;
533 		     i < trans->nr_updates && trans->updates[i].btree_id <= btree_id;
534 		     i++) {
535 			if (trans->updates[i].btree_id < btree_id) {
536 				*btree_id_updates_start = i;
537 				continue;
538 			}
539 
540 			int ret = run_one_trans_trigger(trans, trans->updates + i);
541 			if (ret < 0)
542 				return ret;
543 			if (ret)
544 				trans_trigger_run = true;
545 		}
546 	} while (trans_trigger_run);
547 
548 	trans_for_each_update(trans, i)
549 		BUG_ON(!(i->flags & BTREE_TRIGGER_norun) &&
550 		       i->btree_id == btree_id &&
551 		       btree_node_type_has_trans_triggers(i->bkey_type) &&
552 		       (!i->insert_trigger_run || !i->overwrite_trigger_run));
553 
554 	return 0;
555 }
556 
bch2_trans_commit_run_triggers(struct btree_trans * trans)557 static int bch2_trans_commit_run_triggers(struct btree_trans *trans)
558 {
559 	unsigned btree_id = 0, btree_id_updates_start = 0;
560 	int ret = 0;
561 
562 	/*
563 	 *
564 	 * For a given btree, this algorithm runs insert triggers before
565 	 * overwrite triggers: this is so that when extents are being moved
566 	 * (e.g. by FALLOCATE_FL_INSERT_RANGE), we don't drop references before
567 	 * they are re-added.
568 	 */
569 	for (btree_id = 0; btree_id < BTREE_ID_NR; btree_id++) {
570 		if (btree_id == BTREE_ID_alloc)
571 			continue;
572 
573 		ret = run_btree_triggers(trans, btree_id, &btree_id_updates_start);
574 		if (ret)
575 			return ret;
576 	}
577 
578 	btree_id_updates_start = 0;
579 	ret = run_btree_triggers(trans, BTREE_ID_alloc, &btree_id_updates_start);
580 	if (ret)
581 		return ret;
582 
583 #ifdef CONFIG_BCACHEFS_DEBUG
584 	trans_for_each_update(trans, i)
585 		BUG_ON(!(i->flags & BTREE_TRIGGER_norun) &&
586 		       btree_node_type_has_trans_triggers(i->bkey_type) &&
587 		       (!i->insert_trigger_run || !i->overwrite_trigger_run));
588 #endif
589 	return 0;
590 }
591 
bch2_trans_commit_run_gc_triggers(struct btree_trans * trans)592 static noinline int bch2_trans_commit_run_gc_triggers(struct btree_trans *trans)
593 {
594 	trans_for_each_update(trans, i)
595 		if (btree_node_type_has_triggers(i->bkey_type) &&
596 		    gc_visited(trans->c, gc_pos_btree(i->btree_id, i->level, i->k->k.p))) {
597 			int ret = run_one_mem_trigger(trans, i, i->flags|BTREE_TRIGGER_gc);
598 			if (ret)
599 				return ret;
600 		}
601 
602 	return 0;
603 }
604 
605 static inline int
bch2_trans_commit_write_locked(struct btree_trans * trans,unsigned flags,struct btree_insert_entry ** stopped_at,unsigned long trace_ip)606 bch2_trans_commit_write_locked(struct btree_trans *trans, unsigned flags,
607 			       struct btree_insert_entry **stopped_at,
608 			       unsigned long trace_ip)
609 {
610 	struct bch_fs *c = trans->c;
611 	struct btree_trans_commit_hook *h;
612 	unsigned u64s = 0;
613 	int ret = 0;
614 
615 	bch2_trans_verify_not_unlocked_or_in_restart(trans);
616 
617 	if (race_fault()) {
618 		trace_and_count(c, trans_restart_fault_inject, trans, trace_ip);
619 		return btree_trans_restart(trans, BCH_ERR_transaction_restart_fault_inject);
620 	}
621 
622 	/*
623 	 * Check if the insert will fit in the leaf node with the write lock
624 	 * held, otherwise another thread could write the node changing the
625 	 * amount of space available:
626 	 */
627 
628 	prefetch(&trans->c->journal.flags);
629 
630 	trans_for_each_update(trans, i) {
631 		/* Multiple inserts might go to same leaf: */
632 		if (!same_leaf_as_prev(trans, i))
633 			u64s = 0;
634 
635 		u64s += i->k->k.u64s;
636 		ret = !i->cached
637 			? btree_key_can_insert(trans, insert_l(trans, i)->b, u64s)
638 			: btree_key_can_insert_cached(trans, flags, trans->paths + i->path, u64s);
639 		if (ret) {
640 			*stopped_at = i;
641 			return ret;
642 		}
643 
644 		i->k->k.needs_whiteout = false;
645 	}
646 
647 	/*
648 	 * Don't get journal reservation until after we know insert will
649 	 * succeed:
650 	 */
651 	if (likely(!(flags & BCH_TRANS_COMMIT_no_journal_res))) {
652 		ret = bch2_trans_journal_res_get(trans,
653 				(flags & BCH_WATERMARK_MASK)|
654 				JOURNAL_RES_GET_NONBLOCK);
655 		if (ret)
656 			return ret;
657 
658 		if (unlikely(trans->journal_transaction_names))
659 			journal_transaction_name(trans);
660 	}
661 
662 	/*
663 	 * Not allowed to fail after we've gotten our journal reservation - we
664 	 * have to use it:
665 	 */
666 
667 	if (IS_ENABLED(CONFIG_BCACHEFS_DEBUG) &&
668 	    !(flags & BCH_TRANS_COMMIT_no_journal_res)) {
669 		if (bch2_journal_seq_verify)
670 			trans_for_each_update(trans, i)
671 				i->k->k.bversion.lo = trans->journal_res.seq;
672 		else if (bch2_inject_invalid_keys)
673 			trans_for_each_update(trans, i)
674 				i->k->k.bversion = MAX_VERSION;
675 	}
676 
677 	h = trans->hooks;
678 	while (h) {
679 		ret = h->fn(trans, h);
680 		if (ret)
681 			return ret;
682 		h = h->next;
683 	}
684 
685 	struct jset_entry *entry = trans->journal_entries;
686 
687 	percpu_down_read(&c->mark_lock);
688 	for (entry = trans->journal_entries;
689 	     entry != (void *) ((u64 *) trans->journal_entries + trans->journal_entries_u64s);
690 	     entry = vstruct_next(entry))
691 		if (entry->type == BCH_JSET_ENTRY_write_buffer_keys &&
692 		    entry->start->k.type == KEY_TYPE_accounting) {
693 			ret = bch2_accounting_trans_commit_hook(trans, bkey_i_to_accounting(entry->start), flags);
694 			if (ret)
695 				goto revert_fs_usage;
696 		}
697 	percpu_up_read(&c->mark_lock);
698 
699 	/* XXX: we only want to run this if deltas are nonzero */
700 	bch2_trans_account_disk_usage_change(trans);
701 
702 	trans_for_each_update(trans, i)
703 		if (btree_node_type_has_atomic_triggers(i->bkey_type)) {
704 			ret = run_one_mem_trigger(trans, i, BTREE_TRIGGER_atomic|i->flags);
705 			if (ret)
706 				goto fatal_err;
707 		}
708 
709 	if (unlikely(c->gc_pos.phase)) {
710 		ret = bch2_trans_commit_run_gc_triggers(trans);
711 		if  (ret)
712 			goto fatal_err;
713 	}
714 
715 	struct bkey_validate_context validate_context = { .from	= BKEY_VALIDATE_commit };
716 
717 	if (!(flags & BCH_TRANS_COMMIT_no_journal_res))
718 		validate_context.flags = BCH_VALIDATE_write|BCH_VALIDATE_commit;
719 
720 	for (struct jset_entry *i = trans->journal_entries;
721 	     i != (void *) ((u64 *) trans->journal_entries + trans->journal_entries_u64s);
722 	     i = vstruct_next(i)) {
723 		ret = bch2_journal_entry_validate(c, NULL, i,
724 						  bcachefs_metadata_version_current,
725 						  CPU_BIG_ENDIAN, validate_context);
726 		if (unlikely(ret)) {
727 			bch2_trans_inconsistent(trans, "invalid journal entry on insert from %s\n",
728 						trans->fn);
729 			goto fatal_err;
730 		}
731 	}
732 
733 	trans_for_each_update(trans, i) {
734 		validate_context.level	= i->level;
735 		validate_context.btree	= i->btree_id;
736 
737 		ret = bch2_bkey_validate(c, bkey_i_to_s_c(i->k), validate_context);
738 		if (unlikely(ret)){
739 			bch2_trans_inconsistent(trans, "invalid bkey on insert from %s -> %ps\n",
740 						trans->fn, (void *) i->ip_allocated);
741 			goto fatal_err;
742 		}
743 		btree_insert_entry_checks(trans, i);
744 	}
745 
746 	if (likely(!(flags & BCH_TRANS_COMMIT_no_journal_res))) {
747 		struct journal *j = &c->journal;
748 		struct jset_entry *entry;
749 
750 		trans_for_each_update(trans, i) {
751 			if (i->key_cache_already_flushed)
752 				continue;
753 
754 			if (i->flags & BTREE_UPDATE_nojournal)
755 				continue;
756 
757 			verify_update_old_key(trans, i);
758 
759 			if (trans->journal_transaction_names) {
760 				entry = bch2_journal_add_entry(j, &trans->journal_res,
761 						       BCH_JSET_ENTRY_overwrite,
762 						       i->btree_id, i->level,
763 						       i->old_k.u64s);
764 				bkey_reassemble((struct bkey_i *) entry->start,
765 						(struct bkey_s_c) { &i->old_k, i->old_v });
766 			}
767 
768 			entry = bch2_journal_add_entry(j, &trans->journal_res,
769 					       BCH_JSET_ENTRY_btree_keys,
770 					       i->btree_id, i->level,
771 					       i->k->k.u64s);
772 			bkey_copy((struct bkey_i *) entry->start, i->k);
773 		}
774 
775 		memcpy_u64s_small(journal_res_entry(&c->journal, &trans->journal_res),
776 				  trans->journal_entries,
777 				  trans->journal_entries_u64s);
778 
779 		trans->journal_res.offset	+= trans->journal_entries_u64s;
780 		trans->journal_res.u64s		-= trans->journal_entries_u64s;
781 
782 		if (trans->journal_seq)
783 			*trans->journal_seq = trans->journal_res.seq;
784 	}
785 
786 	trans_for_each_update(trans, i) {
787 		struct btree_path *path = trans->paths + i->path;
788 
789 		if (!i->cached)
790 			bch2_btree_insert_key_leaf(trans, path, i->k, trans->journal_res.seq);
791 		else if (!i->key_cache_already_flushed)
792 			bch2_btree_insert_key_cached(trans, flags, i);
793 		else
794 			bch2_btree_key_cache_drop(trans, path);
795 	}
796 
797 	return 0;
798 fatal_err:
799 	bch2_fs_fatal_error(c, "fatal error in transaction commit: %s", bch2_err_str(ret));
800 	percpu_down_read(&c->mark_lock);
801 revert_fs_usage:
802 	for (struct jset_entry *entry2 = trans->journal_entries;
803 	     entry2 != entry;
804 	     entry2 = vstruct_next(entry2))
805 		if (entry2->type == BCH_JSET_ENTRY_write_buffer_keys &&
806 		    entry2->start->k.type == KEY_TYPE_accounting)
807 			bch2_accounting_trans_commit_revert(trans,
808 					bkey_i_to_accounting(entry2->start), flags);
809 	percpu_up_read(&c->mark_lock);
810 	return ret;
811 }
812 
bch2_drop_overwrites_from_journal(struct btree_trans * trans)813 static noinline void bch2_drop_overwrites_from_journal(struct btree_trans *trans)
814 {
815 	/*
816 	 * Accounting keys aren't deduped in the journal: we have to compare
817 	 * each individual update against what's in the btree to see if it has
818 	 * been applied yet, and accounting updates also don't overwrite,
819 	 * they're deltas that accumulate.
820 	 */
821 	trans_for_each_update(trans, i)
822 		if (i->k->k.type != KEY_TYPE_accounting)
823 			bch2_journal_key_overwritten(trans->c, i->btree_id, i->level, i->k->k.p);
824 }
825 
bch2_trans_commit_journal_pin_flush(struct journal * j,struct journal_entry_pin * _pin,u64 seq)826 static int bch2_trans_commit_journal_pin_flush(struct journal *j,
827 				struct journal_entry_pin *_pin, u64 seq)
828 {
829 	return 0;
830 }
831 
832 /*
833  * Get journal reservation, take write locks, and attempt to do btree update(s):
834  */
do_bch2_trans_commit(struct btree_trans * trans,unsigned flags,struct btree_insert_entry ** stopped_at,unsigned long trace_ip)835 static inline int do_bch2_trans_commit(struct btree_trans *trans, unsigned flags,
836 				       struct btree_insert_entry **stopped_at,
837 				       unsigned long trace_ip)
838 {
839 	struct bch_fs *c = trans->c;
840 	int ret = 0, u64s_delta = 0;
841 
842 	for (unsigned idx = 0; idx < trans->nr_updates; idx++) {
843 		struct btree_insert_entry *i = trans->updates + idx;
844 		if (i->cached)
845 			continue;
846 
847 		u64s_delta += !bkey_deleted(&i->k->k) ? i->k->k.u64s : 0;
848 		u64s_delta -= i->old_btree_u64s;
849 
850 		if (!same_leaf_as_next(trans, i)) {
851 			if (u64s_delta <= 0) {
852 				ret = bch2_foreground_maybe_merge(trans, i->path,
853 							i->level, flags);
854 				if (unlikely(ret))
855 					return ret;
856 			}
857 
858 			u64s_delta = 0;
859 		}
860 	}
861 
862 	ret = bch2_trans_lock_write(trans);
863 	if (unlikely(ret))
864 		return ret;
865 
866 	ret = bch2_trans_commit_write_locked(trans, flags, stopped_at, trace_ip);
867 
868 	if (!ret && unlikely(trans->journal_replay_not_finished))
869 		bch2_drop_overwrites_from_journal(trans);
870 
871 	bch2_trans_unlock_updates_write(trans);
872 
873 	if (!ret && trans->journal_pin)
874 		bch2_journal_pin_add(&c->journal, trans->journal_res.seq,
875 				     trans->journal_pin,
876 				     bch2_trans_commit_journal_pin_flush);
877 
878 	/*
879 	 * Drop journal reservation after dropping write locks, since dropping
880 	 * the journal reservation may kick off a journal write:
881 	 */
882 	if (likely(!(flags & BCH_TRANS_COMMIT_no_journal_res)))
883 		bch2_journal_res_put(&c->journal, &trans->journal_res);
884 
885 	return ret;
886 }
887 
journal_reclaim_wait_done(struct bch_fs * c)888 static int journal_reclaim_wait_done(struct bch_fs *c)
889 {
890 	int ret = bch2_journal_error(&c->journal) ?:
891 		bch2_btree_key_cache_wait_done(c);
892 
893 	if (!ret)
894 		journal_reclaim_kick(&c->journal);
895 	return ret;
896 }
897 
898 static noinline
bch2_trans_commit_error(struct btree_trans * trans,unsigned flags,struct btree_insert_entry * i,int ret,unsigned long trace_ip)899 int bch2_trans_commit_error(struct btree_trans *trans, unsigned flags,
900 			    struct btree_insert_entry *i,
901 			    int ret, unsigned long trace_ip)
902 {
903 	struct bch_fs *c = trans->c;
904 	enum bch_watermark watermark = flags & BCH_WATERMARK_MASK;
905 
906 	switch (ret) {
907 	case -BCH_ERR_btree_insert_btree_node_full:
908 		ret = bch2_btree_split_leaf(trans, i->path, flags);
909 		if (bch2_err_matches(ret, BCH_ERR_transaction_restart))
910 			trace_and_count(c, trans_restart_btree_node_split, trans,
911 					trace_ip, trans->paths + i->path);
912 		break;
913 	case -BCH_ERR_btree_insert_need_mark_replicas:
914 		ret = drop_locks_do(trans,
915 			bch2_accounting_update_sb(trans));
916 		break;
917 	case -BCH_ERR_journal_res_get_blocked:
918 		/*
919 		 * XXX: this should probably be a separate BTREE_INSERT_NONBLOCK
920 		 * flag
921 		 */
922 		if ((flags & BCH_TRANS_COMMIT_journal_reclaim) &&
923 		    watermark < BCH_WATERMARK_reclaim) {
924 			ret = -BCH_ERR_journal_reclaim_would_deadlock;
925 			break;
926 		}
927 
928 		ret = drop_locks_do(trans,
929 			bch2_trans_journal_res_get(trans,
930 					(flags & BCH_WATERMARK_MASK)|
931 					JOURNAL_RES_GET_CHECK));
932 		break;
933 	case -BCH_ERR_btree_insert_need_journal_reclaim:
934 		bch2_trans_unlock(trans);
935 
936 		trace_and_count(c, trans_blocked_journal_reclaim, trans, trace_ip);
937 		track_event_change(&c->times[BCH_TIME_blocked_key_cache_flush], true);
938 
939 		wait_event_freezable(c->journal.reclaim_wait,
940 				     (ret = journal_reclaim_wait_done(c)));
941 
942 		track_event_change(&c->times[BCH_TIME_blocked_key_cache_flush], false);
943 
944 		if (ret < 0)
945 			break;
946 
947 		ret = bch2_trans_relock(trans);
948 		break;
949 	default:
950 		BUG_ON(ret >= 0);
951 		break;
952 	}
953 
954 	BUG_ON(bch2_err_matches(ret, BCH_ERR_transaction_restart) != !!trans->restarted);
955 
956 	bch2_fs_inconsistent_on(bch2_err_matches(ret, ENOSPC) &&
957 				(flags & BCH_TRANS_COMMIT_no_enospc), c,
958 		"%s: incorrectly got %s\n", __func__, bch2_err_str(ret));
959 
960 	return ret;
961 }
962 
963 /*
964  * This is for updates done in the early part of fsck - btree_gc - before we've
965  * gone RW. we only add the new key to the list of keys for journal replay to
966  * do.
967  */
968 static noinline int
do_bch2_trans_commit_to_journal_replay(struct btree_trans * trans)969 do_bch2_trans_commit_to_journal_replay(struct btree_trans *trans)
970 {
971 	struct bch_fs *c = trans->c;
972 
973 	BUG_ON(current != c->recovery_task);
974 
975 	trans_for_each_update(trans, i) {
976 		int ret = bch2_journal_key_insert(c, i->btree_id, i->level, i->k);
977 		if (ret)
978 			return ret;
979 	}
980 
981 	for (struct jset_entry *i = trans->journal_entries;
982 	     i != (void *) ((u64 *) trans->journal_entries + trans->journal_entries_u64s);
983 	     i = vstruct_next(i))
984 		if (i->type == BCH_JSET_ENTRY_btree_keys ||
985 		    i->type == BCH_JSET_ENTRY_write_buffer_keys) {
986 			int ret = bch2_journal_key_insert(c, i->btree_id, i->level, i->start);
987 			if (ret)
988 				return ret;
989 		}
990 
991 	return 0;
992 }
993 
__bch2_trans_commit(struct btree_trans * trans,unsigned flags)994 int __bch2_trans_commit(struct btree_trans *trans, unsigned flags)
995 {
996 	struct btree_insert_entry *errored_at = NULL;
997 	struct bch_fs *c = trans->c;
998 	int ret = 0;
999 
1000 	bch2_trans_verify_not_unlocked_or_in_restart(trans);
1001 
1002 	ret = trans_maybe_inject_restart(trans, _RET_IP_);
1003 	if (unlikely(ret))
1004 		goto out_reset;
1005 
1006 	if (!trans->nr_updates &&
1007 	    !trans->journal_entries_u64s)
1008 		goto out_reset;
1009 
1010 	ret = bch2_trans_commit_run_triggers(trans);
1011 	if (ret)
1012 		goto out_reset;
1013 
1014 	if (!(flags & BCH_TRANS_COMMIT_no_check_rw) &&
1015 	    unlikely(!bch2_write_ref_tryget(c, BCH_WRITE_REF_trans))) {
1016 		if (unlikely(!test_bit(BCH_FS_may_go_rw, &c->flags)))
1017 			ret = do_bch2_trans_commit_to_journal_replay(trans);
1018 		else
1019 			ret = -BCH_ERR_erofs_trans_commit;
1020 		goto out_reset;
1021 	}
1022 
1023 	EBUG_ON(test_bit(BCH_FS_clean_shutdown, &c->flags));
1024 
1025 	trans->journal_u64s		= trans->journal_entries_u64s;
1026 	trans->journal_transaction_names = READ_ONCE(c->opts.journal_transaction_names);
1027 	if (trans->journal_transaction_names)
1028 		trans->journal_u64s += jset_u64s(JSET_ENTRY_LOG_U64s);
1029 
1030 	trans_for_each_update(trans, i) {
1031 		struct btree_path *path = trans->paths + i->path;
1032 
1033 		EBUG_ON(!path->should_be_locked);
1034 
1035 		ret = bch2_btree_path_upgrade(trans, path, i->level + 1);
1036 		if (unlikely(ret))
1037 			goto out;
1038 
1039 		EBUG_ON(!btree_node_intent_locked(path, i->level));
1040 
1041 		if (i->key_cache_already_flushed)
1042 			continue;
1043 
1044 		if (i->flags & BTREE_UPDATE_nojournal)
1045 			continue;
1046 
1047 		/* we're going to journal the key being updated: */
1048 		trans->journal_u64s += jset_u64s(i->k->k.u64s);
1049 
1050 		/* and we're also going to log the overwrite: */
1051 		if (trans->journal_transaction_names)
1052 			trans->journal_u64s += jset_u64s(i->old_k.u64s);
1053 	}
1054 
1055 	if (trans->extra_disk_res) {
1056 		ret = bch2_disk_reservation_add(c, trans->disk_res,
1057 				trans->extra_disk_res,
1058 				(flags & BCH_TRANS_COMMIT_no_enospc)
1059 				? BCH_DISK_RESERVATION_NOFAIL : 0);
1060 		if (ret)
1061 			goto err;
1062 	}
1063 retry:
1064 	errored_at = NULL;
1065 	bch2_trans_verify_not_unlocked_or_in_restart(trans);
1066 	if (likely(!(flags & BCH_TRANS_COMMIT_no_journal_res)))
1067 		memset(&trans->journal_res, 0, sizeof(trans->journal_res));
1068 	memset(&trans->fs_usage_delta, 0, sizeof(trans->fs_usage_delta));
1069 
1070 	ret = do_bch2_trans_commit(trans, flags, &errored_at, _RET_IP_);
1071 
1072 	/* make sure we didn't drop or screw up locks: */
1073 	bch2_trans_verify_locks(trans);
1074 
1075 	if (ret)
1076 		goto err;
1077 
1078 	trace_and_count(c, transaction_commit, trans, _RET_IP_);
1079 out:
1080 	if (likely(!(flags & BCH_TRANS_COMMIT_no_check_rw)))
1081 		bch2_write_ref_put(c, BCH_WRITE_REF_trans);
1082 out_reset:
1083 	if (!ret)
1084 		bch2_trans_downgrade(trans);
1085 	bch2_trans_reset_updates(trans);
1086 
1087 	return ret;
1088 err:
1089 	ret = bch2_trans_commit_error(trans, flags, errored_at, ret, _RET_IP_);
1090 	if (ret)
1091 		goto out;
1092 
1093 	/*
1094 	 * We might have done another transaction commit in the error path -
1095 	 * i.e. btree write buffer flush - which will have made use of
1096 	 * trans->journal_res, but with BCH_TRANS_COMMIT_no_journal_res that is
1097 	 * how the journal sequence number to pin is passed in - so we must
1098 	 * restart:
1099 	 */
1100 	if (flags & BCH_TRANS_COMMIT_no_journal_res) {
1101 		ret = -BCH_ERR_transaction_restart_nested;
1102 		goto out;
1103 	}
1104 
1105 	goto retry;
1106 }
1107