44 static void verify_update_old_key(struct btree_trans *trans, struct btree_insert_entry *i)  in verify_update_old_key()  argument
47 	struct bch_fs *c = trans->c;  in verify_update_old_key()
49 	struct bkey_s_c k = bch2_btree_path_peek_slot_exact(trans->paths + i->path, &u);  in verify_update_old_key()
51 	if (unlikely(trans->journal_replay_not_finished)) {  in verify_update_old_key()
67 static inline struct btree_path_level *insert_l(struct btree_trans *trans, struct btree_insert_entr…  in insert_l()  argument
69 	return (trans->paths + i->path)->l + i->level;  in insert_l()
72 static inline bool same_leaf_as_prev(struct btree_trans *trans,  in same_leaf_as_prev()  argument
75 	return i != trans->updates &&  in same_leaf_as_prev()
76 		insert_l(trans, &i[0])->b == insert_l(trans, &i[-1])->b;  in same_leaf_as_prev()
79 static inline bool same_leaf_as_next(struct btree_trans *trans,  in same_leaf_as_next()  argument
82 	return i + 1 < trans->updates + trans->nr_updates &&  in same_leaf_as_next()
83 		insert_l(trans, &i[0])->b == insert_l(trans, &i[1])->b;  in same_leaf_as_next()
86 inline void bch2_btree_node_prep_for_write(struct btree_trans *trans,  in bch2_btree_node_prep_for_write()  argument
90 	struct bch_fs *c = trans->c;  in bch2_btree_node_prep_for_write()
94 		bch2_trans_node_reinit_iter(trans, b);  in bch2_btree_node_prep_for_write()
101 		bch2_btree_init_next(trans, b);  in bch2_btree_node_prep_for_write()
104 static noinline int trans_lock_write_fail(struct btree_trans *trans, struct btree_insert_entry *i)  in trans_lock_write_fail()  argument
106 	while (--i >= trans->updates) {  in trans_lock_write_fail()
107 		if (same_leaf_as_prev(trans, i))  in trans_lock_write_fail()
110 		bch2_btree_node_unlock_write(trans, trans->paths + i->path, insert_l(trans, i)->b);  in trans_lock_write_fail()
113 	trace_and_count(trans->c, trans_restart_would_deadlock_write, trans);  in trans_lock_write_fail()
114 	return btree_trans_restart(trans, BCH_ERR_transaction_restart_would_deadlock_write);  in trans_lock_write_fail()
117 static inline int bch2_trans_lock_write(struct btree_trans *trans)  in bch2_trans_lock_write()  argument
119 	EBUG_ON(trans->write_locked);  in bch2_trans_lock_write()
121 	trans_for_each_update(trans, i) {  in bch2_trans_lock_write()
122 		if (same_leaf_as_prev(trans, i))  in bch2_trans_lock_write()
125 		if (bch2_btree_node_lock_write(trans, trans->paths + i->path, &insert_l(trans, i)->b->c))  in bch2_trans_lock_write()
126 			return trans_lock_write_fail(trans, i);  in bch2_trans_lock_write()
129 			bch2_btree_node_prep_for_write(trans, trans->paths + i->path, insert_l(trans, i)->b);  in bch2_trans_lock_write()
132 	trans->write_locked = true;  in bch2_trans_lock_write()
136 static inline void bch2_trans_unlock_updates_write(struct btree_trans *trans)  in bch2_trans_unlock_updates_write()  argument
138 	if (likely(trans->write_locked)) {  in bch2_trans_unlock_updates_write()
139 		trans_for_each_update(trans, i)  in bch2_trans_unlock_updates_write()
140 			if (btree_node_locked_type(trans->paths + i->path, i->level) ==  in bch2_trans_unlock_updates_write()
142 				bch2_btree_node_unlock_write_inlined(trans,  in bch2_trans_unlock_updates_write()
143 						trans->paths + i->path, insert_l(trans, i)->b);  in bch2_trans_unlock_updates_write()
144 		trans->write_locked = false;  in bch2_trans_unlock_updates_write()
151 bool bch2_btree_bset_insert_key(struct btree_trans *trans,  in bch2_btree_bset_insert_key()  argument
193 			bch2_btree_path_fix_key_modified(trans, b, k);  in bch2_btree_bset_insert_key()
211 			bch2_btree_path_fix_key_modified(trans, b, k);  in bch2_btree_bset_insert_key()
221 		bch2_btree_node_iter_fix(trans, path, b, node_iter, k,  in bch2_btree_bset_insert_key()
232 	struct btree_trans *trans = bch2_trans_get(c);  in __btree_node_flush()  local
236 	btree_node_lock_nopath_nofail(trans, &b->c, SIX_LOCK_read);  in __btree_node_flush()
252 	btree_node_write_if_need(trans, b, SIX_LOCK_read);  in __btree_node_flush()
255 	bch2_trans_put(trans);  in __btree_node_flush()
287 inline void bch2_btree_insert_key_leaf(struct btree_trans *trans,  in bch2_btree_insert_key_leaf()  argument
292 	struct bch_fs *c = trans->c;  in bch2_btree_insert_key_leaf()
300 	if (unlikely(!bch2_btree_bset_insert_key(trans, path, b,  in bch2_btree_insert_key_leaf()
323 		bch2_trans_node_reinit_iter(trans, b);  in bch2_btree_insert_key_leaf()
330 static inline void btree_insert_entry_checks(struct btree_trans *trans,  in btree_insert_entry_checks()  argument
333 	struct btree_path *path = trans->paths + i->path;  in btree_insert_entry_checks()
342 		test_bit(JOURNAL_replay_done, &trans->c->journal.flags) &&  in btree_insert_entry_checks()
344 		bch2_snapshot_is_internal_node(trans->c, i->k->k.p.snapshot) > 0);  in btree_insert_entry_checks()
347 static __always_inline int bch2_trans_journal_res_get(struct btree_trans *trans,  in bch2_trans_journal_res_get()  argument
350 	return bch2_journal_res_get(&trans->c->journal, &trans->journal_res,  in bch2_trans_journal_res_get()
351 				    trans->journal_u64s, flags, trans);  in bch2_trans_journal_res_get()
356 static noinline void journal_transaction_name(struct btree_trans *trans)  in journal_transaction_name()  argument
358 	struct bch_fs *c = trans->c;  in journal_transaction_name()
361 		bch2_journal_add_entry(j, &trans->journal_res,  in journal_transaction_name()
367 	strncpy(l->d, trans->fn, JSET_ENTRY_LOG_U64s * sizeof(u64));  in journal_transaction_name()
370 static inline int btree_key_can_insert(struct btree_trans *trans,  in btree_key_can_insert()  argument
380 btree_key_can_insert_cached_slowpath(struct btree_trans *trans, unsigned flags,  in btree_key_can_insert_cached_slowpath()  argument
387 	bch2_trans_unlock_updates_write(trans);  in btree_key_can_insert_cached_slowpath()
388 	bch2_trans_unlock(trans);  in btree_key_can_insert_cached_slowpath()
392 		bch_err(trans->c, "error allocating memory for key cache key, btree %s u64s %u",  in btree_key_can_insert_cached_slowpath()
397 	ret =   bch2_trans_relock(trans) ?:  in btree_key_can_insert_cached_slowpath()
398 		bch2_trans_lock_write(trans);  in btree_key_can_insert_cached_slowpath()
406 	trans_for_each_update(trans, i)  in btree_key_can_insert_cached_slowpath()
416 static int btree_key_can_insert_cached(struct btree_trans *trans, unsigned flags,  in btree_key_can_insert_cached()  argument
419 	struct bch_fs *c = trans->c;  in btree_key_can_insert_cached()
444 		return btree_key_can_insert_cached_slowpath(trans, flags, path, new_u64s);  in btree_key_can_insert_cached()
446 	trans_for_each_update(trans, i)  in btree_key_can_insert_cached()
457 static int run_one_mem_trigger(struct btree_trans *trans,  in run_one_mem_trigger()  argument
461 	verify_update_old_key(trans, i);  in run_one_mem_trigger()
472 		return bch2_key_trigger(trans, i->btree_id, i->level,  in run_one_mem_trigger()
476 		return bch2_key_trigger_new(trans, i->btree_id, i->level,  in run_one_mem_trigger()
478 		       bch2_key_trigger_old(trans, i->btree_id, i->level,  in run_one_mem_trigger()
482 static int run_one_trans_trigger(struct btree_trans *trans, struct btree_insert_entry *i)  in run_one_trans_trigger()  argument
484 	verify_update_old_key(trans, i);  in run_one_trans_trigger()
506 		return bch2_key_trigger(trans, i->btree_id, i->level, old, bkey_i_to_s(i->k),  in run_one_trans_trigger()
511 		return bch2_key_trigger_old(trans, i->btree_id, i->level, old, flags) ?: 1;  in run_one_trans_trigger()
514 		return bch2_key_trigger_new(trans, i->btree_id, i->level, bkey_i_to_s(i->k), flags) ?: 1;  in run_one_trans_trigger()
520 static int run_btree_triggers(struct btree_trans *trans, enum btree_id btree_id,  in run_btree_triggers()  argument
533 		     i < trans->nr_updates && trans->updates[i].btree_id <= btree_id;  in run_btree_triggers()
535 			if (trans->updates[i].btree_id < btree_id) {  in run_btree_triggers()
540 			int ret = run_one_trans_trigger(trans, trans->updates + i);  in run_btree_triggers()
548 	trans_for_each_update(trans, i)  in run_btree_triggers()
557 static int bch2_trans_commit_run_triggers(struct btree_trans *trans)  in bch2_trans_commit_run_triggers()  argument
573 		ret = run_btree_triggers(trans, btree_id, &btree_id_updates_start);  in bch2_trans_commit_run_triggers()
579 	ret = run_btree_triggers(trans, BTREE_ID_alloc, &btree_id_updates_start);  in bch2_trans_commit_run_triggers()
584 	trans_for_each_update(trans, i)  in bch2_trans_commit_run_triggers()
592 static noinline int bch2_trans_commit_run_gc_triggers(struct btree_trans *trans)  in bch2_trans_commit_run_gc_triggers()  argument
594 	trans_for_each_update(trans, i)  in bch2_trans_commit_run_gc_triggers()
596 		    gc_visited(trans->c, gc_pos_btree(i->btree_id, i->level, i->k->k.p))) {  in bch2_trans_commit_run_gc_triggers()
597 			int ret = run_one_mem_trigger(trans, i, i->flags|BTREE_TRIGGER_gc);  in bch2_trans_commit_run_gc_triggers()
606 bch2_trans_commit_write_locked(struct btree_trans *trans, unsigned flags,  in bch2_trans_commit_write_locked()  argument
610 	struct bch_fs *c = trans->c;  in bch2_trans_commit_write_locked()
615 	bch2_trans_verify_not_unlocked_or_in_restart(trans);  in bch2_trans_commit_write_locked()
618 		trace_and_count(c, trans_restart_fault_inject, trans, trace_ip);  in bch2_trans_commit_write_locked()
619 		return btree_trans_restart(trans, BCH_ERR_transaction_restart_fault_inject);  in bch2_trans_commit_write_locked()
628 	prefetch(&trans->c->journal.flags);  in bch2_trans_commit_write_locked()
630 	trans_for_each_update(trans, i) {  in bch2_trans_commit_write_locked()
632 		if (!same_leaf_as_prev(trans, i))  in bch2_trans_commit_write_locked()
637 			? btree_key_can_insert(trans, insert_l(trans, i)->b, u64s)  in bch2_trans_commit_write_locked()
638 			: btree_key_can_insert_cached(trans, flags, trans->paths + i->path, u64s);  in bch2_trans_commit_write_locked()
652 		ret = bch2_trans_journal_res_get(trans,  in bch2_trans_commit_write_locked()
658 		if (unlikely(trans->journal_transaction_names))  in bch2_trans_commit_write_locked()
659 			journal_transaction_name(trans);  in bch2_trans_commit_write_locked()
670 			trans_for_each_update(trans, i)  in bch2_trans_commit_write_locked()
671 				i->k->k.bversion.lo = trans->journal_res.seq;  in bch2_trans_commit_write_locked()
673 			trans_for_each_update(trans, i)  in bch2_trans_commit_write_locked()
677 	h = trans->hooks;  in bch2_trans_commit_write_locked()
679 		ret = h->fn(trans, h);  in bch2_trans_commit_write_locked()
685 	struct jset_entry *entry = trans->journal_entries;  in bch2_trans_commit_write_locked()
688 	for (entry = trans->journal_entries;  in bch2_trans_commit_write_locked()
689 	     entry != (void *) ((u64 *) trans->journal_entries + trans->journal_entries_u64s);  in bch2_trans_commit_write_locked()
693 			ret = bch2_accounting_trans_commit_hook(trans, bkey_i_to_accounting(entry->start), flags);  in bch2_trans_commit_write_locked()
700 	bch2_trans_account_disk_usage_change(trans);  in bch2_trans_commit_write_locked()
702 	trans_for_each_update(trans, i)  in bch2_trans_commit_write_locked()
704 			ret = run_one_mem_trigger(trans, i, BTREE_TRIGGER_atomic|i->flags);  in bch2_trans_commit_write_locked()
710 		ret = bch2_trans_commit_run_gc_triggers(trans);  in bch2_trans_commit_write_locked()
720 	for (struct jset_entry *i = trans->journal_entries;  in bch2_trans_commit_write_locked()
721 	     i != (void *) ((u64 *) trans->journal_entries + trans->journal_entries_u64s);  in bch2_trans_commit_write_locked()
727 			bch2_trans_inconsistent(trans, "invalid journal entry on insert from %s\n",  in bch2_trans_commit_write_locked()
728 						trans->fn);  in bch2_trans_commit_write_locked()
733 	trans_for_each_update(trans, i) {  in bch2_trans_commit_write_locked()
739 			bch2_trans_inconsistent(trans, "invalid bkey on insert from %s -> %ps\n",  in bch2_trans_commit_write_locked()
740 						trans->fn, (void *) i->ip_allocated);  in bch2_trans_commit_write_locked()
743 		btree_insert_entry_checks(trans, i);  in bch2_trans_commit_write_locked()
750 		trans_for_each_update(trans, i) {  in bch2_trans_commit_write_locked()
757 			verify_update_old_key(trans, i);  in bch2_trans_commit_write_locked()
759 			if (trans->journal_transaction_names) {  in bch2_trans_commit_write_locked()
760 				entry = bch2_journal_add_entry(j, &trans->journal_res,  in bch2_trans_commit_write_locked()
768 			entry = bch2_journal_add_entry(j, &trans->journal_res,  in bch2_trans_commit_write_locked()
775 		memcpy_u64s_small(journal_res_entry(&c->journal, &trans->journal_res),  in bch2_trans_commit_write_locked()
776 				  trans->journal_entries,  in bch2_trans_commit_write_locked()
777 				  trans->journal_entries_u64s);  in bch2_trans_commit_write_locked()
779 		trans->journal_res.offset	+= trans->journal_entries_u64s;  in bch2_trans_commit_write_locked()
780 		trans->journal_res.u64s		-= trans->journal_entries_u64s;  in bch2_trans_commit_write_locked()
782 		if (trans->journal_seq)  in bch2_trans_commit_write_locked()
783 			*trans->journal_seq = trans->journal_res.seq;  in bch2_trans_commit_write_locked()
786 	trans_for_each_update(trans, i) {  in bch2_trans_commit_write_locked()
787 		struct btree_path *path = trans->paths + i->path;  in bch2_trans_commit_write_locked()
790 			bch2_btree_insert_key_leaf(trans, path, i->k, trans->journal_res.seq);  in bch2_trans_commit_write_locked()
792 			bch2_btree_insert_key_cached(trans, flags, i);  in bch2_trans_commit_write_locked()
794 			bch2_btree_key_cache_drop(trans, path);  in bch2_trans_commit_write_locked()
802 	for (struct jset_entry *entry2 = trans->journal_entries;  in bch2_trans_commit_write_locked()
807 			bch2_accounting_trans_commit_revert(trans,  in bch2_trans_commit_write_locked()
813 static noinline void bch2_drop_overwrites_from_journal(struct btree_trans *trans)  in bch2_drop_overwrites_from_journal()  argument
821 	trans_for_each_update(trans, i)  in bch2_drop_overwrites_from_journal()
823 			bch2_journal_key_overwritten(trans->c, i->btree_id, i->level, i->k->k.p);  in bch2_drop_overwrites_from_journal()
835 static inline int do_bch2_trans_commit(struct btree_trans *trans, unsigned flags,  in do_bch2_trans_commit()  argument
839 	struct bch_fs *c = trans->c;  in do_bch2_trans_commit()
842 	for (unsigned idx = 0; idx < trans->nr_updates; idx++) {  in do_bch2_trans_commit()
843 		struct btree_insert_entry *i = trans->updates + idx;  in do_bch2_trans_commit()
850 		if (!same_leaf_as_next(trans, i)) {  in do_bch2_trans_commit()
852 				ret = bch2_foreground_maybe_merge(trans, i->path,  in do_bch2_trans_commit()
862 	ret = bch2_trans_lock_write(trans);  in do_bch2_trans_commit()
866 	ret = bch2_trans_commit_write_locked(trans, flags, stopped_at, trace_ip);  in do_bch2_trans_commit()
868 	if (!ret && unlikely(trans->journal_replay_not_finished))  in do_bch2_trans_commit()
869 		bch2_drop_overwrites_from_journal(trans);  in do_bch2_trans_commit()
871 	bch2_trans_unlock_updates_write(trans);  in do_bch2_trans_commit()
873 	if (!ret && trans->journal_pin)  in do_bch2_trans_commit()
874 		bch2_journal_pin_add(&c->journal, trans->journal_res.seq,  in do_bch2_trans_commit()
875 				     trans->journal_pin,  in do_bch2_trans_commit()
883 		bch2_journal_res_put(&c->journal, &trans->journal_res);  in do_bch2_trans_commit()
899 int bch2_trans_commit_error(struct btree_trans *trans, unsigned flags,  in bch2_trans_commit_error()  argument
903 	struct bch_fs *c = trans->c;  in bch2_trans_commit_error()
908 		ret = bch2_btree_split_leaf(trans, i->path, flags);  in bch2_trans_commit_error()
910 			trace_and_count(c, trans_restart_btree_node_split, trans,  in bch2_trans_commit_error()
911 					trace_ip, trans->paths + i->path);  in bch2_trans_commit_error()
914 		ret = drop_locks_do(trans,  in bch2_trans_commit_error()
915 			bch2_accounting_update_sb(trans));  in bch2_trans_commit_error()
928 		ret = drop_locks_do(trans,  in bch2_trans_commit_error()
929 			bch2_trans_journal_res_get(trans,  in bch2_trans_commit_error()
934 		bch2_trans_unlock(trans);  in bch2_trans_commit_error()
936 		trace_and_count(c, trans_blocked_journal_reclaim, trans, trace_ip);  in bch2_trans_commit_error()
947 		ret = bch2_trans_relock(trans);  in bch2_trans_commit_error()
954 	BUG_ON(bch2_err_matches(ret, BCH_ERR_transaction_restart) != !!trans->restarted);  in bch2_trans_commit_error()
969 do_bch2_trans_commit_to_journal_replay(struct btree_trans *trans)  in do_bch2_trans_commit_to_journal_replay()  argument
971 	struct bch_fs *c = trans->c;  in do_bch2_trans_commit_to_journal_replay()
975 	trans_for_each_update(trans, i) {  in do_bch2_trans_commit_to_journal_replay()
981 	for (struct jset_entry *i = trans->journal_entries;  in do_bch2_trans_commit_to_journal_replay()
982 	     i != (void *) ((u64 *) trans->journal_entries + trans->journal_entries_u64s);  in do_bch2_trans_commit_to_journal_replay()
994 int __bch2_trans_commit(struct btree_trans *trans, unsigned flags)  in __bch2_trans_commit()  argument
997 	struct bch_fs *c = trans->c;  in __bch2_trans_commit()
1000 	bch2_trans_verify_not_unlocked_or_in_restart(trans);  in __bch2_trans_commit()
1002 	ret = trans_maybe_inject_restart(trans, _RET_IP_);  in __bch2_trans_commit()
1006 	if (!trans->nr_updates &&  in __bch2_trans_commit()
1007 	    !trans->journal_entries_u64s)  in __bch2_trans_commit()
1010 	ret = bch2_trans_commit_run_triggers(trans);  in __bch2_trans_commit()
1017 			ret = do_bch2_trans_commit_to_journal_replay(trans);  in __bch2_trans_commit()
1025 	trans->journal_u64s		= trans->journal_entries_u64s;  in __bch2_trans_commit()
1026 	trans->journal_transaction_names = READ_ONCE(c->opts.journal_transaction_names);  in __bch2_trans_commit()
1027 	if (trans->journal_transaction_names)  in __bch2_trans_commit()
1028 		trans->journal_u64s += jset_u64s(JSET_ENTRY_LOG_U64s);  in __bch2_trans_commit()
1030 	trans_for_each_update(trans, i) {  in __bch2_trans_commit()
1031 		struct btree_path *path = trans->paths + i->path;  in __bch2_trans_commit()
1035 		ret = bch2_btree_path_upgrade(trans, path, i->level + 1);  in __bch2_trans_commit()
1048 		trans->journal_u64s += jset_u64s(i->k->k.u64s);  in __bch2_trans_commit()
1051 		if (trans->journal_transaction_names)  in __bch2_trans_commit()
1052 			trans->journal_u64s += jset_u64s(i->old_k.u64s);  in __bch2_trans_commit()
1055 	if (trans->extra_disk_res) {  in __bch2_trans_commit()
1056 		ret = bch2_disk_reservation_add(c, trans->disk_res,  in __bch2_trans_commit()
1057 				trans->extra_disk_res,  in __bch2_trans_commit()
1065 	bch2_trans_verify_not_unlocked_or_in_restart(trans);  in __bch2_trans_commit()
1067 		memset(&trans->journal_res, 0, sizeof(trans->journal_res));  in __bch2_trans_commit()
1068 	memset(&trans->fs_usage_delta, 0, sizeof(trans->fs_usage_delta));  in __bch2_trans_commit()
1070 	ret = do_bch2_trans_commit(trans, flags, &errored_at, _RET_IP_);  in __bch2_trans_commit()
1073 	bch2_trans_verify_locks(trans);  in __bch2_trans_commit()
1078 	trace_and_count(c, transaction_commit, trans, _RET_IP_);  in __bch2_trans_commit()
1084 		bch2_trans_downgrade(trans);  in __bch2_trans_commit()
1085 	bch2_trans_reset_updates(trans);  in __bch2_trans_commit()
1089 	ret = bch2_trans_commit_error(trans, flags, errored_at, ret, _RET_IP_);  in __bch2_trans_commit()