22 static inline void __btree_path_get(struct btree_trans *trans, struct btree_path *path, bool intent)  in __btree_path_get()  argument
24 	unsigned idx = path - trans->paths;  in __btree_path_get()
26 	EBUG_ON(!test_bit(idx, trans->paths_allocated));  in __btree_path_get()
28 		bch2_dump_trans_paths_updates(trans);  in __btree_path_get()
34 	trace_btree_path_get_ll(trans, path);  in __btree_path_get()
37 static inline bool __btree_path_put(struct btree_trans *trans, struct btree_path *path, bool intent)  in __btree_path_put()  argument
39 	EBUG_ON(!test_bit(path - trans->paths, trans->paths_allocated));  in __btree_path_put()
43 	trace_btree_path_put_ll(trans, path);  in __btree_path_put()
76 static inline void btree_trans_sort_paths(struct btree_trans *trans)  in btree_trans_sort_paths()  argument
79 	    trans->paths_sorted)  in btree_trans_sort_paths()
81 	__bch2_btree_trans_sort_paths(trans);  in btree_trans_sort_paths()
101 __trans_next_path(struct btree_trans *trans, unsigned *idx)  in __trans_next_path()  argument
103 	unsigned long *w = trans->paths_allocated + *idx / BITS_PER_LONG;  in __trans_next_path()
109 	while (*idx < trans->nr_paths) {  in __trans_next_path()
113 			return trans->paths + *idx;  in __trans_next_path()
136 static inline struct btree_path *next_btree_path(struct btree_trans *trans, struct btree_path *path)  in next_btree_path()  argument
140 	EBUG_ON(idx > trans->nr_sorted);  in next_btree_path()
142 	return idx < trans->nr_sorted  in next_btree_path()
143 		? trans->paths + trans->sorted[idx]  in next_btree_path()
147 static inline struct btree_path *prev_btree_path(struct btree_trans *trans, struct btree_path *path)  in prev_btree_path()  argument
149 	unsigned idx = path ? path->sorted_idx : trans->nr_sorted;  in prev_btree_path()
152 		? trans->paths + trans->sorted[idx - 1]  in prev_btree_path()
158 	     (_iter.path_idx = trans->sorted[_iter.sorted_idx],		\
169 	     (_iter.path_idx = trans->sorted[_iter.sorted_idx],		\
175 	for (_i = trans->nr_sorted - 1;					\
176 	     ((_path) = (_trans)->paths + trans->sorted[_i]), (_i) >= 0;\
187 __trans_next_path_with_node(struct btree_trans *trans, struct btree *b,  in __trans_next_path_with_node()  argument
192 	while ((path = __trans_next_path(trans, idx)) &&  in __trans_next_path_with_node()
208 bch2_btree_path_make_mut(struct btree_trans *trans,  in bch2_btree_path_make_mut()  argument
212 	if (trans->paths[path].ref > 1 ||  in bch2_btree_path_make_mut()
213 	    trans->paths[path].preserve)  in bch2_btree_path_make_mut()
214 		path = __bch2_btree_path_make_mut(trans, path, intent, ip);  in bch2_btree_path_make_mut()
215 	trans->paths[path].should_be_locked = false;  in bch2_btree_path_make_mut()
224 bch2_btree_path_set_pos(struct btree_trans *trans,  in bch2_btree_path_set_pos()  argument
228 	return !bpos_eq(new_pos, trans->paths[path].pos)  in bch2_btree_path_set_pos()
229 		? __bch2_btree_path_set_pos(trans, path, new_pos, intent, ip)  in bch2_btree_path_set_pos()
239 static inline int __must_check bch2_btree_path_traverse(struct btree_trans *trans,  in bch2_btree_path_traverse()  argument
242 	bch2_trans_verify_not_unlocked(trans);  in bch2_btree_path_traverse()
244 	if (trans->paths[path].uptodate < BTREE_ITER_NEED_RELOCK)  in bch2_btree_path_traverse()
247 	return bch2_btree_path_traverse_one(trans, path, flags, _RET_IP_);  in bch2_btree_path_traverse()
280 static inline int bch2_trans_mutex_lock(struct btree_trans *trans, struct mutex *lock)  in bch2_trans_mutex_lock()  argument
284 		: __bch2_trans_mutex_lock(trans, lock);  in bch2_trans_mutex_lock()
291 static inline void bch2_trans_verify_paths(struct btree_trans *trans) {}  in bch2_trans_verify_paths()  argument
292 static inline void bch2_assert_pos_locked(struct btree_trans *trans, enum btree_id id,  in bch2_assert_pos_locked()  argument
296 void bch2_btree_path_fix_key_modified(struct btree_trans *trans,
298 void bch2_btree_node_iter_fix(struct btree_trans *trans, struct btree_path *,
311 static inline int trans_was_restarted(struct btree_trans *trans, u32 restart_count)  in trans_was_restarted()  argument
313 	return restart_count != trans->restart_count  in trans_was_restarted()
320 static inline void bch2_trans_verify_not_restarted(struct btree_trans *trans,  in bch2_trans_verify_not_restarted()  argument
323 	if (trans_was_restarted(trans, restart_count))  in bch2_trans_verify_not_restarted()
324 		bch2_trans_restart_error(trans, restart_count);  in bch2_trans_verify_not_restarted()
329 static inline void bch2_trans_verify_not_in_restart(struct btree_trans *trans)  in bch2_trans_verify_not_in_restart()  argument
331 	if (trans->restarted)  in bch2_trans_verify_not_in_restart()
332 		bch2_trans_in_restart_error(trans);  in bch2_trans_verify_not_in_restart()
337 static inline void bch2_trans_verify_not_unlocked(struct btree_trans *trans)  in bch2_trans_verify_not_unlocked()  argument
339 	if (!trans->locked)  in bch2_trans_verify_not_unlocked()
340 		bch2_trans_unlocked_error(trans);  in bch2_trans_verify_not_unlocked()
344 static int btree_trans_restart_nounlock(struct btree_trans *trans, int err)  in btree_trans_restart_nounlock()  argument
349 	trans->restarted = err;  in btree_trans_restart_nounlock()
350 	trans->last_restarted_ip = _THIS_IP_;  in btree_trans_restart_nounlock()
355 static int btree_trans_restart(struct btree_trans *trans, int err)  in btree_trans_restart()  argument
357 	btree_trans_restart_nounlock(trans, err);  in btree_trans_restart()
366 static inline void bch2_btree_path_downgrade(struct btree_trans *trans,  in bch2_btree_path_downgrade()  argument
372 		__bch2_btree_path_downgrade(trans, path, new_locks_want);  in bch2_btree_path_downgrade()
377 void bch2_trans_node_add(struct btree_trans *trans, struct btree_path *, struct btree *);
416 	struct btree_trans *trans = iter->trans;  in bch2_btree_iter_set_pos()  local
419 		bch2_path_put(trans, iter->update_path,  in bch2_btree_iter_set_pos()
446 static inline unsigned __bch2_btree_iter_flags(struct btree_trans *trans,  in __bch2_btree_iter_flags()  argument
462 	if (trans->journal_replay_not_finished)  in __bch2_btree_iter_flags()
468 static inline unsigned bch2_btree_iter_flags(struct btree_trans *trans,  in bch2_btree_iter_flags()  argument
472 	if (!btree_id_cached(trans->c, btree_id)) {  in bch2_btree_iter_flags()
478 	return __bch2_btree_iter_flags(trans, btree_id, flags);  in bch2_btree_iter_flags()
481 static inline void bch2_trans_iter_init_common(struct btree_trans *trans,  in bch2_trans_iter_init_common()  argument
489 	iter->trans		= trans;  in bch2_trans_iter_init_common()
502 	iter->path = bch2_path_get(trans, btree_id, iter->pos,  in bch2_trans_iter_init_common()
509 static inline void bch2_trans_iter_init(struct btree_trans *trans,  in bch2_trans_iter_init()  argument
516 		bch2_trans_iter_init_common(trans, iter, btree_id, pos, 0, 0,  in bch2_trans_iter_init()
517 				bch2_btree_iter_flags(trans, btree_id, flags),  in bch2_trans_iter_init()
520 		bch2_trans_iter_init_outlined(trans, iter, btree_id, pos, flags);  in bch2_trans_iter_init()
538 static inline void *bch2_trans_kmalloc(struct btree_trans *trans, size_t size)  in bch2_trans_kmalloc()  argument
542 	if (likely(trans->mem_top + size <= trans->mem_bytes)) {  in bch2_trans_kmalloc()
543 		void *p = trans->mem + trans->mem_top;  in bch2_trans_kmalloc()
545 		trans->mem_top += size;  in bch2_trans_kmalloc()
549 		return __bch2_trans_kmalloc(trans, size);  in bch2_trans_kmalloc()
553 static inline void *bch2_trans_kmalloc_nomemzero(struct btree_trans *trans, size_t size)  in bch2_trans_kmalloc_nomemzero()  argument
557 	if (likely(trans->mem_top + size <= trans->mem_bytes)) {  in bch2_trans_kmalloc_nomemzero()
558 		void *p = trans->mem + trans->mem_top;  in bch2_trans_kmalloc_nomemzero()
560 		trans->mem_top += size;  in bch2_trans_kmalloc_nomemzero()
563 		return __bch2_trans_kmalloc(trans, size);  in bch2_trans_kmalloc_nomemzero()
567 static inline struct bkey_s_c __bch2_bkey_get_iter(struct btree_trans *trans,  in __bch2_bkey_get_iter()  argument
574 	bch2_trans_iter_init(trans, iter, btree_id, pos, flags);  in __bch2_bkey_get_iter()
580 		bch2_trans_iter_exit(trans, iter);  in __bch2_bkey_get_iter()
584 static inline struct bkey_s_c bch2_bkey_get_iter(struct btree_trans *trans,  in bch2_bkey_get_iter()  argument
589 	return __bch2_bkey_get_iter(trans, iter, btree_id, pos, flags, 0);  in bch2_bkey_get_iter()
605 static inline int __bch2_bkey_get_val_typed(struct btree_trans *trans,  in __bch2_bkey_get_val_typed()  argument
614 	k = __bch2_bkey_get_iter(trans, &iter, btree_id, pos, flags, type);  in __bch2_bkey_get_val_typed()
622 		bch2_trans_iter_exit(trans, &iter);  in __bch2_bkey_get_val_typed()
695 static inline int btree_trans_too_many_iters(struct btree_trans *trans)  in btree_trans_too_many_iters()  argument
697 	if (bitmap_weight(trans->paths_allocated, trans->nr_paths) > BTREE_ITER_NORMAL_LIMIT - 8)  in btree_trans_too_many_iters()
698 		return __bch2_btree_trans_too_many_iters(trans);  in btree_trans_too_many_iters()
776 	bch2_trans_begin(trans);					\
909 	struct btree_trans *trans = bch2_trans_get(_c);			\
911 	bch2_trans_put(trans);						\
915 #define bch2_trans_do(_c, _do)	bch2_trans_run(_c, lockrestart_do(trans, _do))