26 static inline struct btree_transaction_stats *btree_trans_stats(struct btree_trans *trans)  in btree_trans_stats()  argument
28 	return trans->fn_idx < ARRAY_SIZE(trans->c->btree_transaction_stats)  in btree_trans_stats()
29 		? &trans->c->btree_transaction_stats[trans->fn_idx]  in btree_trans_stats()
79 static inline void mark_btree_node_locked(struct btree_trans *trans,  in mark_btree_node_locked()  argument
109 static void btree_trans_lock_hold_time_update(struct btree_trans *trans,  in btree_trans_lock_hold_time_update()  argument
113 	__bch2_time_stats_update(&btree_trans_stats(trans)->lock_hold_times,  in btree_trans_lock_hold_time_update()
124 static inline void btree_node_unlock(struct btree_trans *trans,  in btree_node_unlock()  argument
133 			bch2_btree_node_unlock_write(trans, path, path->l[level].b);  in btree_node_unlock()
137 		btree_trans_lock_hold_time_update(trans, path, level);  in btree_node_unlock()
152 static inline void __bch2_btree_path_unlock(struct btree_trans *trans,  in __bch2_btree_path_unlock()  argument
158 		btree_node_unlock(trans, path, btree_path_lowest_level_locked(path));  in __bch2_btree_path_unlock()
166 __bch2_btree_node_unlock_write(struct btree_trans *trans, struct btree *b)  in __bch2_btree_node_unlock_write()  argument
172 		trans_for_each_path_with_node(trans, b, linked, i)  in __bch2_btree_node_unlock_write()
180 bch2_btree_node_unlock_write_inlined(struct btree_trans *trans, struct btree_path *path,  in bch2_btree_node_unlock_write_inlined()  argument
188 	__bch2_btree_node_unlock_write(trans, b);  in bch2_btree_node_unlock_write_inlined()
195 static inline void trans_set_locked(struct btree_trans *trans, bool try)  in trans_set_locked()  argument
197 	if (!trans->locked) {  in trans_set_locked()
198 		lock_acquire_exclusive(&trans->dep_map, 0, try, NULL, _THIS_IP_);  in trans_set_locked()
199 		trans->locked = true;  in trans_set_locked()
200 		trans->last_unlock_ip = 0;  in trans_set_locked()
202 		trans->pf_memalloc_nofs = (current->flags & PF_MEMALLOC_NOFS) != 0;  in trans_set_locked()
207 static inline void trans_set_unlocked(struct btree_trans *trans)  in trans_set_unlocked()  argument
209 	if (trans->locked) {  in trans_set_unlocked()
210 		lock_release(&trans->dep_map, _THIS_IP_);  in trans_set_unlocked()
211 		trans->locked = false;  in trans_set_unlocked()
212 		trans->last_unlock_ip = _RET_IP_;  in trans_set_unlocked()
214 		if (!trans->pf_memalloc_nofs)  in trans_set_unlocked()
219 static inline int __btree_node_lock_nopath(struct btree_trans *trans,  in __btree_node_lock_nopath()  argument
225 	trans->lock_may_not_fail = lock_may_not_fail;  in __btree_node_lock_nopath()
226 	trans->lock_must_abort	= false;  in __btree_node_lock_nopath()
227 	trans->locking		= b;  in __btree_node_lock_nopath()
229 	int ret = six_lock_ip_waiter(&b->lock, type, &trans->locking_wait,  in __btree_node_lock_nopath()
230 				     bch2_six_check_for_deadlock, trans, ip);  in __btree_node_lock_nopath()
231 	WRITE_ONCE(trans->locking, NULL);  in __btree_node_lock_nopath()
232 	WRITE_ONCE(trans->locking_wait.start_time, 0);  in __btree_node_lock_nopath()
235 		trace_btree_path_lock(trans, _THIS_IP_, b);  in __btree_node_lock_nopath()
240 btree_node_lock_nopath(struct btree_trans *trans,  in btree_node_lock_nopath()  argument
245 	return __btree_node_lock_nopath(trans, b, type, false, ip);  in btree_node_lock_nopath()
248 static inline void btree_node_lock_nopath_nofail(struct btree_trans *trans,  in btree_node_lock_nopath_nofail()  argument
252 	int ret = __btree_node_lock_nopath(trans, b, type, true, _THIS_IP_);  in btree_node_lock_nopath_nofail()
261 static inline bool btree_node_lock_increment(struct btree_trans *trans,  in btree_node_lock_increment()  argument
269 	trans_for_each_path(trans, path, i)  in btree_node_lock_increment()
279 static inline int btree_node_lock(struct btree_trans *trans,  in btree_node_lock()  argument
289 	bch2_trans_verify_not_unlocked_or_in_restart(trans);  in btree_node_lock()
292 	    btree_node_lock_increment(trans, b, level, (enum btree_node_locked_type) type) ||  in btree_node_lock()
293 	    !(ret = btree_node_lock_nopath(trans, b, type, btree_path_ip_allocated(path)))) {  in btree_node_lock()
305 static inline int __btree_node_lock_write(struct btree_trans *trans,  in __btree_node_lock_write()  argument
323 		: __bch2_btree_node_lock_write(trans, path, b, lock_may_not_fail);  in __btree_node_lock_write()
327 bch2_btree_node_lock_write(struct btree_trans *trans,  in bch2_btree_node_lock_write()  argument
331 	return __btree_node_lock_write(trans, path, b, false);  in bch2_btree_node_lock_write()
344 static inline int bch2_btree_path_relock(struct btree_trans *trans,  in bch2_btree_path_relock()  argument
349 		: __bch2_btree_path_relock(trans, path, trace_ip);  in bch2_btree_path_relock()
354 static inline bool bch2_btree_node_relock(struct btree_trans *trans,  in bch2_btree_node_relock()  argument
363 		 __bch2_btree_node_relock(trans, path, level, true));  in bch2_btree_node_relock()
366 static inline bool bch2_btree_node_relock_notrace(struct btree_trans *trans,  in bch2_btree_node_relock_notrace()  argument
375 		 __bch2_btree_node_relock(trans, path, level, false));  in bch2_btree_node_relock_notrace()
388 static inline int bch2_btree_path_upgrade(struct btree_trans *trans,  in bch2_btree_path_upgrade()  argument
398 	    ? __bch2_btree_path_upgrade(trans, path, new_locks_want, &f)  in bch2_btree_path_upgrade()
402 	trace_and_count(trans->c, trans_restart_upgrade, trans, _THIS_IP_, path,  in bch2_btree_path_upgrade()
404 	return btree_trans_restart(trans, BCH_ERR_transaction_restart_upgrade);  in bch2_btree_path_upgrade()
409 static inline void btree_path_set_should_be_locked(struct btree_trans *trans, struct btree_path *pa…  in btree_path_set_should_be_locked()  argument
415 	trace_btree_path_should_be_locked(trans, path);  in btree_path_set_should_be_locked()
418 static inline void __btree_path_set_level_up(struct btree_trans *trans,  in __btree_path_set_level_up()  argument
422 	btree_node_unlock(trans, path, l);  in __btree_path_set_level_up()
426 static inline void btree_path_set_level_up(struct btree_trans *trans,  in btree_path_set_level_up()  argument
429 	__btree_path_set_level_up(trans, path, path->level++);  in btree_path_set_level_up()
447 static inline void bch2_trans_verify_locks(struct btree_trans *trans) {}  in bch2_trans_verify_locks()  argument