Lines Matching refs:iter
28 static inline unsigned long btree_iter_ip_allocated(struct btree_iter *iter) in btree_iter_ip_allocated() argument
31 return iter->ip_allocated; in btree_iter_ip_allocated()
61 static inline struct bpos bkey_successor(struct btree_iter *iter, struct bpos p) in bkey_successor() argument
64 if (iter->flags & BTREE_ITER_all_snapshots) { in bkey_successor()
68 p.snapshot = iter->snapshot; in bkey_successor()
74 static inline struct bpos bkey_predecessor(struct btree_iter *iter, struct bpos p) in bkey_predecessor() argument
77 if (iter->flags & BTREE_ITER_all_snapshots) { in bkey_predecessor()
81 p.snapshot = iter->snapshot; in bkey_predecessor()
87 static inline struct bpos btree_iter_search_key(struct btree_iter *iter) in btree_iter_search_key() argument
89 struct bpos pos = iter->pos; in btree_iter_search_key()
91 if ((iter->flags & BTREE_ITER_is_extents) && in btree_iter_search_key()
93 pos = bkey_successor(iter, pos); in btree_iter_search_key()
154 tmp = l->iter; in bch2_btree_path_verify_level()
171 bch2_btree_node_iter_verify(&l->iter, l->b); in bch2_btree_path_verify_level()
179 k = bch2_btree_node_iter_peek_all(&l->iter, l->b); in bch2_btree_path_verify_level()
241 unsigned iter; in bch2_trans_verify_paths() local
243 trans_for_each_path(trans, path, iter) in bch2_trans_verify_paths()
247 static void bch2_btree_iter_verify(struct btree_iter *iter) in bch2_btree_iter_verify() argument
249 struct btree_trans *trans = iter->trans; in bch2_btree_iter_verify()
251 BUG_ON(!!(iter->flags & BTREE_ITER_cached) != btree_iter_path(trans, iter)->cached); in bch2_btree_iter_verify()
253 BUG_ON((iter->flags & BTREE_ITER_is_extents) && in bch2_btree_iter_verify()
254 (iter->flags & BTREE_ITER_all_snapshots)); in bch2_btree_iter_verify()
256 BUG_ON(!(iter->flags & BTREE_ITER_snapshot_field) && in bch2_btree_iter_verify()
257 (iter->flags & BTREE_ITER_all_snapshots) && in bch2_btree_iter_verify()
258 !btree_type_has_snapshot_field(iter->btree_id)); in bch2_btree_iter_verify()
260 if (iter->update_path) in bch2_btree_iter_verify()
261 bch2_btree_path_verify(trans, &trans->paths[iter->update_path]); in bch2_btree_iter_verify()
262 bch2_btree_path_verify(trans, btree_iter_path(trans, iter)); in bch2_btree_iter_verify()
265 static void bch2_btree_iter_verify_entry_exit(struct btree_iter *iter) in bch2_btree_iter_verify_entry_exit() argument
267 BUG_ON((iter->flags & BTREE_ITER_filter_snapshots) && in bch2_btree_iter_verify_entry_exit()
268 !iter->pos.snapshot); in bch2_btree_iter_verify_entry_exit()
270 BUG_ON(!(iter->flags & BTREE_ITER_all_snapshots) && in bch2_btree_iter_verify_entry_exit()
271 iter->pos.snapshot != iter->snapshot); in bch2_btree_iter_verify_entry_exit()
273 BUG_ON(bkey_lt(iter->pos, bkey_start_pos(&iter->k)) || in bch2_btree_iter_verify_entry_exit()
274 bkey_gt(iter->pos, iter->k.p)); in bch2_btree_iter_verify_entry_exit()
277 static int bch2_btree_iter_verify_ret(struct btree_iter *iter, struct bkey_s_c k) in bch2_btree_iter_verify_ret() argument
279 struct btree_trans *trans = iter->trans; in bch2_btree_iter_verify_ret()
287 if (!(iter->flags & BTREE_ITER_filter_snapshots)) in bch2_btree_iter_verify_ret()
294 iter->snapshot, in bch2_btree_iter_verify_ret()
297 bch2_trans_iter_init(trans, ©, iter->btree_id, iter->pos, in bch2_btree_iter_verify_ret()
309 bch2_snapshot_is_ancestor(trans->c, iter->snapshot, in bch2_btree_iter_verify_ret()
319 iter->snapshot, in bch2_btree_iter_verify_ret()
333 struct trans_for_each_path_inorder_iter iter; in bch2_assert_pos_locked() local
338 trans_for_each_path_inorder(trans, path, iter) { in bch2_assert_pos_locked()
366 static inline void bch2_btree_iter_verify(struct btree_iter *iter) {} in bch2_btree_iter_verify() argument
367 static inline void bch2_btree_iter_verify_entry_exit(struct btree_iter *iter) {} in bch2_btree_iter_verify_entry_exit() argument
368 static inline int bch2_btree_iter_verify_ret(struct btree_iter *iter, struct bkey_s_c k) { return 0… in bch2_btree_iter_verify_ret() argument
374 static void btree_node_iter_set_set_pos(struct btree_node_iter *iter, in btree_node_iter_set_set_pos() argument
381 btree_node_iter_for_each(iter, set) in btree_node_iter_set_set_pos()
384 bch2_btree_node_iter_sort(iter, b); in btree_node_iter_set_set_pos()
388 bch2_btree_node_iter_push(iter, b, k, btree_bkey_last(b, t)); in btree_node_iter_set_set_pos()
397 if (where != bch2_btree_node_iter_peek_all(&l->iter, l->b)) in __bch2_btree_path_fix_key_modified()
401 bch2_btree_node_iter_advance(&l->iter, l->b); in __bch2_btree_path_fix_key_modified()
520 if (node_iter != &path->l[b->c.level].iter) { in bch2_btree_node_iter_fix()
530 &linked->l[b->c.level].iter, t, in bch2_btree_node_iter_fix()
560 bch2_btree_node_iter_peek_all(&l->iter, l->b)); in btree_path_level_peek_all()
569 bch2_btree_node_iter_peek(&l->iter, l->b)); in btree_path_level_peek()
583 bch2_btree_node_iter_prev(&l->iter, l->b)); in btree_path_level_prev()
598 while ((k = bch2_btree_node_iter_peek_all(&l->iter, l->b)) && in btree_path_advance_to_pos()
603 bch2_btree_node_iter_advance(&l->iter, l->b); in btree_path_advance_to_pos()
615 bch2_btree_node_iter_init(&l->iter, l->b, &path->pos); in __btree_path_level_init()
622 bch2_btree_node_iter_peek(&l->iter, l->b); in __btree_path_level_init()
782 struct btree_node_iter node_iter = l->iter; in btree_path_prefetch()
861 k = bch2_btree_node_iter_peek_all(&l->iter, l->b); in btree_node_mem_ptr_set()
882 __bch2_btree_and_journal_iter_init_node_iter(trans, &jiter, l->b, l->iter, path->pos); in btree_node_iter_and_journal_peek()
931 struct bkey_packed *k = bch2_btree_node_iter_peek(&l->iter, l->b); in btree_path_down()
1203 unsigned iter; in bch2_btree_path_traverse_one() local
1205 trans_for_each_path_with_node(trans, path_l(path)->b, linked, iter) in bch2_btree_path_traverse_one()
1302 bch2_btree_node_iter_init(&l->iter, l->b, &path->pos); in __bch2_btree_path_set_pos()
1309 bch2_btree_node_iter_peek(&l->iter, l->b); in __bch2_btree_path_set_pos()
1556 struct trans_for_each_path_inorder_iter iter; in __bch2_trans_paths_to_text() local
1561 trans_for_each_path_idx_inorder(trans, iter) { in __bch2_trans_paths_to_text()
1562 bch2_btree_path_to_text_short(out, trans, iter.path_idx); in __bch2_trans_paths_to_text()
1717 struct trans_for_each_path_inorder_iter iter; in bch2_path_get() local
1726 trans_for_each_path_inorder(trans, path, iter) { in bch2_path_get()
1734 path_pos = iter.path_idx; in bch2_path_get()
1820 _k = bch2_btree_node_iter_peek_all(&l->iter, l->b); in bch2_btree_path_peek_slot()
1847 void bch2_set_btree_iter_dontneed(struct btree_iter *iter) in bch2_set_btree_iter_dontneed() argument
1849 struct btree_trans *trans = iter->trans; in bch2_set_btree_iter_dontneed()
1851 if (!iter->path || trans->restarted) in bch2_set_btree_iter_dontneed()
1854 struct btree_path *path = btree_iter_path(trans, iter); in bch2_set_btree_iter_dontneed()
1862 __bch2_btree_iter_traverse(struct btree_iter *iter) in __bch2_btree_iter_traverse() argument
1864 return bch2_btree_path_traverse(iter->trans, iter->path, iter->flags); in __bch2_btree_iter_traverse()
1868 bch2_btree_iter_traverse(struct btree_iter *iter) in bch2_btree_iter_traverse() argument
1870 struct btree_trans *trans = iter->trans; in bch2_btree_iter_traverse()
1875 iter->path = bch2_btree_path_set_pos(trans, iter->path, in bch2_btree_iter_traverse()
1876 btree_iter_search_key(iter), in bch2_btree_iter_traverse()
1877 iter->flags & BTREE_ITER_intent, in bch2_btree_iter_traverse()
1878 btree_iter_ip_allocated(iter)); in bch2_btree_iter_traverse()
1880 ret = bch2_btree_path_traverse(iter->trans, iter->path, iter->flags); in bch2_btree_iter_traverse()
1884 struct btree_path *path = btree_iter_path(trans, iter); in bch2_btree_iter_traverse()
1892 struct btree *bch2_btree_iter_peek_node(struct btree_iter *iter) in bch2_btree_iter_peek_node() argument
1894 struct btree_trans *trans = iter->trans; in bch2_btree_iter_peek_node()
1898 EBUG_ON(trans->paths[iter->path].cached); in bch2_btree_iter_peek_node()
1899 bch2_btree_iter_verify(iter); in bch2_btree_iter_peek_node()
1901 ret = bch2_btree_path_traverse(trans, iter->path, iter->flags); in bch2_btree_iter_peek_node()
1905 struct btree_path *path = btree_iter_path(trans, iter); in bch2_btree_iter_peek_node()
1910 BUG_ON(bpos_lt(b->key.k.p, iter->pos)); in bch2_btree_iter_peek_node()
1912 bkey_init(&iter->k); in bch2_btree_iter_peek_node()
1913 iter->k.p = iter->pos = b->key.k.p; in bch2_btree_iter_peek_node()
1915 iter->path = bch2_btree_path_set_pos(trans, iter->path, b->key.k.p, in bch2_btree_iter_peek_node()
1916 iter->flags & BTREE_ITER_intent, in bch2_btree_iter_peek_node()
1917 btree_iter_ip_allocated(iter)); in bch2_btree_iter_peek_node()
1918 btree_path_set_should_be_locked(trans, btree_iter_path(trans, iter)); in bch2_btree_iter_peek_node()
1920 bch2_btree_iter_verify_entry_exit(iter); in bch2_btree_iter_peek_node()
1921 bch2_btree_iter_verify(iter); in bch2_btree_iter_peek_node()
1930 struct btree *bch2_btree_iter_peek_node_and_restart(struct btree_iter *iter) in bch2_btree_iter_peek_node_and_restart() argument
1934 while (b = bch2_btree_iter_peek_node(iter), in bch2_btree_iter_peek_node_and_restart()
1936 bch2_trans_begin(iter->trans); in bch2_btree_iter_peek_node_and_restart()
1941 struct btree *bch2_btree_iter_next_node(struct btree_iter *iter) in bch2_btree_iter_next_node() argument
1943 struct btree_trans *trans = iter->trans; in bch2_btree_iter_next_node()
1947 EBUG_ON(trans->paths[iter->path].cached); in bch2_btree_iter_next_node()
1949 bch2_btree_iter_verify(iter); in bch2_btree_iter_next_node()
1951 ret = bch2_btree_path_traverse(trans, iter->path, iter->flags); in bch2_btree_iter_next_node()
1956 struct btree_path *path = btree_iter_path(trans, iter); in bch2_btree_iter_next_node()
1980 if (bpos_eq(iter->pos, b->key.k.p)) { in bch2_btree_iter_next_node()
1990 iter->path = bch2_btree_path_set_pos(trans, iter->path, in bch2_btree_iter_next_node()
1991 bpos_successor(iter->pos), in bch2_btree_iter_next_node()
1992 iter->flags & BTREE_ITER_intent, in bch2_btree_iter_next_node()
1993 btree_iter_ip_allocated(iter)); in bch2_btree_iter_next_node()
1995 path = btree_iter_path(trans, iter); in bch2_btree_iter_next_node()
1996 btree_path_set_level_down(trans, path, iter->min_depth); in bch2_btree_iter_next_node()
1998 ret = bch2_btree_path_traverse(trans, iter->path, iter->flags); in bch2_btree_iter_next_node()
2002 path = btree_iter_path(trans, iter); in bch2_btree_iter_next_node()
2006 bkey_init(&iter->k); in bch2_btree_iter_next_node()
2007 iter->k.p = iter->pos = b->key.k.p; in bch2_btree_iter_next_node()
2009 iter->path = bch2_btree_path_set_pos(trans, iter->path, b->key.k.p, in bch2_btree_iter_next_node()
2010 iter->flags & BTREE_ITER_intent, in bch2_btree_iter_next_node()
2011 btree_iter_ip_allocated(iter)); in bch2_btree_iter_next_node()
2012 btree_path_set_should_be_locked(trans, btree_iter_path(trans, iter)); in bch2_btree_iter_next_node()
2013 EBUG_ON(btree_iter_path(trans, iter)->uptodate); in bch2_btree_iter_next_node()
2015 bch2_btree_iter_verify_entry_exit(iter); in bch2_btree_iter_next_node()
2016 bch2_btree_iter_verify(iter); in bch2_btree_iter_next_node()
2026 inline bool bch2_btree_iter_advance(struct btree_iter *iter) in bch2_btree_iter_advance() argument
2028 struct bpos pos = iter->k.p; in bch2_btree_iter_advance()
2029 bool ret = !(iter->flags & BTREE_ITER_all_snapshots in bch2_btree_iter_advance()
2033 if (ret && !(iter->flags & BTREE_ITER_is_extents)) in bch2_btree_iter_advance()
2034 pos = bkey_successor(iter, pos); in bch2_btree_iter_advance()
2035 bch2_btree_iter_set_pos(iter, pos); in bch2_btree_iter_advance()
2039 inline bool bch2_btree_iter_rewind(struct btree_iter *iter) in bch2_btree_iter_rewind() argument
2041 struct bpos pos = bkey_start_pos(&iter->k); in bch2_btree_iter_rewind()
2042 bool ret = !(iter->flags & BTREE_ITER_all_snapshots in bch2_btree_iter_rewind()
2046 if (ret && !(iter->flags & BTREE_ITER_is_extents)) in bch2_btree_iter_rewind()
2047 pos = bkey_predecessor(iter, pos); in bch2_btree_iter_rewind()
2048 bch2_btree_iter_set_pos(iter, pos); in bch2_btree_iter_rewind()
2053 void bch2_btree_trans_peek_prev_updates(struct btree_trans *trans, struct btree_iter *iter, in bch2_btree_trans_peek_prev_updates() argument
2056 struct bpos end = path_l(btree_iter_path(trans, iter))->b->data->min_key; in bch2_btree_trans_peek_prev_updates()
2060 i->btree_id == iter->btree_id && in bch2_btree_trans_peek_prev_updates()
2061 bpos_le(i->k->k.p, iter->pos) && in bch2_btree_trans_peek_prev_updates()
2063 iter->k = i->k->k; in bch2_btree_trans_peek_prev_updates()
2069 void bch2_btree_trans_peek_updates(struct btree_trans *trans, struct btree_iter *iter, in bch2_btree_trans_peek_updates() argument
2072 struct btree_path *path = btree_iter_path(trans, iter); in bch2_btree_trans_peek_updates()
2077 i->btree_id == iter->btree_id && in bch2_btree_trans_peek_updates()
2080 iter->k = i->k->k; in bch2_btree_trans_peek_updates()
2086 void bch2_btree_trans_peek_slot_updates(struct btree_trans *trans, struct btree_iter *iter, in bch2_btree_trans_peek_slot_updates() argument
2091 i->btree_id == iter->btree_id && in bch2_btree_trans_peek_slot_updates()
2092 bpos_eq(i->k->k.p, iter->pos)) { in bch2_btree_trans_peek_slot_updates()
2093 iter->k = i->k->k; in bch2_btree_trans_peek_slot_updates()
2099 struct btree_iter *iter, in bch2_btree_journal_peek() argument
2102 struct btree_path *path = btree_iter_path(trans, iter); in bch2_btree_journal_peek()
2104 return bch2_journal_keys_peek_upto(trans->c, iter->btree_id, in bch2_btree_journal_peek()
2108 &iter->journal_idx); in bch2_btree_journal_peek()
2113 struct btree_iter *iter) in btree_trans_peek_slot_journal() argument
2115 struct btree_path *path = btree_iter_path(trans, iter); in btree_trans_peek_slot_journal()
2116 struct bkey_i *k = bch2_btree_journal_peek(trans, iter, path->pos); in btree_trans_peek_slot_journal()
2119 iter->k = k->k; in btree_trans_peek_slot_journal()
2128 struct btree_iter *iter, in btree_trans_peek_journal() argument
2131 struct btree_path *path = btree_iter_path(trans, iter); in btree_trans_peek_journal()
2133 bch2_btree_journal_peek(trans, iter, in btree_trans_peek_journal()
2137 iter->k = next_journal->k; in btree_trans_peek_journal()
2149 struct bkey_s_c btree_trans_peek_key_cache(struct btree_iter *iter, struct bpos pos) in btree_trans_peek_key_cache() argument
2151 struct btree_trans *trans = iter->trans; in btree_trans_peek_key_cache()
2160 if ((iter->flags & BTREE_ITER_key_cache_fill) && in btree_trans_peek_key_cache()
2161 bpos_eq(iter->pos, pos)) in btree_trans_peek_key_cache()
2164 if (!bch2_btree_key_cache_find(c, iter->btree_id, pos)) in btree_trans_peek_key_cache()
2167 if (!iter->key_cache_path) in btree_trans_peek_key_cache()
2168 iter->key_cache_path = bch2_path_get(trans, iter->btree_id, pos, in btree_trans_peek_key_cache()
2169 iter->flags & BTREE_ITER_intent, 0, in btree_trans_peek_key_cache()
2170 iter->flags|BTREE_ITER_cached| in btree_trans_peek_key_cache()
2174 iter->key_cache_path = bch2_btree_path_set_pos(trans, iter->key_cache_path, pos, in btree_trans_peek_key_cache()
2175 iter->flags & BTREE_ITER_intent, in btree_trans_peek_key_cache()
2176 btree_iter_ip_allocated(iter)); in btree_trans_peek_key_cache()
2178 ret = bch2_btree_path_traverse(trans, iter->key_cache_path, in btree_trans_peek_key_cache()
2179 iter->flags|BTREE_ITER_cached) ?: in btree_trans_peek_key_cache()
2180 bch2_btree_path_relock(trans, btree_iter_path(trans, iter), _THIS_IP_); in btree_trans_peek_key_cache()
2184 btree_path_set_should_be_locked(trans, trans->paths + iter->key_cache_path); in btree_trans_peek_key_cache()
2186 k = bch2_btree_path_peek_slot(trans->paths + iter->key_cache_path, &u); in btree_trans_peek_key_cache()
2188 iter->k = u; in btree_trans_peek_key_cache()
2189 k.k = &iter->k; in btree_trans_peek_key_cache()
2194 static struct bkey_s_c __bch2_btree_iter_peek(struct btree_iter *iter, struct bpos search_key) in __bch2_btree_iter_peek() argument
2196 struct btree_trans *trans = iter->trans; in __bch2_btree_iter_peek()
2200 EBUG_ON(btree_iter_path(trans, iter)->cached); in __bch2_btree_iter_peek()
2201 bch2_btree_iter_verify(iter); in __bch2_btree_iter_peek()
2206 iter->path = bch2_btree_path_set_pos(trans, iter->path, search_key, in __bch2_btree_iter_peek()
2207 iter->flags & BTREE_ITER_intent, in __bch2_btree_iter_peek()
2208 btree_iter_ip_allocated(iter)); in __bch2_btree_iter_peek()
2210 ret = bch2_btree_path_traverse(trans, iter->path, iter->flags); in __bch2_btree_iter_peek()
2213 bch2_btree_iter_set_pos(iter, iter->pos); in __bch2_btree_iter_peek()
2218 struct btree_path *path = btree_iter_path(trans, iter); in __bch2_btree_iter_peek()
2223 bch2_btree_iter_set_pos(iter, SPOS_MAX); in __bch2_btree_iter_peek()
2230 k = btree_path_level_peek_all(trans->c, l, &iter->k); in __bch2_btree_iter_peek()
2232 if (unlikely(iter->flags & BTREE_ITER_with_key_cache) && in __bch2_btree_iter_peek()
2234 (k2 = btree_trans_peek_key_cache(iter, k.k->p)).k) { in __bch2_btree_iter_peek()
2238 bch2_btree_iter_set_pos(iter, iter->pos); in __bch2_btree_iter_peek()
2243 if (unlikely(iter->flags & BTREE_ITER_with_journal)) in __bch2_btree_iter_peek()
2244 k = btree_trans_peek_journal(trans, iter, k); in __bch2_btree_iter_peek()
2246 if (unlikely((iter->flags & BTREE_ITER_with_updates) && in __bch2_btree_iter_peek()
2248 bch2_btree_trans_peek_updates(trans, iter, &k); in __bch2_btree_iter_peek()
2271 bch2_btree_iter_set_pos(iter, SPOS_MAX); in __bch2_btree_iter_peek()
2277 bch2_btree_iter_verify(iter); in __bch2_btree_iter_peek()
2290 struct bkey_s_c bch2_btree_iter_peek_upto(struct btree_iter *iter, struct bpos end) in bch2_btree_iter_peek_upto() argument
2292 struct btree_trans *trans = iter->trans; in bch2_btree_iter_peek_upto()
2293 struct bpos search_key = btree_iter_search_key(iter); in bch2_btree_iter_peek_upto()
2299 EBUG_ON((iter->flags & BTREE_ITER_filter_snapshots) && bkey_eq(end, POS_MAX)); in bch2_btree_iter_peek_upto()
2301 if (iter->update_path) { in bch2_btree_iter_peek_upto()
2302 bch2_path_put_nokeep(trans, iter->update_path, in bch2_btree_iter_peek_upto()
2303 iter->flags & BTREE_ITER_intent); in bch2_btree_iter_peek_upto()
2304 iter->update_path = 0; in bch2_btree_iter_peek_upto()
2307 bch2_btree_iter_verify_entry_exit(iter); in bch2_btree_iter_peek_upto()
2310 k = __bch2_btree_iter_peek(iter, search_key); in bch2_btree_iter_peek_upto()
2326 if (unlikely(!(iter->flags & BTREE_ITER_is_extents) in bch2_btree_iter_peek_upto()
2331 if (iter->update_path && in bch2_btree_iter_peek_upto()
2332 !bkey_eq(trans->paths[iter->update_path].pos, k.k->p)) { in bch2_btree_iter_peek_upto()
2333 bch2_path_put_nokeep(trans, iter->update_path, in bch2_btree_iter_peek_upto()
2334 iter->flags & BTREE_ITER_intent); in bch2_btree_iter_peek_upto()
2335 iter->update_path = 0; in bch2_btree_iter_peek_upto()
2338 if ((iter->flags & BTREE_ITER_filter_snapshots) && in bch2_btree_iter_peek_upto()
2339 (iter->flags & BTREE_ITER_intent) && in bch2_btree_iter_peek_upto()
2340 !(iter->flags & BTREE_ITER_is_extents) && in bch2_btree_iter_peek_upto()
2341 !iter->update_path) { in bch2_btree_iter_peek_upto()
2344 if (pos.snapshot < iter->snapshot) { in bch2_btree_iter_peek_upto()
2349 pos.snapshot = iter->snapshot; in bch2_btree_iter_peek_upto()
2355 __btree_path_get(trans, trans->paths + iter->path, iter->flags & BTREE_ITER_intent); in bch2_btree_iter_peek_upto()
2356 iter->update_path = iter->path; in bch2_btree_iter_peek_upto()
2358 iter->update_path = bch2_btree_path_set_pos(trans, in bch2_btree_iter_peek_upto()
2359 iter->update_path, pos, in bch2_btree_iter_peek_upto()
2360 iter->flags & BTREE_ITER_intent, in bch2_btree_iter_peek_upto()
2362 ret = bch2_btree_path_traverse(trans, iter->update_path, iter->flags); in bch2_btree_iter_peek_upto()
2373 if ((iter->flags & BTREE_ITER_filter_snapshots) && in bch2_btree_iter_peek_upto()
2375 iter->snapshot, in bch2_btree_iter_peek_upto()
2382 !(iter->flags & BTREE_ITER_all_snapshots)) { in bch2_btree_iter_peek_upto()
2383 search_key = bkey_successor(iter, k.k->p); in bch2_btree_iter_peek_upto()
2392 if (!(iter->flags & BTREE_ITER_is_extents)) in bch2_btree_iter_peek_upto()
2395 iter_pos = bkey_max(iter->pos, bkey_start_pos(k.k)); in bch2_btree_iter_peek_upto()
2397 if (unlikely(iter->flags & BTREE_ITER_all_snapshots ? bpos_gt(iter_pos, end) : in bch2_btree_iter_peek_upto()
2398 iter->flags & BTREE_ITER_is_extents ? bkey_ge(iter_pos, end) : in bch2_btree_iter_peek_upto()
2405 iter->pos = iter_pos; in bch2_btree_iter_peek_upto()
2407 iter->path = bch2_btree_path_set_pos(trans, iter->path, k.k->p, in bch2_btree_iter_peek_upto()
2408 iter->flags & BTREE_ITER_intent, in bch2_btree_iter_peek_upto()
2409 btree_iter_ip_allocated(iter)); in bch2_btree_iter_peek_upto()
2411 btree_path_set_should_be_locked(trans, btree_iter_path(trans, iter)); in bch2_btree_iter_peek_upto()
2413 if (iter->update_path) { in bch2_btree_iter_peek_upto()
2414 ret = bch2_btree_path_relock(trans, trans->paths + iter->update_path, _THIS_IP_); in bch2_btree_iter_peek_upto()
2418 btree_path_set_should_be_locked(trans, trans->paths + iter->update_path); in bch2_btree_iter_peek_upto()
2421 if (!(iter->flags & BTREE_ITER_all_snapshots)) in bch2_btree_iter_peek_upto()
2422 iter->pos.snapshot = iter->snapshot; in bch2_btree_iter_peek_upto()
2424 ret = bch2_btree_iter_verify_ret(iter, k); in bch2_btree_iter_peek_upto()
2426 bch2_btree_iter_set_pos(iter, iter->pos); in bch2_btree_iter_peek_upto()
2430 bch2_btree_iter_verify_entry_exit(iter); in bch2_btree_iter_peek_upto()
2434 bch2_btree_iter_set_pos(iter, end); in bch2_btree_iter_peek_upto()
2446 struct bkey_s_c bch2_btree_iter_next(struct btree_iter *iter) in bch2_btree_iter_next() argument
2448 if (!bch2_btree_iter_advance(iter)) in bch2_btree_iter_next()
2451 return bch2_btree_iter_peek(iter); in bch2_btree_iter_next()
2461 struct bkey_s_c bch2_btree_iter_peek_prev(struct btree_iter *iter) in bch2_btree_iter_peek_prev() argument
2463 struct btree_trans *trans = iter->trans; in bch2_btree_iter_peek_prev()
2464 struct bpos search_key = iter->pos; in bch2_btree_iter_peek_prev()
2472 EBUG_ON(btree_iter_path(trans, iter)->cached || in bch2_btree_iter_peek_prev()
2473 btree_iter_path(trans, iter)->level); in bch2_btree_iter_peek_prev()
2475 if (iter->flags & BTREE_ITER_with_journal) in bch2_btree_iter_peek_prev()
2478 bch2_btree_iter_verify(iter); in bch2_btree_iter_peek_prev()
2479 bch2_btree_iter_verify_entry_exit(iter); in bch2_btree_iter_peek_prev()
2481 if (iter->flags & BTREE_ITER_filter_snapshots) in bch2_btree_iter_peek_prev()
2485 iter->path = bch2_btree_path_set_pos(trans, iter->path, search_key, in bch2_btree_iter_peek_prev()
2486 iter->flags & BTREE_ITER_intent, in bch2_btree_iter_peek_prev()
2487 btree_iter_ip_allocated(iter)); in bch2_btree_iter_peek_prev()
2489 ret = bch2_btree_path_traverse(trans, iter->path, iter->flags); in bch2_btree_iter_peek_prev()
2492 bch2_btree_iter_set_pos(iter, iter->pos); in bch2_btree_iter_peek_prev()
2497 struct btree_path *path = btree_iter_path(trans, iter); in bch2_btree_iter_peek_prev()
2499 k = btree_path_level_peek(trans, path, &path->l[0], &iter->k); in bch2_btree_iter_peek_prev()
2501 ((iter->flags & BTREE_ITER_is_extents) in bch2_btree_iter_peek_prev()
2504 k = btree_path_level_prev(trans, path, &path->l[0], &iter->k); in bch2_btree_iter_peek_prev()
2506 if (unlikely((iter->flags & BTREE_ITER_with_updates) && in bch2_btree_iter_peek_prev()
2508 bch2_btree_trans_peek_prev_updates(trans, iter, &k); in bch2_btree_iter_peek_prev()
2511 if (iter->flags & BTREE_ITER_filter_snapshots) { in bch2_btree_iter_peek_prev()
2512 if (k.k->p.snapshot == iter->snapshot) in bch2_btree_iter_peek_prev()
2521 bch2_path_put_nokeep(trans, iter->path, in bch2_btree_iter_peek_prev()
2522 iter->flags & BTREE_ITER_intent); in bch2_btree_iter_peek_prev()
2523 iter->path = saved_path; in bch2_btree_iter_peek_prev()
2525 iter->k = saved_k; in bch2_btree_iter_peek_prev()
2531 iter->snapshot, in bch2_btree_iter_peek_prev()
2535 iter->flags & BTREE_ITER_intent); in bch2_btree_iter_peek_prev()
2536 saved_path = btree_path_clone(trans, iter->path, in bch2_btree_iter_peek_prev()
2537 iter->flags & BTREE_ITER_intent, in bch2_btree_iter_peek_prev()
2539 path = btree_iter_path(trans, iter); in bch2_btree_iter_peek_prev()
2550 !(iter->flags & BTREE_ITER_all_snapshots)) { in bch2_btree_iter_peek_prev()
2551 search_key = bkey_predecessor(iter, k.k->p); in bch2_btree_iter_peek_prev()
2552 if (iter->flags & BTREE_ITER_filter_snapshots) in bch2_btree_iter_peek_prev()
2564 bch2_btree_iter_set_pos(iter, POS_MIN); in bch2_btree_iter_peek_prev()
2570 EBUG_ON(bkey_gt(bkey_start_pos(k.k), iter->pos)); in bch2_btree_iter_peek_prev()
2573 if (bkey_lt(k.k->p, iter->pos)) in bch2_btree_iter_peek_prev()
2574 iter->pos = k.k->p; in bch2_btree_iter_peek_prev()
2576 if (iter->flags & BTREE_ITER_filter_snapshots) in bch2_btree_iter_peek_prev()
2577 iter->pos.snapshot = iter->snapshot; in bch2_btree_iter_peek_prev()
2580 bch2_path_put_nokeep(trans, saved_path, iter->flags & BTREE_ITER_intent); in bch2_btree_iter_peek_prev()
2582 bch2_btree_iter_verify_entry_exit(iter); in bch2_btree_iter_peek_prev()
2583 bch2_btree_iter_verify(iter); in bch2_btree_iter_peek_prev()
2595 struct bkey_s_c bch2_btree_iter_prev(struct btree_iter *iter) in bch2_btree_iter_prev() argument
2597 if (!bch2_btree_iter_rewind(iter)) in bch2_btree_iter_prev()
2600 return bch2_btree_iter_peek_prev(iter); in bch2_btree_iter_prev()
2603 struct bkey_s_c bch2_btree_iter_peek_slot(struct btree_iter *iter) in bch2_btree_iter_peek_slot() argument
2605 struct btree_trans *trans = iter->trans; in bch2_btree_iter_peek_slot()
2611 bch2_btree_iter_verify(iter); in bch2_btree_iter_peek_slot()
2612 bch2_btree_iter_verify_entry_exit(iter); in bch2_btree_iter_peek_slot()
2613 EBUG_ON(btree_iter_path(trans, iter)->level && (iter->flags & BTREE_ITER_with_key_cache)); in bch2_btree_iter_peek_slot()
2616 if ((iter->flags & BTREE_ITER_is_extents) && in bch2_btree_iter_peek_slot()
2617 unlikely(iter->pos.offset == KEY_OFFSET_MAX)) { in bch2_btree_iter_peek_slot()
2618 if (iter->pos.inode == KEY_INODE_MAX) in bch2_btree_iter_peek_slot()
2621 bch2_btree_iter_set_pos(iter, bpos_nosnap_successor(iter->pos)); in bch2_btree_iter_peek_slot()
2624 search_key = btree_iter_search_key(iter); in bch2_btree_iter_peek_slot()
2625 iter->path = bch2_btree_path_set_pos(trans, iter->path, search_key, in bch2_btree_iter_peek_slot()
2626 iter->flags & BTREE_ITER_intent, in bch2_btree_iter_peek_slot()
2627 btree_iter_ip_allocated(iter)); in bch2_btree_iter_peek_slot()
2629 ret = bch2_btree_path_traverse(trans, iter->path, iter->flags); in bch2_btree_iter_peek_slot()
2635 if ((iter->flags & BTREE_ITER_cached) || in bch2_btree_iter_peek_slot()
2636 !(iter->flags & (BTREE_ITER_is_extents|BTREE_ITER_filter_snapshots))) { in bch2_btree_iter_peek_slot()
2639 if (unlikely((iter->flags & BTREE_ITER_with_updates) && in bch2_btree_iter_peek_slot()
2641 bch2_btree_trans_peek_slot_updates(trans, iter, &k); in bch2_btree_iter_peek_slot()
2646 if (unlikely(iter->flags & BTREE_ITER_with_journal) && in bch2_btree_iter_peek_slot()
2647 (k = btree_trans_peek_slot_journal(trans, iter)).k) in bch2_btree_iter_peek_slot()
2650 if (unlikely(iter->flags & BTREE_ITER_with_key_cache) && in bch2_btree_iter_peek_slot()
2651 (k = btree_trans_peek_key_cache(iter, iter->pos)).k) { in bch2_btree_iter_peek_slot()
2653 iter->k = *k.k; in bch2_btree_iter_peek_slot()
2658 k = bch2_btree_path_peek_slot(trans->paths + iter->path, &iter->k); in bch2_btree_iter_peek_slot()
2663 struct bpos end = iter->pos; in bch2_btree_iter_peek_slot()
2665 if (iter->flags & BTREE_ITER_is_extents) in bch2_btree_iter_peek_slot()
2668 EBUG_ON(btree_iter_path(trans, iter)->level); in bch2_btree_iter_peek_slot()
2670 if (iter->flags & BTREE_ITER_intent) { in bch2_btree_iter_peek_slot()
2673 bch2_trans_copy_iter(&iter2, iter); in bch2_btree_iter_peek_slot()
2677 swap(iter->key_cache_path, iter2.key_cache_path); in bch2_btree_iter_peek_slot()
2678 iter->k = iter2.k; in bch2_btree_iter_peek_slot()
2679 k.k = &iter->k; in bch2_btree_iter_peek_slot()
2683 struct bpos pos = iter->pos; in bch2_btree_iter_peek_slot()
2685 k = bch2_btree_iter_peek_upto(iter, end); in bch2_btree_iter_peek_slot()
2687 bch2_btree_iter_set_pos(iter, pos); in bch2_btree_iter_peek_slot()
2689 iter->pos = pos; in bch2_btree_iter_peek_slot()
2697 if (bkey_lt(iter->pos, next)) { in bch2_btree_iter_peek_slot()
2698 bkey_init(&iter->k); in bch2_btree_iter_peek_slot()
2699 iter->k.p = iter->pos; in bch2_btree_iter_peek_slot()
2701 if (iter->flags & BTREE_ITER_is_extents) { in bch2_btree_iter_peek_slot()
2702 bch2_key_resize(&iter->k, in bch2_btree_iter_peek_slot()
2704 (next.inode == iter->pos.inode in bch2_btree_iter_peek_slot()
2707 iter->pos.offset)); in bch2_btree_iter_peek_slot()
2708 EBUG_ON(!iter->k.size); in bch2_btree_iter_peek_slot()
2711 k = (struct bkey_s_c) { &iter->k, NULL }; in bch2_btree_iter_peek_slot()
2715 btree_path_set_should_be_locked(trans, btree_iter_path(trans, iter)); in bch2_btree_iter_peek_slot()
2717 bch2_btree_iter_verify_entry_exit(iter); in bch2_btree_iter_peek_slot()
2718 bch2_btree_iter_verify(iter); in bch2_btree_iter_peek_slot()
2719 ret = bch2_btree_iter_verify_ret(iter, k); in bch2_btree_iter_peek_slot()
2726 struct bkey_s_c bch2_btree_iter_next_slot(struct btree_iter *iter) in bch2_btree_iter_next_slot() argument
2728 if (!bch2_btree_iter_advance(iter)) in bch2_btree_iter_next_slot()
2731 return bch2_btree_iter_peek_slot(iter); in bch2_btree_iter_next_slot()
2734 struct bkey_s_c bch2_btree_iter_prev_slot(struct btree_iter *iter) in bch2_btree_iter_prev_slot() argument
2736 if (!bch2_btree_iter_rewind(iter)) in bch2_btree_iter_prev_slot()
2739 return bch2_btree_iter_peek_slot(iter); in bch2_btree_iter_prev_slot()
2743 struct bkey_s_c bch2_btree_iter_peek_and_restart_outlined(struct btree_iter *iter) in bch2_btree_iter_peek_and_restart_outlined() argument
2747 while (btree_trans_too_many_iters(iter->trans) || in bch2_btree_iter_peek_and_restart_outlined()
2748 (k = bch2_btree_iter_peek_type(iter, iter->flags), in bch2_btree_iter_peek_and_restart_outlined()
2750 bch2_trans_begin(iter->trans); in bch2_btree_iter_peek_and_restart_outlined()
2781 struct trans_for_each_path_inorder_iter iter; in btree_trans_verify_sorted() local
2786 trans_for_each_path_inorder(trans, path, iter) { in btree_trans_verify_sorted()
2882 void bch2_trans_iter_exit(struct btree_trans *trans, struct btree_iter *iter) in bch2_trans_iter_exit() argument
2884 if (iter->update_path) in bch2_trans_iter_exit()
2885 bch2_path_put_nokeep(trans, iter->update_path, in bch2_trans_iter_exit()
2886 iter->flags & BTREE_ITER_intent); in bch2_trans_iter_exit()
2887 if (iter->path) in bch2_trans_iter_exit()
2888 bch2_path_put(trans, iter->path, in bch2_trans_iter_exit()
2889 iter->flags & BTREE_ITER_intent); in bch2_trans_iter_exit()
2890 if (iter->key_cache_path) in bch2_trans_iter_exit()
2891 bch2_path_put(trans, iter->key_cache_path, in bch2_trans_iter_exit()
2892 iter->flags & BTREE_ITER_intent); in bch2_trans_iter_exit()
2893 iter->path = 0; in bch2_trans_iter_exit()
2894 iter->update_path = 0; in bch2_trans_iter_exit()
2895 iter->key_cache_path = 0; in bch2_trans_iter_exit()
2896 iter->trans = NULL; in bch2_trans_iter_exit()
2900 struct btree_iter *iter, in bch2_trans_iter_init_outlined() argument
2904 bch2_trans_iter_init_common(trans, iter, btree_id, pos, 0, 0, in bch2_trans_iter_init_outlined()
2910 struct btree_iter *iter, in bch2_trans_node_iter_init() argument
2921 bch2_trans_iter_init_common(trans, iter, btree_id, pos, locks_want, depth, in bch2_trans_node_iter_init()
2925 iter->min_depth = depth; in bch2_trans_node_iter_init()
2927 struct btree_path *path = btree_iter_path(trans, iter); in bch2_trans_node_iter_init()
2930 BUG_ON(iter->min_depth != depth); in bch2_trans_node_iter_init()