118 static void journal_iter_verify(struct journal_iter *iter)  in journal_iter_verify()  argument
120 	struct journal_keys *keys = iter->keys;  in journal_iter_verify()
123 	BUG_ON(iter->idx >= keys->gap &&  in journal_iter_verify()
124 	       iter->idx <  keys->gap + gap_size);  in journal_iter_verify()
126 	if (iter->idx < keys->size) {  in journal_iter_verify()
127 		struct journal_key *k = keys->data + iter->idx;  in journal_iter_verify()
129 		int cmp = cmp_int(k->btree_id,	iter->btree_id) ?:  in journal_iter_verify()
130 			  cmp_int(k->level,	iter->level);  in journal_iter_verify()
141 	struct journal_iter *iter;  in journal_iters_fix()  local
149 	list_for_each_entry(iter, &c->journal_iters, list) {  in journal_iters_fix()
150 		journal_iter_verify(iter);  in journal_iters_fix()
151 		if (iter->idx		== gap_end &&  in journal_iters_fix()
152 		    new_key->btree_id	== iter->btree_id &&  in journal_iters_fix()
153 		    new_key->level	== iter->level)  in journal_iters_fix()
154 			iter->idx = keys->gap - 1;  in journal_iters_fix()
155 		journal_iter_verify(iter);  in journal_iters_fix()
162 	struct journal_iter *iter;  in journal_iters_move_gap()  local
165 	list_for_each_entry(iter, &c->journal_iters, list) {  in journal_iters_move_gap()
166 		if (iter->idx > old_gap)  in journal_iters_move_gap()
167 			iter->idx -= gap_size;  in journal_iters_move_gap()
168 		if (iter->idx >= new_gap)  in journal_iters_move_gap()
169 			iter->idx += gap_size;  in journal_iters_move_gap()
306 static void bch2_journal_iter_advance(struct journal_iter *iter)  in bch2_journal_iter_advance()  argument
308 	if (iter->idx < iter->keys->size) {  in bch2_journal_iter_advance()
309 		iter->idx++;  in bch2_journal_iter_advance()
310 		if (iter->idx == iter->keys->gap)  in bch2_journal_iter_advance()
311 			iter->idx += iter->keys->size - iter->keys->nr;  in bch2_journal_iter_advance()
315 static struct bkey_s_c bch2_journal_iter_peek(struct journal_iter *iter)  in bch2_journal_iter_peek()  argument
317 	journal_iter_verify(iter);  in bch2_journal_iter_peek()
319 	while (iter->idx < iter->keys->size) {  in bch2_journal_iter_peek()
320 		struct journal_key *k = iter->keys->data + iter->idx;  in bch2_journal_iter_peek()
322 		int cmp = cmp_int(k->btree_id,	iter->btree_id) ?:  in bch2_journal_iter_peek()
323 			  cmp_int(k->level,	iter->level);  in bch2_journal_iter_peek()
331 		bch2_journal_iter_advance(iter);  in bch2_journal_iter_peek()
337 static void bch2_journal_iter_exit(struct journal_iter *iter)  in bch2_journal_iter_exit()  argument
339 	list_del(&iter->list);  in bch2_journal_iter_exit()
343 				   struct journal_iter *iter,  in bch2_journal_iter_init()  argument
347 	iter->btree_id	= id;  in bch2_journal_iter_init()
348 	iter->level	= level;  in bch2_journal_iter_init()
349 	iter->keys	= &c->journal_keys;  in bch2_journal_iter_init()
350 	iter->idx	= bch2_journal_key_search(&c->journal_keys, id, level, pos);  in bch2_journal_iter_init()
352 	journal_iter_verify(iter);  in bch2_journal_iter_init()
355 static struct bkey_s_c bch2_journal_iter_peek_btree(struct btree_and_journal_iter *iter)  in bch2_journal_iter_peek_btree()  argument
357 	return bch2_btree_node_iter_peek_unpack(&iter->node_iter,  in bch2_journal_iter_peek_btree()
358 						iter->b, &iter->unpacked);  in bch2_journal_iter_peek_btree()
361 static void bch2_journal_iter_advance_btree(struct btree_and_journal_iter *iter)  in bch2_journal_iter_advance_btree()  argument
363 	bch2_btree_node_iter_advance(&iter->node_iter, iter->b);  in bch2_journal_iter_advance_btree()
366 void bch2_btree_and_journal_iter_advance(struct btree_and_journal_iter *iter)  in bch2_btree_and_journal_iter_advance()  argument
368 	if (bpos_eq(iter->pos, SPOS_MAX))  in bch2_btree_and_journal_iter_advance()
369 		iter->at_end = true;  in bch2_btree_and_journal_iter_advance()
371 		iter->pos = bpos_successor(iter->pos);  in bch2_btree_and_journal_iter_advance()
376 	struct btree_and_journal_iter iter = *_iter;  in btree_and_journal_iter_prefetch()  local
377 	struct bch_fs *c = iter.trans->c;  in btree_and_journal_iter_prefetch()
378 	unsigned level = iter.journal.level;  in btree_and_journal_iter_prefetch()
384 	iter.prefetch = false;  in btree_and_journal_iter_prefetch()
388 		bch2_btree_and_journal_iter_advance(&iter);  in btree_and_journal_iter_prefetch()
389 		struct bkey_s_c k = bch2_btree_and_journal_iter_peek(&iter);  in btree_and_journal_iter_prefetch()
394 		bch2_btree_node_prefetch(iter.trans, NULL, tmp.k, iter.journal.btree_id, level - 1);  in btree_and_journal_iter_prefetch()
400 struct bkey_s_c bch2_btree_and_journal_iter_peek(struct btree_and_journal_iter *iter)  in bch2_btree_and_journal_iter_peek()  argument
404 	if (iter->prefetch && iter->journal.level)  in bch2_btree_and_journal_iter_peek()
405 		btree_and_journal_iter_prefetch(iter);  in bch2_btree_and_journal_iter_peek()
407 	if (iter->at_end)  in bch2_btree_and_journal_iter_peek()
410 	while ((btree_k = bch2_journal_iter_peek_btree(iter)).k &&  in bch2_btree_and_journal_iter_peek()
411 	       bpos_lt(btree_k.k->p, iter->pos))  in bch2_btree_and_journal_iter_peek()
412 		bch2_journal_iter_advance_btree(iter);  in bch2_btree_and_journal_iter_peek()
414 	if (iter->trans->journal_replay_not_finished)  in bch2_btree_and_journal_iter_peek()
415 		while ((journal_k = bch2_journal_iter_peek(&iter->journal)).k &&  in bch2_btree_and_journal_iter_peek()
416 		       bpos_lt(journal_k.k->p, iter->pos))  in bch2_btree_and_journal_iter_peek()
417 			bch2_journal_iter_advance(&iter->journal);  in bch2_btree_and_journal_iter_peek()
424 	if (ret.k && iter->b && bpos_gt(ret.k->p, iter->b->data->max_key))  in bch2_btree_and_journal_iter_peek()
428 		iter->pos = ret.k->p;  in bch2_btree_and_journal_iter_peek()
430 			bch2_btree_and_journal_iter_advance(iter);  in bch2_btree_and_journal_iter_peek()
434 		iter->pos = SPOS_MAX;  in bch2_btree_and_journal_iter_peek()
435 		iter->at_end = true;  in bch2_btree_and_journal_iter_peek()
441 void bch2_btree_and_journal_iter_exit(struct btree_and_journal_iter *iter)  in bch2_btree_and_journal_iter_exit()  argument
443 	bch2_journal_iter_exit(&iter->journal);  in bch2_btree_and_journal_iter_exit()
447 						  struct btree_and_journal_iter *iter,  in __bch2_btree_and_journal_iter_init_node_iter()  argument
452 	memset(iter, 0, sizeof(*iter));  in __bch2_btree_and_journal_iter_init_node_iter()
454 	iter->trans = trans;  in __bch2_btree_and_journal_iter_init_node_iter()
455 	iter->b = b;  in __bch2_btree_and_journal_iter_init_node_iter()
456 	iter->node_iter = node_iter;  in __bch2_btree_and_journal_iter_init_node_iter()
457 	iter->pos = b->data->min_key;  in __bch2_btree_and_journal_iter_init_node_iter()
458 	iter->at_end = false;  in __bch2_btree_and_journal_iter_init_node_iter()
459 	INIT_LIST_HEAD(&iter->journal.list);  in __bch2_btree_and_journal_iter_init_node_iter()
462 		bch2_journal_iter_init(trans->c, &iter->journal, b->c.btree_id, b->c.level, pos);  in __bch2_btree_and_journal_iter_init_node_iter()
464 			list_add(&iter->journal.list, &trans->c->journal_iters);  in __bch2_btree_and_journal_iter_init_node_iter()
473 						struct btree_and_journal_iter *iter,  in bch2_btree_and_journal_iter_init_node_iter()  argument
479 	__bch2_btree_and_journal_iter_init_node_iter(trans, iter, b, node_iter, b->data->min_key);  in bch2_btree_and_journal_iter_init_node_iter()
487 	struct genradix_iter iter;  in bch2_journal_entries_free()  local
489 	genradix_for_each(&c->journal_entries, iter, i)  in bch2_journal_entries_free()
556 	struct genradix_iter iter;  in bch2_journal_keys_sort()  local
561 	genradix_for_each(&c->journal_entries, iter, _i) {  in bch2_journal_keys_sort()