Lines Matching +full:- +full:b
1 // SPDX-License-Identifier: GPL-2.0
26 unsigned n = ARRAY_SIZE(iter->data); in __btree_node_iter_used()
28 while (n && __btree_node_iter_set_end(iter, n - 1)) in __btree_node_iter_used()
29 --n; in __btree_node_iter_used()
34 struct bset_tree *bch2_bkey_to_bset(struct btree *b, struct bkey_packed *k) in bch2_bkey_to_bset() argument
36 return bch2_bkey_to_bset_inlined(b, k); in bch2_bkey_to_bset()
40 * There are never duplicate live keys in the btree - but including keys that
50 * prior to the first key greater than the key we're inserting - our insert
55 void bch2_dump_bset(struct bch_fs *c, struct btree *b, in bch2_dump_bset() argument
63 if (!i->u64s) in bch2_dump_bset()
66 for (_k = i->start; in bch2_dump_bset()
71 if (!_k->u64s) { in bch2_dump_bset()
72 printk(KERN_ERR "block %u key %5zu - u64s 0? aieee!\n", set, in bch2_dump_bset()
73 _k->_data - i->_data); in bch2_dump_bset()
77 k = bkey_disassemble(b, _k, &uk); in bch2_dump_bset()
85 _k->_data - i->_data, buf.buf); in bch2_dump_bset()
90 n = bkey_unpack_key(b, _n); in bch2_dump_bset()
92 if (bpos_lt(n.p, k.k->p)) { in bch2_dump_bset()
97 if (!bkey_deleted(k.k) && bpos_eq(n.p, k.k->p)) in bch2_dump_bset()
104 void bch2_dump_btree_node(struct bch_fs *c, struct btree *b) in bch2_dump_btree_node() argument
107 for_each_bset(b, t) in bch2_dump_btree_node()
108 bch2_dump_bset(c, b, bset(b, t), t - b->set); in bch2_dump_btree_node()
112 void bch2_dump_btree_node_iter(struct btree *b, in bch2_dump_btree_node_iter() argument
119 __btree_node_iter_used(iter), b->nsets); in bch2_dump_btree_node_iter()
122 struct bkey_packed *k = __btree_node_offset_to_key(b, set->k); in bch2_dump_btree_node_iter()
123 struct bset_tree *t = bch2_bkey_to_bset(b, k); in bch2_dump_btree_node_iter()
124 struct bkey uk = bkey_unpack_key(b, k); in bch2_dump_btree_node_iter()
129 t - b->set, set->k, buf.buf); in bch2_dump_btree_node_iter()
135 struct btree_nr_keys bch2_btree_node_count_keys(struct btree *b) in bch2_btree_node_count_keys() argument
140 for_each_bset(b, t) in bch2_btree_node_count_keys()
141 bset_tree_for_each_key(b, t, k) in bch2_btree_node_count_keys()
143 btree_keys_account_key_add(&nr, t - b->set, k); in bch2_btree_node_count_keys()
149 void __bch2_verify_btree_nr_keys(struct btree *b) in __bch2_verify_btree_nr_keys() argument
151 struct btree_nr_keys nr = bch2_btree_node_count_keys(b); in __bch2_verify_btree_nr_keys()
153 BUG_ON(memcmp(&nr, &b->nr, sizeof(nr))); in __bch2_verify_btree_nr_keys()
157 struct btree *b) in bch2_btree_node_iter_next_check() argument
162 k = bch2_btree_node_iter_peek_all(&iter, b); in bch2_btree_node_iter_next_check()
163 __bch2_btree_node_iter_advance(&iter, b); in bch2_btree_node_iter_next_check()
164 n = bch2_btree_node_iter_peek_all(&iter, b); in bch2_btree_node_iter_next_check()
166 bkey_unpack_key(b, k); in bch2_btree_node_iter_next_check()
169 bkey_iter_cmp(b, k, n) > 0) { in bch2_btree_node_iter_next_check()
171 struct bkey ku = bkey_unpack_key(b, k); in bch2_btree_node_iter_next_check()
172 struct bkey nu = bkey_unpack_key(b, n); in bch2_btree_node_iter_next_check()
176 bch2_dump_btree_node(NULL, b); in bch2_btree_node_iter_next_check()
184 struct bkey_packed *k2 = __btree_node_offset_to_key(b, set->k); in bch2_btree_node_iter_next_check()
185 struct bset_tree *t = bch2_bkey_to_bset(b, k2); in bch2_btree_node_iter_next_check()
186 printk(" [%zi %zi]", t - b->set, in bch2_btree_node_iter_next_check()
187 k2->_data - bset(b, t)->_data); in bch2_btree_node_iter_next_check()
194 struct btree *b) in bch2_btree_node_iter_verify() argument
204 BUG_ON(set->k > set->end); in bch2_btree_node_iter_verify()
206 BUG_ON(set != s2 && set->end == s2->end); in bch2_btree_node_iter_verify()
209 /* Verify that set->end is correct: */ in bch2_btree_node_iter_verify()
211 for_each_bset(b, t) in bch2_btree_node_iter_verify()
212 if (set->end == t->end_offset) { in bch2_btree_node_iter_verify()
213 BUG_ON(set->k < btree_bkey_first_offset(t) || in bch2_btree_node_iter_verify()
214 set->k >= t->end_offset); in bch2_btree_node_iter_verify()
224 BUG_ON(set != iter->data && in bch2_btree_node_iter_verify()
225 btree_node_iter_cmp(b, set[-1], set[0]) > 0); in bch2_btree_node_iter_verify()
227 k = bch2_btree_node_iter_peek_all(iter, b); in bch2_btree_node_iter_verify()
229 for_each_bset(b, t) { in bch2_btree_node_iter_verify()
230 if (iter->data[0].end == t->end_offset) in bch2_btree_node_iter_verify()
233 p = bch2_bkey_prev_all(b, t, in bch2_btree_node_iter_verify()
234 bch2_btree_node_iter_bset_pos(iter, b, t)); in bch2_btree_node_iter_verify()
236 BUG_ON(p && bkey_iter_cmp(b, k, p) < 0); in bch2_btree_node_iter_verify()
240 void bch2_verify_insert_pos(struct btree *b, struct bkey_packed *where, in bch2_verify_insert_pos() argument
243 struct bset_tree *t = bch2_bkey_to_bset(b, where); in bch2_verify_insert_pos()
244 struct bkey_packed *prev = bch2_bkey_prev_all(b, t, where); in bch2_verify_insert_pos()
245 struct bkey_packed *next = (void *) ((u64 *) where->_data + clobber_u64s); in bch2_verify_insert_pos()
250 bkey_iter_cmp(b, prev, insert) > 0); in bch2_verify_insert_pos()
253 bkey_iter_cmp(b, prev, insert) > 0) { in bch2_verify_insert_pos()
254 struct bkey k1 = bkey_unpack_key(b, prev); in bch2_verify_insert_pos()
255 struct bkey k2 = bkey_unpack_key(b, insert); in bch2_verify_insert_pos()
257 bch2_dump_btree_node(NULL, b); in bch2_verify_insert_pos()
268 BUG_ON(next != btree_bkey_last(b, t) && in bch2_verify_insert_pos()
269 bkey_iter_cmp(b, insert, next) > 0); in bch2_verify_insert_pos()
271 if (next != btree_bkey_last(b, t) && in bch2_verify_insert_pos()
272 bkey_iter_cmp(b, insert, next) > 0) { in bch2_verify_insert_pos()
273 struct bkey k1 = bkey_unpack_key(b, insert); in bch2_verify_insert_pos()
274 struct bkey k2 = bkey_unpack_key(b, next); in bch2_verify_insert_pos()
276 bch2_dump_btree_node(NULL, b); in bch2_verify_insert_pos()
291 struct btree *b) {} in bch2_btree_node_iter_next_check() argument
319 BUG_ON(t->aux_data_offset == U16_MAX); in bset_aux_tree_buf_end()
323 return t->aux_data_offset; in bset_aux_tree_buf_end()
325 return t->aux_data_offset + in bset_aux_tree_buf_end()
326 DIV_ROUND_UP(t->size * sizeof(struct bkey_float), 8); in bset_aux_tree_buf_end()
328 return t->aux_data_offset + in bset_aux_tree_buf_end()
329 DIV_ROUND_UP(sizeof(struct rw_aux_tree) * t->size, 8); in bset_aux_tree_buf_end()
335 static unsigned bset_aux_tree_buf_start(const struct btree *b, in bset_aux_tree_buf_start() argument
338 return t == b->set in bset_aux_tree_buf_start()
339 ? DIV_ROUND_UP(b->unpack_fn_len, 8) in bset_aux_tree_buf_start()
340 : bset_aux_tree_buf_end(t - 1); in bset_aux_tree_buf_start()
343 static void *__aux_tree_base(const struct btree *b, in __aux_tree_base() argument
346 return b->aux_data + t->aux_data_offset * 8; in __aux_tree_base()
349 static struct ro_aux_tree *ro_aux_tree_base(const struct btree *b, in ro_aux_tree_base() argument
354 return __aux_tree_base(b, t); in ro_aux_tree_base()
357 static struct bkey_float *bkey_float(const struct btree *b, in bkey_float() argument
361 return ro_aux_tree_base(b, t)->f + idx; in bkey_float()
364 static void bset_aux_tree_verify(struct btree *b) in bset_aux_tree_verify() argument
367 for_each_bset(b, t) { in bset_aux_tree_verify()
368 if (t->aux_data_offset == U16_MAX) in bset_aux_tree_verify()
371 BUG_ON(t != b->set && in bset_aux_tree_verify()
372 t[-1].aux_data_offset == U16_MAX); in bset_aux_tree_verify()
374 BUG_ON(t->aux_data_offset < bset_aux_tree_buf_start(b, t)); in bset_aux_tree_verify()
375 BUG_ON(t->aux_data_offset > btree_aux_data_u64s(b)); in bset_aux_tree_verify()
376 BUG_ON(bset_aux_tree_buf_end(t) > btree_aux_data_u64s(b)); in bset_aux_tree_verify()
381 void bch2_btree_keys_init(struct btree *b) in bch2_btree_keys_init() argument
385 b->nsets = 0; in bch2_btree_keys_init()
386 memset(&b->nr, 0, sizeof(b->nr)); in bch2_btree_keys_init()
389 b->set[i].data_offset = U16_MAX; in bch2_btree_keys_init()
391 bch2_bset_set_no_aux_tree(b, b->set); in bch2_btree_keys_init()
397 * Cacheline/offset <-> bkey pointer arithmetic:
399 * t->tree is a binary search tree in an array; each node corresponds to a key
400 * in one cacheline in t->set (BSET_CACHELINE bytes).
404 * then bkey_float->m gives us the offset within that cacheline, in units of 8
412 * bits we're going to store in bkey_float->mantissa. t->prev[j] stores the size
413 * of the previous key so we can walk backwards to it from t->tree[j]'s key.
416 static inline void *bset_cacheline(const struct btree *b, in bset_cacheline() argument
420 return (void *) round_down((unsigned long) btree_bkey_first(b, t), in bset_cacheline()
425 static struct bkey_packed *cacheline_to_bkey(const struct btree *b, in cacheline_to_bkey() argument
430 return bset_cacheline(b, t, cacheline) + offset * 8; in cacheline_to_bkey()
433 static unsigned bkey_to_cacheline(const struct btree *b, in bkey_to_cacheline() argument
437 return ((void *) k - bset_cacheline(b, t, 0)) / BSET_CACHELINE; in bkey_to_cacheline()
440 static ssize_t __bkey_to_cacheline_offset(const struct btree *b, in __bkey_to_cacheline_offset() argument
445 return (u64 *) k - (u64 *) bset_cacheline(b, t, cacheline); in __bkey_to_cacheline_offset()
448 static unsigned bkey_to_cacheline_offset(const struct btree *b, in bkey_to_cacheline_offset() argument
453 size_t m = __bkey_to_cacheline_offset(b, t, cacheline, k); in bkey_to_cacheline_offset()
459 static inline struct bkey_packed *tree_to_bkey(const struct btree *b, in tree_to_bkey() argument
463 return cacheline_to_bkey(b, t, in tree_to_bkey()
464 __eytzinger1_to_inorder(j, t->size - 1, t->extra), in tree_to_bkey()
465 bkey_float(b, t, j)->key_offset); in tree_to_bkey()
468 static struct rw_aux_tree *rw_aux_tree(const struct btree *b, in rw_aux_tree() argument
473 return __aux_tree_base(b, t); in rw_aux_tree()
477 * For the write set - the one we're currently inserting keys into - we don't
478 * maintain a full search tree, we just keep a simple lookup table in t->prev.
480 static struct bkey_packed *rw_aux_to_bkey(const struct btree *b, in rw_aux_to_bkey() argument
484 return __btree_node_offset_to_key(b, rw_aux_tree(b, t)[j].offset); in rw_aux_to_bkey()
487 static void rw_aux_tree_set(const struct btree *b, struct bset_tree *t, in rw_aux_tree_set() argument
490 EBUG_ON(k >= btree_bkey_last(b, t)); in rw_aux_tree_set()
492 rw_aux_tree(b, t)[j] = (struct rw_aux_tree) { in rw_aux_tree_set()
493 .offset = __btree_node_key_to_offset(b, k), in rw_aux_tree_set()
494 .k = bkey_unpack_pos(b, k), in rw_aux_tree_set()
498 static void bch2_bset_verify_rw_aux_tree(struct btree *b, in bch2_bset_verify_rw_aux_tree() argument
501 struct bkey_packed *k = btree_bkey_first(b, t); in bch2_bset_verify_rw_aux_tree()
512 BUG_ON(t->size < 1); in bch2_bset_verify_rw_aux_tree()
513 BUG_ON(rw_aux_to_bkey(b, t, j) != k); in bch2_bset_verify_rw_aux_tree()
517 if (rw_aux_to_bkey(b, t, j) == k) { in bch2_bset_verify_rw_aux_tree()
518 BUG_ON(!bpos_eq(rw_aux_tree(b, t)[j].k, in bch2_bset_verify_rw_aux_tree()
519 bkey_unpack_pos(b, k))); in bch2_bset_verify_rw_aux_tree()
521 if (++j == t->size) in bch2_bset_verify_rw_aux_tree()
524 BUG_ON(rw_aux_tree(b, t)[j].offset <= in bch2_bset_verify_rw_aux_tree()
525 rw_aux_tree(b, t)[j - 1].offset); in bch2_bset_verify_rw_aux_tree()
529 BUG_ON(k >= btree_bkey_last(b, t)); in bch2_bset_verify_rw_aux_tree()
534 static unsigned rw_aux_tree_bsearch(struct btree *b, in rw_aux_tree_bsearch() argument
538 unsigned bset_offs = offset - btree_bkey_first_offset(t); in rw_aux_tree_bsearch()
539 unsigned bset_u64s = t->end_offset - btree_bkey_first_offset(t); in rw_aux_tree_bsearch()
540 unsigned idx = bset_u64s ? bset_offs * t->size / bset_u64s : 0; in rw_aux_tree_bsearch()
543 EBUG_ON(!t->size); in rw_aux_tree_bsearch()
544 EBUG_ON(idx > t->size); in rw_aux_tree_bsearch()
546 while (idx < t->size && in rw_aux_tree_bsearch()
547 rw_aux_tree(b, t)[idx].offset < offset) in rw_aux_tree_bsearch()
551 rw_aux_tree(b, t)[idx - 1].offset >= offset) in rw_aux_tree_bsearch()
552 idx--; in rw_aux_tree_bsearch()
554 EBUG_ON(idx < t->size && in rw_aux_tree_bsearch()
555 rw_aux_tree(b, t)[idx].offset < offset); in rw_aux_tree_bsearch()
556 EBUG_ON(idx && rw_aux_tree(b, t)[idx - 1].offset >= offset); in rw_aux_tree_bsearch()
557 EBUG_ON(idx + 1 < t->size && in rw_aux_tree_bsearch()
558 rw_aux_tree(b, t)[idx].offset == in rw_aux_tree_bsearch()
559 rw_aux_tree(b, t)[idx + 1].offset); in rw_aux_tree_bsearch()
571 v = get_unaligned((u64 *) (((u8 *) k->_data) + (f->exponent >> 3))); in bkey_mantissa()
580 v >>= f->exponent & 7; in bkey_mantissa()
582 v >>= 64 - (f->exponent & 7) - BKEY_MANTISSA_BITS; in bkey_mantissa()
587 static __always_inline void make_bfloat(struct btree *b, struct bset_tree *t, in make_bfloat() argument
592 struct bkey_float *f = bkey_float(b, t, j); in make_bfloat()
593 struct bkey_packed *m = tree_to_bkey(b, t, j); in make_bfloat()
596 : tree_to_bkey(b, t, j >> ffs(j)); in make_bfloat()
599 : tree_to_bkey(b, t, j >> (ffz(j) + 1)); in make_bfloat()
609 !b->nr_key_bits) { in make_bfloat()
610 f->exponent = BFLOAT_FAILED_UNPACKED; in make_bfloat()
617 * comparisons - that bit always becomes the high bit of in make_bfloat()
618 * bfloat->mantissa, and thus the exponent we're calculating here is in make_bfloat()
619 * the position of what will become the low bit in bfloat->mantissa: in make_bfloat()
621 * Note that this may be negative - we may be running off the low end in make_bfloat()
624 high_bit = max(bch2_bkey_greatest_differing_bit(b, l, r), in make_bfloat()
625 min_t(unsigned, BKEY_MANTISSA_BITS, b->nr_key_bits) - 1); in make_bfloat()
626 exponent = high_bit - (BKEY_MANTISSA_BITS - 1); in make_bfloat()
630 * (k->_data), to get the key bits starting at exponent: in make_bfloat()
633 shift = (int) (b->format.key_u64s * 64 - b->nr_key_bits) + exponent; in make_bfloat()
635 EBUG_ON(shift + BKEY_MANTISSA_BITS > b->format.key_u64s * 64); in make_bfloat()
638 b->nr_key_bits - in make_bfloat()
639 exponent - in make_bfloat()
646 f->exponent = shift; in make_bfloat()
650 * If we've got garbage bits, set them to all 1s - it's legal for the in make_bfloat()
654 mantissa |= ~(~0U << -exponent); in make_bfloat()
656 f->mantissa = mantissa; in make_bfloat()
659 /* bytes remaining - only valid for last bset: */
660 static unsigned __bset_tree_capacity(struct btree *b, const struct bset_tree *t) in __bset_tree_capacity() argument
662 bset_aux_tree_verify(b); in __bset_tree_capacity()
664 return btree_aux_data_bytes(b) - t->aux_data_offset * sizeof(u64); in __bset_tree_capacity()
667 static unsigned bset_ro_tree_capacity(struct btree *b, const struct bset_tree *t) in bset_ro_tree_capacity() argument
669 return __bset_tree_capacity(b, t) / sizeof(struct bkey_float); in bset_ro_tree_capacity()
672 static unsigned bset_rw_tree_capacity(struct btree *b, const struct bset_tree *t) in bset_rw_tree_capacity() argument
674 return __bset_tree_capacity(b, t) / sizeof(struct rw_aux_tree); in bset_rw_tree_capacity()
677 static noinline void __build_rw_aux_tree(struct btree *b, struct bset_tree *t) in __build_rw_aux_tree() argument
681 t->size = 1; in __build_rw_aux_tree()
682 t->extra = BSET_RW_AUX_TREE_VAL; in __build_rw_aux_tree()
683 rw_aux_tree(b, t)[0].offset = in __build_rw_aux_tree() local
684 __btree_node_key_to_offset(b, btree_bkey_first(b, t)); in __build_rw_aux_tree()
686 bset_tree_for_each_key(b, t, k) { in __build_rw_aux_tree()
687 if (t->size == bset_rw_tree_capacity(b, t)) in __build_rw_aux_tree()
690 if ((void *) k - (void *) rw_aux_to_bkey(b, t, t->size - 1) > in __build_rw_aux_tree()
692 rw_aux_tree_set(b, t, t->size++, k); in __build_rw_aux_tree()
696 static noinline void __build_ro_aux_tree(struct btree *b, struct bset_tree *t) in __build_ro_aux_tree() argument
698 struct bkey_packed *k = btree_bkey_first(b, t); in __build_ro_aux_tree()
702 t->size = min(bkey_to_cacheline(b, t, btree_bkey_last(b, t)), in __build_ro_aux_tree()
703 bset_ro_tree_capacity(b, t)); in __build_ro_aux_tree()
705 if (t->size < 2) { in __build_ro_aux_tree()
706 t->size = 0; in __build_ro_aux_tree()
707 t->extra = BSET_NO_AUX_TREE_VAL; in __build_ro_aux_tree()
711 t->extra = eytzinger1_extra(t->size - 1); in __build_ro_aux_tree()
714 eytzinger1_for_each(j, t->size - 1) { in __build_ro_aux_tree()
715 while (bkey_to_cacheline(b, t, k) < cacheline) in __build_ro_aux_tree()
718 if (k >= btree_bkey_last(b, t)) { in __build_ro_aux_tree()
720 t->size--; in __build_ro_aux_tree()
724 bkey_float(b, t, j)->key_offset = in __build_ro_aux_tree()
725 bkey_to_cacheline_offset(b, t, cacheline++, k); in __build_ro_aux_tree()
727 EBUG_ON(tree_to_bkey(b, t, j) != k); in __build_ro_aux_tree()
730 if (!bkey_pack_pos(bkey_to_packed(&min_key), b->data->min_key, b)) { in __build_ro_aux_tree()
732 min_key.k.p = b->data->min_key; in __build_ro_aux_tree()
735 if (!bkey_pack_pos(bkey_to_packed(&max_key), b->data->max_key, b)) { in __build_ro_aux_tree()
737 max_key.k.p = b->data->max_key; in __build_ro_aux_tree()
741 eytzinger1_for_each(j, t->size - 1) in __build_ro_aux_tree()
742 make_bfloat(b, t, j, in __build_ro_aux_tree()
747 static void bset_alloc_tree(struct btree *b, struct bset_tree *t) in bset_alloc_tree() argument
751 for (i = b->set; i != t; i++) in bset_alloc_tree()
754 bch2_bset_set_no_aux_tree(b, t); in bset_alloc_tree()
757 t->aux_data_offset = round_up(bset_aux_tree_buf_start(b, t), in bset_alloc_tree()
760 bset_aux_tree_verify(b); in bset_alloc_tree()
763 void bch2_bset_build_aux_tree(struct btree *b, struct bset_tree *t, in bch2_bset_build_aux_tree() argument
771 bset_alloc_tree(b, t); in bch2_bset_build_aux_tree()
773 if (!__bset_tree_capacity(b, t)) in bch2_bset_build_aux_tree()
777 __build_rw_aux_tree(b, t); in bch2_bset_build_aux_tree()
779 __build_ro_aux_tree(b, t); in bch2_bset_build_aux_tree()
781 bset_aux_tree_verify(b); in bch2_bset_build_aux_tree()
784 void bch2_bset_init_first(struct btree *b, struct bset *i) in bch2_bset_init_first() argument
788 BUG_ON(b->nsets); in bch2_bset_init_first()
791 get_random_bytes(&i->seq, sizeof(i->seq)); in bch2_bset_init_first()
794 t = &b->set[b->nsets++]; in bch2_bset_init_first()
795 set_btree_bset(b, t, i); in bch2_bset_init_first()
798 void bch2_bset_init_next(struct btree *b, struct btree_node_entry *bne) in bch2_bset_init_next() argument
800 struct bset *i = &bne->keys; in bch2_bset_init_next()
803 BUG_ON(bset_byte_offset(b, bne) >= btree_buf_bytes(b)); in bch2_bset_init_next()
804 BUG_ON((void *) bne < (void *) btree_bkey_last(b, bset_tree_last(b))); in bch2_bset_init_next()
805 BUG_ON(b->nsets >= MAX_BSETS); in bch2_bset_init_next()
808 i->seq = btree_bset_first(b)->seq; in bch2_bset_init_next()
811 t = &b->set[b->nsets++]; in bch2_bset_init_next()
812 set_btree_bset(b, t, i); in bch2_bset_init_next()
816 * find _some_ key in the same bset as @k that precedes @k - not necessarily the
819 static struct bkey_packed *__bkey_prev(struct btree *b, struct bset_tree *t, in __bkey_prev() argument
826 EBUG_ON(k < btree_bkey_first(b, t) || in __bkey_prev()
827 k > btree_bkey_last(b, t)); in __bkey_prev()
829 if (k == btree_bkey_first(b, t)) in __bkey_prev()
834 p = btree_bkey_first(b, t); in __bkey_prev()
837 j = min_t(unsigned, t->size - 1, bkey_to_cacheline(b, t, k)); in __bkey_prev()
840 p = j ? tree_to_bkey(b, t, in __bkey_prev()
841 __inorder_to_eytzinger1(j--, in __bkey_prev()
842 t->size - 1, t->extra)) in __bkey_prev()
843 : btree_bkey_first(b, t); in __bkey_prev()
847 offset = __btree_node_key_to_offset(b, k); in __bkey_prev()
848 j = rw_aux_tree_bsearch(b, t, offset); in __bkey_prev()
849 p = j ? rw_aux_to_bkey(b, t, j - 1) in __bkey_prev()
850 : btree_bkey_first(b, t); in __bkey_prev()
857 struct bkey_packed *bch2_bkey_prev_filter(struct btree *b, in bch2_bkey_prev_filter() argument
864 while ((p = __bkey_prev(b, t, k)) && !ret) { in bch2_bkey_prev_filter()
866 if (i->type >= min_key_type) in bch2_bkey_prev_filter()
877 : btree_bkey_first(b, t); in bch2_bkey_prev_filter()
880 BUG_ON(i->type >= min_key_type); in bch2_bkey_prev_filter()
888 static void rw_aux_tree_insert_entry(struct btree *b, in rw_aux_tree_insert_entry() argument
892 EBUG_ON(!idx || idx > t->size); in rw_aux_tree_insert_entry()
893 struct bkey_packed *start = rw_aux_to_bkey(b, t, idx - 1); in rw_aux_tree_insert_entry()
894 struct bkey_packed *end = idx < t->size in rw_aux_tree_insert_entry()
895 ? rw_aux_to_bkey(b, t, idx) in rw_aux_tree_insert_entry()
896 : btree_bkey_last(b, t); in rw_aux_tree_insert_entry()
898 if (t->size < bset_rw_tree_capacity(b, t) && in rw_aux_tree_insert_entry()
899 (void *) end - (void *) start > L1_CACHE_BYTES) { in rw_aux_tree_insert_entry()
907 if ((void *) k - (void *) start >= L1_CACHE_BYTES) { in rw_aux_tree_insert_entry()
908 memmove(&rw_aux_tree(b, t)[idx + 1], in rw_aux_tree_insert_entry()
909 &rw_aux_tree(b, t)[idx], in rw_aux_tree_insert_entry()
910 (void *) &rw_aux_tree(b, t)[t->size] - in rw_aux_tree_insert_entry()
911 (void *) &rw_aux_tree(b, t)[idx]); in rw_aux_tree_insert_entry()
912 t->size++; in rw_aux_tree_insert_entry()
913 rw_aux_tree_set(b, t, idx, k); in rw_aux_tree_insert_entry()
920 static void bch2_bset_fix_lookup_table(struct btree *b, in bch2_bset_fix_lookup_table() argument
926 int shift = new_u64s - clobber_u64s; in bch2_bset_fix_lookup_table()
927 unsigned idx, j, where = __btree_node_key_to_offset(b, _where); in bch2_bset_fix_lookup_table()
934 if (where > rw_aux_tree(b, t)[t->size - 1].offset) { in bch2_bset_fix_lookup_table()
935 rw_aux_tree_insert_entry(b, t, t->size); in bch2_bset_fix_lookup_table()
940 idx = rw_aux_tree_bsearch(b, t, where); in bch2_bset_fix_lookup_table()
942 if (rw_aux_tree(b, t)[idx].offset == where) { in bch2_bset_fix_lookup_table() local
945 } else if (where < t->end_offset) { in bch2_bset_fix_lookup_table()
946 rw_aux_tree_set(b, t, idx++, _where); in bch2_bset_fix_lookup_table()
948 EBUG_ON(where != t->end_offset); in bch2_bset_fix_lookup_table()
949 rw_aux_tree_insert_entry(b, t, --t->size); in bch2_bset_fix_lookup_table()
954 EBUG_ON(idx < t->size && rw_aux_tree(b, t)[idx].offset <= where); in bch2_bset_fix_lookup_table()
955 if (idx < t->size && in bch2_bset_fix_lookup_table()
956 rw_aux_tree(b, t)[idx].offset + shift == in bch2_bset_fix_lookup_table()
957 rw_aux_tree(b, t)[idx - 1].offset) { in bch2_bset_fix_lookup_table()
958 memmove(&rw_aux_tree(b, t)[idx], in bch2_bset_fix_lookup_table()
959 &rw_aux_tree(b, t)[idx + 1], in bch2_bset_fix_lookup_table()
960 (void *) &rw_aux_tree(b, t)[t->size] - in bch2_bset_fix_lookup_table()
961 (void *) &rw_aux_tree(b, t)[idx + 1]); in bch2_bset_fix_lookup_table()
962 t->size -= 1; in bch2_bset_fix_lookup_table()
965 for (j = idx; j < t->size; j++) in bch2_bset_fix_lookup_table()
966 rw_aux_tree(b, t)[j].offset += shift; in bch2_bset_fix_lookup_table() local
968 EBUG_ON(idx < t->size && in bch2_bset_fix_lookup_table()
969 rw_aux_tree(b, t)[idx].offset == in bch2_bset_fix_lookup_table()
970 rw_aux_tree(b, t)[idx - 1].offset); in bch2_bset_fix_lookup_table()
972 rw_aux_tree_insert_entry(b, t, idx); in bch2_bset_fix_lookup_table()
975 bch2_bset_verify_rw_aux_tree(b, t); in bch2_bset_fix_lookup_table()
976 bset_aux_tree_verify(b); in bch2_bset_fix_lookup_table()
979 void bch2_bset_insert(struct btree *b, in bch2_bset_insert() argument
984 struct bkey_format *f = &b->format; in bch2_bset_insert()
985 struct bset_tree *t = bset_tree_last(b); in bch2_bset_insert()
988 bch2_bset_verify_rw_aux_tree(b, t); in bch2_bset_insert()
989 bch2_verify_insert_pos(b, where, bkey_to_packed(insert), clobber_u64s); in bch2_bset_insert()
991 if (bch2_bkey_pack_key(&packed, &insert->k, f)) in bch2_bset_insert()
994 if (!bkey_deleted(&insert->k)) in bch2_bset_insert()
995 btree_keys_account_key_add(&b->nr, t - b->set, src); in bch2_bset_insert()
997 if (src->u64s != clobber_u64s) { in bch2_bset_insert()
998 u64 *src_p = (u64 *) where->_data + clobber_u64s; in bch2_bset_insert()
999 u64 *dst_p = (u64 *) where->_data + src->u64s; in bch2_bset_insert()
1001 EBUG_ON((int) le16_to_cpu(bset(b, t)->u64s) < in bch2_bset_insert()
1002 (int) clobber_u64s - src->u64s); in bch2_bset_insert()
1004 memmove_u64s(dst_p, src_p, btree_bkey_last(b, t)->_data - src_p); in bch2_bset_insert()
1005 le16_add_cpu(&bset(b, t)->u64s, src->u64s - clobber_u64s); in bch2_bset_insert()
1006 set_btree_bset_end(b, t); in bch2_bset_insert()
1011 memcpy_u64s(bkeyp_val(f, where), &insert->v, in bch2_bset_insert()
1014 if (src->u64s != clobber_u64s) in bch2_bset_insert()
1015 bch2_bset_fix_lookup_table(b, t, where, clobber_u64s, src->u64s); in bch2_bset_insert()
1017 bch2_verify_btree_nr_keys(b); in bch2_bset_insert()
1020 void bch2_bset_delete(struct btree *b, in bch2_bset_delete() argument
1024 struct bset_tree *t = bset_tree_last(b); in bch2_bset_delete()
1025 u64 *src_p = (u64 *) where->_data + clobber_u64s; in bch2_bset_delete()
1026 u64 *dst_p = where->_data; in bch2_bset_delete()
1028 bch2_bset_verify_rw_aux_tree(b, t); in bch2_bset_delete()
1030 EBUG_ON(le16_to_cpu(bset(b, t)->u64s) < clobber_u64s); in bch2_bset_delete()
1032 memmove_u64s_down(dst_p, src_p, btree_bkey_last(b, t)->_data - src_p); in bch2_bset_delete()
1033 le16_add_cpu(&bset(b, t)->u64s, -clobber_u64s); in bch2_bset_delete()
1034 set_btree_bset_end(b, t); in bch2_bset_delete()
1036 bch2_bset_fix_lookup_table(b, t, where, clobber_u64s, 0); in bch2_bset_delete()
1042 static struct bkey_packed *bset_search_write_set(const struct btree *b, in bset_search_write_set() argument
1046 unsigned l = 0, r = t->size; in bset_search_write_set()
1051 if (bpos_lt(rw_aux_tree(b, t)[m].k, *search)) in bset_search_write_set()
1057 return rw_aux_to_bkey(b, t, l); in bset_search_write_set()
1063 asm("prefetcht0 (-127 + 64 * 0)(%0);" in prefetch_four_cachelines()
1064 "prefetcht0 (-127 + 64 * 1)(%0);" in prefetch_four_cachelines()
1065 "prefetcht0 (-127 + 64 * 2)(%0);" in prefetch_four_cachelines()
1066 "prefetcht0 (-127 + 64 * 3)(%0);" in prefetch_four_cachelines()
1077 static inline bool bkey_mantissa_bits_dropped(const struct btree *b, in bkey_mantissa_bits_dropped() argument
1081 unsigned key_bits_start = b->format.key_u64s * 64 - b->nr_key_bits; in bkey_mantissa_bits_dropped()
1083 return f->exponent > key_bits_start; in bkey_mantissa_bits_dropped()
1085 unsigned key_bits_end = high_bit_offset + b->nr_key_bits; in bkey_mantissa_bits_dropped()
1087 return f->exponent + BKEY_MANTISSA_BITS < key_bits_end; in bkey_mantissa_bits_dropped()
1092 static struct bkey_packed *bset_search_tree(const struct btree *b, in bset_search_tree() argument
1097 struct ro_aux_tree *base = ro_aux_tree_base(b, t); in bset_search_tree()
1104 if (likely(n << 4 < t->size)) in bset_search_tree()
1105 prefetch(&base->f[n << 4]); in bset_search_tree()
1107 f = &base->f[n]; in bset_search_tree()
1108 if (unlikely(f->exponent >= BFLOAT_FAILED)) in bset_search_tree()
1111 l = f->mantissa; in bset_search_tree()
1114 if (unlikely(l == r) && bkey_mantissa_bits_dropped(b, f)) in bset_search_tree()
1120 k = tree_to_bkey(b, t, n); in bset_search_tree()
1121 cmp = bkey_cmp_p_or_unp(b, k, packed_search, search); in bset_search_tree()
1126 } while (n < t->size); in bset_search_tree()
1128 inorder = __eytzinger1_to_inorder(n >> 1, t->size - 1, t->extra); in bset_search_tree()
1131 * n would have been the node we recursed to - the low bit tells us if in bset_search_tree()
1135 --inorder; in bset_search_tree()
1137 return btree_bkey_first(b, t); in bset_search_tree()
1139 f = &base->f[eytzinger1_prev(n >> 1, t->size - 1)]; in bset_search_tree()
1142 return cacheline_to_bkey(b, t, inorder, f->key_offset); in bset_search_tree()
1146 struct bkey_packed *__bch2_bset_search(struct btree *b, in __bch2_bset_search() argument
1161 * use a much simpler lookup table to do a binary search - in __bch2_bset_search()
1163 * * Or we use the auxiliary search tree we constructed earlier - in __bch2_bset_search()
1169 return btree_bkey_first(b, t); in __bch2_bset_search()
1171 return bset_search_write_set(b, t, search); in __bch2_bset_search()
1173 return bset_search_tree(b, t, search, lossy_packed_search); in __bch2_bset_search()
1180 struct bkey_packed *bch2_bset_search_linear(struct btree *b, in bch2_bset_search_linear() argument
1188 while (m != btree_bkey_last(b, t) && in bch2_bset_search_linear()
1189 bkey_iter_cmp_p_or_unp(b, m, in bch2_bset_search_linear()
1194 while (m != btree_bkey_last(b, t) && in bch2_bset_search_linear()
1195 bkey_iter_pos_cmp(b, m, search) < 0) in bch2_bset_search_linear()
1199 struct bkey_packed *prev = bch2_bkey_prev_all(b, t, m); in bch2_bset_search_linear()
1202 bkey_iter_cmp_p_or_unp(b, prev, in bch2_bset_search_linear()
1212 struct btree *b, in __bch2_btree_node_iter_push() argument
1222 BUG_ON(pos >= iter->data + ARRAY_SIZE(iter->data)); in __bch2_btree_node_iter_push()
1224 __btree_node_key_to_offset(b, k), in __bch2_btree_node_iter_push()
1225 __btree_node_key_to_offset(b, end) in __bch2_btree_node_iter_push()
1231 struct btree *b, in bch2_btree_node_iter_push() argument
1235 __bch2_btree_node_iter_push(iter, b, k, end); in bch2_btree_node_iter_push()
1236 bch2_btree_node_iter_sort(iter, b); in bch2_btree_node_iter_push()
1241 struct btree *b, struct bpos *search) in btree_node_iter_init_pack_failed() argument
1247 bch2_btree_node_iter_init_from_start(iter, b); in btree_node_iter_init_pack_failed()
1249 while ((k = bch2_btree_node_iter_peek(iter, b)) && in btree_node_iter_init_pack_failed()
1250 bkey_iter_pos_cmp(b, k, search) < 0) in btree_node_iter_init_pack_failed()
1251 bch2_btree_node_iter_advance(iter, b); in btree_node_iter_init_pack_failed()
1255 * bch2_btree_node_iter_init - initialize a btree node iterator, starting from a
1259 * @b: btree node to search
1279 * - For non extents, we guarantee that the live key comes last - see
1283 * - For extents, deleted keys sort last (see the comment at the top of this
1285 * key strictly greater than your search key - an extent that compares equal
1290 * the range we want - if we're unlucky and there's an extent that ends
1300 struct btree *b, struct bpos *search) in bch2_btree_node_iter_init() argument
1303 struct btree_node_iter_set *pos = iter->data; in bch2_btree_node_iter_init()
1307 EBUG_ON(bpos_lt(*search, b->data->min_key)); in bch2_btree_node_iter_init()
1308 EBUG_ON(bpos_gt(*search, b->data->max_key)); in bch2_btree_node_iter_init()
1309 bset_aux_tree_verify(b); in bch2_btree_node_iter_init()
1313 switch (bch2_bkey_pack_pos_lossy(&p, *search, b)) { in bch2_btree_node_iter_init()
1321 btree_node_iter_init_pack_failed(iter, b, search); in bch2_btree_node_iter_init()
1325 for (i = 0; i < b->nsets; i++) { in bch2_btree_node_iter_init()
1326 k[i] = __bch2_bset_search(b, b->set + i, search, &p); in bch2_btree_node_iter_init()
1330 for (i = 0; i < b->nsets; i++) { in bch2_btree_node_iter_init()
1331 struct bset_tree *t = b->set + i; in bch2_btree_node_iter_init()
1332 struct bkey_packed *end = btree_bkey_last(b, t); in bch2_btree_node_iter_init()
1334 k[i] = bch2_bset_search_linear(b, t, search, in bch2_btree_node_iter_init()
1338 __btree_node_key_to_offset(b, k[i]), in bch2_btree_node_iter_init()
1339 __btree_node_key_to_offset(b, end) in bch2_btree_node_iter_init()
1343 bch2_btree_node_iter_sort(iter, b); in bch2_btree_node_iter_init()
1347 struct btree *b) in bch2_btree_node_iter_init_from_start() argument
1351 for_each_bset(b, t) in bch2_btree_node_iter_init_from_start()
1352 __bch2_btree_node_iter_push(iter, b, in bch2_btree_node_iter_init_from_start()
1353 btree_bkey_first(b, t), in bch2_btree_node_iter_init_from_start()
1354 btree_bkey_last(b, t)); in bch2_btree_node_iter_init_from_start()
1355 bch2_btree_node_iter_sort(iter, b); in bch2_btree_node_iter_init_from_start()
1359 struct btree *b, in bch2_btree_node_iter_bset_pos() argument
1365 if (set->end == t->end_offset) in bch2_btree_node_iter_bset_pos()
1366 return __btree_node_offset_to_key(b, set->k); in bch2_btree_node_iter_bset_pos()
1368 return btree_bkey_last(b, t); in bch2_btree_node_iter_bset_pos()
1372 struct btree *b, in btree_node_iter_sort_two() argument
1377 if ((ret = (btree_node_iter_cmp(b, in btree_node_iter_sort_two()
1378 iter->data[first], in btree_node_iter_sort_two()
1379 iter->data[first + 1]) > 0))) in btree_node_iter_sort_two()
1380 swap(iter->data[first], iter->data[first + 1]); in btree_node_iter_sort_two()
1385 struct btree *b) in bch2_btree_node_iter_sort() argument
1390 btree_node_iter_sort_two(iter, b, 0); in bch2_btree_node_iter_sort()
1391 btree_node_iter_sort_two(iter, b, 1); in bch2_btree_node_iter_sort()
1395 btree_node_iter_sort_two(iter, b, 0); in bch2_btree_node_iter_sort()
1402 iter->data + ARRAY_SIZE(iter->data) - 1; in bch2_btree_node_iter_set_drop()
1404 memmove(&set[0], &set[1], (void *) last - (void *) set); in bch2_btree_node_iter_set_drop()
1409 struct btree *b) in __bch2_btree_node_iter_advance() argument
1411 iter->data->k += __bch2_btree_node_iter_peek_all(iter, b)->u64s; in __bch2_btree_node_iter_advance()
1413 EBUG_ON(iter->data->k > iter->data->end); in __bch2_btree_node_iter_advance()
1417 iter->data[0] = iter->data[1]; in __bch2_btree_node_iter_advance()
1418 iter->data[1] = iter->data[2]; in __bch2_btree_node_iter_advance()
1419 iter->data[2] = (struct btree_node_iter_set) { 0, 0 }; in __bch2_btree_node_iter_advance()
1426 if (!btree_node_iter_sort_two(iter, b, 0)) in __bch2_btree_node_iter_advance()
1432 btree_node_iter_sort_two(iter, b, 1); in __bch2_btree_node_iter_advance()
1436 struct btree *b) in bch2_btree_node_iter_advance() argument
1439 bch2_btree_node_iter_verify(iter, b); in bch2_btree_node_iter_advance()
1440 bch2_btree_node_iter_next_check(iter, b); in bch2_btree_node_iter_advance()
1443 __bch2_btree_node_iter_advance(iter, b); in bch2_btree_node_iter_advance()
1450 struct btree *b) in bch2_btree_node_iter_prev_all() argument
1457 bch2_btree_node_iter_verify(iter, b); in bch2_btree_node_iter_prev_all()
1459 for_each_bset(b, t) { in bch2_btree_node_iter_prev_all()
1460 k = bch2_bkey_prev_all(b, t, in bch2_btree_node_iter_prev_all()
1461 bch2_btree_node_iter_bset_pos(iter, b, t)); in bch2_btree_node_iter_prev_all()
1463 (!prev || bkey_iter_cmp(b, k, prev) > 0)) { in bch2_btree_node_iter_prev_all()
1465 end = t->end_offset; in bch2_btree_node_iter_prev_all()
1474 * prev we picked ends up in slot 0 - sort won't necessarily put it in bch2_btree_node_iter_prev_all()
1478 if (set->end == end) in bch2_btree_node_iter_prev_all()
1481 BUG_ON(set != &iter->data[__btree_node_iter_used(iter)]); in bch2_btree_node_iter_prev_all()
1483 BUG_ON(set >= iter->data + ARRAY_SIZE(iter->data)); in bch2_btree_node_iter_prev_all()
1485 memmove(&iter->data[1], in bch2_btree_node_iter_prev_all()
1486 &iter->data[0], in bch2_btree_node_iter_prev_all()
1487 (void *) set - (void *) &iter->data[0]); in bch2_btree_node_iter_prev_all()
1489 iter->data[0].k = __btree_node_key_to_offset(b, prev); in bch2_btree_node_iter_prev_all()
1490 iter->data[0].end = end; in bch2_btree_node_iter_prev_all()
1493 bch2_btree_node_iter_verify(iter, b); in bch2_btree_node_iter_prev_all()
1498 struct btree *b) in bch2_btree_node_iter_prev() argument
1503 prev = bch2_btree_node_iter_prev_all(iter, b); in bch2_btree_node_iter_prev()
1510 struct btree *b, in bch2_btree_node_iter_peek_unpack() argument
1513 struct bkey_packed *k = bch2_btree_node_iter_peek(iter, b); in bch2_btree_node_iter_peek_unpack()
1515 return k ? bkey_disassemble(b, k, u) : bkey_s_c_null; in bch2_btree_node_iter_peek_unpack()
1520 void bch2_btree_keys_stats(const struct btree *b, struct bset_stats *stats) in bch2_btree_keys_stats() argument
1522 for_each_bset_c(b, t) { in bch2_btree_keys_stats()
1526 stats->sets[type].nr++; in bch2_btree_keys_stats()
1527 stats->sets[type].bytes += le16_to_cpu(bset(b, t)->u64s) * in bch2_btree_keys_stats()
1531 stats->floats += t->size - 1; in bch2_btree_keys_stats()
1533 for (j = 1; j < t->size; j++) in bch2_btree_keys_stats()
1534 stats->failed += in bch2_btree_keys_stats()
1535 bkey_float(b, t, j)->exponent == in bch2_btree_keys_stats()
1541 void bch2_bfloat_to_text(struct printbuf *out, struct btree *b, in bch2_bfloat_to_text() argument
1544 struct bset_tree *t = bch2_bkey_to_bset(b, k); in bch2_bfloat_to_text()
1551 inorder = bkey_to_cacheline(b, t, k); in bch2_bfloat_to_text()
1552 if (!inorder || inorder >= t->size) in bch2_bfloat_to_text()
1555 j = __inorder_to_eytzinger1(inorder, t->size - 1, t->extra); in bch2_bfloat_to_text()
1556 if (k != tree_to_bkey(b, t, j)) in bch2_bfloat_to_text()
1559 switch (bkey_float(b, t, j)->exponent) { in bch2_bfloat_to_text()
1561 uk = bkey_unpack_key(b, k); in bch2_bfloat_to_text()