Lines Matching +full:- +full:b
1 // SPDX-License-Identifier: GPL-2.0
25 #include "sb-members.h"
26 #include "super-io.h"
45 int bch2_btree_node_check_topology(struct btree_trans *trans, struct btree *b) in bch2_btree_node_check_topology() argument
47 struct bch_fs *c = trans->c; in bch2_btree_node_check_topology()
48 struct bpos node_min = b->key.k.type == KEY_TYPE_btree_ptr_v2 in bch2_btree_node_check_topology()
49 ? bkey_i_to_btree_ptr_v2(&b->key)->v.min_key in bch2_btree_node_check_topology()
50 : b->data->min_key; in bch2_btree_node_check_topology()
57 BUG_ON(b->key.k.type == KEY_TYPE_btree_ptr_v2 && in bch2_btree_node_check_topology()
58 !bpos_eq(bkey_i_to_btree_ptr_v2(&b->key)->v.min_key, in bch2_btree_node_check_topology()
59 b->data->min_key)); in bch2_btree_node_check_topology()
61 if (b == btree_node_root(c, b)) { in bch2_btree_node_check_topology()
62 if (!bpos_eq(b->data->min_key, POS_MIN)) { in bch2_btree_node_check_topology()
64 bch2_bpos_to_text(&buf, b->data->min_key); in bch2_btree_node_check_topology()
70 if (!bpos_eq(b->data->max_key, SPOS_MAX)) { in bch2_btree_node_check_topology()
72 bch2_bpos_to_text(&buf, b->data->max_key); in bch2_btree_node_check_topology()
79 if (!b->c.level) in bch2_btree_node_check_topology()
83 bkey_init(&prev.k->k); in bch2_btree_node_check_topology()
84 bch2_btree_and_journal_iter_init_node_iter(trans, &iter, b); in bch2_btree_node_check_topology()
87 if (k.k->type != KEY_TYPE_btree_ptr_v2) in bch2_btree_node_check_topology()
92 struct bpos expected_min = bkey_deleted(&prev.k->k) in bch2_btree_node_check_topology()
94 : bpos_successor(prev.k->k.p); in bch2_btree_node_check_topology()
96 if (!bpos_eq(expected_min, bp.v->min_key)) { in bch2_btree_node_check_topology()
102 bch2_btree_id_str(b->c.btree_id), b->c.level); in bch2_btree_node_check_topology()
103 bch2_bkey_val_to_text(&buf, c, bkey_i_to_s_c(&b->key)); in bch2_btree_node_check_topology()
117 if (bkey_deleted(&prev.k->k)) { in bch2_btree_node_check_topology()
123 bch2_btree_id_str(b->c.btree_id), b->c.level); in bch2_btree_node_check_topology()
124 bch2_bkey_val_to_text(&buf, c, bkey_i_to_s_c(&b->key)); in bch2_btree_node_check_topology()
128 } else if (!bpos_eq(prev.k->k.p, b->key.k.p)) { in bch2_btree_node_check_topology()
134 bch2_btree_id_str(b->c.btree_id), b->c.level); in bch2_btree_node_check_topology()
135 bch2_bkey_val_to_text(&buf, c, bkey_i_to_s_c(&b->key)); in bch2_btree_node_check_topology()
149 if ((c->opts.recovery_passes & BIT_ULL(BCH_RECOVERY_PASS_check_topology)) && in bch2_btree_node_check_topology()
150 c->curr_recovery_pass > BCH_RECOVERY_PASS_check_topology) { in bch2_btree_node_check_topology()
152 ret = -BCH_ERR_btree_need_topology_repair; in bch2_btree_node_check_topology()
161 static void __bch2_btree_calc_format(struct bkey_format_state *s, struct btree *b) in __bch2_btree_calc_format() argument
166 for_each_bset(b, t) in __bch2_btree_calc_format()
167 bset_tree_for_each_key(b, t, k) in __bch2_btree_calc_format()
169 uk = bkey_unpack_key(b, k); in __bch2_btree_calc_format()
174 static struct bkey_format bch2_btree_calc_format(struct btree *b) in bch2_btree_calc_format() argument
179 bch2_bkey_format_add_pos(&s, b->data->min_key); in bch2_btree_calc_format()
180 bch2_bkey_format_add_pos(&s, b->data->max_key); in bch2_btree_calc_format()
181 __bch2_btree_calc_format(&s, b); in bch2_btree_calc_format()
192 (((int) new_f->key_u64s - old_f->key_u64s) * in btree_node_u64s_with_format()
194 (((int) new_f->key_u64s - BKEY_U64s) * in btree_node_u64s_with_format()
203 * bch2_btree_node_format_fits - check if we could rewrite node with a new format
206 * @b: btree node to rewrite
207 * @nr: number of keys for new node (i.e. b->nr)
210 * Returns: true if all re-packed keys will be able to fit in a new node.
214 static bool bch2_btree_node_format_fits(struct bch_fs *c, struct btree *b, in bch2_btree_node_format_fits() argument
218 size_t u64s = btree_node_u64s_with_format(nr, &b->format, new_f); in bch2_btree_node_format_fits()
220 return __vstruct_bytes(struct btree_node, u64s) < btree_buf_bytes(b); in bch2_btree_node_format_fits()
225 static void __btree_node_free(struct btree_trans *trans, struct btree *b) in __btree_node_free() argument
227 struct bch_fs *c = trans->c; in __btree_node_free()
229 trace_and_count(c, btree_node_free, trans, b); in __btree_node_free()
231 BUG_ON(btree_node_write_blocked(b)); in __btree_node_free()
232 BUG_ON(btree_node_dirty(b)); in __btree_node_free()
233 BUG_ON(btree_node_need_write(b)); in __btree_node_free()
234 BUG_ON(b == btree_node_root(c, b)); in __btree_node_free()
235 BUG_ON(b->ob.nr); in __btree_node_free()
236 BUG_ON(!list_empty(&b->write_blocked)); in __btree_node_free()
237 BUG_ON(b->will_make_reachable); in __btree_node_free()
239 clear_btree_node_noevict(b); in __btree_node_free()
241 mutex_lock(&c->btree_cache.lock); in __btree_node_free()
242 list_move(&b->list, &c->btree_cache.freeable); in __btree_node_free()
243 mutex_unlock(&c->btree_cache.lock); in __btree_node_free()
248 struct btree *b) in bch2_btree_node_free_inmem() argument
250 struct bch_fs *c = trans->c; in bch2_btree_node_free_inmem()
251 unsigned i, level = b->c.level; in bch2_btree_node_free_inmem()
253 bch2_btree_node_lock_write_nofail(trans, path, &b->c); in bch2_btree_node_free_inmem()
255 mutex_lock(&c->btree_cache.lock); in bch2_btree_node_free_inmem()
256 bch2_btree_node_hash_remove(&c->btree_cache, b); in bch2_btree_node_free_inmem()
257 mutex_unlock(&c->btree_cache.lock); in bch2_btree_node_free_inmem()
259 __btree_node_free(trans, b); in bch2_btree_node_free_inmem()
261 six_unlock_write(&b->c.lock); in bch2_btree_node_free_inmem()
265 if (path->l[level].b == b) { in bch2_btree_node_free_inmem()
267 path->l[level].b = ERR_PTR(-BCH_ERR_no_btree_node_init); in bch2_btree_node_free_inmem()
273 struct btree *b) in bch2_btree_node_free_never_used() argument
275 struct bch_fs *c = as->c; in bch2_btree_node_free_never_used()
276 struct prealloc_nodes *p = &as->prealloc_nodes[b->c.lock.readers != NULL]; in bch2_btree_node_free_never_used()
278 unsigned i, level = b->c.level; in bch2_btree_node_free_never_used()
280 BUG_ON(!list_empty(&b->write_blocked)); in bch2_btree_node_free_never_used()
281 BUG_ON(b->will_make_reachable != (1UL|(unsigned long) as)); in bch2_btree_node_free_never_used()
283 b->will_make_reachable = 0; in bch2_btree_node_free_never_used()
284 closure_put(&as->cl); in bch2_btree_node_free_never_used()
286 clear_btree_node_will_make_reachable(b); in bch2_btree_node_free_never_used()
287 clear_btree_node_accessed(b); in bch2_btree_node_free_never_used()
288 clear_btree_node_dirty_acct(c, b); in bch2_btree_node_free_never_used()
289 clear_btree_node_need_write(b); in bch2_btree_node_free_never_used()
291 mutex_lock(&c->btree_cache.lock); in bch2_btree_node_free_never_used()
292 bch2_btree_node_hash_remove(&c->btree_cache, b); in bch2_btree_node_free_never_used()
293 mutex_unlock(&c->btree_cache.lock); in bch2_btree_node_free_never_used()
295 BUG_ON(p->nr >= ARRAY_SIZE(p->b)); in bch2_btree_node_free_never_used()
296 p->b[p->nr++] = b; in bch2_btree_node_free_never_used()
298 six_unlock_intent(&b->c.lock); in bch2_btree_node_free_never_used()
301 if (path->l[level].b == b) { in bch2_btree_node_free_never_used()
303 path->l[level].b = ERR_PTR(-BCH_ERR_no_btree_node_init); in bch2_btree_node_free_never_used()
313 struct bch_fs *c = trans->c; in __bch2_btree_node_alloc()
315 struct btree *b; in __bch2_btree_node_alloc() local
325 b = bch2_btree_node_mem_alloc(trans, interior_node); in __bch2_btree_node_alloc()
326 if (IS_ERR(b)) in __bch2_btree_node_alloc()
327 return b; in __bch2_btree_node_alloc()
329 BUG_ON(b->ob.nr); in __bch2_btree_node_alloc()
331 mutex_lock(&c->btree_reserve_cache_lock); in __bch2_btree_node_alloc()
332 if (c->btree_reserve_cache_nr > nr_reserve) { in __bch2_btree_node_alloc()
334 &c->btree_reserve_cache[--c->btree_reserve_cache_nr]; in __bch2_btree_node_alloc()
336 obs = a->ob; in __bch2_btree_node_alloc()
337 bkey_copy(&tmp.k, &a->k); in __bch2_btree_node_alloc()
338 mutex_unlock(&c->btree_reserve_cache_lock); in __bch2_btree_node_alloc()
341 mutex_unlock(&c->btree_reserve_cache_lock); in __bch2_btree_node_alloc()
344 c->opts.metadata_target ?: in __bch2_btree_node_alloc()
345 c->opts.foreground_target, in __bch2_btree_node_alloc()
347 writepoint_ptr(&c->btree_write_point), in __bch2_btree_node_alloc()
349 res->nr_replicas, in __bch2_btree_node_alloc()
350 min(res->nr_replicas, in __bch2_btree_node_alloc()
351 c->opts.metadata_replicas_required), in __bch2_btree_node_alloc()
356 if (wp->sectors_free < btree_sectors(c)) { in __bch2_btree_node_alloc()
360 open_bucket_for_each(c, &wp->ptrs, ob, i) in __bch2_btree_node_alloc()
361 if (ob->sectors_free < btree_sectors(c)) in __bch2_btree_node_alloc()
362 ob->sectors_free = 0; in __bch2_btree_node_alloc()
374 bkey_copy(&b->key, &tmp.k); in __bch2_btree_node_alloc()
375 b->ob = obs; in __bch2_btree_node_alloc()
376 six_unlock_write(&b->c.lock); in __bch2_btree_node_alloc()
377 six_unlock_intent(&b->c.lock); in __bch2_btree_node_alloc()
379 return b; in __bch2_btree_node_alloc()
381 bch2_btree_node_to_freelist(c, b); in __bch2_btree_node_alloc()
389 struct bch_fs *c = as->c; in bch2_btree_node_alloc()
390 struct btree *b; in bch2_btree_node_alloc() local
391 struct prealloc_nodes *p = &as->prealloc_nodes[!!level]; in bch2_btree_node_alloc()
395 BUG_ON(!p->nr); in bch2_btree_node_alloc()
397 b = p->b[--p->nr]; in bch2_btree_node_alloc()
399 btree_node_lock_nopath_nofail(trans, &b->c, SIX_LOCK_intent); in bch2_btree_node_alloc()
400 btree_node_lock_nopath_nofail(trans, &b->c, SIX_LOCK_write); in bch2_btree_node_alloc()
402 set_btree_node_accessed(b); in bch2_btree_node_alloc()
403 set_btree_node_dirty_acct(c, b); in bch2_btree_node_alloc()
404 set_btree_node_need_write(b); in bch2_btree_node_alloc()
406 bch2_bset_init_first(b, &b->data->keys); in bch2_btree_node_alloc()
407 b->c.level = level; in bch2_btree_node_alloc()
408 b->c.btree_id = as->btree_id; in bch2_btree_node_alloc()
409 b->version_ondisk = c->sb.version; in bch2_btree_node_alloc()
411 memset(&b->nr, 0, sizeof(b->nr)); in bch2_btree_node_alloc()
412 b->data->magic = cpu_to_le64(bset_magic(c)); in bch2_btree_node_alloc()
413 memset(&b->data->_ptr, 0, sizeof(b->data->_ptr)); in bch2_btree_node_alloc()
414 b->data->flags = 0; in bch2_btree_node_alloc()
415 SET_BTREE_NODE_ID(b->data, as->btree_id); in bch2_btree_node_alloc()
416 SET_BTREE_NODE_LEVEL(b->data, level); in bch2_btree_node_alloc()
418 if (b->key.k.type == KEY_TYPE_btree_ptr_v2) { in bch2_btree_node_alloc()
419 struct bkey_i_btree_ptr_v2 *bp = bkey_i_to_btree_ptr_v2(&b->key); in bch2_btree_node_alloc()
421 bp->v.mem_ptr = 0; in bch2_btree_node_alloc()
422 bp->v.seq = b->data->keys.seq; in bch2_btree_node_alloc()
423 bp->v.sectors_written = 0; in bch2_btree_node_alloc()
426 SET_BTREE_NODE_NEW_EXTENT_OVERWRITE(b->data, true); in bch2_btree_node_alloc()
428 bch2_btree_build_aux_trees(b); in bch2_btree_node_alloc()
430 ret = bch2_btree_node_hash_insert(&c->btree_cache, b, level, as->btree_id); in bch2_btree_node_alloc()
433 trace_and_count(c, btree_node_alloc, trans, b); in bch2_btree_node_alloc()
435 return b; in bch2_btree_node_alloc()
438 static void btree_set_min(struct btree *b, struct bpos pos) in btree_set_min() argument
440 if (b->key.k.type == KEY_TYPE_btree_ptr_v2) in btree_set_min()
441 bkey_i_to_btree_ptr_v2(&b->key)->v.min_key = pos; in btree_set_min()
442 b->data->min_key = pos; in btree_set_min()
445 static void btree_set_max(struct btree *b, struct bpos pos) in btree_set_max() argument
447 b->key.k.p = pos; in btree_set_max()
448 b->data->max_key = pos; in btree_set_max()
453 struct btree *b) in bch2_btree_node_alloc_replacement()
455 struct btree *n = bch2_btree_node_alloc(as, trans, b->c.level); in bch2_btree_node_alloc_replacement()
456 struct bkey_format format = bch2_btree_calc_format(b); in bch2_btree_node_alloc_replacement()
459 * The keys might expand with the new format - if they wouldn't fit in in bch2_btree_node_alloc_replacement()
462 if (!bch2_btree_node_format_fits(as->c, b, b->nr, &format)) in bch2_btree_node_alloc_replacement()
463 format = b->format; in bch2_btree_node_alloc_replacement()
465 SET_BTREE_NODE_SEQ(n->data, BTREE_NODE_SEQ(b->data) + 1); in bch2_btree_node_alloc_replacement()
467 btree_set_min(n, b->data->min_key); in bch2_btree_node_alloc_replacement()
468 btree_set_max(n, b->data->max_key); in bch2_btree_node_alloc_replacement()
470 n->data->format = format; in bch2_btree_node_alloc_replacement()
473 bch2_btree_sort_into(as->c, n, b); in bch2_btree_node_alloc_replacement()
482 struct btree *b = bch2_btree_node_alloc(as, trans, level); in __btree_root_alloc() local
484 btree_set_min(b, POS_MIN); in __btree_root_alloc()
485 btree_set_max(b, SPOS_MAX); in __btree_root_alloc()
486 b->data->format = bch2_btree_calc_format(b); in __btree_root_alloc()
488 btree_node_set_format(b, b->data->format); in __btree_root_alloc()
489 bch2_btree_build_aux_trees(b); in __btree_root_alloc()
491 return b; in __btree_root_alloc()
496 struct bch_fs *c = as->c; in bch2_btree_reserve_put()
499 for (p = as->prealloc_nodes; in bch2_btree_reserve_put()
500 p < as->prealloc_nodes + ARRAY_SIZE(as->prealloc_nodes); in bch2_btree_reserve_put()
502 while (p->nr) { in bch2_btree_reserve_put()
503 struct btree *b = p->b[--p->nr]; in bch2_btree_reserve_put() local
505 mutex_lock(&c->btree_reserve_cache_lock); in bch2_btree_reserve_put()
507 if (c->btree_reserve_cache_nr < in bch2_btree_reserve_put()
508 ARRAY_SIZE(c->btree_reserve_cache)) { in bch2_btree_reserve_put()
510 &c->btree_reserve_cache[c->btree_reserve_cache_nr++]; in bch2_btree_reserve_put()
512 a->ob = b->ob; in bch2_btree_reserve_put()
513 b->ob.nr = 0; in bch2_btree_reserve_put()
514 bkey_copy(&a->k, &b->key); in bch2_btree_reserve_put()
516 bch2_open_buckets_put(c, &b->ob); in bch2_btree_reserve_put()
519 mutex_unlock(&c->btree_reserve_cache_lock); in bch2_btree_reserve_put()
521 btree_node_lock_nopath_nofail(trans, &b->c, SIX_LOCK_intent); in bch2_btree_reserve_put()
522 btree_node_lock_nopath_nofail(trans, &b->c, SIX_LOCK_write); in bch2_btree_reserve_put()
523 __btree_node_free(trans, b); in bch2_btree_reserve_put()
524 six_unlock_write(&b->c.lock); in bch2_btree_reserve_put()
525 six_unlock_intent(&b->c.lock); in bch2_btree_reserve_put()
536 struct btree *b; in bch2_btree_reserve_get() local
551 struct prealloc_nodes *p = as->prealloc_nodes + interior; in bch2_btree_reserve_get()
553 while (p->nr < nr_nodes[interior]) { in bch2_btree_reserve_get()
554 b = __bch2_btree_node_alloc(trans, &as->disk_res, cl, in bch2_btree_reserve_get()
556 if (IS_ERR(b)) { in bch2_btree_reserve_get()
557 ret = PTR_ERR(b); in bch2_btree_reserve_get()
561 p->b[p->nr++] = b; in bch2_btree_reserve_get()
573 struct bch_fs *c = as->c; in bch2_btree_update_free()
575 if (as->took_gc_lock) in bch2_btree_update_free()
576 up_read(&c->gc_lock); in bch2_btree_update_free()
577 as->took_gc_lock = false; in bch2_btree_update_free()
579 bch2_journal_pin_drop(&c->journal, &as->journal); in bch2_btree_update_free()
580 bch2_journal_pin_flush(&c->journal, &as->journal); in bch2_btree_update_free()
581 bch2_disk_reservation_put(c, &as->disk_res); in bch2_btree_update_free()
584 bch2_time_stats_update(&c->times[BCH_TIME_btree_interior_update_total], in bch2_btree_update_free()
585 as->start_time); in bch2_btree_update_free()
587 mutex_lock(&c->btree_interior_update_lock); in bch2_btree_update_free()
588 list_del(&as->unwritten_list); in bch2_btree_update_free()
589 list_del(&as->list); in bch2_btree_update_free()
591 closure_debug_destroy(&as->cl); in bch2_btree_update_free()
592 mempool_free(as, &c->btree_interior_update_pool); in bch2_btree_update_free()
598 closure_wake_up(&c->btree_interior_update_wait); in bch2_btree_update_free()
600 mutex_unlock(&c->btree_interior_update_lock); in bch2_btree_update_free()
604 struct keylist *keys, struct btree *b) in btree_update_add_key() argument
606 struct bkey_i *k = &b->key; in btree_update_add_key()
608 BUG_ON(bch2_keylist_u64s(keys) + k->k.u64s > in btree_update_add_key()
609 ARRAY_SIZE(as->_old_keys)); in btree_update_add_key()
611 bkey_copy(keys->top, k); in btree_update_add_key()
612 bkey_i_to_btree_ptr_v2(keys->top)->v.mem_ptr = b->c.level + 1; in btree_update_add_key()
619 for_each_keylist_key(&as->new_keys, k) in btree_update_new_nodes_marked_sb()
620 if (!bch2_dev_btree_bitmap_marked(as->c, bkey_i_to_s_c(k))) in btree_update_new_nodes_marked_sb()
627 struct bch_fs *c = as->c; in btree_update_new_nodes_mark_sb()
629 mutex_lock(&c->sb_lock); in btree_update_new_nodes_mark_sb()
630 for_each_keylist_key(&as->new_keys, k) in btree_update_new_nodes_mark_sb()
634 mutex_unlock(&c->sb_lock); in btree_update_new_nodes_mark_sb()
644 struct jset_entry *e = bch2_trans_jset_entry_alloc(trans, as->journal_u64s); in btree_update_nodes_written_trans()
649 memcpy(e, as->journal_entries, as->journal_u64s * sizeof(u64)); in btree_update_nodes_written_trans()
651 trans->journal_pin = &as->journal; in btree_update_nodes_written_trans()
653 for_each_keylist_key(&as->old_keys, k) { in btree_update_nodes_written_trans()
654 unsigned level = bkey_i_to_btree_ptr_v2(k)->v.mem_ptr; in btree_update_nodes_written_trans()
656 ret = bch2_key_trigger_old(trans, as->btree_id, level, bkey_i_to_s_c(k), in btree_update_nodes_written_trans()
662 for_each_keylist_key(&as->new_keys, k) { in btree_update_nodes_written_trans()
663 unsigned level = bkey_i_to_btree_ptr_v2(k)->v.mem_ptr; in btree_update_nodes_written_trans()
665 ret = bch2_key_trigger_new(trans, as->btree_id, level, bkey_i_to_s(k), in btree_update_nodes_written_trans()
676 struct bch_fs *c = as->c; in btree_update_nodes_written()
677 struct btree *b; in btree_update_nodes_written() local
690 ret = bch2_journal_error(&c->journal); in btree_update_nodes_written()
701 for (i = 0; i < as->nr_old_nodes; i++) { in btree_update_nodes_written()
704 b = as->old_nodes[i]; in btree_update_nodes_written()
706 btree_node_lock_nopath_nofail(trans, &b->c, SIX_LOCK_read); in btree_update_nodes_written()
707 seq = b->data ? b->data->keys.seq : 0; in btree_update_nodes_written()
708 six_unlock_read(&b->c.lock); in btree_update_nodes_written()
710 if (seq == as->old_nodes_seq[i]) in btree_update_nodes_written()
711 wait_on_bit_io(&b->flags, BTREE_NODE_write_in_flight_inner, in btree_update_nodes_written()
728 ret = commit_do(trans, &as->disk_res, &journal_seq, in btree_update_nodes_written()
736 bch2_fs_fatal_err_on(ret && !bch2_journal_error(&c->journal), c, in btree_update_nodes_written()
744 * It should be, but bch2_path_get_unlocked_mut() -> bch2_path_get() in btree_update_nodes_written()
753 * to free as->b and calling btree_update_reparent() on us - we'll in btree_update_nodes_written()
756 b = READ_ONCE(as->b); in btree_update_nodes_written()
757 if (b) { in btree_update_nodes_written()
759 * @b is the node we did the final insert into: in btree_update_nodes_written()
763 * so that shutdown works, but the i->journal_seq mechanism in btree_update_nodes_written()
765 * didn't get a journal sequence number) - instead in btree_update_nodes_written()
771 as->btree_id, b->c.level, b->key.k.p); in btree_update_nodes_written()
772 struct btree_path *path = trans->paths + path_idx; in btree_update_nodes_written()
773 btree_node_lock_nopath_nofail(trans, &b->c, SIX_LOCK_intent); in btree_update_nodes_written()
774 mark_btree_node_locked(trans, path, b->c.level, BTREE_NODE_INTENT_LOCKED); in btree_update_nodes_written()
775 path->l[b->c.level].lock_seq = six_lock_seq(&b->c.lock); in btree_update_nodes_written()
776 path->l[b->c.level].b = b; in btree_update_nodes_written()
778 bch2_btree_node_lock_write_nofail(trans, path, &b->c); in btree_update_nodes_written()
780 mutex_lock(&c->btree_interior_update_lock); in btree_update_nodes_written()
782 list_del(&as->write_blocked_list); in btree_update_nodes_written()
783 if (list_empty(&b->write_blocked)) in btree_update_nodes_written()
784 clear_btree_node_write_blocked(b); in btree_update_nodes_written()
790 if (as->b == b) { in btree_update_nodes_written()
791 BUG_ON(!b->c.level); in btree_update_nodes_written()
792 BUG_ON(!btree_node_dirty(b)); in btree_update_nodes_written()
795 struct bset *last = btree_bset_last(b); in btree_update_nodes_written()
797 last->journal_seq = cpu_to_le64( in btree_update_nodes_written()
799 le64_to_cpu(last->journal_seq))); in btree_update_nodes_written()
801 bch2_btree_add_journal_pin(c, b, journal_seq); in btree_update_nodes_written()
808 set_btree_node_never_write(b); in btree_update_nodes_written()
812 mutex_unlock(&c->btree_interior_update_lock); in btree_update_nodes_written()
814 mark_btree_node_locked_noreset(path, b->c.level, BTREE_NODE_INTENT_LOCKED); in btree_update_nodes_written()
815 six_unlock_write(&b->c.lock); in btree_update_nodes_written()
817 btree_node_write_if_need(c, b, SIX_LOCK_intent); in btree_update_nodes_written()
818 btree_node_unlock(trans, path, b->c.level); in btree_update_nodes_written()
822 bch2_journal_pin_drop(&c->journal, &as->journal); in btree_update_nodes_written()
824 mutex_lock(&c->btree_interior_update_lock); in btree_update_nodes_written()
825 for (i = 0; i < as->nr_new_nodes; i++) { in btree_update_nodes_written()
826 b = as->new_nodes[i]; in btree_update_nodes_written()
828 BUG_ON(b->will_make_reachable != (unsigned long) as); in btree_update_nodes_written()
829 b->will_make_reachable = 0; in btree_update_nodes_written()
830 clear_btree_node_will_make_reachable(b); in btree_update_nodes_written()
832 mutex_unlock(&c->btree_interior_update_lock); in btree_update_nodes_written()
834 for (i = 0; i < as->nr_new_nodes; i++) { in btree_update_nodes_written()
835 b = as->new_nodes[i]; in btree_update_nodes_written()
837 btree_node_lock_nopath_nofail(trans, &b->c, SIX_LOCK_read); in btree_update_nodes_written()
838 btree_node_write_if_need(c, b, SIX_LOCK_read); in btree_update_nodes_written()
839 six_unlock_read(&b->c.lock); in btree_update_nodes_written()
842 for (i = 0; i < as->nr_open_buckets; i++) in btree_update_nodes_written()
843 bch2_open_bucket_put(c, c->open_buckets + as->open_buckets[i]); in btree_update_nodes_written()
856 mutex_lock(&c->btree_interior_update_lock); in btree_interior_update_work()
857 as = list_first_entry_or_null(&c->btree_interior_updates_unwritten, in btree_interior_update_work()
859 if (as && !as->nodes_written) in btree_interior_update_work()
861 mutex_unlock(&c->btree_interior_update_lock); in btree_interior_update_work()
873 struct bch_fs *c = as->c; in CLOSURE_CALLBACK()
875 mutex_lock(&c->btree_interior_update_lock); in CLOSURE_CALLBACK()
876 as->nodes_written = true; in CLOSURE_CALLBACK()
877 mutex_unlock(&c->btree_interior_update_lock); in CLOSURE_CALLBACK()
879 queue_work(c->btree_interior_update_worker, &c->btree_interior_update_work); in CLOSURE_CALLBACK()
883 * We're updating @b with pointers to nodes that haven't finished writing yet:
884 * block @b from being written until @as completes
886 static void btree_update_updated_node(struct btree_update *as, struct btree *b) in btree_update_updated_node() argument
888 struct bch_fs *c = as->c; in btree_update_updated_node()
890 BUG_ON(as->mode != BTREE_UPDATE_none); in btree_update_updated_node()
891 BUG_ON(as->update_level_end < b->c.level); in btree_update_updated_node()
892 BUG_ON(!btree_node_dirty(b)); in btree_update_updated_node()
893 BUG_ON(!b->c.level); in btree_update_updated_node()
895 mutex_lock(&c->btree_interior_update_lock); in btree_update_updated_node()
896 list_add_tail(&as->unwritten_list, &c->btree_interior_updates_unwritten); in btree_update_updated_node()
898 as->mode = BTREE_UPDATE_node; in btree_update_updated_node()
899 as->b = b; in btree_update_updated_node()
900 as->update_level_end = b->c.level; in btree_update_updated_node()
902 set_btree_node_write_blocked(b); in btree_update_updated_node()
903 list_add(&as->write_blocked_list, &b->write_blocked); in btree_update_updated_node()
905 mutex_unlock(&c->btree_interior_update_lock); in btree_update_updated_node()
917 struct bch_fs *c = as->c; in btree_update_reparent()
919 lockdep_assert_held(&c->btree_interior_update_lock); in btree_update_reparent()
921 child->b = NULL; in btree_update_reparent()
922 child->mode = BTREE_UPDATE_update; in btree_update_reparent()
924 bch2_journal_pin_copy(&c->journal, &as->journal, &child->journal, in btree_update_reparent()
928 static void btree_update_updated_root(struct btree_update *as, struct btree *b) in btree_update_updated_root() argument
930 struct bkey_i *insert = &b->key; in btree_update_updated_root()
931 struct bch_fs *c = as->c; in btree_update_updated_root()
933 BUG_ON(as->mode != BTREE_UPDATE_none); in btree_update_updated_root()
935 BUG_ON(as->journal_u64s + jset_u64s(insert->k.u64s) > in btree_update_updated_root()
936 ARRAY_SIZE(as->journal_entries)); in btree_update_updated_root()
938 as->journal_u64s += in btree_update_updated_root()
939 journal_entry_set((void *) &as->journal_entries[as->journal_u64s], in btree_update_updated_root()
941 b->c.btree_id, b->c.level, in btree_update_updated_root()
942 insert, insert->k.u64s); in btree_update_updated_root()
944 mutex_lock(&c->btree_interior_update_lock); in btree_update_updated_root()
945 list_add_tail(&as->unwritten_list, &c->btree_interior_updates_unwritten); in btree_update_updated_root()
947 as->mode = BTREE_UPDATE_root; in btree_update_updated_root()
948 mutex_unlock(&c->btree_interior_update_lock); in btree_update_updated_root()
954 * This causes @as to wait on @b to be written, before it gets to
957 * Additionally, it sets b->will_make_reachable to prevent any additional writes
958 * to @b from happening besides the first until @b is reachable on disk
960 * And it adds @b to the list of @as's new nodes, so that we can update sector
963 static void bch2_btree_update_add_new_node(struct btree_update *as, struct btree *b) in bch2_btree_update_add_new_node() argument
965 struct bch_fs *c = as->c; in bch2_btree_update_add_new_node()
967 closure_get(&as->cl); in bch2_btree_update_add_new_node()
969 mutex_lock(&c->btree_interior_update_lock); in bch2_btree_update_add_new_node()
970 BUG_ON(as->nr_new_nodes >= ARRAY_SIZE(as->new_nodes)); in bch2_btree_update_add_new_node()
971 BUG_ON(b->will_make_reachable); in bch2_btree_update_add_new_node()
973 as->new_nodes[as->nr_new_nodes++] = b; in bch2_btree_update_add_new_node()
974 b->will_make_reachable = 1UL|(unsigned long) as; in bch2_btree_update_add_new_node()
975 set_btree_node_will_make_reachable(b); in bch2_btree_update_add_new_node()
977 mutex_unlock(&c->btree_interior_update_lock); in bch2_btree_update_add_new_node()
979 btree_update_add_key(as, &as->new_keys, b); in bch2_btree_update_add_new_node()
981 if (b->key.k.type == KEY_TYPE_btree_ptr_v2) { in bch2_btree_update_add_new_node()
982 unsigned bytes = vstruct_end(&b->data->keys) - (void *) b->data; in bch2_btree_update_add_new_node()
985 bkey_i_to_btree_ptr_v2(&b->key)->v.sectors_written = in bch2_btree_update_add_new_node()
991 * returns true if @b was a new node
993 static void btree_update_drop_new_node(struct bch_fs *c, struct btree *b) in btree_update_drop_new_node() argument
999 mutex_lock(&c->btree_interior_update_lock); in btree_update_drop_new_node()
1001 * When b->will_make_reachable != 0, it owns a ref on as->cl that's in btree_update_drop_new_node()
1002 * dropped when it gets written by bch2_btree_complete_write - the in btree_update_drop_new_node()
1005 v = xchg(&b->will_make_reachable, 0); in btree_update_drop_new_node()
1006 clear_btree_node_will_make_reachable(b); in btree_update_drop_new_node()
1010 mutex_unlock(&c->btree_interior_update_lock); in btree_update_drop_new_node()
1014 for (i = 0; i < as->nr_new_nodes; i++) in btree_update_drop_new_node()
1015 if (as->new_nodes[i] == b) in btree_update_drop_new_node()
1020 array_remove_item(as->new_nodes, as->nr_new_nodes, i); in btree_update_drop_new_node()
1021 mutex_unlock(&c->btree_interior_update_lock); in btree_update_drop_new_node()
1024 closure_put(&as->cl); in btree_update_drop_new_node()
1027 static void bch2_btree_update_get_open_buckets(struct btree_update *as, struct btree *b) in bch2_btree_update_get_open_buckets() argument
1029 while (b->ob.nr) in bch2_btree_update_get_open_buckets()
1030 as->open_buckets[as->nr_open_buckets++] = in bch2_btree_update_get_open_buckets()
1031 b->ob.v[--b->ob.nr]; in bch2_btree_update_get_open_buckets()
1041 * @b is being split/rewritten: it may have pointers to not-yet-written btree
1042 * nodes and thus outstanding btree_updates - redirect @b's
1046 struct btree *b) in bch2_btree_interior_update_will_free_node() argument
1048 struct bch_fs *c = as->c; in bch2_btree_interior_update_will_free_node()
1052 set_btree_node_dying(b); in bch2_btree_interior_update_will_free_node()
1054 if (btree_node_fake(b)) in bch2_btree_interior_update_will_free_node()
1057 mutex_lock(&c->btree_interior_update_lock); in bch2_btree_interior_update_will_free_node()
1067 list_for_each_entry_safe(p, n, &b->write_blocked, write_blocked_list) { in bch2_btree_interior_update_will_free_node()
1068 list_del_init(&p->write_blocked_list); in bch2_btree_interior_update_will_free_node()
1075 closure_wake_up(&c->btree_interior_update_wait); in bch2_btree_interior_update_will_free_node()
1078 clear_btree_node_dirty_acct(c, b); in bch2_btree_interior_update_will_free_node()
1079 clear_btree_node_need_write(b); in bch2_btree_interior_update_will_free_node()
1080 clear_btree_node_write_blocked(b); in bch2_btree_interior_update_will_free_node()
1085 * If so, transfer that pin to the btree_update operation - in bch2_btree_interior_update_will_free_node()
1090 w = btree_current_write(b); in bch2_btree_interior_update_will_free_node()
1091 bch2_journal_pin_copy(&c->journal, &as->journal, &w->journal, in bch2_btree_interior_update_will_free_node()
1093 bch2_journal_pin_drop(&c->journal, &w->journal); in bch2_btree_interior_update_will_free_node()
1095 w = btree_prev_write(b); in bch2_btree_interior_update_will_free_node()
1096 bch2_journal_pin_copy(&c->journal, &as->journal, &w->journal, in bch2_btree_interior_update_will_free_node()
1098 bch2_journal_pin_drop(&c->journal, &w->journal); in bch2_btree_interior_update_will_free_node()
1100 mutex_unlock(&c->btree_interior_update_lock); in bch2_btree_interior_update_will_free_node()
1106 * reachable - now that we've cancelled any pending writes and moved in bch2_btree_interior_update_will_free_node()
1111 btree_update_drop_new_node(c, b); in bch2_btree_interior_update_will_free_node()
1113 btree_update_add_key(as, &as->old_keys, b); in bch2_btree_interior_update_will_free_node()
1115 as->old_nodes[as->nr_old_nodes] = b; in bch2_btree_interior_update_will_free_node()
1116 as->old_nodes_seq[as->nr_old_nodes] = b->data->keys.seq; in bch2_btree_interior_update_will_free_node()
1117 as->nr_old_nodes++; in bch2_btree_interior_update_will_free_node()
1122 struct bch_fs *c = as->c; in bch2_btree_update_done()
1123 u64 start_time = as->start_time; in bch2_btree_update_done()
1125 BUG_ON(as->mode == BTREE_UPDATE_none); in bch2_btree_update_done()
1127 if (as->took_gc_lock) in bch2_btree_update_done()
1128 up_read(&as->c->gc_lock); in bch2_btree_update_done()
1129 as->took_gc_lock = false; in bch2_btree_update_done()
1133 continue_at(&as->cl, btree_update_set_nodes_written, in bch2_btree_update_done()
1134 as->c->btree_interior_update_worker); in bch2_btree_update_done()
1136 bch2_time_stats_update(&c->times[BCH_TIME_btree_interior_update_foreground], in bch2_btree_update_done()
1144 struct bch_fs *c = trans->c; in bch2_btree_update_start()
1153 u32 restart_count = trans->restart_count; in bch2_btree_update_start()
1155 BUG_ON(!path->should_be_locked); in bch2_btree_update_start()
1166 test_bit(JOURNAL_space_low, &c->journal.flags)) { in bch2_btree_update_start()
1168 return ERR_PTR(-BCH_ERR_journal_reclaim_would_deadlock); in bch2_btree_update_start()
1171 ({ wait_event(c->journal.wait, !test_bit(JOURNAL_space_low, &c->journal.flags)); 0; })); in bch2_btree_update_start()
1193 * split at prior level - it might have been a merge instead: in bch2_btree_update_start()
1195 if (bch2_btree_node_insert_fits(path->l[level_end].b, in bch2_btree_update_start()
1199 split = path->l[level_end].b->nr.live_u64s > BTREE_SPLIT_THRESHOLD(c); in bch2_btree_update_start()
1202 if (!down_read_trylock(&c->gc_lock)) { in bch2_btree_update_start()
1203 ret = drop_locks_do(trans, (down_read(&c->gc_lock), 0)); in bch2_btree_update_start()
1205 up_read(&c->gc_lock); in bch2_btree_update_start()
1210 as = mempool_alloc(&c->btree_interior_update_pool, GFP_NOFS); in bch2_btree_update_start()
1212 closure_init(&as->cl, NULL); in bch2_btree_update_start()
1213 as->c = c; in bch2_btree_update_start()
1214 as->start_time = start_time; in bch2_btree_update_start()
1215 as->ip_started = _RET_IP_; in bch2_btree_update_start()
1216 as->mode = BTREE_UPDATE_none; in bch2_btree_update_start()
1217 as->flags = flags; in bch2_btree_update_start()
1218 as->took_gc_lock = true; in bch2_btree_update_start()
1219 as->btree_id = path->btree_id; in bch2_btree_update_start()
1220 as->update_level_start = level_start; in bch2_btree_update_start()
1221 as->update_level_end = level_end; in bch2_btree_update_start()
1222 INIT_LIST_HEAD(&as->list); in bch2_btree_update_start()
1223 INIT_LIST_HEAD(&as->unwritten_list); in bch2_btree_update_start()
1224 INIT_LIST_HEAD(&as->write_blocked_list); in bch2_btree_update_start()
1225 bch2_keylist_init(&as->old_keys, as->_old_keys); in bch2_btree_update_start()
1226 bch2_keylist_init(&as->new_keys, as->_new_keys); in bch2_btree_update_start()
1227 bch2_keylist_init(&as->parent_keys, as->inline_keys); in bch2_btree_update_start()
1229 mutex_lock(&c->btree_interior_update_lock); in bch2_btree_update_start()
1230 list_add_tail(&as->list, &c->btree_interior_update_list); in bch2_btree_update_start()
1231 mutex_unlock(&c->btree_interior_update_lock); in bch2_btree_update_start()
1241 ret = bch2_journal_error(&c->journal); in bch2_btree_update_start()
1245 ret = bch2_disk_reservation_get(c, &as->disk_res, in bch2_btree_update_start()
1247 c->opts.metadata_replicas, in bch2_btree_update_start()
1264 ret = -BCH_ERR_journal_reclaim_would_deadlock; in bch2_btree_update_start()
1279 trace_and_count(c, btree_reserve_get_fail, trans->fn, in bch2_btree_update_start()
1294 ret != -BCH_ERR_journal_reclaim_would_deadlock) in bch2_btree_update_start()
1301 static void bch2_btree_set_root_inmem(struct bch_fs *c, struct btree *b) in bch2_btree_set_root_inmem() argument
1304 mutex_lock(&c->btree_cache.lock); in bch2_btree_set_root_inmem()
1305 list_del_init(&b->list); in bch2_btree_set_root_inmem()
1306 mutex_unlock(&c->btree_cache.lock); in bch2_btree_set_root_inmem()
1308 mutex_lock(&c->btree_root_lock); in bch2_btree_set_root_inmem()
1309 bch2_btree_id_root(c, b->c.btree_id)->b = b; in bch2_btree_set_root_inmem()
1310 mutex_unlock(&c->btree_root_lock); in bch2_btree_set_root_inmem()
1318 struct btree *b, in bch2_btree_set_root() argument
1321 struct bch_fs *c = as->c; in bch2_btree_set_root()
1323 trace_and_count(c, btree_node_set_root, trans, b); in bch2_btree_set_root()
1325 struct btree *old = btree_node_root(c, b); in bch2_btree_set_root()
1332 bch2_btree_node_lock_write_nofail(trans, path, &old->c); in bch2_btree_set_root()
1334 int ret = bch2_btree_node_lock_write(trans, path, &old->c); in bch2_btree_set_root()
1339 bch2_btree_set_root_inmem(c, b); in bch2_btree_set_root()
1341 btree_update_updated_root(as, b); in bch2_btree_set_root()
1359 struct btree *b, in bch2_insert_fixup_btree_ptr() argument
1363 struct bch_fs *c = as->c; in bch2_insert_fixup_btree_ptr()
1368 BUG_ON(insert->k.type == KEY_TYPE_btree_ptr_v2 && in bch2_insert_fixup_btree_ptr()
1371 if (unlikely(!test_bit(JOURNAL_replay_done, &c->journal.flags))) in bch2_insert_fixup_btree_ptr()
1372 bch2_journal_key_overwritten(c, b->c.btree_id, b->c.level, insert->k.p); in bch2_insert_fixup_btree_ptr()
1375 btree_node_type(b), BCH_VALIDATE_write) ?: in bch2_insert_fixup_btree_ptr()
1376 bch2_bkey_in_btree_node(c, b, bkey_i_to_s_c(insert), BCH_VALIDATE_write)) { in bch2_insert_fixup_btree_ptr()
1381 BUG_ON(as->journal_u64s + jset_u64s(insert->k.u64s) > in bch2_insert_fixup_btree_ptr()
1382 ARRAY_SIZE(as->journal_entries)); in bch2_insert_fixup_btree_ptr()
1384 as->journal_u64s += in bch2_insert_fixup_btree_ptr()
1385 journal_entry_set((void *) &as->journal_entries[as->journal_u64s], in bch2_insert_fixup_btree_ptr()
1387 b->c.btree_id, b->c.level, in bch2_insert_fixup_btree_ptr()
1388 insert, insert->k.u64s); in bch2_insert_fixup_btree_ptr()
1390 while ((k = bch2_btree_node_iter_peek_all(node_iter, b)) && in bch2_insert_fixup_btree_ptr()
1391 bkey_iter_pos_cmp(b, k, &insert->k.p) < 0) in bch2_insert_fixup_btree_ptr()
1392 bch2_btree_node_iter_advance(node_iter, b); in bch2_insert_fixup_btree_ptr()
1394 bch2_btree_bset_insert_key(trans, path, b, node_iter, insert); in bch2_insert_fixup_btree_ptr()
1395 set_btree_node_dirty_acct(c, b); in bch2_insert_fixup_btree_ptr()
1397 old = READ_ONCE(b->flags); in bch2_insert_fixup_btree_ptr()
1404 } while (!try_cmpxchg(&b->flags, &old, new)); in bch2_insert_fixup_btree_ptr()
1413 struct btree *b, in bch2_btree_insert_keys_interior() argument
1420 BUG_ON(btree_node_type(b) != BKEY_TYPE_btree); in bch2_btree_insert_keys_interior()
1422 while ((k = bch2_btree_node_iter_prev_all(&node_iter, b)) && in bch2_btree_insert_keys_interior()
1423 (bkey_cmp_left_packed(b, k, &insert->k.p) >= 0)) in bch2_btree_insert_keys_interior()
1429 if (bpos_gt(insert->k.p, b->key.k.p)) in bch2_btree_insert_keys_interior()
1432 bch2_insert_fixup_btree_ptr(as, trans, path, b, &node_iter, insert); in bch2_btree_insert_keys_interior()
1443 struct btree *b, in __btree_split_node() argument
1453 unsigned u64s, n1_u64s = (b->nr.live_u64s * 3) / 5; in __btree_split_node()
1460 BUG_ON(n[i]->nsets != 1); in __btree_split_node()
1463 out[i] = bsets[i]->start; in __btree_split_node()
1465 SET_BTREE_NODE_SEQ(n[i]->data, BTREE_NODE_SEQ(b->data) + 1); in __btree_split_node()
1470 for_each_btree_node_key(b, k, &iter) { in __btree_split_node()
1474 uk = bkey_unpack_key(b, k); in __btree_split_node()
1476 if (b->c.level && in __btree_split_node()
1478 u64s + k->u64s >= n1_u64s && in __btree_split_node()
1479 bch2_key_deleted_in_journal(trans, b->c.btree_id, b->c.level, uk.p)) in __btree_split_node()
1480 n1_u64s += k->u64s; in __btree_split_node()
1483 u64s += k->u64s; in __btree_split_node()
1489 nr_keys[i].val_u64s += bkeyp_val_u64s(&b->format, k); in __btree_split_node()
1492 btree_set_min(n[0], b->data->min_key); in __btree_split_node()
1495 btree_set_max(n[1], b->data->max_key); in __btree_split_node()
1498 bch2_bkey_format_add_pos(&format[i], n[i]->data->min_key); in __btree_split_node()
1499 bch2_bkey_format_add_pos(&format[i], n[i]->data->max_key); in __btree_split_node()
1501 n[i]->data->format = bch2_bkey_format_done(&format[i]); in __btree_split_node()
1503 unsigned u64s = nr_keys[i].nr_keys * n[i]->data->format.key_u64s + in __btree_split_node()
1505 if (__vstruct_bytes(struct btree_node, u64s) > btree_buf_bytes(b)) in __btree_split_node()
1506 n[i]->data->format = b->format; in __btree_split_node()
1508 btree_node_set_format(n[i], n[i]->data->format); in __btree_split_node()
1512 for_each_btree_node_key(b, k, &iter) { in __btree_split_node()
1517 u64s += k->u64s; in __btree_split_node()
1519 if (bch2_bkey_transform(&n[i]->format, out[i], bkey_packed(k) in __btree_split_node()
1520 ? &b->format: &bch2_bkey_format_current, k)) in __btree_split_node()
1521 out[i]->format = KEY_FORMAT_LOCAL_BTREE; in __btree_split_node()
1523 bch2_bkey_unpack(b, (void *) out[i], k); in __btree_split_node()
1525 out[i]->needs_whiteout = false; in __btree_split_node()
1527 btree_keys_account_key_add(&n[i]->nr, 0, out[i]); in __btree_split_node()
1532 bsets[i]->u64s = cpu_to_le16((u64 *) out[i] - bsets[i]->_data); in __btree_split_node()
1534 BUG_ON(!bsets[i]->u64s); in __btree_split_node()
1536 set_btree_bset_end(n[i], n[i]->set); in __btree_split_node()
1553 * we do the split (and pick the pivot) - the pivot we pick might be between
1560 struct btree *b, in btree_split_insert_keys() argument
1563 struct btree_path *path = trans->paths + path_idx; in btree_split_insert_keys()
1566 bpos_le(bch2_keylist_front(keys)->k.p, b->data->max_key)) { in btree_split_insert_keys()
1569 bch2_btree_node_iter_init(&node_iter, b, &bch2_keylist_front(keys)->k.p); in btree_split_insert_keys()
1571 bch2_btree_insert_keys_interior(as, trans, path, b, node_iter, keys); in btree_split_insert_keys()
1573 BUG_ON(bch2_btree_node_check_topology(trans, b)); in btree_split_insert_keys()
1578 btree_path_idx_t path, struct btree *b, in btree_split() argument
1581 struct bch_fs *c = as->c; in btree_split()
1582 struct btree *parent = btree_node_parent(trans->paths + path, b); in btree_split()
1588 bch2_verify_btree_nr_keys(b); in btree_split()
1589 BUG_ON(!parent && (b != btree_node_root(c, b))); in btree_split()
1590 BUG_ON(parent && !btree_node_intent_locked(trans->paths + path, b->c.level + 1)); in btree_split()
1592 ret = bch2_btree_node_check_topology(trans, b); in btree_split()
1596 bch2_btree_interior_update_will_free_node(as, b); in btree_split()
1598 if (b->nr.live_u64s > BTREE_SPLIT_THRESHOLD(c)) { in btree_split()
1601 trace_and_count(c, btree_node_split, trans, b); in btree_split()
1603 n[0] = n1 = bch2_btree_node_alloc(as, trans, b->c.level); in btree_split()
1604 n[1] = n2 = bch2_btree_node_alloc(as, trans, b->c.level); in btree_split()
1606 __btree_split_node(as, trans, b, n); in btree_split()
1619 six_unlock_write(&n2->c.lock); in btree_split()
1620 six_unlock_write(&n1->c.lock); in btree_split()
1622 path1 = bch2_path_get_unlocked_mut(trans, as->btree_id, n1->c.level, n1->key.k.p); in btree_split()
1623 six_lock_increment(&n1->c.lock, SIX_LOCK_intent); in btree_split()
1624 mark_btree_node_locked(trans, trans->paths + path1, n1->c.level, BTREE_NODE_INTENT_LOCKED); in btree_split()
1625 bch2_btree_path_level_init(trans, trans->paths + path1, n1); in btree_split()
1627 path2 = bch2_path_get_unlocked_mut(trans, as->btree_id, n2->c.level, n2->key.k.p); in btree_split()
1628 six_lock_increment(&n2->c.lock, SIX_LOCK_intent); in btree_split()
1629 mark_btree_node_locked(trans, trans->paths + path2, n2->c.level, BTREE_NODE_INTENT_LOCKED); in btree_split()
1630 bch2_btree_path_level_init(trans, trans->paths + path2, n2); in btree_split()
1637 bch2_keylist_add(&as->parent_keys, &n1->key); in btree_split()
1638 bch2_keylist_add(&as->parent_keys, &n2->key); in btree_split()
1642 n3 = __btree_root_alloc(as, trans, b->c.level + 1); in btree_split()
1645 six_unlock_write(&n3->c.lock); in btree_split()
1647 trans->paths[path2].locks_want++; in btree_split()
1648 BUG_ON(btree_node_locked(trans->paths + path2, n3->c.level)); in btree_split()
1649 six_lock_increment(&n3->c.lock, SIX_LOCK_intent); in btree_split()
1650 mark_btree_node_locked(trans, trans->paths + path2, n3->c.level, BTREE_NODE_INTENT_LOCKED); in btree_split()
1651 bch2_btree_path_level_init(trans, trans->paths + path2, n3); in btree_split()
1653 n3->sib_u64s[0] = U16_MAX; in btree_split()
1654 n3->sib_u64s[1] = U16_MAX; in btree_split()
1656 btree_split_insert_keys(as, trans, path, n3, &as->parent_keys); in btree_split()
1659 trace_and_count(c, btree_node_compact, trans, b); in btree_split()
1661 n1 = bch2_btree_node_alloc_replacement(as, trans, b); in btree_split()
1670 six_unlock_write(&n1->c.lock); in btree_split()
1672 path1 = bch2_path_get_unlocked_mut(trans, as->btree_id, n1->c.level, n1->key.k.p); in btree_split()
1673 six_lock_increment(&n1->c.lock, SIX_LOCK_intent); in btree_split()
1674 mark_btree_node_locked(trans, trans->paths + path1, n1->c.level, BTREE_NODE_INTENT_LOCKED); in btree_split()
1675 bch2_btree_path_level_init(trans, trans->paths + path1, n1); in btree_split()
1678 bch2_keylist_add(&as->parent_keys, &n1->key); in btree_split()
1685 ret = bch2_btree_insert_node(as, trans, path, parent, &as->parent_keys); in btree_split()
1687 ret = bch2_btree_set_root(as, trans, trans->paths + path, n3, false); in btree_split()
1690 ret = bch2_btree_set_root(as, trans, trans->paths + path, n1, false); in btree_split()
1709 * nodes - else another thread could re-acquire a read lock on the old in btree_split()
1713 bch2_btree_node_free_inmem(trans, trans->paths + path, b); in btree_split()
1716 bch2_trans_node_add(trans, trans->paths + path, n3); in btree_split()
1718 bch2_trans_node_add(trans, trans->paths + path2, n2); in btree_split()
1719 bch2_trans_node_add(trans, trans->paths + path1, n1); in btree_split()
1722 six_unlock_intent(&n3->c.lock); in btree_split()
1724 six_unlock_intent(&n2->c.lock); in btree_split()
1725 six_unlock_intent(&n1->c.lock); in btree_split()
1728 __bch2_btree_path_unlock(trans, trans->paths + path2); in btree_split()
1732 __bch2_btree_path_unlock(trans, trans->paths + path1); in btree_split()
1738 bch2_time_stats_update(&c->times[n2 in btree_split()
1753 * bch2_btree_insert_node - insert bkeys into a given btree node
1758 * @b: node to insert keys into
1765 * for leaf nodes -- inserts into interior nodes have to be atomic.
1768 btree_path_idx_t path_idx, struct btree *b, in bch2_btree_insert_node() argument
1771 struct bch_fs *c = as->c; in bch2_btree_insert_node()
1772 struct btree_path *path = trans->paths + path_idx, *linked; in bch2_btree_insert_node()
1774 int old_u64s = le16_to_cpu(btree_bset_last(b)->u64s); in bch2_btree_insert_node()
1775 int old_live_u64s = b->nr.live_u64s; in bch2_btree_insert_node()
1779 lockdep_assert_held(&c->gc_lock); in bch2_btree_insert_node()
1780 BUG_ON(!btree_node_intent_locked(path, b->c.level)); in bch2_btree_insert_node()
1781 BUG_ON(!b->c.level); in bch2_btree_insert_node()
1782 BUG_ON(!as || as->b); in bch2_btree_insert_node()
1785 ret = bch2_btree_node_lock_write(trans, path, &b->c); in bch2_btree_insert_node()
1789 bch2_btree_node_prep_for_write(trans, path, b); in bch2_btree_insert_node()
1791 if (!bch2_btree_node_insert_fits(b, bch2_keylist_u64s(keys))) { in bch2_btree_insert_node()
1792 bch2_btree_node_unlock_write(trans, path, b); in bch2_btree_insert_node()
1796 ret = bch2_btree_node_check_topology(trans, b); in bch2_btree_insert_node()
1798 bch2_btree_node_unlock_write(trans, path, b); in bch2_btree_insert_node()
1802 bch2_btree_insert_keys_interior(as, trans, path, b, in bch2_btree_insert_node()
1803 path->l[b->c.level].iter, keys); in bch2_btree_insert_node()
1805 trans_for_each_path_with_node(trans, b, linked, i) in bch2_btree_insert_node()
1806 bch2_btree_node_iter_peek(&linked->l[b->c.level].iter, b); in bch2_btree_insert_node()
1810 live_u64s_added = (int) b->nr.live_u64s - old_live_u64s; in bch2_btree_insert_node()
1811 u64s_added = (int) le16_to_cpu(btree_bset_last(b)->u64s) - old_u64s; in bch2_btree_insert_node()
1813 if (b->sib_u64s[0] != U16_MAX && live_u64s_added < 0) in bch2_btree_insert_node()
1814 b->sib_u64s[0] = max(0, (int) b->sib_u64s[0] + live_u64s_added); in bch2_btree_insert_node()
1815 if (b->sib_u64s[1] != U16_MAX && live_u64s_added < 0) in bch2_btree_insert_node()
1816 b->sib_u64s[1] = max(0, (int) b->sib_u64s[1] + live_u64s_added); in bch2_btree_insert_node()
1819 bch2_maybe_compact_whiteouts(c, b)) in bch2_btree_insert_node()
1820 bch2_trans_node_reinit_iter(trans, b); in bch2_btree_insert_node()
1822 btree_update_updated_node(as, b); in bch2_btree_insert_node()
1823 bch2_btree_node_unlock_write(trans, path, b); in bch2_btree_insert_node()
1825 BUG_ON(bch2_btree_node_check_topology(trans, b)); in bch2_btree_insert_node()
1832 if (b->c.level >= as->update_level_end) { in bch2_btree_insert_node()
1833 trace_and_count(c, trans_restart_split_race, trans, _THIS_IP_, b); in bch2_btree_insert_node()
1837 return btree_split(as, trans, path_idx, b, keys); in bch2_btree_insert_node()
1845 struct btree *b = path_l(trans->paths + path)->b; in bch2_btree_split_leaf() local
1850 as = bch2_btree_update_start(trans, trans->paths + path, in bch2_btree_split_leaf()
1851 trans->paths[path].level, in bch2_btree_split_leaf()
1856 ret = btree_split(as, trans, path, b, NULL); in bch2_btree_split_leaf()
1864 for (l = trans->paths[path].level + 1; in bch2_btree_split_leaf()
1865 btree_node_intent_locked(&trans->paths[path], l) && !ret; in bch2_btree_split_leaf()
1875 struct bch_fs *c = as->c; in __btree_increase_depth()
1876 struct btree_path *path = trans->paths + path_idx; in __btree_increase_depth()
1877 struct btree *n, *b = bch2_btree_id_root(c, path->btree_id)->b; in __btree_increase_depth() local
1879 BUG_ON(!btree_node_locked(path, b->c.level)); in __btree_increase_depth()
1881 n = __btree_root_alloc(as, trans, b->c.level + 1); in __btree_increase_depth()
1884 six_unlock_write(&n->c.lock); in __btree_increase_depth()
1886 path->locks_want++; in __btree_increase_depth()
1887 BUG_ON(btree_node_locked(path, n->c.level)); in __btree_increase_depth()
1888 six_lock_increment(&n->c.lock, SIX_LOCK_intent); in __btree_increase_depth()
1889 mark_btree_node_locked(trans, path, n->c.level, BTREE_NODE_INTENT_LOCKED); in __btree_increase_depth()
1892 n->sib_u64s[0] = U16_MAX; in __btree_increase_depth()
1893 n->sib_u64s[1] = U16_MAX; in __btree_increase_depth()
1895 bch2_keylist_add(&as->parent_keys, &b->key); in __btree_increase_depth()
1896 btree_split_insert_keys(as, trans, path_idx, n, &as->parent_keys); in __btree_increase_depth()
1904 six_unlock_intent(&n->c.lock); in __btree_increase_depth()
1906 mutex_lock(&c->btree_cache.lock); in __btree_increase_depth()
1907 list_add_tail(&b->list, &c->btree_cache.live[btree_node_pinned(b)].list); in __btree_increase_depth()
1908 mutex_unlock(&c->btree_cache.lock); in __btree_increase_depth()
1915 struct bch_fs *c = trans->c; in bch2_btree_increase_depth()
1916 struct btree *b = bch2_btree_id_root(c, trans->paths[path].btree_id)->b; in bch2_btree_increase_depth() local
1918 if (btree_node_fake(b)) in bch2_btree_increase_depth()
1922 bch2_btree_update_start(trans, trans->paths + path, b->c.level, true, flags); in bch2_btree_increase_depth()
1937 struct bch_fs *c = trans->c; in __bch2_foreground_maybe_merge()
1942 struct btree *b, *m, *n, *prev, *next, *parent; in __bch2_foreground_maybe_merge() local
1945 enum btree_id btree = trans->paths[path].btree_id; in __bch2_foreground_maybe_merge()
1952 BUG_ON(!trans->paths[path].should_be_locked); in __bch2_foreground_maybe_merge()
1953 BUG_ON(!btree_node_locked(&trans->paths[path], level)); in __bch2_foreground_maybe_merge()
1957 * merges and leaving tons of merges for us to do - we really don't need in __bch2_foreground_maybe_merge()
1971 b = trans->paths[path].l[level].b; in __bch2_foreground_maybe_merge()
1973 if ((sib == btree_prev_sib && bpos_eq(b->data->min_key, POS_MIN)) || in __bch2_foreground_maybe_merge()
1974 (sib == btree_next_sib && bpos_eq(b->data->max_key, SPOS_MAX))) { in __bch2_foreground_maybe_merge()
1975 b->sib_u64s[sib] = U16_MAX; in __bch2_foreground_maybe_merge()
1980 ? bpos_predecessor(b->data->min_key) in __bch2_foreground_maybe_merge()
1981 : bpos_successor(b->data->max_key); in __bch2_foreground_maybe_merge()
1989 btree_path_set_should_be_locked(trans, trans->paths + sib_path); in __bch2_foreground_maybe_merge()
1991 m = trans->paths[sib_path].l[level].b; in __bch2_foreground_maybe_merge()
1993 if (btree_node_parent(trans->paths + path, b) != in __bch2_foreground_maybe_merge()
1994 btree_node_parent(trans->paths + sib_path, m)) { in __bch2_foreground_maybe_merge()
1995 b->sib_u64s[sib] = U16_MAX; in __bch2_foreground_maybe_merge()
2001 next = b; in __bch2_foreground_maybe_merge()
2003 prev = b; in __bch2_foreground_maybe_merge()
2007 if (!bpos_eq(bpos_successor(prev->data->max_key), next->data->min_key)) { in __bch2_foreground_maybe_merge()
2010 bch2_bpos_to_text(&buf1, prev->data->max_key); in __bch2_foreground_maybe_merge()
2011 bch2_bpos_to_text(&buf2, next->data->min_key); in __bch2_foreground_maybe_merge()
2024 bch2_bkey_format_add_pos(&new_s, prev->data->min_key); in __bch2_foreground_maybe_merge()
2027 bch2_bkey_format_add_pos(&new_s, next->data->max_key); in __bch2_foreground_maybe_merge()
2030 sib_u64s = btree_node_u64s_with_format(b->nr, &b->format, &new_f) + in __bch2_foreground_maybe_merge()
2031 btree_node_u64s_with_format(m->nr, &m->format, &new_f); in __bch2_foreground_maybe_merge()
2034 sib_u64s -= BTREE_FOREGROUND_MERGE_HYSTERESIS(c); in __bch2_foreground_maybe_merge()
2040 sib_u64s = min(sib_u64s, (size_t) U16_MAX - 1); in __bch2_foreground_maybe_merge()
2041 b->sib_u64s[sib] = sib_u64s; in __bch2_foreground_maybe_merge()
2043 if (b->sib_u64s[sib] > c->btree_foreground_merge_threshold) in __bch2_foreground_maybe_merge()
2046 parent = btree_node_parent(trans->paths + path, b); in __bch2_foreground_maybe_merge()
2047 as = bch2_btree_update_start(trans, trans->paths + path, level, false, in __bch2_foreground_maybe_merge()
2053 trace_and_count(c, btree_node_merge, trans, b); in __bch2_foreground_maybe_merge()
2055 bch2_btree_interior_update_will_free_node(as, b); in __bch2_foreground_maybe_merge()
2058 n = bch2_btree_node_alloc(as, trans, b->c.level); in __bch2_foreground_maybe_merge()
2060 SET_BTREE_NODE_SEQ(n->data, in __bch2_foreground_maybe_merge()
2061 max(BTREE_NODE_SEQ(b->data), in __bch2_foreground_maybe_merge()
2062 BTREE_NODE_SEQ(m->data)) + 1); in __bch2_foreground_maybe_merge()
2064 btree_set_min(n, prev->data->min_key); in __bch2_foreground_maybe_merge()
2065 btree_set_max(n, next->data->max_key); in __bch2_foreground_maybe_merge()
2067 n->data->format = new_f; in __bch2_foreground_maybe_merge()
2075 six_unlock_write(&n->c.lock); in __bch2_foreground_maybe_merge()
2077 new_path = bch2_path_get_unlocked_mut(trans, btree, n->c.level, n->key.k.p); in __bch2_foreground_maybe_merge()
2078 six_lock_increment(&n->c.lock, SIX_LOCK_intent); in __bch2_foreground_maybe_merge()
2079 mark_btree_node_locked(trans, trans->paths + new_path, n->c.level, BTREE_NODE_INTENT_LOCKED); in __bch2_foreground_maybe_merge()
2080 bch2_btree_path_level_init(trans, trans->paths + new_path, n); in __bch2_foreground_maybe_merge()
2083 delete.k.p = prev->key.k.p; in __bch2_foreground_maybe_merge()
2084 bch2_keylist_add(&as->parent_keys, &delete); in __bch2_foreground_maybe_merge()
2085 bch2_keylist_add(&as->parent_keys, &n->key); in __bch2_foreground_maybe_merge()
2089 ret = bch2_btree_insert_node(as, trans, path, parent, &as->parent_keys); in __bch2_foreground_maybe_merge()
2098 bch2_btree_node_free_inmem(trans, trans->paths + path, b); in __bch2_foreground_maybe_merge()
2099 bch2_btree_node_free_inmem(trans, trans->paths + sib_path, m); in __bch2_foreground_maybe_merge()
2101 bch2_trans_node_add(trans, trans->paths + path, n); in __bch2_foreground_maybe_merge()
2105 six_unlock_intent(&n->c.lock); in __bch2_foreground_maybe_merge()
2109 bch2_time_stats_update(&c->times[BCH_TIME_btree_node_merge], start_time); in __bch2_foreground_maybe_merge()
2116 if (ret == -BCH_ERR_journal_reclaim_would_deadlock) in __bch2_foreground_maybe_merge()
2129 struct btree *b, in bch2_btree_node_rewrite() argument
2132 struct bch_fs *c = trans->c; in bch2_btree_node_rewrite()
2141 parent = btree_node_parent(path, b); in bch2_btree_node_rewrite()
2142 as = bch2_btree_update_start(trans, path, b->c.level, false, flags); in bch2_btree_node_rewrite()
2147 bch2_btree_interior_update_will_free_node(as, b); in bch2_btree_node_rewrite()
2149 n = bch2_btree_node_alloc_replacement(as, trans, b); in bch2_btree_node_rewrite()
2153 six_unlock_write(&n->c.lock); in bch2_btree_node_rewrite()
2155 new_path = bch2_path_get_unlocked_mut(trans, iter->btree_id, n->c.level, n->key.k.p); in bch2_btree_node_rewrite()
2156 six_lock_increment(&n->c.lock, SIX_LOCK_intent); in bch2_btree_node_rewrite()
2157 mark_btree_node_locked(trans, trans->paths + new_path, n->c.level, BTREE_NODE_INTENT_LOCKED); in bch2_btree_node_rewrite()
2158 bch2_btree_path_level_init(trans, trans->paths + new_path, n); in bch2_btree_node_rewrite()
2160 trace_and_count(c, btree_node_rewrite, trans, b); in bch2_btree_node_rewrite()
2163 bch2_keylist_add(&as->parent_keys, &n->key); in bch2_btree_node_rewrite()
2164 ret = bch2_btree_insert_node(as, trans, iter->path, parent, &as->parent_keys); in bch2_btree_node_rewrite()
2175 bch2_btree_node_free_inmem(trans, btree_iter_path(trans, iter), b); in bch2_btree_node_rewrite()
2177 bch2_trans_node_add(trans, trans->paths + iter->path, n); in bch2_btree_node_rewrite()
2178 six_unlock_intent(&n->c.lock); in bch2_btree_node_rewrite()
2205 struct bch_fs *c = trans->c; in async_btree_node_rewrite_trans()
2207 struct btree *b; in async_btree_node_rewrite_trans() local
2210 bch2_trans_node_iter_init(trans, &iter, a->btree_id, a->pos, in async_btree_node_rewrite_trans()
2211 BTREE_MAX_DEPTH, a->level, 0); in async_btree_node_rewrite_trans()
2212 b = bch2_btree_iter_peek_node(&iter); in async_btree_node_rewrite_trans()
2213 ret = PTR_ERR_OR_ZERO(b); in async_btree_node_rewrite_trans()
2217 if (!b || b->data->keys.seq != a->seq) { in async_btree_node_rewrite_trans()
2220 if (b) in async_btree_node_rewrite_trans()
2221 bch2_bkey_val_to_text(&buf, c, bkey_i_to_s_c(&b->key)); in async_btree_node_rewrite_trans()
2225 __func__, a->seq, buf.buf); in async_btree_node_rewrite_trans()
2230 ret = bch2_btree_node_rewrite(trans, &iter, b, 0); in async_btree_node_rewrite_trans()
2241 struct bch_fs *c = a->c; in async_btree_node_rewrite_work()
2249 void bch2_btree_node_rewrite_async(struct bch_fs *c, struct btree *b) in bch2_btree_node_rewrite_async() argument
2260 a->c = c; in bch2_btree_node_rewrite_async()
2261 a->btree_id = b->c.btree_id; in bch2_btree_node_rewrite_async()
2262 a->level = b->c.level; in bch2_btree_node_rewrite_async()
2263 a->pos = b->key.k.p; in bch2_btree_node_rewrite_async()
2264 a->seq = b->data->keys.seq; in bch2_btree_node_rewrite_async()
2265 INIT_WORK(&a->work, async_btree_node_rewrite_work); in bch2_btree_node_rewrite_async()
2267 if (unlikely(!test_bit(BCH_FS_may_go_rw, &c->flags))) { in bch2_btree_node_rewrite_async()
2268 mutex_lock(&c->pending_node_rewrites_lock); in bch2_btree_node_rewrite_async()
2269 list_add(&a->list, &c->pending_node_rewrites); in bch2_btree_node_rewrite_async()
2270 mutex_unlock(&c->pending_node_rewrites_lock); in bch2_btree_node_rewrite_async()
2275 if (test_bit(BCH_FS_started, &c->flags)) { in bch2_btree_node_rewrite_async()
2276 bch_err(c, "%s: error getting c->writes ref", __func__); in bch2_btree_node_rewrite_async()
2282 bch_err_msg(c, ret, "going read-write"); in bch2_btree_node_rewrite_async()
2291 queue_work(c->btree_node_rewrite_worker, &a->work); in bch2_btree_node_rewrite_async()
2298 mutex_lock(&c->pending_node_rewrites_lock); in bch2_do_pending_node_rewrites()
2299 list_for_each_entry_safe(a, n, &c->pending_node_rewrites, list) { in bch2_do_pending_node_rewrites()
2300 list_del(&a->list); in bch2_do_pending_node_rewrites()
2303 queue_work(c->btree_node_rewrite_worker, &a->work); in bch2_do_pending_node_rewrites()
2305 mutex_unlock(&c->pending_node_rewrites_lock); in bch2_do_pending_node_rewrites()
2312 mutex_lock(&c->pending_node_rewrites_lock); in bch2_free_pending_node_rewrites()
2313 list_for_each_entry_safe(a, n, &c->pending_node_rewrites, list) { in bch2_free_pending_node_rewrites()
2314 list_del(&a->list); in bch2_free_pending_node_rewrites()
2318 mutex_unlock(&c->pending_node_rewrites_lock); in bch2_free_pending_node_rewrites()
2323 struct btree *b, struct btree *new_hash, in __bch2_btree_node_update_key() argument
2328 struct bch_fs *c = trans->c; in __bch2_btree_node_update_key()
2334 ret = bch2_key_trigger_old(trans, b->c.btree_id, b->c.level + 1, in __bch2_btree_node_update_key()
2335 bkey_i_to_s_c(&b->key), in __bch2_btree_node_update_key()
2337 bch2_key_trigger_new(trans, b->c.btree_id, b->c.level + 1, in __bch2_btree_node_update_key()
2345 bkey_copy(&new_hash->key, new_key); in __bch2_btree_node_update_key()
2346 ret = bch2_btree_node_hash_insert(&c->btree_cache, in __bch2_btree_node_update_key()
2347 new_hash, b->c.level, b->c.btree_id); in __bch2_btree_node_update_key()
2351 parent = btree_node_parent(btree_iter_path(trans, iter), b); in __bch2_btree_node_update_key()
2360 BUG_ON(path2->level != b->c.level); in __bch2_btree_node_update_key()
2361 BUG_ON(!bpos_eq(path2->pos, new_key->k.p)); in __bch2_btree_node_update_key()
2365 trans->paths_sorted = false; in __bch2_btree_node_update_key()
2372 BUG_ON(btree_node_root(c, b) != b); in __bch2_btree_node_update_key()
2375 jset_u64s(new_key->k.u64s)); in __bch2_btree_node_update_key()
2382 b->c.btree_id, b->c.level, in __bch2_btree_node_update_key()
2383 new_key, new_key->k.u64s); in __bch2_btree_node_update_key()
2390 bch2_btree_node_lock_write_nofail(trans, btree_iter_path(trans, iter), &b->c); in __bch2_btree_node_update_key()
2393 mutex_lock(&c->btree_cache.lock); in __bch2_btree_node_update_key()
2394 bch2_btree_node_hash_remove(&c->btree_cache, new_hash); in __bch2_btree_node_update_key()
2395 bch2_btree_node_hash_remove(&c->btree_cache, b); in __bch2_btree_node_update_key()
2397 bkey_copy(&b->key, new_key); in __bch2_btree_node_update_key()
2398 ret = __bch2_btree_node_hash_insert(&c->btree_cache, b); in __bch2_btree_node_update_key()
2400 mutex_unlock(&c->btree_cache.lock); in __bch2_btree_node_update_key()
2402 bkey_copy(&b->key, new_key); in __bch2_btree_node_update_key()
2405 bch2_btree_node_unlock_write(trans, btree_iter_path(trans, iter), b); in __bch2_btree_node_update_key()
2411 mutex_lock(&c->btree_cache.lock); in __bch2_btree_node_update_key()
2412 bch2_btree_node_hash_remove(&c->btree_cache, b); in __bch2_btree_node_update_key()
2413 mutex_unlock(&c->btree_cache.lock); in __bch2_btree_node_update_key()
2419 struct btree *b, struct bkey_i *new_key, in bch2_btree_node_update_key() argument
2422 struct bch_fs *c = trans->c; in bch2_btree_node_update_key()
2428 ret = bch2_btree_path_upgrade(trans, path, b->c.level + 1); in bch2_btree_node_update_key()
2435 * check btree_ptr_hash_val() after @b is locked by in bch2_btree_node_update_key()
2438 if (btree_ptr_hash_val(new_key) != b->hash_val) { in bch2_btree_node_update_key()
2452 path->intent_ref++; in bch2_btree_node_update_key()
2453 ret = __bch2_btree_node_update_key(trans, iter, b, new_hash, new_key, in bch2_btree_node_update_key()
2455 --path->intent_ref; in bch2_btree_node_update_key()
2466 struct btree *b, struct bkey_i *new_key, in bch2_btree_node_update_key_get_iter() argument
2472 bch2_trans_node_iter_init(trans, &iter, b->c.btree_id, b->key.k.p, in bch2_btree_node_update_key_get_iter()
2473 BTREE_MAX_DEPTH, b->c.level, in bch2_btree_node_update_key_get_iter()
2480 if (btree_iter_path(trans, &iter)->l[b->c.level].b != b) { in bch2_btree_node_update_key_get_iter()
2482 BUG_ON(!btree_node_dying(b)); in bch2_btree_node_update_key_get_iter()
2486 BUG_ON(!btree_node_hashed(b)); in bch2_btree_node_update_key_get_iter()
2489 !bch2_bkey_has_device(bkey_i_to_s(&b->key), ptr->dev)); in bch2_btree_node_update_key_get_iter()
2491 ret = bch2_btree_node_update_key(trans, &iter, b, new_key, in bch2_btree_node_update_key_get_iter()
2504 void bch2_btree_set_root_for_read(struct bch_fs *c, struct btree *b) in bch2_btree_set_root_for_read() argument
2506 BUG_ON(btree_node_root(c, b)); in bch2_btree_set_root_for_read()
2508 bch2_btree_set_root_inmem(c, b); in bch2_btree_set_root_for_read()
2513 struct bch_fs *c = trans->c; in bch2_btree_root_alloc_fake_trans()
2515 struct btree *b; in bch2_btree_root_alloc_fake_trans() local
2525 b = bch2_btree_node_mem_alloc(trans, false); in bch2_btree_root_alloc_fake_trans()
2528 ret = PTR_ERR_OR_ZERO(b); in bch2_btree_root_alloc_fake_trans()
2532 set_btree_node_fake(b); in bch2_btree_root_alloc_fake_trans()
2533 set_btree_node_need_rewrite(b); in bch2_btree_root_alloc_fake_trans()
2534 b->c.level = level; in bch2_btree_root_alloc_fake_trans()
2535 b->c.btree_id = id; in bch2_btree_root_alloc_fake_trans()
2537 bkey_btree_ptr_init(&b->key); in bch2_btree_root_alloc_fake_trans()
2538 b->key.k.p = SPOS_MAX; in bch2_btree_root_alloc_fake_trans()
2539 *((u64 *) bkey_i_to_btree_ptr(&b->key)->v.start) = U64_MAX - id; in bch2_btree_root_alloc_fake_trans()
2541 bch2_bset_init_first(b, &b->data->keys); in bch2_btree_root_alloc_fake_trans()
2542 bch2_btree_build_aux_trees(b); in bch2_btree_root_alloc_fake_trans()
2544 b->data->flags = 0; in bch2_btree_root_alloc_fake_trans()
2545 btree_set_min(b, POS_MIN); in bch2_btree_root_alloc_fake_trans()
2546 btree_set_max(b, SPOS_MAX); in bch2_btree_root_alloc_fake_trans()
2547 b->data->format = bch2_btree_calc_format(b); in bch2_btree_root_alloc_fake_trans()
2548 btree_node_set_format(b, b->data->format); in bch2_btree_root_alloc_fake_trans()
2550 ret = bch2_btree_node_hash_insert(&c->btree_cache, b, in bch2_btree_root_alloc_fake_trans()
2551 b->c.level, b->c.btree_id); in bch2_btree_root_alloc_fake_trans()
2554 bch2_btree_set_root_inmem(c, b); in bch2_btree_root_alloc_fake_trans()
2556 six_unlock_write(&b->c.lock); in bch2_btree_root_alloc_fake_trans()
2557 six_unlock_intent(&b->c.lock); in bch2_btree_root_alloc_fake_trans()
2568 prt_printf(out, "%ps: ", (void *) as->ip_started); in bch2_btree_update_to_text()
2569 bch2_trans_commit_flags_to_text(out, as->flags); in bch2_btree_update_to_text()
2571 prt_printf(out, " btree=%s l=%u-%u mode=%s nodes_written=%u cl.remaining=%u journal_seq=%llu\n", in bch2_btree_update_to_text()
2572 bch2_btree_id_str(as->btree_id), in bch2_btree_update_to_text()
2573 as->update_level_start, in bch2_btree_update_to_text()
2574 as->update_level_end, in bch2_btree_update_to_text()
2575 bch2_btree_update_modes[as->mode], in bch2_btree_update_to_text()
2576 as->nodes_written, in bch2_btree_update_to_text()
2577 closure_nr_remaining(&as->cl), in bch2_btree_update_to_text()
2578 as->journal.seq); in bch2_btree_update_to_text()
2585 mutex_lock(&c->btree_interior_update_lock); in bch2_btree_updates_to_text()
2586 list_for_each_entry(as, &c->btree_interior_update_list, list) in bch2_btree_updates_to_text()
2588 mutex_unlock(&c->btree_interior_update_lock); in bch2_btree_updates_to_text()
2595 mutex_lock(&c->btree_interior_update_lock); in bch2_btree_interior_updates_pending()
2596 ret = !list_empty(&c->btree_interior_update_list); in bch2_btree_interior_updates_pending()
2597 mutex_unlock(&c->btree_interior_update_lock); in bch2_btree_interior_updates_pending()
2607 closure_wait_event(&c->btree_interior_update_wait, in bch2_btree_interior_updates_flush()
2614 struct btree_root *r = bch2_btree_id_root(c, entry->btree_id); in bch2_journal_entry_to_btree_root()
2616 mutex_lock(&c->btree_root_lock); in bch2_journal_entry_to_btree_root()
2618 r->level = entry->level; in bch2_journal_entry_to_btree_root()
2619 r->alive = true; in bch2_journal_entry_to_btree_root()
2620 bkey_copy(&r->key, (struct bkey_i *) entry->start); in bch2_journal_entry_to_btree_root()
2622 mutex_unlock(&c->btree_root_lock); in bch2_journal_entry_to_btree_root()
2632 mutex_lock(&c->btree_root_lock); in bch2_btree_roots_to_journal_entries()
2637 if (r->alive && !test_bit(i, &skip)) { in bch2_btree_roots_to_journal_entries()
2639 i, r->level, &r->key, r->key.k.u64s); in bch2_btree_roots_to_journal_entries()
2644 mutex_unlock(&c->btree_root_lock); in bch2_btree_roots_to_journal_entries()
2654 bch2_bkey_val_to_text(out, c, bkey_i_to_s_c(&a->k)); in bch2_btree_alloc_to_text()
2659 open_bucket_for_each(c, &a->ob, ob, i) in bch2_btree_alloc_to_text()
2667 for (unsigned i = 0; i < c->btree_reserve_cache_nr; i++) in bch2_btree_reserve_cache_to_text()
2668 bch2_btree_alloc_to_text(out, c, &c->btree_reserve_cache[i]); in bch2_btree_reserve_cache_to_text()
2673 if (c->btree_node_rewrite_worker) in bch2_fs_btree_interior_update_exit()
2674 destroy_workqueue(c->btree_node_rewrite_worker); in bch2_fs_btree_interior_update_exit()
2675 if (c->btree_interior_update_worker) in bch2_fs_btree_interior_update_exit()
2676 destroy_workqueue(c->btree_interior_update_worker); in bch2_fs_btree_interior_update_exit()
2677 mempool_exit(&c->btree_interior_update_pool); in bch2_fs_btree_interior_update_exit()
2682 mutex_init(&c->btree_reserve_cache_lock); in bch2_fs_btree_interior_update_init_early()
2683 INIT_LIST_HEAD(&c->btree_interior_update_list); in bch2_fs_btree_interior_update_init_early()
2684 INIT_LIST_HEAD(&c->btree_interior_updates_unwritten); in bch2_fs_btree_interior_update_init_early()
2685 mutex_init(&c->btree_interior_update_lock); in bch2_fs_btree_interior_update_init_early()
2686 INIT_WORK(&c->btree_interior_update_work, btree_interior_update_work); in bch2_fs_btree_interior_update_init_early()
2688 INIT_LIST_HEAD(&c->pending_node_rewrites); in bch2_fs_btree_interior_update_init_early()
2689 mutex_init(&c->pending_node_rewrites_lock); in bch2_fs_btree_interior_update_init_early()
2694 c->btree_interior_update_worker = in bch2_fs_btree_interior_update_init()
2696 if (!c->btree_interior_update_worker) in bch2_fs_btree_interior_update_init()
2697 return -BCH_ERR_ENOMEM_btree_interior_update_worker_init; in bch2_fs_btree_interior_update_init()
2699 c->btree_node_rewrite_worker = in bch2_fs_btree_interior_update_init()
2701 if (!c->btree_node_rewrite_worker) in bch2_fs_btree_interior_update_init()
2702 return -BCH_ERR_ENOMEM_btree_interior_update_worker_init; in bch2_fs_btree_interior_update_init()
2704 if (mempool_init_kmalloc_pool(&c->btree_interior_update_pool, 1, in bch2_fs_btree_interior_update_init()
2706 return -BCH_ERR_ENOMEM_btree_interior_update_pool_init; in bch2_fs_btree_interior_update_init()