1 // SPDX-License-Identifier: GPL-2.0 2 3 #include "bcachefs.h" 4 #include "alloc_foreground.h" 5 #include "btree_gc.h" 6 #include "btree_io.h" 7 #include "btree_iter.h" 8 #include "btree_journal_iter.h" 9 #include "btree_key_cache.h" 10 #include "btree_update_interior.h" 11 #include "btree_write_buffer.h" 12 #include "buckets.h" 13 #include "disk_accounting.h" 14 #include "errcode.h" 15 #include "error.h" 16 #include "journal.h" 17 #include "journal_io.h" 18 #include "journal_reclaim.h" 19 #include "replicas.h" 20 #include "snapshot.h" 21 22 #include <linux/prefetch.h> 23 24 static const char * const trans_commit_flags_strs[] = { 25 #define x(n, ...) #n, 26 BCH_TRANS_COMMIT_FLAGS() 27 #undef x 28 NULL 29 }; 30 31 void bch2_trans_commit_flags_to_text(struct printbuf *out, enum bch_trans_commit_flags flags) 32 { 33 enum bch_watermark watermark = flags & BCH_WATERMARK_MASK; 34 35 prt_printf(out, "watermark=%s", bch2_watermarks[watermark]); 36 37 flags >>= BCH_WATERMARK_BITS; 38 if (flags) { 39 prt_char(out, ' '); 40 bch2_prt_bitflags(out, trans_commit_flags_strs, flags); 41 } 42 } 43 44 static void verify_update_old_key(struct btree_trans *trans, struct btree_insert_entry *i) 45 { 46 #ifdef CONFIG_BCACHEFS_DEBUG 47 struct bch_fs *c = trans->c; 48 struct bkey u; 49 struct bkey_s_c k = bch2_btree_path_peek_slot_exact(trans->paths + i->path, &u); 50 51 if (unlikely(trans->journal_replay_not_finished)) { 52 struct bkey_i *j_k = 53 bch2_journal_keys_peek_slot(c, i->btree_id, i->level, i->k->k.p); 54 55 if (j_k) 56 k = bkey_i_to_s_c(j_k); 57 } 58 59 u = *k.k; 60 u.needs_whiteout = i->old_k.needs_whiteout; 61 62 BUG_ON(memcmp(&i->old_k, &u, sizeof(struct bkey))); 63 BUG_ON(i->old_v != k.v); 64 #endif 65 } 66 67 static inline struct btree_path_level *insert_l(struct btree_trans *trans, struct btree_insert_entry *i) 68 { 69 return (trans->paths + i->path)->l + i->level; 70 } 71 72 static inline bool same_leaf_as_prev(struct btree_trans *trans, 73 struct btree_insert_entry *i) 74 { 75 return i != trans->updates && 76 insert_l(trans, &i[0])->b == insert_l(trans, &i[-1])->b; 77 } 78 79 static inline bool same_leaf_as_next(struct btree_trans *trans, 80 struct btree_insert_entry *i) 81 { 82 return i + 1 < trans->updates + trans->nr_updates && 83 insert_l(trans, &i[0])->b == insert_l(trans, &i[1])->b; 84 } 85 86 inline void bch2_btree_node_prep_for_write(struct btree_trans *trans, 87 struct btree_path *path, 88 struct btree *b) 89 { 90 struct bch_fs *c = trans->c; 91 92 if (unlikely(btree_node_just_written(b)) && 93 bch2_btree_post_write_cleanup(c, b)) 94 bch2_trans_node_reinit_iter(trans, b); 95 96 /* 97 * If the last bset has been written, or if it's gotten too big - start 98 * a new bset to insert into: 99 */ 100 if (want_new_bset(c, b)) 101 bch2_btree_init_next(trans, b); 102 } 103 104 static noinline int trans_lock_write_fail(struct btree_trans *trans, struct btree_insert_entry *i) 105 { 106 while (--i >= trans->updates) { 107 if (same_leaf_as_prev(trans, i)) 108 continue; 109 110 bch2_btree_node_unlock_write(trans, trans->paths + i->path, insert_l(trans, i)->b); 111 } 112 113 trace_and_count(trans->c, trans_restart_would_deadlock_write, trans); 114 return btree_trans_restart(trans, BCH_ERR_transaction_restart_would_deadlock_write); 115 } 116 117 static inline int bch2_trans_lock_write(struct btree_trans *trans) 118 { 119 EBUG_ON(trans->write_locked); 120 121 trans_for_each_update(trans, i) { 122 if (same_leaf_as_prev(trans, i)) 123 continue; 124 125 if (bch2_btree_node_lock_write(trans, trans->paths + i->path, &insert_l(trans, i)->b->c)) 126 return trans_lock_write_fail(trans, i); 127 128 if (!i->cached) 129 bch2_btree_node_prep_for_write(trans, trans->paths + i->path, insert_l(trans, i)->b); 130 } 131 132 trans->write_locked = true; 133 return 0; 134 } 135 136 static inline void bch2_trans_unlock_updates_write(struct btree_trans *trans) 137 { 138 if (likely(trans->write_locked)) { 139 trans_for_each_update(trans, i) 140 if (btree_node_locked_type(trans->paths + i->path, i->level) == 141 BTREE_NODE_WRITE_LOCKED) 142 bch2_btree_node_unlock_write_inlined(trans, 143 trans->paths + i->path, insert_l(trans, i)->b); 144 trans->write_locked = false; 145 } 146 } 147 148 /* Inserting into a given leaf node (last stage of insert): */ 149 150 /* Handle overwrites and do insert, for non extents: */ 151 bool bch2_btree_bset_insert_key(struct btree_trans *trans, 152 struct btree_path *path, 153 struct btree *b, 154 struct btree_node_iter *node_iter, 155 struct bkey_i *insert) 156 { 157 struct bkey_packed *k; 158 unsigned clobber_u64s = 0, new_u64s = 0; 159 160 EBUG_ON(btree_node_just_written(b)); 161 EBUG_ON(bset_written(b, btree_bset_last(b))); 162 EBUG_ON(bkey_deleted(&insert->k) && bkey_val_u64s(&insert->k)); 163 EBUG_ON(bpos_lt(insert->k.p, b->data->min_key)); 164 EBUG_ON(bpos_gt(insert->k.p, b->data->max_key)); 165 EBUG_ON(insert->k.u64s > bch2_btree_keys_u64s_remaining(b)); 166 EBUG_ON(!b->c.level && !bpos_eq(insert->k.p, path->pos)); 167 kmsan_check_memory(insert, bkey_bytes(&insert->k)); 168 169 k = bch2_btree_node_iter_peek_all(node_iter, b); 170 if (k && bkey_cmp_left_packed(b, k, &insert->k.p)) 171 k = NULL; 172 173 /* @k is the key being overwritten/deleted, if any: */ 174 EBUG_ON(k && bkey_deleted(k)); 175 176 /* Deleting, but not found? nothing to do: */ 177 if (bkey_deleted(&insert->k) && !k) 178 return false; 179 180 if (bkey_deleted(&insert->k)) { 181 /* Deleting: */ 182 btree_account_key_drop(b, k); 183 k->type = KEY_TYPE_deleted; 184 185 if (k->needs_whiteout) 186 push_whiteout(b, insert->k.p); 187 k->needs_whiteout = false; 188 189 if (k >= btree_bset_last(b)->start) { 190 clobber_u64s = k->u64s; 191 bch2_bset_delete(b, k, clobber_u64s); 192 goto fix_iter; 193 } else { 194 bch2_btree_path_fix_key_modified(trans, b, k); 195 } 196 197 return true; 198 } 199 200 if (k) { 201 /* Overwriting: */ 202 btree_account_key_drop(b, k); 203 k->type = KEY_TYPE_deleted; 204 205 insert->k.needs_whiteout = k->needs_whiteout; 206 k->needs_whiteout = false; 207 208 if (k >= btree_bset_last(b)->start) { 209 clobber_u64s = k->u64s; 210 goto overwrite; 211 } else { 212 bch2_btree_path_fix_key_modified(trans, b, k); 213 } 214 } 215 216 k = bch2_btree_node_iter_bset_pos(node_iter, b, bset_tree_last(b)); 217 overwrite: 218 bch2_bset_insert(b, k, insert, clobber_u64s); 219 new_u64s = k->u64s; 220 fix_iter: 221 if (clobber_u64s != new_u64s) 222 bch2_btree_node_iter_fix(trans, path, b, node_iter, k, 223 clobber_u64s, new_u64s); 224 return true; 225 } 226 227 static int __btree_node_flush(struct journal *j, struct journal_entry_pin *pin, 228 unsigned i, u64 seq) 229 { 230 struct bch_fs *c = container_of(j, struct bch_fs, journal); 231 struct btree_write *w = container_of(pin, struct btree_write, journal); 232 struct btree *b = container_of(w, struct btree, writes[i]); 233 struct btree_trans *trans = bch2_trans_get(c); 234 unsigned long old, new; 235 unsigned idx = w - b->writes; 236 237 btree_node_lock_nopath_nofail(trans, &b->c, SIX_LOCK_read); 238 239 old = READ_ONCE(b->flags); 240 do { 241 new = old; 242 243 if (!(old & (1 << BTREE_NODE_dirty)) || 244 !!(old & (1 << BTREE_NODE_write_idx)) != idx || 245 w->journal.seq != seq) 246 break; 247 248 new &= ~BTREE_WRITE_TYPE_MASK; 249 new |= BTREE_WRITE_journal_reclaim; 250 new |= 1 << BTREE_NODE_need_write; 251 } while (!try_cmpxchg(&b->flags, &old, new)); 252 253 btree_node_write_if_need(trans, b, SIX_LOCK_read); 254 six_unlock_read(&b->c.lock); 255 256 bch2_trans_put(trans); 257 return 0; 258 } 259 260 int bch2_btree_node_flush0(struct journal *j, struct journal_entry_pin *pin, u64 seq) 261 { 262 return __btree_node_flush(j, pin, 0, seq); 263 } 264 265 int bch2_btree_node_flush1(struct journal *j, struct journal_entry_pin *pin, u64 seq) 266 { 267 return __btree_node_flush(j, pin, 1, seq); 268 } 269 270 inline void bch2_btree_add_journal_pin(struct bch_fs *c, 271 struct btree *b, u64 seq) 272 { 273 struct btree_write *w = btree_current_write(b); 274 275 bch2_journal_pin_add(&c->journal, seq, &w->journal, 276 btree_node_write_idx(b) == 0 277 ? bch2_btree_node_flush0 278 : bch2_btree_node_flush1); 279 } 280 281 /** 282 * bch2_btree_insert_key_leaf() - insert a key one key into a leaf node 283 * @trans: btree transaction object 284 * @path: path pointing to @insert's pos 285 * @insert: key to insert 286 * @journal_seq: sequence number of journal reservation 287 */ 288 inline void bch2_btree_insert_key_leaf(struct btree_trans *trans, 289 struct btree_path *path, 290 struct bkey_i *insert, 291 u64 journal_seq) 292 { 293 struct bch_fs *c = trans->c; 294 struct btree *b = path_l(path)->b; 295 struct bset_tree *t = bset_tree_last(b); 296 struct bset *i = bset(b, t); 297 int old_u64s = bset_u64s(t); 298 int old_live_u64s = b->nr.live_u64s; 299 int live_u64s_added, u64s_added; 300 301 if (unlikely(!bch2_btree_bset_insert_key(trans, path, b, 302 &path_l(path)->iter, insert))) 303 return; 304 305 i->journal_seq = cpu_to_le64(max(journal_seq, le64_to_cpu(i->journal_seq))); 306 307 bch2_btree_add_journal_pin(c, b, journal_seq); 308 309 if (unlikely(!btree_node_dirty(b))) { 310 EBUG_ON(test_bit(BCH_FS_clean_shutdown, &c->flags)); 311 set_btree_node_dirty_acct(c, b); 312 } 313 314 live_u64s_added = (int) b->nr.live_u64s - old_live_u64s; 315 u64s_added = (int) bset_u64s(t) - old_u64s; 316 317 if (b->sib_u64s[0] != U16_MAX && live_u64s_added < 0) 318 b->sib_u64s[0] = max(0, (int) b->sib_u64s[0] + live_u64s_added); 319 if (b->sib_u64s[1] != U16_MAX && live_u64s_added < 0) 320 b->sib_u64s[1] = max(0, (int) b->sib_u64s[1] + live_u64s_added); 321 322 if (u64s_added > live_u64s_added && 323 bch2_maybe_compact_whiteouts(c, b)) 324 bch2_trans_node_reinit_iter(trans, b); 325 } 326 327 /* Cached btree updates: */ 328 329 /* Normal update interface: */ 330 331 static inline void btree_insert_entry_checks(struct btree_trans *trans, 332 struct btree_insert_entry *i) 333 { 334 struct btree_path *path = trans->paths + i->path; 335 336 BUG_ON(!bpos_eq(i->k->k.p, path->pos)); 337 BUG_ON(i->cached != path->cached); 338 BUG_ON(i->level != path->level); 339 BUG_ON(i->btree_id != path->btree_id); 340 BUG_ON(i->bkey_type != __btree_node_type(path->level, path->btree_id)); 341 EBUG_ON(!i->level && 342 btree_type_has_snapshots(i->btree_id) && 343 !(i->flags & BTREE_UPDATE_internal_snapshot_node) && 344 test_bit(JOURNAL_replay_done, &trans->c->journal.flags) && 345 i->k->k.p.snapshot && 346 bch2_snapshot_is_internal_node(trans->c, i->k->k.p.snapshot) > 0); 347 } 348 349 static __always_inline int bch2_trans_journal_res_get(struct btree_trans *trans, 350 unsigned flags) 351 { 352 return bch2_journal_res_get(&trans->c->journal, &trans->journal_res, 353 trans->journal_u64s, flags, trans); 354 } 355 356 #define JSET_ENTRY_LOG_U64s 4 357 358 static noinline void journal_transaction_name(struct btree_trans *trans) 359 { 360 struct bch_fs *c = trans->c; 361 struct journal *j = &c->journal; 362 struct jset_entry *entry = 363 bch2_journal_add_entry(j, &trans->journal_res, 364 BCH_JSET_ENTRY_log, 0, 0, 365 JSET_ENTRY_LOG_U64s); 366 struct jset_entry_log *l = 367 container_of(entry, struct jset_entry_log, entry); 368 369 strncpy(l->d, trans->fn, JSET_ENTRY_LOG_U64s * sizeof(u64)); 370 } 371 372 static inline int btree_key_can_insert(struct btree_trans *trans, 373 struct btree *b, unsigned u64s) 374 { 375 if (!bch2_btree_node_insert_fits(b, u64s)) 376 return -BCH_ERR_btree_insert_btree_node_full; 377 378 return 0; 379 } 380 381 noinline static int 382 btree_key_can_insert_cached_slowpath(struct btree_trans *trans, unsigned flags, 383 struct btree_path *path, unsigned new_u64s) 384 { 385 struct bkey_cached *ck = (void *) path->l[0].b; 386 struct bkey_i *new_k; 387 int ret; 388 389 bch2_trans_unlock_updates_write(trans); 390 bch2_trans_unlock(trans); 391 392 new_k = kmalloc(new_u64s * sizeof(u64), GFP_KERNEL); 393 if (!new_k) { 394 bch_err(trans->c, "error allocating memory for key cache key, btree %s u64s %u", 395 bch2_btree_id_str(path->btree_id), new_u64s); 396 return -BCH_ERR_ENOMEM_btree_key_cache_insert; 397 } 398 399 ret = bch2_trans_relock(trans) ?: 400 bch2_trans_lock_write(trans); 401 if (unlikely(ret)) { 402 kfree(new_k); 403 return ret; 404 } 405 406 memcpy(new_k, ck->k, ck->u64s * sizeof(u64)); 407 408 trans_for_each_update(trans, i) 409 if (i->old_v == &ck->k->v) 410 i->old_v = &new_k->v; 411 412 kfree(ck->k); 413 ck->u64s = new_u64s; 414 ck->k = new_k; 415 return 0; 416 } 417 418 static int btree_key_can_insert_cached(struct btree_trans *trans, unsigned flags, 419 struct btree_path *path, unsigned u64s) 420 { 421 struct bch_fs *c = trans->c; 422 struct bkey_cached *ck = (void *) path->l[0].b; 423 unsigned new_u64s; 424 struct bkey_i *new_k; 425 unsigned watermark = flags & BCH_WATERMARK_MASK; 426 427 EBUG_ON(path->level); 428 429 if (watermark < BCH_WATERMARK_reclaim && 430 !test_bit(BKEY_CACHED_DIRTY, &ck->flags) && 431 bch2_btree_key_cache_must_wait(c)) 432 return -BCH_ERR_btree_insert_need_journal_reclaim; 433 434 /* 435 * bch2_varint_decode can read past the end of the buffer by at most 7 436 * bytes (it won't be used): 437 */ 438 u64s += 1; 439 440 if (u64s <= ck->u64s) 441 return 0; 442 443 new_u64s = roundup_pow_of_two(u64s); 444 new_k = krealloc(ck->k, new_u64s * sizeof(u64), GFP_NOWAIT|__GFP_NOWARN); 445 if (unlikely(!new_k)) 446 return btree_key_can_insert_cached_slowpath(trans, flags, path, new_u64s); 447 448 trans_for_each_update(trans, i) 449 if (i->old_v == &ck->k->v) 450 i->old_v = &new_k->v; 451 452 ck->u64s = new_u64s; 453 ck->k = new_k; 454 return 0; 455 } 456 457 /* Triggers: */ 458 459 static int run_one_mem_trigger(struct btree_trans *trans, 460 struct btree_insert_entry *i, 461 unsigned flags) 462 { 463 verify_update_old_key(trans, i); 464 465 if (unlikely(flags & BTREE_TRIGGER_norun)) 466 return 0; 467 468 struct bkey_s_c old = { &i->old_k, i->old_v }; 469 struct bkey_i *new = i->k; 470 const struct bkey_ops *old_ops = bch2_bkey_type_ops(old.k->type); 471 const struct bkey_ops *new_ops = bch2_bkey_type_ops(i->k->k.type); 472 473 if (old_ops->trigger == new_ops->trigger) 474 return bch2_key_trigger(trans, i->btree_id, i->level, 475 old, bkey_i_to_s(new), 476 BTREE_TRIGGER_insert|BTREE_TRIGGER_overwrite|flags); 477 else 478 return bch2_key_trigger_new(trans, i->btree_id, i->level, 479 bkey_i_to_s(new), flags) ?: 480 bch2_key_trigger_old(trans, i->btree_id, i->level, 481 old, flags); 482 } 483 484 static int run_one_trans_trigger(struct btree_trans *trans, struct btree_insert_entry *i) 485 { 486 verify_update_old_key(trans, i); 487 488 if ((i->flags & BTREE_TRIGGER_norun) || 489 !btree_node_type_has_trans_triggers(i->bkey_type)) 490 return 0; 491 492 /* 493 * Transactional triggers create new btree_insert_entries, so we can't 494 * pass them a pointer to a btree_insert_entry, that memory is going to 495 * move: 496 */ 497 struct bkey old_k = i->old_k; 498 struct bkey_s_c old = { &old_k, i->old_v }; 499 const struct bkey_ops *old_ops = bch2_bkey_type_ops(old.k->type); 500 const struct bkey_ops *new_ops = bch2_bkey_type_ops(i->k->k.type); 501 unsigned flags = i->flags|BTREE_TRIGGER_transactional; 502 503 if (!i->insert_trigger_run && 504 !i->overwrite_trigger_run && 505 old_ops->trigger == new_ops->trigger) { 506 i->overwrite_trigger_run = true; 507 i->insert_trigger_run = true; 508 return bch2_key_trigger(trans, i->btree_id, i->level, old, bkey_i_to_s(i->k), 509 BTREE_TRIGGER_insert| 510 BTREE_TRIGGER_overwrite|flags) ?: 1; 511 } else if (!i->overwrite_trigger_run) { 512 i->overwrite_trigger_run = true; 513 return bch2_key_trigger_old(trans, i->btree_id, i->level, old, flags) ?: 1; 514 } else if (!i->insert_trigger_run) { 515 i->insert_trigger_run = true; 516 return bch2_key_trigger_new(trans, i->btree_id, i->level, bkey_i_to_s(i->k), flags) ?: 1; 517 } else { 518 return 0; 519 } 520 } 521 522 static int bch2_trans_commit_run_triggers(struct btree_trans *trans) 523 { 524 unsigned sort_id_start = 0; 525 526 while (sort_id_start < trans->nr_updates) { 527 unsigned i, sort_id = trans->updates[sort_id_start].sort_order; 528 bool trans_trigger_run; 529 530 /* 531 * For a given btree, this algorithm runs insert triggers before 532 * overwrite triggers: this is so that when extents are being 533 * moved (e.g. by FALLOCATE_FL_INSERT_RANGE), we don't drop 534 * references before they are re-added. 535 * 536 * Running triggers will append more updates to the list of 537 * updates as we're walking it: 538 */ 539 do { 540 trans_trigger_run = false; 541 542 for (i = sort_id_start; 543 i < trans->nr_updates && trans->updates[i].sort_order <= sort_id; 544 i++) { 545 if (trans->updates[i].sort_order < sort_id) { 546 sort_id_start = i; 547 continue; 548 } 549 550 int ret = run_one_trans_trigger(trans, trans->updates + i); 551 if (ret < 0) 552 return ret; 553 if (ret) 554 trans_trigger_run = true; 555 } 556 } while (trans_trigger_run); 557 558 sort_id_start = i; 559 } 560 561 #ifdef CONFIG_BCACHEFS_DEBUG 562 trans_for_each_update(trans, i) 563 BUG_ON(!(i->flags & BTREE_TRIGGER_norun) && 564 btree_node_type_has_trans_triggers(i->bkey_type) && 565 (!i->insert_trigger_run || !i->overwrite_trigger_run)); 566 #endif 567 return 0; 568 } 569 570 static noinline int bch2_trans_commit_run_gc_triggers(struct btree_trans *trans) 571 { 572 trans_for_each_update(trans, i) 573 if (btree_node_type_has_triggers(i->bkey_type) && 574 gc_visited(trans->c, gc_pos_btree(i->btree_id, i->level, i->k->k.p))) { 575 int ret = run_one_mem_trigger(trans, i, i->flags|BTREE_TRIGGER_gc); 576 if (ret) 577 return ret; 578 } 579 580 return 0; 581 } 582 583 static inline int 584 bch2_trans_commit_write_locked(struct btree_trans *trans, unsigned flags, 585 struct btree_insert_entry **stopped_at, 586 unsigned long trace_ip) 587 { 588 struct bch_fs *c = trans->c; 589 struct btree_trans_commit_hook *h; 590 unsigned u64s = 0; 591 int ret = 0; 592 593 bch2_trans_verify_not_unlocked_or_in_restart(trans); 594 595 if (race_fault()) { 596 trace_and_count(c, trans_restart_fault_inject, trans, trace_ip); 597 return btree_trans_restart(trans, BCH_ERR_transaction_restart_fault_inject); 598 } 599 600 /* 601 * Check if the insert will fit in the leaf node with the write lock 602 * held, otherwise another thread could write the node changing the 603 * amount of space available: 604 */ 605 606 prefetch(&trans->c->journal.flags); 607 608 trans_for_each_update(trans, i) { 609 /* Multiple inserts might go to same leaf: */ 610 if (!same_leaf_as_prev(trans, i)) 611 u64s = 0; 612 613 u64s += i->k->k.u64s; 614 ret = !i->cached 615 ? btree_key_can_insert(trans, insert_l(trans, i)->b, u64s) 616 : btree_key_can_insert_cached(trans, flags, trans->paths + i->path, u64s); 617 if (ret) { 618 *stopped_at = i; 619 return ret; 620 } 621 622 i->k->k.needs_whiteout = false; 623 } 624 625 /* 626 * Don't get journal reservation until after we know insert will 627 * succeed: 628 */ 629 if (likely(!(flags & BCH_TRANS_COMMIT_no_journal_res))) { 630 ret = bch2_trans_journal_res_get(trans, 631 (flags & BCH_WATERMARK_MASK)| 632 JOURNAL_RES_GET_NONBLOCK); 633 if (ret) 634 return ret; 635 636 if (unlikely(trans->journal_transaction_names)) 637 journal_transaction_name(trans); 638 } 639 640 /* 641 * Not allowed to fail after we've gotten our journal reservation - we 642 * have to use it: 643 */ 644 645 if (IS_ENABLED(CONFIG_BCACHEFS_DEBUG) && 646 !(flags & BCH_TRANS_COMMIT_no_journal_res)) { 647 if (bch2_journal_seq_verify) 648 trans_for_each_update(trans, i) 649 i->k->k.bversion.lo = trans->journal_res.seq; 650 else if (bch2_inject_invalid_keys) 651 trans_for_each_update(trans, i) 652 i->k->k.bversion = MAX_VERSION; 653 } 654 655 h = trans->hooks; 656 while (h) { 657 ret = h->fn(trans, h); 658 if (ret) 659 return ret; 660 h = h->next; 661 } 662 663 struct jset_entry *entry = trans->journal_entries; 664 665 percpu_down_read(&c->mark_lock); 666 for (entry = trans->journal_entries; 667 entry != (void *) ((u64 *) trans->journal_entries + trans->journal_entries_u64s); 668 entry = vstruct_next(entry)) 669 if (entry->type == BCH_JSET_ENTRY_write_buffer_keys && 670 entry->start->k.type == KEY_TYPE_accounting) { 671 ret = bch2_accounting_trans_commit_hook(trans, bkey_i_to_accounting(entry->start), flags); 672 if (ret) 673 goto revert_fs_usage; 674 } 675 percpu_up_read(&c->mark_lock); 676 677 /* XXX: we only want to run this if deltas are nonzero */ 678 bch2_trans_account_disk_usage_change(trans); 679 680 trans_for_each_update(trans, i) 681 if (btree_node_type_has_atomic_triggers(i->bkey_type)) { 682 ret = run_one_mem_trigger(trans, i, BTREE_TRIGGER_atomic|i->flags); 683 if (ret) 684 goto fatal_err; 685 } 686 687 if (unlikely(c->gc_pos.phase)) { 688 ret = bch2_trans_commit_run_gc_triggers(trans); 689 if (ret) 690 goto fatal_err; 691 } 692 693 struct bkey_validate_context validate_context = { .from = BKEY_VALIDATE_commit }; 694 695 if (!(flags & BCH_TRANS_COMMIT_no_journal_res)) 696 validate_context.flags = BCH_VALIDATE_write|BCH_VALIDATE_commit; 697 698 for (struct jset_entry *i = trans->journal_entries; 699 i != (void *) ((u64 *) trans->journal_entries + trans->journal_entries_u64s); 700 i = vstruct_next(i)) { 701 ret = bch2_journal_entry_validate(c, NULL, i, 702 bcachefs_metadata_version_current, 703 CPU_BIG_ENDIAN, validate_context); 704 if (unlikely(ret)) { 705 bch2_trans_inconsistent(trans, "invalid journal entry on insert from %s\n", 706 trans->fn); 707 goto fatal_err; 708 } 709 } 710 711 trans_for_each_update(trans, i) { 712 validate_context.level = i->level; 713 validate_context.btree = i->btree_id; 714 715 ret = bch2_bkey_validate(c, bkey_i_to_s_c(i->k), validate_context); 716 if (unlikely(ret)){ 717 bch2_trans_inconsistent(trans, "invalid bkey on insert from %s -> %ps\n", 718 trans->fn, (void *) i->ip_allocated); 719 goto fatal_err; 720 } 721 btree_insert_entry_checks(trans, i); 722 } 723 724 if (likely(!(flags & BCH_TRANS_COMMIT_no_journal_res))) { 725 struct journal *j = &c->journal; 726 struct jset_entry *entry; 727 728 trans_for_each_update(trans, i) { 729 if (i->key_cache_already_flushed) 730 continue; 731 732 if (i->flags & BTREE_UPDATE_nojournal) 733 continue; 734 735 verify_update_old_key(trans, i); 736 737 if (trans->journal_transaction_names) { 738 entry = bch2_journal_add_entry(j, &trans->journal_res, 739 BCH_JSET_ENTRY_overwrite, 740 i->btree_id, i->level, 741 i->old_k.u64s); 742 bkey_reassemble((struct bkey_i *) entry->start, 743 (struct bkey_s_c) { &i->old_k, i->old_v }); 744 } 745 746 entry = bch2_journal_add_entry(j, &trans->journal_res, 747 BCH_JSET_ENTRY_btree_keys, 748 i->btree_id, i->level, 749 i->k->k.u64s); 750 bkey_copy((struct bkey_i *) entry->start, i->k); 751 } 752 753 memcpy_u64s_small(journal_res_entry(&c->journal, &trans->journal_res), 754 trans->journal_entries, 755 trans->journal_entries_u64s); 756 757 trans->journal_res.offset += trans->journal_entries_u64s; 758 trans->journal_res.u64s -= trans->journal_entries_u64s; 759 760 if (trans->journal_seq) 761 *trans->journal_seq = trans->journal_res.seq; 762 } 763 764 trans_for_each_update(trans, i) { 765 struct btree_path *path = trans->paths + i->path; 766 767 if (!i->cached) 768 bch2_btree_insert_key_leaf(trans, path, i->k, trans->journal_res.seq); 769 else if (!i->key_cache_already_flushed) 770 bch2_btree_insert_key_cached(trans, flags, i); 771 else 772 bch2_btree_key_cache_drop(trans, path); 773 } 774 775 return 0; 776 fatal_err: 777 bch2_fs_fatal_error(c, "fatal error in transaction commit: %s", bch2_err_str(ret)); 778 percpu_down_read(&c->mark_lock); 779 revert_fs_usage: 780 for (struct jset_entry *entry2 = trans->journal_entries; 781 entry2 != entry; 782 entry2 = vstruct_next(entry2)) 783 if (entry2->type == BCH_JSET_ENTRY_write_buffer_keys && 784 entry2->start->k.type == KEY_TYPE_accounting) 785 bch2_accounting_trans_commit_revert(trans, 786 bkey_i_to_accounting(entry2->start), flags); 787 percpu_up_read(&c->mark_lock); 788 return ret; 789 } 790 791 static noinline void bch2_drop_overwrites_from_journal(struct btree_trans *trans) 792 { 793 /* 794 * Accounting keys aren't deduped in the journal: we have to compare 795 * each individual update against what's in the btree to see if it has 796 * been applied yet, and accounting updates also don't overwrite, 797 * they're deltas that accumulate. 798 */ 799 trans_for_each_update(trans, i) 800 if (i->k->k.type != KEY_TYPE_accounting) 801 bch2_journal_key_overwritten(trans->c, i->btree_id, i->level, i->k->k.p); 802 } 803 804 static int bch2_trans_commit_journal_pin_flush(struct journal *j, 805 struct journal_entry_pin *_pin, u64 seq) 806 { 807 return 0; 808 } 809 810 /* 811 * Get journal reservation, take write locks, and attempt to do btree update(s): 812 */ 813 static inline int do_bch2_trans_commit(struct btree_trans *trans, unsigned flags, 814 struct btree_insert_entry **stopped_at, 815 unsigned long trace_ip) 816 { 817 struct bch_fs *c = trans->c; 818 int ret = 0, u64s_delta = 0; 819 820 for (unsigned idx = 0; idx < trans->nr_updates; idx++) { 821 struct btree_insert_entry *i = trans->updates + idx; 822 if (i->cached) 823 continue; 824 825 u64s_delta += !bkey_deleted(&i->k->k) ? i->k->k.u64s : 0; 826 u64s_delta -= i->old_btree_u64s; 827 828 if (!same_leaf_as_next(trans, i)) { 829 if (u64s_delta <= 0) { 830 ret = bch2_foreground_maybe_merge(trans, i->path, 831 i->level, flags); 832 if (unlikely(ret)) 833 return ret; 834 } 835 836 u64s_delta = 0; 837 } 838 } 839 840 ret = bch2_trans_lock_write(trans); 841 if (unlikely(ret)) 842 return ret; 843 844 ret = bch2_trans_commit_write_locked(trans, flags, stopped_at, trace_ip); 845 846 if (!ret && unlikely(trans->journal_replay_not_finished)) 847 bch2_drop_overwrites_from_journal(trans); 848 849 bch2_trans_unlock_updates_write(trans); 850 851 if (!ret && trans->journal_pin) 852 bch2_journal_pin_add(&c->journal, trans->journal_res.seq, 853 trans->journal_pin, 854 bch2_trans_commit_journal_pin_flush); 855 856 /* 857 * Drop journal reservation after dropping write locks, since dropping 858 * the journal reservation may kick off a journal write: 859 */ 860 if (likely(!(flags & BCH_TRANS_COMMIT_no_journal_res))) 861 bch2_journal_res_put(&c->journal, &trans->journal_res); 862 863 return ret; 864 } 865 866 static int journal_reclaim_wait_done(struct bch_fs *c) 867 { 868 int ret = bch2_journal_error(&c->journal) ?: 869 bch2_btree_key_cache_wait_done(c); 870 871 if (!ret) 872 journal_reclaim_kick(&c->journal); 873 return ret; 874 } 875 876 static noinline 877 int bch2_trans_commit_error(struct btree_trans *trans, unsigned flags, 878 struct btree_insert_entry *i, 879 int ret, unsigned long trace_ip) 880 { 881 struct bch_fs *c = trans->c; 882 enum bch_watermark watermark = flags & BCH_WATERMARK_MASK; 883 884 if (bch2_err_matches(ret, BCH_ERR_journal_res_blocked)) { 885 /* 886 * XXX: this should probably be a separate BTREE_INSERT_NONBLOCK 887 * flag 888 */ 889 if ((flags & BCH_TRANS_COMMIT_journal_reclaim) && 890 watermark < BCH_WATERMARK_reclaim) { 891 ret = -BCH_ERR_journal_reclaim_would_deadlock; 892 goto out; 893 } 894 895 ret = drop_locks_do(trans, 896 bch2_trans_journal_res_get(trans, 897 (flags & BCH_WATERMARK_MASK)| 898 JOURNAL_RES_GET_CHECK)); 899 goto out; 900 } 901 902 switch (ret) { 903 case -BCH_ERR_btree_insert_btree_node_full: 904 ret = bch2_btree_split_leaf(trans, i->path, flags); 905 if (bch2_err_matches(ret, BCH_ERR_transaction_restart)) 906 trace_and_count(c, trans_restart_btree_node_split, trans, 907 trace_ip, trans->paths + i->path); 908 break; 909 case -BCH_ERR_btree_insert_need_mark_replicas: 910 ret = drop_locks_do(trans, 911 bch2_accounting_update_sb(trans)); 912 break; 913 case -BCH_ERR_btree_insert_need_journal_reclaim: 914 bch2_trans_unlock(trans); 915 916 trace_and_count(c, trans_blocked_journal_reclaim, trans, trace_ip); 917 track_event_change(&c->times[BCH_TIME_blocked_key_cache_flush], true); 918 919 wait_event_freezable(c->journal.reclaim_wait, 920 (ret = journal_reclaim_wait_done(c))); 921 922 track_event_change(&c->times[BCH_TIME_blocked_key_cache_flush], false); 923 924 if (ret < 0) 925 break; 926 927 ret = bch2_trans_relock(trans); 928 break; 929 default: 930 BUG_ON(ret >= 0); 931 break; 932 } 933 out: 934 BUG_ON(bch2_err_matches(ret, BCH_ERR_transaction_restart) != !!trans->restarted); 935 936 bch2_fs_inconsistent_on(bch2_err_matches(ret, ENOSPC) && 937 (flags & BCH_TRANS_COMMIT_no_enospc), c, 938 "%s: incorrectly got %s\n", __func__, bch2_err_str(ret)); 939 940 return ret; 941 } 942 943 /* 944 * This is for updates done in the early part of fsck - btree_gc - before we've 945 * gone RW. we only add the new key to the list of keys for journal replay to 946 * do. 947 */ 948 static noinline int 949 do_bch2_trans_commit_to_journal_replay(struct btree_trans *trans) 950 { 951 struct bch_fs *c = trans->c; 952 953 BUG_ON(current != c->recovery_task); 954 955 trans_for_each_update(trans, i) { 956 int ret = bch2_journal_key_insert(c, i->btree_id, i->level, i->k); 957 if (ret) 958 return ret; 959 } 960 961 for (struct jset_entry *i = trans->journal_entries; 962 i != (void *) ((u64 *) trans->journal_entries + trans->journal_entries_u64s); 963 i = vstruct_next(i)) 964 if (i->type == BCH_JSET_ENTRY_btree_keys || 965 i->type == BCH_JSET_ENTRY_write_buffer_keys) { 966 int ret = bch2_journal_key_insert(c, i->btree_id, i->level, i->start); 967 if (ret) 968 return ret; 969 } 970 971 return 0; 972 } 973 974 int __bch2_trans_commit(struct btree_trans *trans, unsigned flags) 975 { 976 struct btree_insert_entry *errored_at = NULL; 977 struct bch_fs *c = trans->c; 978 int ret = 0; 979 980 bch2_trans_verify_not_unlocked_or_in_restart(trans); 981 982 ret = trans_maybe_inject_restart(trans, _RET_IP_); 983 if (unlikely(ret)) 984 goto out_reset; 985 986 if (!trans->nr_updates && 987 !trans->journal_entries_u64s) 988 goto out_reset; 989 990 ret = bch2_trans_commit_run_triggers(trans); 991 if (ret) 992 goto out_reset; 993 994 if (!(flags & BCH_TRANS_COMMIT_no_check_rw) && 995 unlikely(!bch2_write_ref_tryget(c, BCH_WRITE_REF_trans))) { 996 if (unlikely(!test_bit(BCH_FS_may_go_rw, &c->flags))) 997 ret = do_bch2_trans_commit_to_journal_replay(trans); 998 else 999 ret = -BCH_ERR_erofs_trans_commit; 1000 goto out_reset; 1001 } 1002 1003 EBUG_ON(test_bit(BCH_FS_clean_shutdown, &c->flags)); 1004 1005 trans->journal_u64s = trans->journal_entries_u64s; 1006 trans->journal_transaction_names = READ_ONCE(c->opts.journal_transaction_names); 1007 if (trans->journal_transaction_names) 1008 trans->journal_u64s += jset_u64s(JSET_ENTRY_LOG_U64s); 1009 1010 trans_for_each_update(trans, i) { 1011 struct btree_path *path = trans->paths + i->path; 1012 1013 EBUG_ON(!path->should_be_locked); 1014 1015 ret = bch2_btree_path_upgrade(trans, path, i->level + 1); 1016 if (unlikely(ret)) 1017 goto out; 1018 1019 EBUG_ON(!btree_node_intent_locked(path, i->level)); 1020 1021 if (i->key_cache_already_flushed) 1022 continue; 1023 1024 if (i->flags & BTREE_UPDATE_nojournal) 1025 continue; 1026 1027 /* we're going to journal the key being updated: */ 1028 trans->journal_u64s += jset_u64s(i->k->k.u64s); 1029 1030 /* and we're also going to log the overwrite: */ 1031 if (trans->journal_transaction_names) 1032 trans->journal_u64s += jset_u64s(i->old_k.u64s); 1033 } 1034 1035 if (trans->extra_disk_res) { 1036 ret = bch2_disk_reservation_add(c, trans->disk_res, 1037 trans->extra_disk_res, 1038 (flags & BCH_TRANS_COMMIT_no_enospc) 1039 ? BCH_DISK_RESERVATION_NOFAIL : 0); 1040 if (ret) 1041 goto err; 1042 } 1043 retry: 1044 errored_at = NULL; 1045 bch2_trans_verify_not_unlocked_or_in_restart(trans); 1046 if (likely(!(flags & BCH_TRANS_COMMIT_no_journal_res))) 1047 memset(&trans->journal_res, 0, sizeof(trans->journal_res)); 1048 memset(&trans->fs_usage_delta, 0, sizeof(trans->fs_usage_delta)); 1049 1050 ret = do_bch2_trans_commit(trans, flags, &errored_at, _RET_IP_); 1051 1052 /* make sure we didn't drop or screw up locks: */ 1053 bch2_trans_verify_locks(trans); 1054 1055 if (ret) 1056 goto err; 1057 1058 trace_and_count(c, transaction_commit, trans, _RET_IP_); 1059 out: 1060 if (likely(!(flags & BCH_TRANS_COMMIT_no_check_rw))) 1061 bch2_write_ref_put(c, BCH_WRITE_REF_trans); 1062 out_reset: 1063 if (!ret) 1064 bch2_trans_downgrade(trans); 1065 bch2_trans_reset_updates(trans); 1066 1067 return ret; 1068 err: 1069 ret = bch2_trans_commit_error(trans, flags, errored_at, ret, _RET_IP_); 1070 if (ret) 1071 goto out; 1072 1073 /* 1074 * We might have done another transaction commit in the error path - 1075 * i.e. btree write buffer flush - which will have made use of 1076 * trans->journal_res, but with BCH_TRANS_COMMIT_no_journal_res that is 1077 * how the journal sequence number to pin is passed in - so we must 1078 * restart: 1079 */ 1080 if (flags & BCH_TRANS_COMMIT_no_journal_res) { 1081 ret = -BCH_ERR_transaction_restart_nested; 1082 goto out; 1083 } 1084 1085 goto retry; 1086 } 1087