1 // SPDX-License-Identifier: GPL-2.0 2 3 #include "bcachefs.h" 4 #include "btree_gc.h" 5 #include "btree_io.h" 6 #include "btree_iter.h" 7 #include "btree_journal_iter.h" 8 #include "btree_key_cache.h" 9 #include "btree_update_interior.h" 10 #include "btree_write_buffer.h" 11 #include "buckets.h" 12 #include "errcode.h" 13 #include "error.h" 14 #include "journal.h" 15 #include "journal_reclaim.h" 16 #include "replicas.h" 17 #include "snapshot.h" 18 19 #include <linux/prefetch.h> 20 21 static void verify_update_old_key(struct btree_trans *trans, struct btree_insert_entry *i) 22 { 23 #ifdef CONFIG_BCACHEFS_DEBUG 24 struct bch_fs *c = trans->c; 25 struct bkey u; 26 struct bkey_s_c k = bch2_btree_path_peek_slot_exact(i->path, &u); 27 28 if (unlikely(trans->journal_replay_not_finished)) { 29 struct bkey_i *j_k = 30 bch2_journal_keys_peek_slot(c, i->btree_id, i->level, i->k->k.p); 31 32 if (j_k) 33 k = bkey_i_to_s_c(j_k); 34 } 35 36 u = *k.k; 37 u.needs_whiteout = i->old_k.needs_whiteout; 38 39 BUG_ON(memcmp(&i->old_k, &u, sizeof(struct bkey))); 40 BUG_ON(i->old_v != k.v); 41 #endif 42 } 43 44 static inline struct btree_path_level *insert_l(struct btree_insert_entry *i) 45 { 46 return i->path->l + i->level; 47 } 48 49 static inline bool same_leaf_as_prev(struct btree_trans *trans, 50 struct btree_insert_entry *i) 51 { 52 return i != trans->updates && 53 insert_l(&i[0])->b == insert_l(&i[-1])->b; 54 } 55 56 static inline bool same_leaf_as_next(struct btree_trans *trans, 57 struct btree_insert_entry *i) 58 { 59 return i + 1 < trans->updates + trans->nr_updates && 60 insert_l(&i[0])->b == insert_l(&i[1])->b; 61 } 62 63 inline void bch2_btree_node_prep_for_write(struct btree_trans *trans, 64 struct btree_path *path, 65 struct btree *b) 66 { 67 struct bch_fs *c = trans->c; 68 69 if (unlikely(btree_node_just_written(b)) && 70 bch2_btree_post_write_cleanup(c, b)) 71 bch2_trans_node_reinit_iter(trans, b); 72 73 /* 74 * If the last bset has been written, or if it's gotten too big - start 75 * a new bset to insert into: 76 */ 77 if (want_new_bset(c, b)) 78 bch2_btree_init_next(trans, b); 79 } 80 81 /* Inserting into a given leaf node (last stage of insert): */ 82 83 /* Handle overwrites and do insert, for non extents: */ 84 bool bch2_btree_bset_insert_key(struct btree_trans *trans, 85 struct btree_path *path, 86 struct btree *b, 87 struct btree_node_iter *node_iter, 88 struct bkey_i *insert) 89 { 90 struct bkey_packed *k; 91 unsigned clobber_u64s = 0, new_u64s = 0; 92 93 EBUG_ON(btree_node_just_written(b)); 94 EBUG_ON(bset_written(b, btree_bset_last(b))); 95 EBUG_ON(bkey_deleted(&insert->k) && bkey_val_u64s(&insert->k)); 96 EBUG_ON(bpos_lt(insert->k.p, b->data->min_key)); 97 EBUG_ON(bpos_gt(insert->k.p, b->data->max_key)); 98 EBUG_ON(insert->k.u64s > 99 bch_btree_keys_u64s_remaining(trans->c, b)); 100 EBUG_ON(!b->c.level && !bpos_eq(insert->k.p, path->pos)); 101 102 k = bch2_btree_node_iter_peek_all(node_iter, b); 103 if (k && bkey_cmp_left_packed(b, k, &insert->k.p)) 104 k = NULL; 105 106 /* @k is the key being overwritten/deleted, if any: */ 107 EBUG_ON(k && bkey_deleted(k)); 108 109 /* Deleting, but not found? nothing to do: */ 110 if (bkey_deleted(&insert->k) && !k) 111 return false; 112 113 if (bkey_deleted(&insert->k)) { 114 /* Deleting: */ 115 btree_account_key_drop(b, k); 116 k->type = KEY_TYPE_deleted; 117 118 if (k->needs_whiteout) 119 push_whiteout(trans->c, b, insert->k.p); 120 k->needs_whiteout = false; 121 122 if (k >= btree_bset_last(b)->start) { 123 clobber_u64s = k->u64s; 124 bch2_bset_delete(b, k, clobber_u64s); 125 goto fix_iter; 126 } else { 127 bch2_btree_path_fix_key_modified(trans, b, k); 128 } 129 130 return true; 131 } 132 133 if (k) { 134 /* Overwriting: */ 135 btree_account_key_drop(b, k); 136 k->type = KEY_TYPE_deleted; 137 138 insert->k.needs_whiteout = k->needs_whiteout; 139 k->needs_whiteout = false; 140 141 if (k >= btree_bset_last(b)->start) { 142 clobber_u64s = k->u64s; 143 goto overwrite; 144 } else { 145 bch2_btree_path_fix_key_modified(trans, b, k); 146 } 147 } 148 149 k = bch2_btree_node_iter_bset_pos(node_iter, b, bset_tree_last(b)); 150 overwrite: 151 bch2_bset_insert(b, node_iter, k, insert, clobber_u64s); 152 new_u64s = k->u64s; 153 fix_iter: 154 if (clobber_u64s != new_u64s) 155 bch2_btree_node_iter_fix(trans, path, b, node_iter, k, 156 clobber_u64s, new_u64s); 157 return true; 158 } 159 160 static int __btree_node_flush(struct journal *j, struct journal_entry_pin *pin, 161 unsigned i, u64 seq) 162 { 163 struct bch_fs *c = container_of(j, struct bch_fs, journal); 164 struct btree_write *w = container_of(pin, struct btree_write, journal); 165 struct btree *b = container_of(w, struct btree, writes[i]); 166 struct btree_trans *trans = bch2_trans_get(c); 167 unsigned long old, new, v; 168 unsigned idx = w - b->writes; 169 170 btree_node_lock_nopath_nofail(trans, &b->c, SIX_LOCK_read); 171 v = READ_ONCE(b->flags); 172 173 do { 174 old = new = v; 175 176 if (!(old & (1 << BTREE_NODE_dirty)) || 177 !!(old & (1 << BTREE_NODE_write_idx)) != idx || 178 w->journal.seq != seq) 179 break; 180 181 new &= ~BTREE_WRITE_TYPE_MASK; 182 new |= BTREE_WRITE_journal_reclaim; 183 new |= 1 << BTREE_NODE_need_write; 184 } while ((v = cmpxchg(&b->flags, old, new)) != old); 185 186 btree_node_write_if_need(c, b, SIX_LOCK_read); 187 six_unlock_read(&b->c.lock); 188 189 bch2_trans_put(trans); 190 return 0; 191 } 192 193 int bch2_btree_node_flush0(struct journal *j, struct journal_entry_pin *pin, u64 seq) 194 { 195 return __btree_node_flush(j, pin, 0, seq); 196 } 197 198 int bch2_btree_node_flush1(struct journal *j, struct journal_entry_pin *pin, u64 seq) 199 { 200 return __btree_node_flush(j, pin, 1, seq); 201 } 202 203 inline void bch2_btree_add_journal_pin(struct bch_fs *c, 204 struct btree *b, u64 seq) 205 { 206 struct btree_write *w = btree_current_write(b); 207 208 bch2_journal_pin_add(&c->journal, seq, &w->journal, 209 btree_node_write_idx(b) == 0 210 ? bch2_btree_node_flush0 211 : bch2_btree_node_flush1); 212 } 213 214 /** 215 * bch2_btree_insert_key_leaf() - insert a key one key into a leaf node 216 * @trans: btree transaction object 217 * @path: path pointing to @insert's pos 218 * @insert: key to insert 219 * @journal_seq: sequence number of journal reservation 220 */ 221 inline void bch2_btree_insert_key_leaf(struct btree_trans *trans, 222 struct btree_path *path, 223 struct bkey_i *insert, 224 u64 journal_seq) 225 { 226 struct bch_fs *c = trans->c; 227 struct btree *b = path_l(path)->b; 228 struct bset_tree *t = bset_tree_last(b); 229 struct bset *i = bset(b, t); 230 int old_u64s = bset_u64s(t); 231 int old_live_u64s = b->nr.live_u64s; 232 int live_u64s_added, u64s_added; 233 234 if (unlikely(!bch2_btree_bset_insert_key(trans, path, b, 235 &path_l(path)->iter, insert))) 236 return; 237 238 i->journal_seq = cpu_to_le64(max(journal_seq, le64_to_cpu(i->journal_seq))); 239 240 bch2_btree_add_journal_pin(c, b, journal_seq); 241 242 if (unlikely(!btree_node_dirty(b))) { 243 EBUG_ON(test_bit(BCH_FS_CLEAN_SHUTDOWN, &c->flags)); 244 set_btree_node_dirty_acct(c, b); 245 } 246 247 live_u64s_added = (int) b->nr.live_u64s - old_live_u64s; 248 u64s_added = (int) bset_u64s(t) - old_u64s; 249 250 if (b->sib_u64s[0] != U16_MAX && live_u64s_added < 0) 251 b->sib_u64s[0] = max(0, (int) b->sib_u64s[0] + live_u64s_added); 252 if (b->sib_u64s[1] != U16_MAX && live_u64s_added < 0) 253 b->sib_u64s[1] = max(0, (int) b->sib_u64s[1] + live_u64s_added); 254 255 if (u64s_added > live_u64s_added && 256 bch2_maybe_compact_whiteouts(c, b)) 257 bch2_trans_node_reinit_iter(trans, b); 258 } 259 260 /* Cached btree updates: */ 261 262 /* Normal update interface: */ 263 264 static inline void btree_insert_entry_checks(struct btree_trans *trans, 265 struct btree_insert_entry *i) 266 { 267 BUG_ON(!bpos_eq(i->k->k.p, i->path->pos)); 268 BUG_ON(i->cached != i->path->cached); 269 BUG_ON(i->level != i->path->level); 270 BUG_ON(i->btree_id != i->path->btree_id); 271 EBUG_ON(!i->level && 272 !(i->flags & BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE) && 273 test_bit(JOURNAL_REPLAY_DONE, &trans->c->journal.flags) && 274 i->k->k.p.snapshot && 275 bch2_snapshot_is_internal_node(trans->c, i->k->k.p.snapshot)); 276 } 277 278 static noinline int 279 bch2_trans_journal_preres_get_cold(struct btree_trans *trans, unsigned flags, 280 unsigned long trace_ip) 281 { 282 return drop_locks_do(trans, 283 bch2_journal_preres_get(&trans->c->journal, 284 &trans->journal_preres, 285 trans->journal_preres_u64s, 286 (flags & BCH_WATERMARK_MASK))); 287 } 288 289 static __always_inline int bch2_trans_journal_res_get(struct btree_trans *trans, 290 unsigned flags) 291 { 292 return bch2_journal_res_get(&trans->c->journal, &trans->journal_res, 293 trans->journal_u64s, flags); 294 } 295 296 #define JSET_ENTRY_LOG_U64s 4 297 298 static noinline void journal_transaction_name(struct btree_trans *trans) 299 { 300 struct bch_fs *c = trans->c; 301 struct journal *j = &c->journal; 302 struct jset_entry *entry = 303 bch2_journal_add_entry(j, &trans->journal_res, 304 BCH_JSET_ENTRY_log, 0, 0, 305 JSET_ENTRY_LOG_U64s); 306 struct jset_entry_log *l = 307 container_of(entry, struct jset_entry_log, entry); 308 309 strncpy(l->d, trans->fn, JSET_ENTRY_LOG_U64s * sizeof(u64)); 310 } 311 312 static inline int btree_key_can_insert(struct btree_trans *trans, 313 struct btree *b, unsigned u64s) 314 { 315 struct bch_fs *c = trans->c; 316 317 if (!bch2_btree_node_insert_fits(c, b, u64s)) 318 return -BCH_ERR_btree_insert_btree_node_full; 319 320 return 0; 321 } 322 323 static int btree_key_can_insert_cached(struct btree_trans *trans, unsigned flags, 324 struct btree_path *path, unsigned u64s) 325 { 326 struct bch_fs *c = trans->c; 327 struct bkey_cached *ck = (void *) path->l[0].b; 328 struct btree_insert_entry *i; 329 unsigned new_u64s; 330 struct bkey_i *new_k; 331 332 EBUG_ON(path->level); 333 334 if (!test_bit(BKEY_CACHED_DIRTY, &ck->flags) && 335 bch2_btree_key_cache_must_wait(c) && 336 !(flags & BTREE_INSERT_JOURNAL_RECLAIM)) 337 return -BCH_ERR_btree_insert_need_journal_reclaim; 338 339 /* 340 * bch2_varint_decode can read past the end of the buffer by at most 7 341 * bytes (it won't be used): 342 */ 343 u64s += 1; 344 345 if (u64s <= ck->u64s) 346 return 0; 347 348 new_u64s = roundup_pow_of_two(u64s); 349 new_k = krealloc(ck->k, new_u64s * sizeof(u64), GFP_NOFS); 350 if (!new_k) { 351 bch_err(c, "error allocating memory for key cache key, btree %s u64s %u", 352 bch2_btree_ids[path->btree_id], new_u64s); 353 return -BCH_ERR_ENOMEM_btree_key_cache_insert; 354 } 355 356 trans_for_each_update(trans, i) 357 if (i->old_v == &ck->k->v) 358 i->old_v = &new_k->v; 359 360 ck->u64s = new_u64s; 361 ck->k = new_k; 362 return 0; 363 } 364 365 /* Triggers: */ 366 367 static int run_one_mem_trigger(struct btree_trans *trans, 368 struct btree_insert_entry *i, 369 unsigned flags) 370 { 371 struct bkey_s_c old = { &i->old_k, i->old_v }; 372 struct bkey_i *new = i->k; 373 const struct bkey_ops *old_ops = bch2_bkey_type_ops(old.k->type); 374 const struct bkey_ops *new_ops = bch2_bkey_type_ops(i->k->k.type); 375 int ret; 376 377 verify_update_old_key(trans, i); 378 379 if (unlikely(flags & BTREE_TRIGGER_NORUN)) 380 return 0; 381 382 if (!btree_node_type_needs_gc((enum btree_node_type) i->btree_id)) 383 return 0; 384 385 if (old_ops->atomic_trigger == new_ops->atomic_trigger && 386 ((1U << old.k->type) & BTREE_TRIGGER_WANTS_OLD_AND_NEW)) { 387 ret = bch2_mark_key(trans, i->btree_id, i->level, 388 old, bkey_i_to_s_c(new), 389 BTREE_TRIGGER_INSERT|BTREE_TRIGGER_OVERWRITE|flags); 390 } else { 391 struct bkey _deleted = KEY(0, 0, 0); 392 struct bkey_s_c deleted = (struct bkey_s_c) { &_deleted, NULL }; 393 394 _deleted.p = i->path->pos; 395 396 ret = bch2_mark_key(trans, i->btree_id, i->level, 397 deleted, bkey_i_to_s_c(new), 398 BTREE_TRIGGER_INSERT|flags) ?: 399 bch2_mark_key(trans, i->btree_id, i->level, 400 old, deleted, 401 BTREE_TRIGGER_OVERWRITE|flags); 402 } 403 404 return ret; 405 } 406 407 static int run_one_trans_trigger(struct btree_trans *trans, struct btree_insert_entry *i, 408 bool overwrite) 409 { 410 /* 411 * Transactional triggers create new btree_insert_entries, so we can't 412 * pass them a pointer to a btree_insert_entry, that memory is going to 413 * move: 414 */ 415 struct bkey old_k = i->old_k; 416 struct bkey_s_c old = { &old_k, i->old_v }; 417 const struct bkey_ops *old_ops = bch2_bkey_type_ops(old.k->type); 418 const struct bkey_ops *new_ops = bch2_bkey_type_ops(i->k->k.type); 419 420 verify_update_old_key(trans, i); 421 422 if ((i->flags & BTREE_TRIGGER_NORUN) || 423 !(BTREE_NODE_TYPE_HAS_TRANS_TRIGGERS & (1U << i->bkey_type))) 424 return 0; 425 426 if (!i->insert_trigger_run && 427 !i->overwrite_trigger_run && 428 old_ops->trans_trigger == new_ops->trans_trigger && 429 ((1U << old.k->type) & BTREE_TRIGGER_WANTS_OLD_AND_NEW)) { 430 i->overwrite_trigger_run = true; 431 i->insert_trigger_run = true; 432 return bch2_trans_mark_key(trans, i->btree_id, i->level, old, i->k, 433 BTREE_TRIGGER_INSERT| 434 BTREE_TRIGGER_OVERWRITE| 435 i->flags) ?: 1; 436 } else if (overwrite && !i->overwrite_trigger_run) { 437 i->overwrite_trigger_run = true; 438 return bch2_trans_mark_old(trans, i->btree_id, i->level, old, i->flags) ?: 1; 439 } else if (!overwrite && !i->insert_trigger_run) { 440 i->insert_trigger_run = true; 441 return bch2_trans_mark_new(trans, i->btree_id, i->level, i->k, i->flags) ?: 1; 442 } else { 443 return 0; 444 } 445 } 446 447 static int run_btree_triggers(struct btree_trans *trans, enum btree_id btree_id, 448 struct btree_insert_entry *btree_id_start) 449 { 450 struct btree_insert_entry *i; 451 bool trans_trigger_run; 452 int ret, overwrite; 453 454 for (overwrite = 1; overwrite >= 0; --overwrite) { 455 456 /* 457 * Running triggers will append more updates to the list of updates as 458 * we're walking it: 459 */ 460 do { 461 trans_trigger_run = false; 462 463 for (i = btree_id_start; 464 i < trans->updates + trans->nr_updates && i->btree_id <= btree_id; 465 i++) { 466 if (i->btree_id != btree_id) 467 continue; 468 469 ret = run_one_trans_trigger(trans, i, overwrite); 470 if (ret < 0) 471 return ret; 472 if (ret) 473 trans_trigger_run = true; 474 } 475 } while (trans_trigger_run); 476 } 477 478 return 0; 479 } 480 481 static int bch2_trans_commit_run_triggers(struct btree_trans *trans) 482 { 483 struct btree_insert_entry *i = NULL, *btree_id_start = trans->updates; 484 unsigned btree_id = 0; 485 int ret = 0; 486 487 /* 488 * 489 * For a given btree, this algorithm runs insert triggers before 490 * overwrite triggers: this is so that when extents are being moved 491 * (e.g. by FALLOCATE_FL_INSERT_RANGE), we don't drop references before 492 * they are re-added. 493 */ 494 for (btree_id = 0; btree_id < BTREE_ID_NR; btree_id++) { 495 if (btree_id == BTREE_ID_alloc) 496 continue; 497 498 while (btree_id_start < trans->updates + trans->nr_updates && 499 btree_id_start->btree_id < btree_id) 500 btree_id_start++; 501 502 ret = run_btree_triggers(trans, btree_id, btree_id_start); 503 if (ret) 504 return ret; 505 } 506 507 trans_for_each_update(trans, i) { 508 if (i->btree_id > BTREE_ID_alloc) 509 break; 510 if (i->btree_id == BTREE_ID_alloc) { 511 ret = run_btree_triggers(trans, BTREE_ID_alloc, i); 512 if (ret) 513 return ret; 514 break; 515 } 516 } 517 518 #ifdef CONFIG_BCACHEFS_DEBUG 519 trans_for_each_update(trans, i) 520 BUG_ON(!(i->flags & BTREE_TRIGGER_NORUN) && 521 (BTREE_NODE_TYPE_HAS_TRANS_TRIGGERS & (1U << i->bkey_type)) && 522 (!i->insert_trigger_run || !i->overwrite_trigger_run)); 523 #endif 524 return 0; 525 } 526 527 static noinline int bch2_trans_commit_run_gc_triggers(struct btree_trans *trans) 528 { 529 struct bch_fs *c = trans->c; 530 struct btree_insert_entry *i; 531 int ret = 0; 532 533 trans_for_each_update(trans, i) { 534 /* 535 * XXX: synchronization of cached update triggers with gc 536 * XXX: synchronization of interior node updates with gc 537 */ 538 BUG_ON(i->cached || i->level); 539 540 if (gc_visited(c, gc_pos_btree_node(insert_l(i)->b))) { 541 ret = run_one_mem_trigger(trans, i, i->flags|BTREE_TRIGGER_GC); 542 if (ret) 543 break; 544 } 545 } 546 547 return ret; 548 } 549 550 static inline int 551 bch2_trans_commit_write_locked(struct btree_trans *trans, unsigned flags, 552 struct btree_insert_entry **stopped_at, 553 unsigned long trace_ip) 554 { 555 struct bch_fs *c = trans->c; 556 struct btree_insert_entry *i; 557 struct btree_write_buffered_key *wb; 558 struct btree_trans_commit_hook *h; 559 unsigned u64s = 0; 560 int ret; 561 562 if (race_fault()) { 563 trace_and_count(c, trans_restart_fault_inject, trans, trace_ip); 564 return btree_trans_restart_nounlock(trans, BCH_ERR_transaction_restart_fault_inject); 565 } 566 567 /* 568 * Check if the insert will fit in the leaf node with the write lock 569 * held, otherwise another thread could write the node changing the 570 * amount of space available: 571 */ 572 573 prefetch(&trans->c->journal.flags); 574 575 trans_for_each_update(trans, i) { 576 /* Multiple inserts might go to same leaf: */ 577 if (!same_leaf_as_prev(trans, i)) 578 u64s = 0; 579 580 u64s += i->k->k.u64s; 581 ret = !i->cached 582 ? btree_key_can_insert(trans, insert_l(i)->b, u64s) 583 : btree_key_can_insert_cached(trans, flags, i->path, u64s); 584 if (ret) { 585 *stopped_at = i; 586 return ret; 587 } 588 } 589 590 if (trans->nr_wb_updates && 591 trans->nr_wb_updates + c->btree_write_buffer.state.nr > c->btree_write_buffer.size) 592 return -BCH_ERR_btree_insert_need_flush_buffer; 593 594 /* 595 * Don't get journal reservation until after we know insert will 596 * succeed: 597 */ 598 if (likely(!(flags & BTREE_INSERT_JOURNAL_REPLAY))) { 599 ret = bch2_trans_journal_res_get(trans, 600 (flags & BCH_WATERMARK_MASK)| 601 JOURNAL_RES_GET_NONBLOCK); 602 if (ret) 603 return ret; 604 605 if (unlikely(trans->journal_transaction_names)) 606 journal_transaction_name(trans); 607 } else { 608 trans->journal_res.seq = c->journal.replay_journal_seq; 609 } 610 611 /* 612 * Not allowed to fail after we've gotten our journal reservation - we 613 * have to use it: 614 */ 615 616 if (IS_ENABLED(CONFIG_BCACHEFS_DEBUG) && 617 !(flags & BTREE_INSERT_JOURNAL_REPLAY)) { 618 if (bch2_journal_seq_verify) 619 trans_for_each_update(trans, i) 620 i->k->k.version.lo = trans->journal_res.seq; 621 else if (bch2_inject_invalid_keys) 622 trans_for_each_update(trans, i) 623 i->k->k.version = MAX_VERSION; 624 } 625 626 if (trans->fs_usage_deltas && 627 bch2_trans_fs_usage_apply(trans, trans->fs_usage_deltas)) 628 return -BCH_ERR_btree_insert_need_mark_replicas; 629 630 if (trans->nr_wb_updates) { 631 EBUG_ON(flags & BTREE_INSERT_JOURNAL_REPLAY); 632 633 ret = bch2_btree_insert_keys_write_buffer(trans); 634 if (ret) 635 goto revert_fs_usage; 636 } 637 638 h = trans->hooks; 639 while (h) { 640 ret = h->fn(trans, h); 641 if (ret) 642 goto revert_fs_usage; 643 h = h->next; 644 } 645 646 trans_for_each_update(trans, i) 647 if (BTREE_NODE_TYPE_HAS_MEM_TRIGGERS & (1U << i->bkey_type)) { 648 ret = run_one_mem_trigger(trans, i, i->flags); 649 if (ret) 650 goto fatal_err; 651 } 652 653 if (unlikely(c->gc_pos.phase)) { 654 ret = bch2_trans_commit_run_gc_triggers(trans); 655 if (ret) 656 goto fatal_err; 657 } 658 659 if (unlikely(trans->extra_journal_entries.nr)) { 660 memcpy_u64s_small(journal_res_entry(&c->journal, &trans->journal_res), 661 trans->extra_journal_entries.data, 662 trans->extra_journal_entries.nr); 663 664 trans->journal_res.offset += trans->extra_journal_entries.nr; 665 trans->journal_res.u64s -= trans->extra_journal_entries.nr; 666 } 667 668 if (likely(!(flags & BTREE_INSERT_JOURNAL_REPLAY))) { 669 struct journal *j = &c->journal; 670 struct jset_entry *entry; 671 672 trans_for_each_update(trans, i) { 673 if (i->key_cache_already_flushed) 674 continue; 675 676 if (i->flags & BTREE_UPDATE_NOJOURNAL) 677 continue; 678 679 verify_update_old_key(trans, i); 680 681 if (trans->journal_transaction_names) { 682 entry = bch2_journal_add_entry(j, &trans->journal_res, 683 BCH_JSET_ENTRY_overwrite, 684 i->btree_id, i->level, 685 i->old_k.u64s); 686 bkey_reassemble(&entry->start[0], 687 (struct bkey_s_c) { &i->old_k, i->old_v }); 688 } 689 690 entry = bch2_journal_add_entry(j, &trans->journal_res, 691 BCH_JSET_ENTRY_btree_keys, 692 i->btree_id, i->level, 693 i->k->k.u64s); 694 bkey_copy(&entry->start[0], i->k); 695 } 696 697 trans_for_each_wb_update(trans, wb) { 698 entry = bch2_journal_add_entry(j, &trans->journal_res, 699 BCH_JSET_ENTRY_btree_keys, 700 wb->btree, 0, 701 wb->k.k.u64s); 702 bkey_copy(&entry->start[0], &wb->k); 703 } 704 705 if (trans->journal_seq) 706 *trans->journal_seq = trans->journal_res.seq; 707 } 708 709 trans_for_each_update(trans, i) { 710 i->k->k.needs_whiteout = false; 711 712 if (!i->cached) { 713 u64 seq = trans->journal_res.seq; 714 715 if (i->flags & BTREE_UPDATE_PREJOURNAL) 716 seq = i->seq; 717 718 bch2_btree_insert_key_leaf(trans, i->path, i->k, seq); 719 } else if (!i->key_cache_already_flushed) 720 bch2_btree_insert_key_cached(trans, flags, i); 721 else { 722 bch2_btree_key_cache_drop(trans, i->path); 723 btree_path_set_dirty(i->path, BTREE_ITER_NEED_TRAVERSE); 724 } 725 } 726 727 return 0; 728 fatal_err: 729 bch2_fatal_error(c); 730 revert_fs_usage: 731 if (trans->fs_usage_deltas) 732 bch2_trans_fs_usage_revert(trans, trans->fs_usage_deltas); 733 return ret; 734 } 735 736 static noinline int trans_lock_write_fail(struct btree_trans *trans, struct btree_insert_entry *i) 737 { 738 while (--i >= trans->updates) { 739 if (same_leaf_as_prev(trans, i)) 740 continue; 741 742 bch2_btree_node_unlock_write(trans, i->path, insert_l(i)->b); 743 } 744 745 trace_and_count(trans->c, trans_restart_would_deadlock_write, trans); 746 return btree_trans_restart(trans, BCH_ERR_transaction_restart_would_deadlock_write); 747 } 748 749 static inline int trans_lock_write(struct btree_trans *trans) 750 { 751 struct btree_insert_entry *i; 752 753 trans_for_each_update(trans, i) { 754 if (same_leaf_as_prev(trans, i)) 755 continue; 756 757 if (bch2_btree_node_lock_write(trans, i->path, &insert_l(i)->b->c)) 758 return trans_lock_write_fail(trans, i); 759 760 if (!i->cached) 761 bch2_btree_node_prep_for_write(trans, i->path, insert_l(i)->b); 762 } 763 764 return 0; 765 } 766 767 static noinline void bch2_drop_overwrites_from_journal(struct btree_trans *trans) 768 { 769 struct btree_insert_entry *i; 770 struct btree_write_buffered_key *wb; 771 772 trans_for_each_update(trans, i) 773 bch2_journal_key_overwritten(trans->c, i->btree_id, i->level, i->k->k.p); 774 775 trans_for_each_wb_update(trans, wb) 776 bch2_journal_key_overwritten(trans->c, wb->btree, 0, wb->k.k.p); 777 } 778 779 static noinline int bch2_trans_commit_bkey_invalid(struct btree_trans *trans, unsigned flags, 780 struct btree_insert_entry *i, 781 struct printbuf *err) 782 { 783 struct bch_fs *c = trans->c; 784 int rw = (flags & BTREE_INSERT_JOURNAL_REPLAY) ? READ : WRITE; 785 786 printbuf_reset(err); 787 prt_printf(err, "invalid bkey on insert from %s -> %ps", 788 trans->fn, (void *) i->ip_allocated); 789 prt_newline(err); 790 printbuf_indent_add(err, 2); 791 792 bch2_bkey_val_to_text(err, c, bkey_i_to_s_c(i->k)); 793 prt_newline(err); 794 795 bch2_bkey_invalid(c, bkey_i_to_s_c(i->k), 796 i->bkey_type, rw, err); 797 bch2_print_string_as_lines(KERN_ERR, err->buf); 798 799 bch2_inconsistent_error(c); 800 bch2_dump_trans_updates(trans); 801 802 return -EINVAL; 803 } 804 805 /* 806 * Get journal reservation, take write locks, and attempt to do btree update(s): 807 */ 808 static inline int do_bch2_trans_commit(struct btree_trans *trans, unsigned flags, 809 struct btree_insert_entry **stopped_at, 810 unsigned long trace_ip) 811 { 812 struct bch_fs *c = trans->c; 813 struct btree_insert_entry *i; 814 int ret = 0, u64s_delta = 0; 815 816 trans_for_each_update(trans, i) { 817 if (i->cached) 818 continue; 819 820 u64s_delta += !bkey_deleted(&i->k->k) ? i->k->k.u64s : 0; 821 u64s_delta -= i->old_btree_u64s; 822 823 if (!same_leaf_as_next(trans, i)) { 824 if (u64s_delta <= 0) { 825 ret = bch2_foreground_maybe_merge(trans, i->path, 826 i->level, flags); 827 if (unlikely(ret)) 828 return ret; 829 } 830 831 u64s_delta = 0; 832 } 833 } 834 835 ret = bch2_journal_preres_get(&c->journal, 836 &trans->journal_preres, trans->journal_preres_u64s, 837 (flags & BCH_WATERMARK_MASK)|JOURNAL_RES_GET_NONBLOCK); 838 if (unlikely(ret == -BCH_ERR_journal_preres_get_blocked)) 839 ret = bch2_trans_journal_preres_get_cold(trans, flags, trace_ip); 840 if (unlikely(ret)) 841 return ret; 842 843 ret = trans_lock_write(trans); 844 if (unlikely(ret)) 845 return ret; 846 847 ret = bch2_trans_commit_write_locked(trans, flags, stopped_at, trace_ip); 848 849 if (!ret && unlikely(trans->journal_replay_not_finished)) 850 bch2_drop_overwrites_from_journal(trans); 851 852 trans_for_each_update(trans, i) 853 if (!same_leaf_as_prev(trans, i)) 854 bch2_btree_node_unlock_write_inlined(trans, i->path, 855 insert_l(i)->b); 856 857 if (!ret && trans->journal_pin) 858 bch2_journal_pin_add(&c->journal, trans->journal_res.seq, 859 trans->journal_pin, NULL); 860 861 /* 862 * Drop journal reservation after dropping write locks, since dropping 863 * the journal reservation may kick off a journal write: 864 */ 865 bch2_journal_res_put(&c->journal, &trans->journal_res); 866 867 if (unlikely(ret)) 868 return ret; 869 870 bch2_trans_downgrade(trans); 871 872 return 0; 873 } 874 875 static int journal_reclaim_wait_done(struct bch_fs *c) 876 { 877 int ret = bch2_journal_error(&c->journal) ?: 878 !bch2_btree_key_cache_must_wait(c); 879 880 if (!ret) 881 journal_reclaim_kick(&c->journal); 882 return ret; 883 } 884 885 static noinline 886 int bch2_trans_commit_error(struct btree_trans *trans, unsigned flags, 887 struct btree_insert_entry *i, 888 int ret, unsigned long trace_ip) 889 { 890 struct bch_fs *c = trans->c; 891 892 switch (ret) { 893 case -BCH_ERR_btree_insert_btree_node_full: 894 ret = bch2_btree_split_leaf(trans, i->path, flags); 895 if (bch2_err_matches(ret, BCH_ERR_transaction_restart)) 896 trace_and_count(c, trans_restart_btree_node_split, trans, trace_ip, i->path); 897 break; 898 case -BCH_ERR_btree_insert_need_mark_replicas: 899 ret = drop_locks_do(trans, 900 bch2_replicas_delta_list_mark(c, trans->fs_usage_deltas)); 901 break; 902 case -BCH_ERR_journal_res_get_blocked: 903 /* 904 * XXX: this should probably be a separate BTREE_INSERT_NONBLOCK 905 * flag 906 */ 907 if ((flags & BTREE_INSERT_JOURNAL_RECLAIM) && 908 (flags & BCH_WATERMARK_MASK) != BCH_WATERMARK_reclaim) { 909 ret = -BCH_ERR_journal_reclaim_would_deadlock; 910 break; 911 } 912 913 ret = drop_locks_do(trans, 914 bch2_trans_journal_res_get(trans, 915 (flags & BCH_WATERMARK_MASK)| 916 JOURNAL_RES_GET_CHECK)); 917 break; 918 case -BCH_ERR_btree_insert_need_journal_reclaim: 919 bch2_trans_unlock(trans); 920 921 trace_and_count(c, trans_blocked_journal_reclaim, trans, trace_ip); 922 923 wait_event_freezable(c->journal.reclaim_wait, 924 (ret = journal_reclaim_wait_done(c))); 925 if (ret < 0) 926 break; 927 928 ret = bch2_trans_relock(trans); 929 break; 930 case -BCH_ERR_btree_insert_need_flush_buffer: { 931 struct btree_write_buffer *wb = &c->btree_write_buffer; 932 933 ret = 0; 934 935 if (wb->state.nr > wb->size * 3 / 4) { 936 bch2_trans_unlock(trans); 937 mutex_lock(&wb->flush_lock); 938 939 if (wb->state.nr > wb->size * 3 / 4) { 940 bch2_trans_begin(trans); 941 ret = __bch2_btree_write_buffer_flush(trans, 942 flags|BTREE_INSERT_NOCHECK_RW, true); 943 if (!ret) { 944 trace_and_count(c, trans_restart_write_buffer_flush, trans, _THIS_IP_); 945 ret = btree_trans_restart(trans, BCH_ERR_transaction_restart_write_buffer_flush); 946 } 947 } else { 948 mutex_unlock(&wb->flush_lock); 949 ret = bch2_trans_relock(trans); 950 } 951 } 952 break; 953 } 954 default: 955 BUG_ON(ret >= 0); 956 break; 957 } 958 959 BUG_ON(bch2_err_matches(ret, BCH_ERR_transaction_restart) != !!trans->restarted); 960 961 bch2_fs_inconsistent_on(bch2_err_matches(ret, ENOSPC) && 962 !(flags & BTREE_INSERT_NOWAIT) && 963 (flags & BTREE_INSERT_NOFAIL), c, 964 "%s: incorrectly got %s\n", __func__, bch2_err_str(ret)); 965 966 return ret; 967 } 968 969 static noinline int 970 bch2_trans_commit_get_rw_cold(struct btree_trans *trans, unsigned flags) 971 { 972 struct bch_fs *c = trans->c; 973 int ret; 974 975 if (likely(!(flags & BTREE_INSERT_LAZY_RW)) || 976 test_bit(BCH_FS_STARTED, &c->flags)) 977 return -BCH_ERR_erofs_trans_commit; 978 979 ret = drop_locks_do(trans, bch2_fs_read_write_early(c)); 980 if (ret) 981 return ret; 982 983 bch2_write_ref_get(c, BCH_WRITE_REF_trans); 984 return 0; 985 } 986 987 /* 988 * This is for updates done in the early part of fsck - btree_gc - before we've 989 * gone RW. we only add the new key to the list of keys for journal replay to 990 * do. 991 */ 992 static noinline int 993 do_bch2_trans_commit_to_journal_replay(struct btree_trans *trans) 994 { 995 struct bch_fs *c = trans->c; 996 struct btree_insert_entry *i; 997 int ret = 0; 998 999 trans_for_each_update(trans, i) { 1000 ret = bch2_journal_key_insert(c, i->btree_id, i->level, i->k); 1001 if (ret) 1002 break; 1003 } 1004 1005 return ret; 1006 } 1007 1008 int __bch2_trans_commit(struct btree_trans *trans, unsigned flags) 1009 { 1010 struct bch_fs *c = trans->c; 1011 struct btree_insert_entry *i = NULL; 1012 struct btree_write_buffered_key *wb; 1013 unsigned u64s; 1014 int ret = 0; 1015 1016 if (!trans->nr_updates && 1017 !trans->nr_wb_updates && 1018 !trans->extra_journal_entries.nr) 1019 goto out_reset; 1020 1021 if (flags & BTREE_INSERT_GC_LOCK_HELD) 1022 lockdep_assert_held(&c->gc_lock); 1023 1024 ret = bch2_trans_commit_run_triggers(trans); 1025 if (ret) 1026 goto out_reset; 1027 1028 trans_for_each_update(trans, i) { 1029 struct printbuf buf = PRINTBUF; 1030 enum bkey_invalid_flags invalid_flags = 0; 1031 1032 if (!(flags & BTREE_INSERT_JOURNAL_REPLAY)) 1033 invalid_flags |= BKEY_INVALID_WRITE|BKEY_INVALID_COMMIT; 1034 1035 if (unlikely(bch2_bkey_invalid(c, bkey_i_to_s_c(i->k), 1036 i->bkey_type, invalid_flags, &buf))) 1037 ret = bch2_trans_commit_bkey_invalid(trans, flags, i, &buf); 1038 btree_insert_entry_checks(trans, i); 1039 printbuf_exit(&buf); 1040 1041 if (ret) 1042 return ret; 1043 } 1044 1045 if (unlikely(!test_bit(BCH_FS_MAY_GO_RW, &c->flags))) { 1046 ret = do_bch2_trans_commit_to_journal_replay(trans); 1047 goto out_reset; 1048 } 1049 1050 if (!(flags & BTREE_INSERT_NOCHECK_RW) && 1051 unlikely(!bch2_write_ref_tryget(c, BCH_WRITE_REF_trans))) { 1052 ret = bch2_trans_commit_get_rw_cold(trans, flags); 1053 if (ret) 1054 goto out_reset; 1055 } 1056 1057 if (c->btree_write_buffer.state.nr > c->btree_write_buffer.size / 2 && 1058 mutex_trylock(&c->btree_write_buffer.flush_lock)) { 1059 bch2_trans_begin(trans); 1060 bch2_trans_unlock(trans); 1061 1062 ret = __bch2_btree_write_buffer_flush(trans, 1063 flags|BTREE_INSERT_NOCHECK_RW, true); 1064 if (!ret) { 1065 trace_and_count(c, trans_restart_write_buffer_flush, trans, _THIS_IP_); 1066 ret = btree_trans_restart(trans, BCH_ERR_transaction_restart_write_buffer_flush); 1067 } 1068 goto out; 1069 } 1070 1071 EBUG_ON(test_bit(BCH_FS_CLEAN_SHUTDOWN, &c->flags)); 1072 1073 memset(&trans->journal_preres, 0, sizeof(trans->journal_preres)); 1074 1075 trans->journal_u64s = trans->extra_journal_entries.nr; 1076 trans->journal_preres_u64s = 0; 1077 1078 trans->journal_transaction_names = READ_ONCE(c->opts.journal_transaction_names); 1079 1080 if (trans->journal_transaction_names) 1081 trans->journal_u64s += jset_u64s(JSET_ENTRY_LOG_U64s); 1082 1083 trans_for_each_update(trans, i) { 1084 EBUG_ON(!i->path->should_be_locked); 1085 1086 ret = bch2_btree_path_upgrade(trans, i->path, i->level + 1); 1087 if (unlikely(ret)) 1088 goto out; 1089 1090 EBUG_ON(!btree_node_intent_locked(i->path, i->level)); 1091 1092 if (i->key_cache_already_flushed) 1093 continue; 1094 1095 /* we're going to journal the key being updated: */ 1096 u64s = jset_u64s(i->k->k.u64s); 1097 if (i->cached && 1098 likely(!(flags & BTREE_INSERT_JOURNAL_REPLAY))) 1099 trans->journal_preres_u64s += u64s; 1100 1101 if (i->flags & BTREE_UPDATE_NOJOURNAL) 1102 continue; 1103 1104 trans->journal_u64s += u64s; 1105 1106 /* and we're also going to log the overwrite: */ 1107 if (trans->journal_transaction_names) 1108 trans->journal_u64s += jset_u64s(i->old_k.u64s); 1109 } 1110 1111 trans_for_each_wb_update(trans, wb) 1112 trans->journal_u64s += jset_u64s(wb->k.k.u64s); 1113 1114 if (trans->extra_journal_res) { 1115 ret = bch2_disk_reservation_add(c, trans->disk_res, 1116 trans->extra_journal_res, 1117 (flags & BTREE_INSERT_NOFAIL) 1118 ? BCH_DISK_RESERVATION_NOFAIL : 0); 1119 if (ret) 1120 goto err; 1121 } 1122 retry: 1123 bch2_trans_verify_not_in_restart(trans); 1124 memset(&trans->journal_res, 0, sizeof(trans->journal_res)); 1125 1126 ret = do_bch2_trans_commit(trans, flags, &i, _RET_IP_); 1127 1128 /* make sure we didn't drop or screw up locks: */ 1129 bch2_trans_verify_locks(trans); 1130 1131 if (ret) 1132 goto err; 1133 1134 trace_and_count(c, transaction_commit, trans, _RET_IP_); 1135 out: 1136 bch2_journal_preres_put(&c->journal, &trans->journal_preres); 1137 1138 if (likely(!(flags & BTREE_INSERT_NOCHECK_RW))) 1139 bch2_write_ref_put(c, BCH_WRITE_REF_trans); 1140 out_reset: 1141 bch2_trans_reset_updates(trans); 1142 1143 return ret; 1144 err: 1145 ret = bch2_trans_commit_error(trans, flags, i, ret, _RET_IP_); 1146 if (ret) 1147 goto out; 1148 1149 goto retry; 1150 } 1151