1 // SPDX-License-Identifier: GPL-2.0 2 #include "bcachefs.h" 3 #include "alloc_background.h" 4 #include "alloc_foreground.h" 5 #include "backpointers.h" 6 #include "bkey_buf.h" 7 #include "btree_cache.h" 8 #include "btree_io.h" 9 #include "btree_key_cache.h" 10 #include "btree_update.h" 11 #include "btree_update_interior.h" 12 #include "btree_gc.h" 13 #include "btree_write_buffer.h" 14 #include "buckets.h" 15 #include "buckets_waiting_for_journal.h" 16 #include "clock.h" 17 #include "debug.h" 18 #include "disk_accounting.h" 19 #include "ec.h" 20 #include "error.h" 21 #include "lru.h" 22 #include "recovery.h" 23 #include "trace.h" 24 #include "varint.h" 25 26 #include <linux/kthread.h> 27 #include <linux/math64.h> 28 #include <linux/random.h> 29 #include <linux/rculist.h> 30 #include <linux/rcupdate.h> 31 #include <linux/sched/task.h> 32 #include <linux/sort.h> 33 #include <linux/jiffies.h> 34 35 static void bch2_discard_one_bucket_fast(struct bch_dev *, u64); 36 37 /* Persistent alloc info: */ 38 39 static const unsigned BCH_ALLOC_V1_FIELD_BYTES[] = { 40 #define x(name, bits) [BCH_ALLOC_FIELD_V1_##name] = bits / 8, 41 BCH_ALLOC_FIELDS_V1() 42 #undef x 43 }; 44 45 struct bkey_alloc_unpacked { 46 u64 journal_seq; 47 u8 gen; 48 u8 oldest_gen; 49 u8 data_type; 50 bool need_discard:1; 51 bool need_inc_gen:1; 52 #define x(_name, _bits) u##_bits _name; 53 BCH_ALLOC_FIELDS_V2() 54 #undef x 55 }; 56 57 static inline u64 alloc_field_v1_get(const struct bch_alloc *a, 58 const void **p, unsigned field) 59 { 60 unsigned bytes = BCH_ALLOC_V1_FIELD_BYTES[field]; 61 u64 v; 62 63 if (!(a->fields & (1 << field))) 64 return 0; 65 66 switch (bytes) { 67 case 1: 68 v = *((const u8 *) *p); 69 break; 70 case 2: 71 v = le16_to_cpup(*p); 72 break; 73 case 4: 74 v = le32_to_cpup(*p); 75 break; 76 case 8: 77 v = le64_to_cpup(*p); 78 break; 79 default: 80 BUG(); 81 } 82 83 *p += bytes; 84 return v; 85 } 86 87 static void bch2_alloc_unpack_v1(struct bkey_alloc_unpacked *out, 88 struct bkey_s_c k) 89 { 90 const struct bch_alloc *in = bkey_s_c_to_alloc(k).v; 91 const void *d = in->data; 92 unsigned idx = 0; 93 94 out->gen = in->gen; 95 96 #define x(_name, _bits) out->_name = alloc_field_v1_get(in, &d, idx++); 97 BCH_ALLOC_FIELDS_V1() 98 #undef x 99 } 100 101 static int bch2_alloc_unpack_v2(struct bkey_alloc_unpacked *out, 102 struct bkey_s_c k) 103 { 104 struct bkey_s_c_alloc_v2 a = bkey_s_c_to_alloc_v2(k); 105 const u8 *in = a.v->data; 106 const u8 *end = bkey_val_end(a); 107 unsigned fieldnr = 0; 108 int ret; 109 u64 v; 110 111 out->gen = a.v->gen; 112 out->oldest_gen = a.v->oldest_gen; 113 out->data_type = a.v->data_type; 114 115 #define x(_name, _bits) \ 116 if (fieldnr < a.v->nr_fields) { \ 117 ret = bch2_varint_decode_fast(in, end, &v); \ 118 if (ret < 0) \ 119 return ret; \ 120 in += ret; \ 121 } else { \ 122 v = 0; \ 123 } \ 124 out->_name = v; \ 125 if (v != out->_name) \ 126 return -1; \ 127 fieldnr++; 128 129 BCH_ALLOC_FIELDS_V2() 130 #undef x 131 return 0; 132 } 133 134 static int bch2_alloc_unpack_v3(struct bkey_alloc_unpacked *out, 135 struct bkey_s_c k) 136 { 137 struct bkey_s_c_alloc_v3 a = bkey_s_c_to_alloc_v3(k); 138 const u8 *in = a.v->data; 139 const u8 *end = bkey_val_end(a); 140 unsigned fieldnr = 0; 141 int ret; 142 u64 v; 143 144 out->gen = a.v->gen; 145 out->oldest_gen = a.v->oldest_gen; 146 out->data_type = a.v->data_type; 147 out->need_discard = BCH_ALLOC_V3_NEED_DISCARD(a.v); 148 out->need_inc_gen = BCH_ALLOC_V3_NEED_INC_GEN(a.v); 149 out->journal_seq = le64_to_cpu(a.v->journal_seq); 150 151 #define x(_name, _bits) \ 152 if (fieldnr < a.v->nr_fields) { \ 153 ret = bch2_varint_decode_fast(in, end, &v); \ 154 if (ret < 0) \ 155 return ret; \ 156 in += ret; \ 157 } else { \ 158 v = 0; \ 159 } \ 160 out->_name = v; \ 161 if (v != out->_name) \ 162 return -1; \ 163 fieldnr++; 164 165 BCH_ALLOC_FIELDS_V2() 166 #undef x 167 return 0; 168 } 169 170 static struct bkey_alloc_unpacked bch2_alloc_unpack(struct bkey_s_c k) 171 { 172 struct bkey_alloc_unpacked ret = { .gen = 0 }; 173 174 switch (k.k->type) { 175 case KEY_TYPE_alloc: 176 bch2_alloc_unpack_v1(&ret, k); 177 break; 178 case KEY_TYPE_alloc_v2: 179 bch2_alloc_unpack_v2(&ret, k); 180 break; 181 case KEY_TYPE_alloc_v3: 182 bch2_alloc_unpack_v3(&ret, k); 183 break; 184 } 185 186 return ret; 187 } 188 189 static unsigned bch_alloc_v1_val_u64s(const struct bch_alloc *a) 190 { 191 unsigned i, bytes = offsetof(struct bch_alloc, data); 192 193 for (i = 0; i < ARRAY_SIZE(BCH_ALLOC_V1_FIELD_BYTES); i++) 194 if (a->fields & (1 << i)) 195 bytes += BCH_ALLOC_V1_FIELD_BYTES[i]; 196 197 return DIV_ROUND_UP(bytes, sizeof(u64)); 198 } 199 200 int bch2_alloc_v1_validate(struct bch_fs *c, struct bkey_s_c k, 201 enum bch_validate_flags flags) 202 { 203 struct bkey_s_c_alloc a = bkey_s_c_to_alloc(k); 204 int ret = 0; 205 206 /* allow for unknown fields */ 207 bkey_fsck_err_on(bkey_val_u64s(a.k) < bch_alloc_v1_val_u64s(a.v), 208 c, alloc_v1_val_size_bad, 209 "incorrect value size (%zu < %u)", 210 bkey_val_u64s(a.k), bch_alloc_v1_val_u64s(a.v)); 211 fsck_err: 212 return ret; 213 } 214 215 int bch2_alloc_v2_validate(struct bch_fs *c, struct bkey_s_c k, 216 enum bch_validate_flags flags) 217 { 218 struct bkey_alloc_unpacked u; 219 int ret = 0; 220 221 bkey_fsck_err_on(bch2_alloc_unpack_v2(&u, k), 222 c, alloc_v2_unpack_error, 223 "unpack error"); 224 fsck_err: 225 return ret; 226 } 227 228 int bch2_alloc_v3_validate(struct bch_fs *c, struct bkey_s_c k, 229 enum bch_validate_flags flags) 230 { 231 struct bkey_alloc_unpacked u; 232 int ret = 0; 233 234 bkey_fsck_err_on(bch2_alloc_unpack_v3(&u, k), 235 c, alloc_v2_unpack_error, 236 "unpack error"); 237 fsck_err: 238 return ret; 239 } 240 241 int bch2_alloc_v4_validate(struct bch_fs *c, struct bkey_s_c k, 242 enum bch_validate_flags flags) 243 { 244 struct bch_alloc_v4 a; 245 int ret = 0; 246 247 bkey_val_copy(&a, bkey_s_c_to_alloc_v4(k)); 248 249 bkey_fsck_err_on(alloc_v4_u64s_noerror(&a) > bkey_val_u64s(k.k), 250 c, alloc_v4_val_size_bad, 251 "bad val size (%u > %zu)", 252 alloc_v4_u64s_noerror(&a), bkey_val_u64s(k.k)); 253 254 bkey_fsck_err_on(!BCH_ALLOC_V4_BACKPOINTERS_START(&a) && 255 BCH_ALLOC_V4_NR_BACKPOINTERS(&a), 256 c, alloc_v4_backpointers_start_bad, 257 "invalid backpointers_start"); 258 259 bkey_fsck_err_on(alloc_data_type(a, a.data_type) != a.data_type, 260 c, alloc_key_data_type_bad, 261 "invalid data type (got %u should be %u)", 262 a.data_type, alloc_data_type(a, a.data_type)); 263 264 for (unsigned i = 0; i < 2; i++) 265 bkey_fsck_err_on(a.io_time[i] > LRU_TIME_MAX, 266 c, alloc_key_io_time_bad, 267 "invalid io_time[%s]: %llu, max %llu", 268 i == READ ? "read" : "write", 269 a.io_time[i], LRU_TIME_MAX); 270 271 unsigned stripe_sectors = BCH_ALLOC_V4_BACKPOINTERS_START(&a) * sizeof(u64) > 272 offsetof(struct bch_alloc_v4, stripe_sectors) 273 ? a.stripe_sectors 274 : 0; 275 276 switch (a.data_type) { 277 case BCH_DATA_free: 278 case BCH_DATA_need_gc_gens: 279 case BCH_DATA_need_discard: 280 bkey_fsck_err_on(stripe_sectors || 281 a.dirty_sectors || 282 a.cached_sectors || 283 a.stripe, 284 c, alloc_key_empty_but_have_data, 285 "empty data type free but have data %u.%u.%u %u", 286 stripe_sectors, 287 a.dirty_sectors, 288 a.cached_sectors, 289 a.stripe); 290 break; 291 case BCH_DATA_sb: 292 case BCH_DATA_journal: 293 case BCH_DATA_btree: 294 case BCH_DATA_user: 295 case BCH_DATA_parity: 296 bkey_fsck_err_on(!a.dirty_sectors && 297 !stripe_sectors, 298 c, alloc_key_dirty_sectors_0, 299 "data_type %s but dirty_sectors==0", 300 bch2_data_type_str(a.data_type)); 301 break; 302 case BCH_DATA_cached: 303 bkey_fsck_err_on(!a.cached_sectors || 304 a.dirty_sectors || 305 stripe_sectors || 306 a.stripe, 307 c, alloc_key_cached_inconsistency, 308 "data type inconsistency"); 309 310 bkey_fsck_err_on(!a.io_time[READ] && 311 c->curr_recovery_pass > BCH_RECOVERY_PASS_check_alloc_to_lru_refs, 312 c, alloc_key_cached_but_read_time_zero, 313 "cached bucket with read_time == 0"); 314 break; 315 case BCH_DATA_stripe: 316 break; 317 } 318 fsck_err: 319 return ret; 320 } 321 322 void bch2_alloc_v4_swab(struct bkey_s k) 323 { 324 struct bch_alloc_v4 *a = bkey_s_to_alloc_v4(k).v; 325 struct bch_backpointer *bp, *bps; 326 327 a->journal_seq = swab64(a->journal_seq); 328 a->flags = swab32(a->flags); 329 a->dirty_sectors = swab32(a->dirty_sectors); 330 a->cached_sectors = swab32(a->cached_sectors); 331 a->io_time[0] = swab64(a->io_time[0]); 332 a->io_time[1] = swab64(a->io_time[1]); 333 a->stripe = swab32(a->stripe); 334 a->nr_external_backpointers = swab32(a->nr_external_backpointers); 335 a->stripe_sectors = swab32(a->stripe_sectors); 336 337 bps = alloc_v4_backpointers(a); 338 for (bp = bps; bp < bps + BCH_ALLOC_V4_NR_BACKPOINTERS(a); bp++) { 339 bp->bucket_offset = swab40(bp->bucket_offset); 340 bp->bucket_len = swab32(bp->bucket_len); 341 bch2_bpos_swab(&bp->pos); 342 } 343 } 344 345 void bch2_alloc_to_text(struct printbuf *out, struct bch_fs *c, struct bkey_s_c k) 346 { 347 struct bch_alloc_v4 _a; 348 const struct bch_alloc_v4 *a = bch2_alloc_to_v4(k, &_a); 349 struct bch_dev *ca = c ? bch2_dev_bucket_tryget_noerror(c, k.k->p) : NULL; 350 351 prt_newline(out); 352 printbuf_indent_add(out, 2); 353 354 prt_printf(out, "gen %u oldest_gen %u data_type ", a->gen, a->oldest_gen); 355 bch2_prt_data_type(out, a->data_type); 356 prt_newline(out); 357 prt_printf(out, "journal_seq %llu\n", a->journal_seq); 358 prt_printf(out, "need_discard %llu\n", BCH_ALLOC_V4_NEED_DISCARD(a)); 359 prt_printf(out, "need_inc_gen %llu\n", BCH_ALLOC_V4_NEED_INC_GEN(a)); 360 prt_printf(out, "dirty_sectors %u\n", a->dirty_sectors); 361 prt_printf(out, "stripe_sectors %u\n", a->stripe_sectors); 362 prt_printf(out, "cached_sectors %u\n", a->cached_sectors); 363 prt_printf(out, "stripe %u\n", a->stripe); 364 prt_printf(out, "stripe_redundancy %u\n", a->stripe_redundancy); 365 prt_printf(out, "io_time[READ] %llu\n", a->io_time[READ]); 366 prt_printf(out, "io_time[WRITE] %llu\n", a->io_time[WRITE]); 367 368 if (ca) 369 prt_printf(out, "fragmentation %llu\n", alloc_lru_idx_fragmentation(*a, ca)); 370 prt_printf(out, "bp_start %llu\n", BCH_ALLOC_V4_BACKPOINTERS_START(a)); 371 printbuf_indent_sub(out, 2); 372 373 bch2_dev_put(ca); 374 } 375 376 void __bch2_alloc_to_v4(struct bkey_s_c k, struct bch_alloc_v4 *out) 377 { 378 if (k.k->type == KEY_TYPE_alloc_v4) { 379 void *src, *dst; 380 381 *out = *bkey_s_c_to_alloc_v4(k).v; 382 383 src = alloc_v4_backpointers(out); 384 SET_BCH_ALLOC_V4_BACKPOINTERS_START(out, BCH_ALLOC_V4_U64s); 385 dst = alloc_v4_backpointers(out); 386 387 if (src < dst) 388 memset(src, 0, dst - src); 389 390 SET_BCH_ALLOC_V4_NR_BACKPOINTERS(out, 0); 391 } else { 392 struct bkey_alloc_unpacked u = bch2_alloc_unpack(k); 393 394 *out = (struct bch_alloc_v4) { 395 .journal_seq = u.journal_seq, 396 .flags = u.need_discard, 397 .gen = u.gen, 398 .oldest_gen = u.oldest_gen, 399 .data_type = u.data_type, 400 .stripe_redundancy = u.stripe_redundancy, 401 .dirty_sectors = u.dirty_sectors, 402 .cached_sectors = u.cached_sectors, 403 .io_time[READ] = u.read_time, 404 .io_time[WRITE] = u.write_time, 405 .stripe = u.stripe, 406 }; 407 408 SET_BCH_ALLOC_V4_BACKPOINTERS_START(out, BCH_ALLOC_V4_U64s); 409 } 410 } 411 412 static noinline struct bkey_i_alloc_v4 * 413 __bch2_alloc_to_v4_mut(struct btree_trans *trans, struct bkey_s_c k) 414 { 415 struct bkey_i_alloc_v4 *ret; 416 417 ret = bch2_trans_kmalloc(trans, max(bkey_bytes(k.k), sizeof(struct bkey_i_alloc_v4))); 418 if (IS_ERR(ret)) 419 return ret; 420 421 if (k.k->type == KEY_TYPE_alloc_v4) { 422 void *src, *dst; 423 424 bkey_reassemble(&ret->k_i, k); 425 426 src = alloc_v4_backpointers(&ret->v); 427 SET_BCH_ALLOC_V4_BACKPOINTERS_START(&ret->v, BCH_ALLOC_V4_U64s); 428 dst = alloc_v4_backpointers(&ret->v); 429 430 if (src < dst) 431 memset(src, 0, dst - src); 432 433 SET_BCH_ALLOC_V4_NR_BACKPOINTERS(&ret->v, 0); 434 set_alloc_v4_u64s(ret); 435 } else { 436 bkey_alloc_v4_init(&ret->k_i); 437 ret->k.p = k.k->p; 438 bch2_alloc_to_v4(k, &ret->v); 439 } 440 return ret; 441 } 442 443 static inline struct bkey_i_alloc_v4 *bch2_alloc_to_v4_mut_inlined(struct btree_trans *trans, struct bkey_s_c k) 444 { 445 struct bkey_s_c_alloc_v4 a; 446 447 if (likely(k.k->type == KEY_TYPE_alloc_v4) && 448 ((a = bkey_s_c_to_alloc_v4(k), true) && 449 BCH_ALLOC_V4_NR_BACKPOINTERS(a.v) == 0)) 450 return bch2_bkey_make_mut_noupdate_typed(trans, k, alloc_v4); 451 452 return __bch2_alloc_to_v4_mut(trans, k); 453 } 454 455 struct bkey_i_alloc_v4 *bch2_alloc_to_v4_mut(struct btree_trans *trans, struct bkey_s_c k) 456 { 457 return bch2_alloc_to_v4_mut_inlined(trans, k); 458 } 459 460 struct bkey_i_alloc_v4 * 461 bch2_trans_start_alloc_update_noupdate(struct btree_trans *trans, struct btree_iter *iter, 462 struct bpos pos) 463 { 464 struct bkey_s_c k = bch2_bkey_get_iter(trans, iter, BTREE_ID_alloc, pos, 465 BTREE_ITER_with_updates| 466 BTREE_ITER_cached| 467 BTREE_ITER_intent); 468 int ret = bkey_err(k); 469 if (unlikely(ret)) 470 return ERR_PTR(ret); 471 472 struct bkey_i_alloc_v4 *a = bch2_alloc_to_v4_mut_inlined(trans, k); 473 ret = PTR_ERR_OR_ZERO(a); 474 if (unlikely(ret)) 475 goto err; 476 return a; 477 err: 478 bch2_trans_iter_exit(trans, iter); 479 return ERR_PTR(ret); 480 } 481 482 __flatten 483 struct bkey_i_alloc_v4 *bch2_trans_start_alloc_update(struct btree_trans *trans, struct bpos pos, 484 enum btree_iter_update_trigger_flags flags) 485 { 486 struct btree_iter iter; 487 struct bkey_i_alloc_v4 *a = bch2_trans_start_alloc_update_noupdate(trans, &iter, pos); 488 int ret = PTR_ERR_OR_ZERO(a); 489 if (ret) 490 return ERR_PTR(ret); 491 492 ret = bch2_trans_update(trans, &iter, &a->k_i, flags); 493 bch2_trans_iter_exit(trans, &iter); 494 return unlikely(ret) ? ERR_PTR(ret) : a; 495 } 496 497 static struct bpos alloc_gens_pos(struct bpos pos, unsigned *offset) 498 { 499 *offset = pos.offset & KEY_TYPE_BUCKET_GENS_MASK; 500 501 pos.offset >>= KEY_TYPE_BUCKET_GENS_BITS; 502 return pos; 503 } 504 505 static struct bpos bucket_gens_pos_to_alloc(struct bpos pos, unsigned offset) 506 { 507 pos.offset <<= KEY_TYPE_BUCKET_GENS_BITS; 508 pos.offset += offset; 509 return pos; 510 } 511 512 static unsigned alloc_gen(struct bkey_s_c k, unsigned offset) 513 { 514 return k.k->type == KEY_TYPE_bucket_gens 515 ? bkey_s_c_to_bucket_gens(k).v->gens[offset] 516 : 0; 517 } 518 519 int bch2_bucket_gens_validate(struct bch_fs *c, struct bkey_s_c k, 520 enum bch_validate_flags flags) 521 { 522 int ret = 0; 523 524 bkey_fsck_err_on(bkey_val_bytes(k.k) != sizeof(struct bch_bucket_gens), 525 c, bucket_gens_val_size_bad, 526 "bad val size (%zu != %zu)", 527 bkey_val_bytes(k.k), sizeof(struct bch_bucket_gens)); 528 fsck_err: 529 return ret; 530 } 531 532 void bch2_bucket_gens_to_text(struct printbuf *out, struct bch_fs *c, struct bkey_s_c k) 533 { 534 struct bkey_s_c_bucket_gens g = bkey_s_c_to_bucket_gens(k); 535 unsigned i; 536 537 for (i = 0; i < ARRAY_SIZE(g.v->gens); i++) { 538 if (i) 539 prt_char(out, ' '); 540 prt_printf(out, "%u", g.v->gens[i]); 541 } 542 } 543 544 int bch2_bucket_gens_init(struct bch_fs *c) 545 { 546 struct btree_trans *trans = bch2_trans_get(c); 547 struct bkey_i_bucket_gens g; 548 bool have_bucket_gens_key = false; 549 int ret; 550 551 ret = for_each_btree_key(trans, iter, BTREE_ID_alloc, POS_MIN, 552 BTREE_ITER_prefetch, k, ({ 553 /* 554 * Not a fsck error because this is checked/repaired by 555 * bch2_check_alloc_key() which runs later: 556 */ 557 if (!bch2_dev_bucket_exists(c, k.k->p)) 558 continue; 559 560 struct bch_alloc_v4 a; 561 u8 gen = bch2_alloc_to_v4(k, &a)->gen; 562 unsigned offset; 563 struct bpos pos = alloc_gens_pos(iter.pos, &offset); 564 int ret2 = 0; 565 566 if (have_bucket_gens_key && !bkey_eq(g.k.p, pos)) { 567 ret2 = bch2_btree_insert_trans(trans, BTREE_ID_bucket_gens, &g.k_i, 0) ?: 568 bch2_trans_commit(trans, NULL, NULL, BCH_TRANS_COMMIT_no_enospc); 569 if (ret2) 570 goto iter_err; 571 have_bucket_gens_key = false; 572 } 573 574 if (!have_bucket_gens_key) { 575 bkey_bucket_gens_init(&g.k_i); 576 g.k.p = pos; 577 have_bucket_gens_key = true; 578 } 579 580 g.v.gens[offset] = gen; 581 iter_err: 582 ret2; 583 })); 584 585 if (have_bucket_gens_key && !ret) 586 ret = commit_do(trans, NULL, NULL, 587 BCH_TRANS_COMMIT_no_enospc, 588 bch2_btree_insert_trans(trans, BTREE_ID_bucket_gens, &g.k_i, 0)); 589 590 bch2_trans_put(trans); 591 592 bch_err_fn(c, ret); 593 return ret; 594 } 595 596 int bch2_alloc_read(struct bch_fs *c) 597 { 598 struct btree_trans *trans = bch2_trans_get(c); 599 struct bch_dev *ca = NULL; 600 int ret; 601 602 if (c->sb.version_upgrade_complete >= bcachefs_metadata_version_bucket_gens) { 603 ret = for_each_btree_key(trans, iter, BTREE_ID_bucket_gens, POS_MIN, 604 BTREE_ITER_prefetch, k, ({ 605 u64 start = bucket_gens_pos_to_alloc(k.k->p, 0).offset; 606 u64 end = bucket_gens_pos_to_alloc(bpos_nosnap_successor(k.k->p), 0).offset; 607 608 if (k.k->type != KEY_TYPE_bucket_gens) 609 continue; 610 611 ca = bch2_dev_iterate(c, ca, k.k->p.inode); 612 /* 613 * Not a fsck error because this is checked/repaired by 614 * bch2_check_alloc_key() which runs later: 615 */ 616 if (!ca) { 617 bch2_btree_iter_set_pos(&iter, POS(k.k->p.inode + 1, 0)); 618 continue; 619 } 620 621 const struct bch_bucket_gens *g = bkey_s_c_to_bucket_gens(k).v; 622 623 for (u64 b = max_t(u64, ca->mi.first_bucket, start); 624 b < min_t(u64, ca->mi.nbuckets, end); 625 b++) 626 *bucket_gen(ca, b) = g->gens[b & KEY_TYPE_BUCKET_GENS_MASK]; 627 0; 628 })); 629 } else { 630 ret = for_each_btree_key(trans, iter, BTREE_ID_alloc, POS_MIN, 631 BTREE_ITER_prefetch, k, ({ 632 ca = bch2_dev_iterate(c, ca, k.k->p.inode); 633 /* 634 * Not a fsck error because this is checked/repaired by 635 * bch2_check_alloc_key() which runs later: 636 */ 637 if (!ca) { 638 bch2_btree_iter_set_pos(&iter, POS(k.k->p.inode + 1, 0)); 639 continue; 640 } 641 642 if (k.k->p.offset < ca->mi.first_bucket) { 643 bch2_btree_iter_set_pos(&iter, POS(k.k->p.inode, ca->mi.first_bucket)); 644 continue; 645 } 646 647 if (k.k->p.offset >= ca->mi.nbuckets) { 648 bch2_btree_iter_set_pos(&iter, POS(k.k->p.inode + 1, 0)); 649 continue; 650 } 651 652 struct bch_alloc_v4 a; 653 *bucket_gen(ca, k.k->p.offset) = bch2_alloc_to_v4(k, &a)->gen; 654 0; 655 })); 656 } 657 658 bch2_dev_put(ca); 659 bch2_trans_put(trans); 660 661 bch_err_fn(c, ret); 662 return ret; 663 } 664 665 /* Free space/discard btree: */ 666 667 static int bch2_bucket_do_index(struct btree_trans *trans, 668 struct bch_dev *ca, 669 struct bkey_s_c alloc_k, 670 const struct bch_alloc_v4 *a, 671 bool set) 672 { 673 struct bch_fs *c = trans->c; 674 struct btree_iter iter; 675 struct bkey_s_c old; 676 struct bkey_i *k; 677 enum btree_id btree; 678 enum bch_bkey_type old_type = !set ? KEY_TYPE_set : KEY_TYPE_deleted; 679 enum bch_bkey_type new_type = set ? KEY_TYPE_set : KEY_TYPE_deleted; 680 struct printbuf buf = PRINTBUF; 681 int ret; 682 683 if (a->data_type != BCH_DATA_free && 684 a->data_type != BCH_DATA_need_discard) 685 return 0; 686 687 k = bch2_trans_kmalloc_nomemzero(trans, sizeof(*k)); 688 if (IS_ERR(k)) 689 return PTR_ERR(k); 690 691 bkey_init(&k->k); 692 k->k.type = new_type; 693 694 switch (a->data_type) { 695 case BCH_DATA_free: 696 btree = BTREE_ID_freespace; 697 k->k.p = alloc_freespace_pos(alloc_k.k->p, *a); 698 bch2_key_resize(&k->k, 1); 699 break; 700 case BCH_DATA_need_discard: 701 btree = BTREE_ID_need_discard; 702 k->k.p = alloc_k.k->p; 703 break; 704 default: 705 return 0; 706 } 707 708 old = bch2_bkey_get_iter(trans, &iter, btree, 709 bkey_start_pos(&k->k), 710 BTREE_ITER_intent); 711 ret = bkey_err(old); 712 if (ret) 713 return ret; 714 715 if (ca->mi.freespace_initialized && 716 c->curr_recovery_pass > BCH_RECOVERY_PASS_check_alloc_info && 717 bch2_trans_inconsistent_on(old.k->type != old_type, trans, 718 "incorrect key when %s %s:%llu:%llu:0 (got %s should be %s)\n" 719 " for %s", 720 set ? "setting" : "clearing", 721 bch2_btree_id_str(btree), 722 iter.pos.inode, 723 iter.pos.offset, 724 bch2_bkey_types[old.k->type], 725 bch2_bkey_types[old_type], 726 (bch2_bkey_val_to_text(&buf, c, alloc_k), buf.buf))) { 727 ret = -EIO; 728 goto err; 729 } 730 731 ret = bch2_trans_update(trans, &iter, k, 0); 732 err: 733 bch2_trans_iter_exit(trans, &iter); 734 printbuf_exit(&buf); 735 return ret; 736 } 737 738 static noinline int bch2_bucket_gen_update(struct btree_trans *trans, 739 struct bpos bucket, u8 gen) 740 { 741 struct btree_iter iter; 742 unsigned offset; 743 struct bpos pos = alloc_gens_pos(bucket, &offset); 744 struct bkey_i_bucket_gens *g; 745 struct bkey_s_c k; 746 int ret; 747 748 g = bch2_trans_kmalloc(trans, sizeof(*g)); 749 ret = PTR_ERR_OR_ZERO(g); 750 if (ret) 751 return ret; 752 753 k = bch2_bkey_get_iter(trans, &iter, BTREE_ID_bucket_gens, pos, 754 BTREE_ITER_intent| 755 BTREE_ITER_with_updates); 756 ret = bkey_err(k); 757 if (ret) 758 return ret; 759 760 if (k.k->type != KEY_TYPE_bucket_gens) { 761 bkey_bucket_gens_init(&g->k_i); 762 g->k.p = iter.pos; 763 } else { 764 bkey_reassemble(&g->k_i, k); 765 } 766 767 g->v.gens[offset] = gen; 768 769 ret = bch2_trans_update(trans, &iter, &g->k_i, 0); 770 bch2_trans_iter_exit(trans, &iter); 771 return ret; 772 } 773 774 static inline int bch2_dev_data_type_accounting_mod(struct btree_trans *trans, struct bch_dev *ca, 775 enum bch_data_type data_type, 776 s64 delta_buckets, 777 s64 delta_sectors, 778 s64 delta_fragmented, unsigned flags) 779 { 780 struct disk_accounting_pos acc = { 781 .type = BCH_DISK_ACCOUNTING_dev_data_type, 782 .dev_data_type.dev = ca->dev_idx, 783 .dev_data_type.data_type = data_type, 784 }; 785 s64 d[3] = { delta_buckets, delta_sectors, delta_fragmented }; 786 787 return bch2_disk_accounting_mod(trans, &acc, d, 3, flags & BTREE_TRIGGER_gc); 788 } 789 790 int bch2_alloc_key_to_dev_counters(struct btree_trans *trans, struct bch_dev *ca, 791 const struct bch_alloc_v4 *old, 792 const struct bch_alloc_v4 *new, 793 unsigned flags) 794 { 795 s64 old_sectors = bch2_bucket_sectors(*old); 796 s64 new_sectors = bch2_bucket_sectors(*new); 797 if (old->data_type != new->data_type) { 798 int ret = bch2_dev_data_type_accounting_mod(trans, ca, new->data_type, 799 1, new_sectors, bch2_bucket_sectors_fragmented(ca, *new), flags) ?: 800 bch2_dev_data_type_accounting_mod(trans, ca, old->data_type, 801 -1, -old_sectors, -bch2_bucket_sectors_fragmented(ca, *old), flags); 802 if (ret) 803 return ret; 804 } else if (old_sectors != new_sectors) { 805 int ret = bch2_dev_data_type_accounting_mod(trans, ca, new->data_type, 806 0, 807 new_sectors - old_sectors, 808 bch2_bucket_sectors_fragmented(ca, *new) - 809 bch2_bucket_sectors_fragmented(ca, *old), flags); 810 if (ret) 811 return ret; 812 } 813 814 s64 old_unstriped = bch2_bucket_sectors_unstriped(*old); 815 s64 new_unstriped = bch2_bucket_sectors_unstriped(*new); 816 if (old_unstriped != new_unstriped) { 817 int ret = bch2_dev_data_type_accounting_mod(trans, ca, BCH_DATA_unstriped, 818 !!new_unstriped - !!old_unstriped, 819 new_unstriped - old_unstriped, 820 0, 821 flags); 822 if (ret) 823 return ret; 824 } 825 826 return 0; 827 } 828 829 int bch2_trigger_alloc(struct btree_trans *trans, 830 enum btree_id btree, unsigned level, 831 struct bkey_s_c old, struct bkey_s new, 832 enum btree_iter_update_trigger_flags flags) 833 { 834 struct bch_fs *c = trans->c; 835 struct printbuf buf = PRINTBUF; 836 int ret = 0; 837 838 struct bch_dev *ca = bch2_dev_bucket_tryget(c, new.k->p); 839 if (!ca) 840 return -EIO; 841 842 struct bch_alloc_v4 old_a_convert; 843 const struct bch_alloc_v4 *old_a = bch2_alloc_to_v4(old, &old_a_convert); 844 845 struct bch_alloc_v4 *new_a; 846 if (likely(new.k->type == KEY_TYPE_alloc_v4)) { 847 new_a = bkey_s_to_alloc_v4(new).v; 848 } else { 849 BUG_ON(!(flags & (BTREE_TRIGGER_gc|BTREE_TRIGGER_check_repair))); 850 851 struct bkey_i_alloc_v4 *new_ka = bch2_alloc_to_v4_mut_inlined(trans, new.s_c); 852 ret = PTR_ERR_OR_ZERO(new_ka); 853 if (unlikely(ret)) 854 goto err; 855 new_a = &new_ka->v; 856 } 857 858 if (flags & BTREE_TRIGGER_transactional) { 859 alloc_data_type_set(new_a, new_a->data_type); 860 861 if (bch2_bucket_sectors_total(*new_a) > bch2_bucket_sectors_total(*old_a)) { 862 new_a->io_time[READ] = bch2_current_io_time(c, READ); 863 new_a->io_time[WRITE]= bch2_current_io_time(c, WRITE); 864 SET_BCH_ALLOC_V4_NEED_INC_GEN(new_a, true); 865 SET_BCH_ALLOC_V4_NEED_DISCARD(new_a, true); 866 } 867 868 if (data_type_is_empty(new_a->data_type) && 869 BCH_ALLOC_V4_NEED_INC_GEN(new_a) && 870 !bch2_bucket_is_open_safe(c, new.k->p.inode, new.k->p.offset)) { 871 new_a->gen++; 872 SET_BCH_ALLOC_V4_NEED_INC_GEN(new_a, false); 873 alloc_data_type_set(new_a, new_a->data_type); 874 } 875 876 if (old_a->data_type != new_a->data_type || 877 (new_a->data_type == BCH_DATA_free && 878 alloc_freespace_genbits(*old_a) != alloc_freespace_genbits(*new_a))) { 879 ret = bch2_bucket_do_index(trans, ca, old, old_a, false) ?: 880 bch2_bucket_do_index(trans, ca, new.s_c, new_a, true); 881 if (ret) 882 goto err; 883 } 884 885 if (new_a->data_type == BCH_DATA_cached && 886 !new_a->io_time[READ]) 887 new_a->io_time[READ] = bch2_current_io_time(c, READ); 888 889 u64 old_lru = alloc_lru_idx_read(*old_a); 890 u64 new_lru = alloc_lru_idx_read(*new_a); 891 if (old_lru != new_lru) { 892 ret = bch2_lru_change(trans, new.k->p.inode, 893 bucket_to_u64(new.k->p), 894 old_lru, new_lru); 895 if (ret) 896 goto err; 897 } 898 899 old_lru = alloc_lru_idx_fragmentation(*old_a, ca); 900 new_lru = alloc_lru_idx_fragmentation(*new_a, ca); 901 if (old_lru != new_lru) { 902 ret = bch2_lru_change(trans, 903 BCH_LRU_FRAGMENTATION_START, 904 bucket_to_u64(new.k->p), 905 old_lru, new_lru); 906 if (ret) 907 goto err; 908 } 909 910 if (old_a->gen != new_a->gen) { 911 ret = bch2_bucket_gen_update(trans, new.k->p, new_a->gen); 912 if (ret) 913 goto err; 914 } 915 916 if ((flags & BTREE_TRIGGER_bucket_invalidate) && 917 old_a->cached_sectors) { 918 ret = bch2_mod_dev_cached_sectors(trans, ca->dev_idx, 919 -((s64) old_a->cached_sectors), 920 flags & BTREE_TRIGGER_gc); 921 if (ret) 922 goto err; 923 } 924 925 ret = bch2_alloc_key_to_dev_counters(trans, ca, old_a, new_a, flags); 926 if (ret) 927 goto err; 928 } 929 930 if ((flags & BTREE_TRIGGER_atomic) && (flags & BTREE_TRIGGER_insert)) { 931 u64 journal_seq = trans->journal_res.seq; 932 u64 bucket_journal_seq = new_a->journal_seq; 933 934 if ((flags & BTREE_TRIGGER_insert) && 935 data_type_is_empty(old_a->data_type) != 936 data_type_is_empty(new_a->data_type) && 937 new.k->type == KEY_TYPE_alloc_v4) { 938 struct bch_alloc_v4 *v = bkey_s_to_alloc_v4(new).v; 939 940 /* 941 * If the btree updates referring to a bucket weren't flushed 942 * before the bucket became empty again, then the we don't have 943 * to wait on a journal flush before we can reuse the bucket: 944 */ 945 v->journal_seq = bucket_journal_seq = 946 data_type_is_empty(new_a->data_type) && 947 (journal_seq == v->journal_seq || 948 bch2_journal_noflush_seq(&c->journal, v->journal_seq)) 949 ? 0 : journal_seq; 950 } 951 952 if (!data_type_is_empty(old_a->data_type) && 953 data_type_is_empty(new_a->data_type) && 954 bucket_journal_seq) { 955 ret = bch2_set_bucket_needs_journal_commit(&c->buckets_waiting_for_journal, 956 c->journal.flushed_seq_ondisk, 957 new.k->p.inode, new.k->p.offset, 958 bucket_journal_seq); 959 if (bch2_fs_fatal_err_on(ret, c, 960 "setting bucket_needs_journal_commit: %s", bch2_err_str(ret))) 961 goto err; 962 } 963 964 if (new_a->gen != old_a->gen) { 965 rcu_read_lock(); 966 u8 *gen = bucket_gen(ca, new.k->p.offset); 967 if (unlikely(!gen)) { 968 rcu_read_unlock(); 969 goto invalid_bucket; 970 } 971 *gen = new_a->gen; 972 rcu_read_unlock(); 973 } 974 975 #define eval_state(_a, expr) ({ const struct bch_alloc_v4 *a = _a; expr; }) 976 #define statechange(expr) !eval_state(old_a, expr) && eval_state(new_a, expr) 977 #define bucket_flushed(a) (!a->journal_seq || a->journal_seq <= c->journal.flushed_seq_ondisk) 978 979 if (statechange(a->data_type == BCH_DATA_free) && 980 bucket_flushed(new_a)) 981 closure_wake_up(&c->freelist_wait); 982 983 if (statechange(a->data_type == BCH_DATA_need_discard) && 984 !bch2_bucket_is_open_safe(c, new.k->p.inode, new.k->p.offset) && 985 bucket_flushed(new_a)) 986 bch2_discard_one_bucket_fast(ca, new.k->p.offset); 987 988 if (statechange(a->data_type == BCH_DATA_cached) && 989 !bch2_bucket_is_open(c, new.k->p.inode, new.k->p.offset) && 990 should_invalidate_buckets(ca, bch2_dev_usage_read(ca))) 991 bch2_dev_do_invalidates(ca); 992 993 if (statechange(a->data_type == BCH_DATA_need_gc_gens)) 994 bch2_gc_gens_async(c); 995 } 996 997 if ((flags & BTREE_TRIGGER_gc) && (flags & BTREE_TRIGGER_insert)) { 998 rcu_read_lock(); 999 struct bucket *g = gc_bucket(ca, new.k->p.offset); 1000 if (unlikely(!g)) { 1001 rcu_read_unlock(); 1002 goto invalid_bucket; 1003 } 1004 g->gen_valid = 1; 1005 g->gen = new_a->gen; 1006 rcu_read_unlock(); 1007 } 1008 err: 1009 printbuf_exit(&buf); 1010 bch2_dev_put(ca); 1011 return ret; 1012 invalid_bucket: 1013 bch2_fs_inconsistent(c, "reference to invalid bucket\n %s", 1014 (bch2_bkey_val_to_text(&buf, c, new.s_c), buf.buf)); 1015 ret = -EIO; 1016 goto err; 1017 } 1018 1019 /* 1020 * This synthesizes deleted extents for holes, similar to BTREE_ITER_slots for 1021 * extents style btrees, but works on non-extents btrees: 1022 */ 1023 static struct bkey_s_c bch2_get_key_or_hole(struct btree_iter *iter, struct bpos end, struct bkey *hole) 1024 { 1025 struct bkey_s_c k = bch2_btree_iter_peek_slot(iter); 1026 1027 if (bkey_err(k)) 1028 return k; 1029 1030 if (k.k->type) { 1031 return k; 1032 } else { 1033 struct btree_iter iter2; 1034 struct bpos next; 1035 1036 bch2_trans_copy_iter(&iter2, iter); 1037 1038 struct btree_path *path = btree_iter_path(iter->trans, iter); 1039 if (!bpos_eq(path->l[0].b->key.k.p, SPOS_MAX)) 1040 end = bkey_min(end, bpos_nosnap_successor(path->l[0].b->key.k.p)); 1041 1042 end = bkey_min(end, POS(iter->pos.inode, iter->pos.offset + U32_MAX - 1)); 1043 1044 /* 1045 * btree node min/max is a closed interval, upto takes a half 1046 * open interval: 1047 */ 1048 k = bch2_btree_iter_peek_upto(&iter2, end); 1049 next = iter2.pos; 1050 bch2_trans_iter_exit(iter->trans, &iter2); 1051 1052 BUG_ON(next.offset >= iter->pos.offset + U32_MAX); 1053 1054 if (bkey_err(k)) 1055 return k; 1056 1057 bkey_init(hole); 1058 hole->p = iter->pos; 1059 1060 bch2_key_resize(hole, next.offset - iter->pos.offset); 1061 return (struct bkey_s_c) { hole, NULL }; 1062 } 1063 } 1064 1065 static bool next_bucket(struct bch_fs *c, struct bch_dev **ca, struct bpos *bucket) 1066 { 1067 if (*ca) { 1068 if (bucket->offset < (*ca)->mi.first_bucket) 1069 bucket->offset = (*ca)->mi.first_bucket; 1070 1071 if (bucket->offset < (*ca)->mi.nbuckets) 1072 return true; 1073 1074 bch2_dev_put(*ca); 1075 *ca = NULL; 1076 bucket->inode++; 1077 bucket->offset = 0; 1078 } 1079 1080 rcu_read_lock(); 1081 *ca = __bch2_next_dev_idx(c, bucket->inode, NULL); 1082 if (*ca) { 1083 *bucket = POS((*ca)->dev_idx, (*ca)->mi.first_bucket); 1084 bch2_dev_get(*ca); 1085 } 1086 rcu_read_unlock(); 1087 1088 return *ca != NULL; 1089 } 1090 1091 static struct bkey_s_c bch2_get_key_or_real_bucket_hole(struct btree_iter *iter, 1092 struct bch_dev **ca, struct bkey *hole) 1093 { 1094 struct bch_fs *c = iter->trans->c; 1095 struct bkey_s_c k; 1096 again: 1097 k = bch2_get_key_or_hole(iter, POS_MAX, hole); 1098 if (bkey_err(k)) 1099 return k; 1100 1101 *ca = bch2_dev_iterate_noerror(c, *ca, k.k->p.inode); 1102 1103 if (!k.k->type) { 1104 struct bpos hole_start = bkey_start_pos(k.k); 1105 1106 if (!*ca || !bucket_valid(*ca, hole_start.offset)) { 1107 if (!next_bucket(c, ca, &hole_start)) 1108 return bkey_s_c_null; 1109 1110 bch2_btree_iter_set_pos(iter, hole_start); 1111 goto again; 1112 } 1113 1114 if (k.k->p.offset > (*ca)->mi.nbuckets) 1115 bch2_key_resize(hole, (*ca)->mi.nbuckets - hole_start.offset); 1116 } 1117 1118 return k; 1119 } 1120 1121 static noinline_for_stack 1122 int bch2_check_alloc_key(struct btree_trans *trans, 1123 struct bkey_s_c alloc_k, 1124 struct btree_iter *alloc_iter, 1125 struct btree_iter *discard_iter, 1126 struct btree_iter *freespace_iter, 1127 struct btree_iter *bucket_gens_iter) 1128 { 1129 struct bch_fs *c = trans->c; 1130 struct bch_alloc_v4 a_convert; 1131 const struct bch_alloc_v4 *a; 1132 unsigned discard_key_type, freespace_key_type; 1133 unsigned gens_offset; 1134 struct bkey_s_c k; 1135 struct printbuf buf = PRINTBUF; 1136 int ret = 0; 1137 1138 struct bch_dev *ca = bch2_dev_bucket_tryget_noerror(c, alloc_k.k->p); 1139 if (fsck_err_on(!ca, 1140 trans, alloc_key_to_missing_dev_bucket, 1141 "alloc key for invalid device:bucket %llu:%llu", 1142 alloc_k.k->p.inode, alloc_k.k->p.offset)) 1143 ret = bch2_btree_delete_at(trans, alloc_iter, 0); 1144 if (!ca) 1145 return ret; 1146 1147 if (!ca->mi.freespace_initialized) 1148 goto out; 1149 1150 a = bch2_alloc_to_v4(alloc_k, &a_convert); 1151 1152 discard_key_type = a->data_type == BCH_DATA_need_discard ? KEY_TYPE_set : 0; 1153 bch2_btree_iter_set_pos(discard_iter, alloc_k.k->p); 1154 k = bch2_btree_iter_peek_slot(discard_iter); 1155 ret = bkey_err(k); 1156 if (ret) 1157 goto err; 1158 1159 if (fsck_err_on(k.k->type != discard_key_type, 1160 trans, need_discard_key_wrong, 1161 "incorrect key in need_discard btree (got %s should be %s)\n" 1162 " %s", 1163 bch2_bkey_types[k.k->type], 1164 bch2_bkey_types[discard_key_type], 1165 (bch2_bkey_val_to_text(&buf, c, alloc_k), buf.buf))) { 1166 struct bkey_i *update = 1167 bch2_trans_kmalloc(trans, sizeof(*update)); 1168 1169 ret = PTR_ERR_OR_ZERO(update); 1170 if (ret) 1171 goto err; 1172 1173 bkey_init(&update->k); 1174 update->k.type = discard_key_type; 1175 update->k.p = discard_iter->pos; 1176 1177 ret = bch2_trans_update(trans, discard_iter, update, 0); 1178 if (ret) 1179 goto err; 1180 } 1181 1182 freespace_key_type = a->data_type == BCH_DATA_free ? KEY_TYPE_set : 0; 1183 bch2_btree_iter_set_pos(freespace_iter, alloc_freespace_pos(alloc_k.k->p, *a)); 1184 k = bch2_btree_iter_peek_slot(freespace_iter); 1185 ret = bkey_err(k); 1186 if (ret) 1187 goto err; 1188 1189 if (fsck_err_on(k.k->type != freespace_key_type, 1190 trans, freespace_key_wrong, 1191 "incorrect key in freespace btree (got %s should be %s)\n" 1192 " %s", 1193 bch2_bkey_types[k.k->type], 1194 bch2_bkey_types[freespace_key_type], 1195 (printbuf_reset(&buf), 1196 bch2_bkey_val_to_text(&buf, c, alloc_k), buf.buf))) { 1197 struct bkey_i *update = 1198 bch2_trans_kmalloc(trans, sizeof(*update)); 1199 1200 ret = PTR_ERR_OR_ZERO(update); 1201 if (ret) 1202 goto err; 1203 1204 bkey_init(&update->k); 1205 update->k.type = freespace_key_type; 1206 update->k.p = freespace_iter->pos; 1207 bch2_key_resize(&update->k, 1); 1208 1209 ret = bch2_trans_update(trans, freespace_iter, update, 0); 1210 if (ret) 1211 goto err; 1212 } 1213 1214 bch2_btree_iter_set_pos(bucket_gens_iter, alloc_gens_pos(alloc_k.k->p, &gens_offset)); 1215 k = bch2_btree_iter_peek_slot(bucket_gens_iter); 1216 ret = bkey_err(k); 1217 if (ret) 1218 goto err; 1219 1220 if (fsck_err_on(a->gen != alloc_gen(k, gens_offset), 1221 trans, bucket_gens_key_wrong, 1222 "incorrect gen in bucket_gens btree (got %u should be %u)\n" 1223 " %s", 1224 alloc_gen(k, gens_offset), a->gen, 1225 (printbuf_reset(&buf), 1226 bch2_bkey_val_to_text(&buf, c, alloc_k), buf.buf))) { 1227 struct bkey_i_bucket_gens *g = 1228 bch2_trans_kmalloc(trans, sizeof(*g)); 1229 1230 ret = PTR_ERR_OR_ZERO(g); 1231 if (ret) 1232 goto err; 1233 1234 if (k.k->type == KEY_TYPE_bucket_gens) { 1235 bkey_reassemble(&g->k_i, k); 1236 } else { 1237 bkey_bucket_gens_init(&g->k_i); 1238 g->k.p = alloc_gens_pos(alloc_k.k->p, &gens_offset); 1239 } 1240 1241 g->v.gens[gens_offset] = a->gen; 1242 1243 ret = bch2_trans_update(trans, bucket_gens_iter, &g->k_i, 0); 1244 if (ret) 1245 goto err; 1246 } 1247 out: 1248 err: 1249 fsck_err: 1250 bch2_dev_put(ca); 1251 printbuf_exit(&buf); 1252 return ret; 1253 } 1254 1255 static noinline_for_stack 1256 int bch2_check_alloc_hole_freespace(struct btree_trans *trans, 1257 struct bch_dev *ca, 1258 struct bpos start, 1259 struct bpos *end, 1260 struct btree_iter *freespace_iter) 1261 { 1262 struct bkey_s_c k; 1263 struct printbuf buf = PRINTBUF; 1264 int ret; 1265 1266 if (!ca->mi.freespace_initialized) 1267 return 0; 1268 1269 bch2_btree_iter_set_pos(freespace_iter, start); 1270 1271 k = bch2_btree_iter_peek_slot(freespace_iter); 1272 ret = bkey_err(k); 1273 if (ret) 1274 goto err; 1275 1276 *end = bkey_min(k.k->p, *end); 1277 1278 if (fsck_err_on(k.k->type != KEY_TYPE_set, 1279 trans, freespace_hole_missing, 1280 "hole in alloc btree missing in freespace btree\n" 1281 " device %llu buckets %llu-%llu", 1282 freespace_iter->pos.inode, 1283 freespace_iter->pos.offset, 1284 end->offset)) { 1285 struct bkey_i *update = 1286 bch2_trans_kmalloc(trans, sizeof(*update)); 1287 1288 ret = PTR_ERR_OR_ZERO(update); 1289 if (ret) 1290 goto err; 1291 1292 bkey_init(&update->k); 1293 update->k.type = KEY_TYPE_set; 1294 update->k.p = freespace_iter->pos; 1295 bch2_key_resize(&update->k, 1296 min_t(u64, U32_MAX, end->offset - 1297 freespace_iter->pos.offset)); 1298 1299 ret = bch2_trans_update(trans, freespace_iter, update, 0); 1300 if (ret) 1301 goto err; 1302 } 1303 err: 1304 fsck_err: 1305 printbuf_exit(&buf); 1306 return ret; 1307 } 1308 1309 static noinline_for_stack 1310 int bch2_check_alloc_hole_bucket_gens(struct btree_trans *trans, 1311 struct bpos start, 1312 struct bpos *end, 1313 struct btree_iter *bucket_gens_iter) 1314 { 1315 struct bkey_s_c k; 1316 struct printbuf buf = PRINTBUF; 1317 unsigned i, gens_offset, gens_end_offset; 1318 int ret; 1319 1320 bch2_btree_iter_set_pos(bucket_gens_iter, alloc_gens_pos(start, &gens_offset)); 1321 1322 k = bch2_btree_iter_peek_slot(bucket_gens_iter); 1323 ret = bkey_err(k); 1324 if (ret) 1325 goto err; 1326 1327 if (bkey_cmp(alloc_gens_pos(start, &gens_offset), 1328 alloc_gens_pos(*end, &gens_end_offset))) 1329 gens_end_offset = KEY_TYPE_BUCKET_GENS_NR; 1330 1331 if (k.k->type == KEY_TYPE_bucket_gens) { 1332 struct bkey_i_bucket_gens g; 1333 bool need_update = false; 1334 1335 bkey_reassemble(&g.k_i, k); 1336 1337 for (i = gens_offset; i < gens_end_offset; i++) { 1338 if (fsck_err_on(g.v.gens[i], trans, 1339 bucket_gens_hole_wrong, 1340 "hole in alloc btree at %llu:%llu with nonzero gen in bucket_gens btree (%u)", 1341 bucket_gens_pos_to_alloc(k.k->p, i).inode, 1342 bucket_gens_pos_to_alloc(k.k->p, i).offset, 1343 g.v.gens[i])) { 1344 g.v.gens[i] = 0; 1345 need_update = true; 1346 } 1347 } 1348 1349 if (need_update) { 1350 struct bkey_i *u = bch2_trans_kmalloc(trans, sizeof(g)); 1351 1352 ret = PTR_ERR_OR_ZERO(u); 1353 if (ret) 1354 goto err; 1355 1356 memcpy(u, &g, sizeof(g)); 1357 1358 ret = bch2_trans_update(trans, bucket_gens_iter, u, 0); 1359 if (ret) 1360 goto err; 1361 } 1362 } 1363 1364 *end = bkey_min(*end, bucket_gens_pos_to_alloc(bpos_nosnap_successor(k.k->p), 0)); 1365 err: 1366 fsck_err: 1367 printbuf_exit(&buf); 1368 return ret; 1369 } 1370 1371 static noinline_for_stack int bch2_check_discard_freespace_key(struct btree_trans *trans, 1372 struct btree_iter *iter) 1373 { 1374 struct bch_fs *c = trans->c; 1375 struct btree_iter alloc_iter; 1376 struct bkey_s_c alloc_k; 1377 struct bch_alloc_v4 a_convert; 1378 const struct bch_alloc_v4 *a; 1379 u64 genbits; 1380 struct bpos pos; 1381 enum bch_data_type state = iter->btree_id == BTREE_ID_need_discard 1382 ? BCH_DATA_need_discard 1383 : BCH_DATA_free; 1384 struct printbuf buf = PRINTBUF; 1385 int ret; 1386 1387 pos = iter->pos; 1388 pos.offset &= ~(~0ULL << 56); 1389 genbits = iter->pos.offset & (~0ULL << 56); 1390 1391 alloc_k = bch2_bkey_get_iter(trans, &alloc_iter, BTREE_ID_alloc, pos, 0); 1392 ret = bkey_err(alloc_k); 1393 if (ret) 1394 return ret; 1395 1396 if (fsck_err_on(!bch2_dev_bucket_exists(c, pos), 1397 trans, need_discard_freespace_key_to_invalid_dev_bucket, 1398 "entry in %s btree for nonexistant dev:bucket %llu:%llu", 1399 bch2_btree_id_str(iter->btree_id), pos.inode, pos.offset)) 1400 goto delete; 1401 1402 a = bch2_alloc_to_v4(alloc_k, &a_convert); 1403 1404 if (fsck_err_on(a->data_type != state || 1405 (state == BCH_DATA_free && 1406 genbits != alloc_freespace_genbits(*a)), 1407 trans, need_discard_freespace_key_bad, 1408 "%s\n incorrectly set at %s:%llu:%llu:0 (free %u, genbits %llu should be %llu)", 1409 (bch2_bkey_val_to_text(&buf, c, alloc_k), buf.buf), 1410 bch2_btree_id_str(iter->btree_id), 1411 iter->pos.inode, 1412 iter->pos.offset, 1413 a->data_type == state, 1414 genbits >> 56, alloc_freespace_genbits(*a) >> 56)) 1415 goto delete; 1416 out: 1417 fsck_err: 1418 bch2_set_btree_iter_dontneed(&alloc_iter); 1419 bch2_trans_iter_exit(trans, &alloc_iter); 1420 printbuf_exit(&buf); 1421 return ret; 1422 delete: 1423 ret = bch2_btree_delete_extent_at(trans, iter, 1424 iter->btree_id == BTREE_ID_freespace ? 1 : 0, 0) ?: 1425 bch2_trans_commit(trans, NULL, NULL, 1426 BCH_TRANS_COMMIT_no_enospc); 1427 goto out; 1428 } 1429 1430 /* 1431 * We've already checked that generation numbers in the bucket_gens btree are 1432 * valid for buckets that exist; this just checks for keys for nonexistent 1433 * buckets. 1434 */ 1435 static noinline_for_stack 1436 int bch2_check_bucket_gens_key(struct btree_trans *trans, 1437 struct btree_iter *iter, 1438 struct bkey_s_c k) 1439 { 1440 struct bch_fs *c = trans->c; 1441 struct bkey_i_bucket_gens g; 1442 u64 start = bucket_gens_pos_to_alloc(k.k->p, 0).offset; 1443 u64 end = bucket_gens_pos_to_alloc(bpos_nosnap_successor(k.k->p), 0).offset; 1444 u64 b; 1445 bool need_update = false; 1446 struct printbuf buf = PRINTBUF; 1447 int ret = 0; 1448 1449 BUG_ON(k.k->type != KEY_TYPE_bucket_gens); 1450 bkey_reassemble(&g.k_i, k); 1451 1452 struct bch_dev *ca = bch2_dev_tryget_noerror(c, k.k->p.inode); 1453 if (!ca) { 1454 if (fsck_err(trans, bucket_gens_to_invalid_dev, 1455 "bucket_gens key for invalid device:\n %s", 1456 (bch2_bkey_val_to_text(&buf, c, k), buf.buf))) 1457 ret = bch2_btree_delete_at(trans, iter, 0); 1458 goto out; 1459 } 1460 1461 if (fsck_err_on(end <= ca->mi.first_bucket || 1462 start >= ca->mi.nbuckets, 1463 trans, bucket_gens_to_invalid_buckets, 1464 "bucket_gens key for invalid buckets:\n %s", 1465 (bch2_bkey_val_to_text(&buf, c, k), buf.buf))) { 1466 ret = bch2_btree_delete_at(trans, iter, 0); 1467 goto out; 1468 } 1469 1470 for (b = start; b < ca->mi.first_bucket; b++) 1471 if (fsck_err_on(g.v.gens[b & KEY_TYPE_BUCKET_GENS_MASK], 1472 trans, bucket_gens_nonzero_for_invalid_buckets, 1473 "bucket_gens key has nonzero gen for invalid bucket")) { 1474 g.v.gens[b & KEY_TYPE_BUCKET_GENS_MASK] = 0; 1475 need_update = true; 1476 } 1477 1478 for (b = ca->mi.nbuckets; b < end; b++) 1479 if (fsck_err_on(g.v.gens[b & KEY_TYPE_BUCKET_GENS_MASK], 1480 trans, bucket_gens_nonzero_for_invalid_buckets, 1481 "bucket_gens key has nonzero gen for invalid bucket")) { 1482 g.v.gens[b & KEY_TYPE_BUCKET_GENS_MASK] = 0; 1483 need_update = true; 1484 } 1485 1486 if (need_update) { 1487 struct bkey_i *u = bch2_trans_kmalloc(trans, sizeof(g)); 1488 1489 ret = PTR_ERR_OR_ZERO(u); 1490 if (ret) 1491 goto out; 1492 1493 memcpy(u, &g, sizeof(g)); 1494 ret = bch2_trans_update(trans, iter, u, 0); 1495 } 1496 out: 1497 fsck_err: 1498 bch2_dev_put(ca); 1499 printbuf_exit(&buf); 1500 return ret; 1501 } 1502 1503 int bch2_check_alloc_info(struct bch_fs *c) 1504 { 1505 struct btree_trans *trans = bch2_trans_get(c); 1506 struct btree_iter iter, discard_iter, freespace_iter, bucket_gens_iter; 1507 struct bch_dev *ca = NULL; 1508 struct bkey hole; 1509 struct bkey_s_c k; 1510 int ret = 0; 1511 1512 bch2_trans_iter_init(trans, &iter, BTREE_ID_alloc, POS_MIN, 1513 BTREE_ITER_prefetch); 1514 bch2_trans_iter_init(trans, &discard_iter, BTREE_ID_need_discard, POS_MIN, 1515 BTREE_ITER_prefetch); 1516 bch2_trans_iter_init(trans, &freespace_iter, BTREE_ID_freespace, POS_MIN, 1517 BTREE_ITER_prefetch); 1518 bch2_trans_iter_init(trans, &bucket_gens_iter, BTREE_ID_bucket_gens, POS_MIN, 1519 BTREE_ITER_prefetch); 1520 1521 while (1) { 1522 struct bpos next; 1523 1524 bch2_trans_begin(trans); 1525 1526 k = bch2_get_key_or_real_bucket_hole(&iter, &ca, &hole); 1527 ret = bkey_err(k); 1528 if (ret) 1529 goto bkey_err; 1530 1531 if (!k.k) 1532 break; 1533 1534 if (k.k->type) { 1535 next = bpos_nosnap_successor(k.k->p); 1536 1537 ret = bch2_check_alloc_key(trans, 1538 k, &iter, 1539 &discard_iter, 1540 &freespace_iter, 1541 &bucket_gens_iter); 1542 if (ret) 1543 goto bkey_err; 1544 } else { 1545 next = k.k->p; 1546 1547 ret = bch2_check_alloc_hole_freespace(trans, ca, 1548 bkey_start_pos(k.k), 1549 &next, 1550 &freespace_iter) ?: 1551 bch2_check_alloc_hole_bucket_gens(trans, 1552 bkey_start_pos(k.k), 1553 &next, 1554 &bucket_gens_iter); 1555 if (ret) 1556 goto bkey_err; 1557 } 1558 1559 ret = bch2_trans_commit(trans, NULL, NULL, 1560 BCH_TRANS_COMMIT_no_enospc); 1561 if (ret) 1562 goto bkey_err; 1563 1564 bch2_btree_iter_set_pos(&iter, next); 1565 bkey_err: 1566 if (bch2_err_matches(ret, BCH_ERR_transaction_restart)) 1567 continue; 1568 if (ret) 1569 break; 1570 } 1571 bch2_trans_iter_exit(trans, &bucket_gens_iter); 1572 bch2_trans_iter_exit(trans, &freespace_iter); 1573 bch2_trans_iter_exit(trans, &discard_iter); 1574 bch2_trans_iter_exit(trans, &iter); 1575 bch2_dev_put(ca); 1576 ca = NULL; 1577 1578 if (ret < 0) 1579 goto err; 1580 1581 ret = for_each_btree_key(trans, iter, 1582 BTREE_ID_need_discard, POS_MIN, 1583 BTREE_ITER_prefetch, k, 1584 bch2_check_discard_freespace_key(trans, &iter)); 1585 if (ret) 1586 goto err; 1587 1588 bch2_trans_iter_init(trans, &iter, BTREE_ID_freespace, POS_MIN, 1589 BTREE_ITER_prefetch); 1590 while (1) { 1591 bch2_trans_begin(trans); 1592 k = bch2_btree_iter_peek(&iter); 1593 if (!k.k) 1594 break; 1595 1596 ret = bkey_err(k) ?: 1597 bch2_check_discard_freespace_key(trans, &iter); 1598 if (bch2_err_matches(ret, BCH_ERR_transaction_restart)) { 1599 ret = 0; 1600 continue; 1601 } 1602 if (ret) { 1603 struct printbuf buf = PRINTBUF; 1604 bch2_bkey_val_to_text(&buf, c, k); 1605 1606 bch_err(c, "while checking %s", buf.buf); 1607 printbuf_exit(&buf); 1608 break; 1609 } 1610 1611 bch2_btree_iter_set_pos(&iter, bpos_nosnap_successor(iter.pos)); 1612 } 1613 bch2_trans_iter_exit(trans, &iter); 1614 if (ret) 1615 goto err; 1616 1617 ret = for_each_btree_key_commit(trans, iter, 1618 BTREE_ID_bucket_gens, POS_MIN, 1619 BTREE_ITER_prefetch, k, 1620 NULL, NULL, BCH_TRANS_COMMIT_no_enospc, 1621 bch2_check_bucket_gens_key(trans, &iter, k)); 1622 err: 1623 bch2_trans_put(trans); 1624 bch_err_fn(c, ret); 1625 return ret; 1626 } 1627 1628 static int bch2_check_alloc_to_lru_ref(struct btree_trans *trans, 1629 struct btree_iter *alloc_iter, 1630 struct bkey_buf *last_flushed) 1631 { 1632 struct bch_fs *c = trans->c; 1633 struct bch_alloc_v4 a_convert; 1634 const struct bch_alloc_v4 *a; 1635 struct bkey_s_c alloc_k; 1636 struct printbuf buf = PRINTBUF; 1637 int ret; 1638 1639 alloc_k = bch2_btree_iter_peek(alloc_iter); 1640 if (!alloc_k.k) 1641 return 0; 1642 1643 ret = bkey_err(alloc_k); 1644 if (ret) 1645 return ret; 1646 1647 struct bch_dev *ca = bch2_dev_tryget_noerror(c, alloc_k.k->p.inode); 1648 if (!ca) 1649 return 0; 1650 1651 a = bch2_alloc_to_v4(alloc_k, &a_convert); 1652 1653 u64 lru_idx = alloc_lru_idx_fragmentation(*a, ca); 1654 if (lru_idx) { 1655 ret = bch2_lru_check_set(trans, BCH_LRU_FRAGMENTATION_START, 1656 lru_idx, alloc_k, last_flushed); 1657 if (ret) 1658 goto err; 1659 } 1660 1661 if (a->data_type != BCH_DATA_cached) 1662 goto err; 1663 1664 if (fsck_err_on(!a->io_time[READ], 1665 trans, alloc_key_cached_but_read_time_zero, 1666 "cached bucket with read_time 0\n" 1667 " %s", 1668 (printbuf_reset(&buf), 1669 bch2_bkey_val_to_text(&buf, c, alloc_k), buf.buf))) { 1670 struct bkey_i_alloc_v4 *a_mut = 1671 bch2_alloc_to_v4_mut(trans, alloc_k); 1672 ret = PTR_ERR_OR_ZERO(a_mut); 1673 if (ret) 1674 goto err; 1675 1676 a_mut->v.io_time[READ] = bch2_current_io_time(c, READ); 1677 ret = bch2_trans_update(trans, alloc_iter, 1678 &a_mut->k_i, BTREE_TRIGGER_norun); 1679 if (ret) 1680 goto err; 1681 1682 a = &a_mut->v; 1683 } 1684 1685 ret = bch2_lru_check_set(trans, alloc_k.k->p.inode, a->io_time[READ], 1686 alloc_k, last_flushed); 1687 if (ret) 1688 goto err; 1689 err: 1690 fsck_err: 1691 bch2_dev_put(ca); 1692 printbuf_exit(&buf); 1693 return ret; 1694 } 1695 1696 int bch2_check_alloc_to_lru_refs(struct bch_fs *c) 1697 { 1698 struct bkey_buf last_flushed; 1699 1700 bch2_bkey_buf_init(&last_flushed); 1701 bkey_init(&last_flushed.k->k); 1702 1703 int ret = bch2_trans_run(c, 1704 for_each_btree_key_commit(trans, iter, BTREE_ID_alloc, 1705 POS_MIN, BTREE_ITER_prefetch, k, 1706 NULL, NULL, BCH_TRANS_COMMIT_no_enospc, 1707 bch2_check_alloc_to_lru_ref(trans, &iter, &last_flushed))); 1708 1709 bch2_bkey_buf_exit(&last_flushed, c); 1710 bch_err_fn(c, ret); 1711 return ret; 1712 } 1713 1714 static int discard_in_flight_add(struct bch_dev *ca, u64 bucket, bool in_progress) 1715 { 1716 int ret; 1717 1718 mutex_lock(&ca->discard_buckets_in_flight_lock); 1719 darray_for_each(ca->discard_buckets_in_flight, i) 1720 if (i->bucket == bucket) { 1721 ret = -BCH_ERR_EEXIST_discard_in_flight_add; 1722 goto out; 1723 } 1724 1725 ret = darray_push(&ca->discard_buckets_in_flight, ((struct discard_in_flight) { 1726 .in_progress = in_progress, 1727 .bucket = bucket, 1728 })); 1729 out: 1730 mutex_unlock(&ca->discard_buckets_in_flight_lock); 1731 return ret; 1732 } 1733 1734 static void discard_in_flight_remove(struct bch_dev *ca, u64 bucket) 1735 { 1736 mutex_lock(&ca->discard_buckets_in_flight_lock); 1737 darray_for_each(ca->discard_buckets_in_flight, i) 1738 if (i->bucket == bucket) { 1739 BUG_ON(!i->in_progress); 1740 darray_remove_item(&ca->discard_buckets_in_flight, i); 1741 goto found; 1742 } 1743 BUG(); 1744 found: 1745 mutex_unlock(&ca->discard_buckets_in_flight_lock); 1746 } 1747 1748 struct discard_buckets_state { 1749 u64 seen; 1750 u64 open; 1751 u64 need_journal_commit; 1752 u64 discarded; 1753 u64 need_journal_commit_this_dev; 1754 }; 1755 1756 static int bch2_discard_one_bucket(struct btree_trans *trans, 1757 struct bch_dev *ca, 1758 struct btree_iter *need_discard_iter, 1759 struct bpos *discard_pos_done, 1760 struct discard_buckets_state *s) 1761 { 1762 struct bch_fs *c = trans->c; 1763 struct bpos pos = need_discard_iter->pos; 1764 struct btree_iter iter = { NULL }; 1765 struct bkey_s_c k; 1766 struct bkey_i_alloc_v4 *a; 1767 struct printbuf buf = PRINTBUF; 1768 bool discard_locked = false; 1769 int ret = 0; 1770 1771 if (bch2_bucket_is_open_safe(c, pos.inode, pos.offset)) { 1772 s->open++; 1773 goto out; 1774 } 1775 1776 if (bch2_bucket_needs_journal_commit(&c->buckets_waiting_for_journal, 1777 c->journal.flushed_seq_ondisk, 1778 pos.inode, pos.offset)) { 1779 s->need_journal_commit++; 1780 s->need_journal_commit_this_dev++; 1781 goto out; 1782 } 1783 1784 k = bch2_bkey_get_iter(trans, &iter, BTREE_ID_alloc, 1785 need_discard_iter->pos, 1786 BTREE_ITER_cached); 1787 ret = bkey_err(k); 1788 if (ret) 1789 goto out; 1790 1791 a = bch2_alloc_to_v4_mut(trans, k); 1792 ret = PTR_ERR_OR_ZERO(a); 1793 if (ret) 1794 goto out; 1795 1796 if (bch2_bucket_sectors_total(a->v)) { 1797 if (bch2_trans_inconsistent_on(c->curr_recovery_pass > BCH_RECOVERY_PASS_check_alloc_info, 1798 trans, "attempting to discard bucket with dirty data\n%s", 1799 (bch2_bkey_val_to_text(&buf, c, k), buf.buf))) 1800 ret = -EIO; 1801 goto out; 1802 } 1803 1804 if (a->v.data_type != BCH_DATA_need_discard) { 1805 if (data_type_is_empty(a->v.data_type) && 1806 BCH_ALLOC_V4_NEED_INC_GEN(&a->v)) { 1807 a->v.gen++; 1808 SET_BCH_ALLOC_V4_NEED_INC_GEN(&a->v, false); 1809 goto write; 1810 } 1811 1812 if (bch2_trans_inconsistent_on(c->curr_recovery_pass > BCH_RECOVERY_PASS_check_alloc_info, 1813 trans, "bucket incorrectly set in need_discard btree\n" 1814 "%s", 1815 (bch2_bkey_val_to_text(&buf, c, k), buf.buf))) 1816 ret = -EIO; 1817 goto out; 1818 } 1819 1820 if (a->v.journal_seq > c->journal.flushed_seq_ondisk) { 1821 if (bch2_trans_inconsistent_on(c->curr_recovery_pass > BCH_RECOVERY_PASS_check_alloc_info, 1822 trans, "clearing need_discard but journal_seq %llu > flushed_seq %llu\n%s", 1823 a->v.journal_seq, 1824 c->journal.flushed_seq_ondisk, 1825 (bch2_bkey_val_to_text(&buf, c, k), buf.buf))) 1826 ret = -EIO; 1827 goto out; 1828 } 1829 1830 if (discard_in_flight_add(ca, iter.pos.offset, true)) 1831 goto out; 1832 1833 discard_locked = true; 1834 1835 if (!bkey_eq(*discard_pos_done, iter.pos) && 1836 ca->mi.discard && !c->opts.nochanges) { 1837 /* 1838 * This works without any other locks because this is the only 1839 * thread that removes items from the need_discard tree 1840 */ 1841 bch2_trans_unlock_long(trans); 1842 blkdev_issue_discard(ca->disk_sb.bdev, 1843 k.k->p.offset * ca->mi.bucket_size, 1844 ca->mi.bucket_size, 1845 GFP_KERNEL); 1846 *discard_pos_done = iter.pos; 1847 1848 ret = bch2_trans_relock_notrace(trans); 1849 if (ret) 1850 goto out; 1851 } 1852 1853 SET_BCH_ALLOC_V4_NEED_DISCARD(&a->v, false); 1854 write: 1855 alloc_data_type_set(&a->v, a->v.data_type); 1856 1857 ret = bch2_trans_update(trans, &iter, &a->k_i, 0) ?: 1858 bch2_trans_commit(trans, NULL, NULL, 1859 BCH_WATERMARK_btree| 1860 BCH_TRANS_COMMIT_no_enospc); 1861 if (ret) 1862 goto out; 1863 1864 count_event(c, bucket_discard); 1865 s->discarded++; 1866 out: 1867 if (discard_locked) 1868 discard_in_flight_remove(ca, iter.pos.offset); 1869 s->seen++; 1870 bch2_trans_iter_exit(trans, &iter); 1871 printbuf_exit(&buf); 1872 return ret; 1873 } 1874 1875 static void bch2_do_discards_work(struct work_struct *work) 1876 { 1877 struct bch_dev *ca = container_of(work, struct bch_dev, discard_work); 1878 struct bch_fs *c = ca->fs; 1879 struct discard_buckets_state s = {}; 1880 struct bpos discard_pos_done = POS_MAX; 1881 int ret; 1882 1883 /* 1884 * We're doing the commit in bch2_discard_one_bucket instead of using 1885 * for_each_btree_key_commit() so that we can increment counters after 1886 * successful commit: 1887 */ 1888 ret = bch2_trans_run(c, 1889 for_each_btree_key_upto(trans, iter, 1890 BTREE_ID_need_discard, 1891 POS(ca->dev_idx, 0), 1892 POS(ca->dev_idx, U64_MAX), 0, k, 1893 bch2_discard_one_bucket(trans, ca, &iter, &discard_pos_done, &s))); 1894 1895 trace_discard_buckets(c, s.seen, s.open, s.need_journal_commit, s.discarded, 1896 bch2_err_str(ret)); 1897 1898 percpu_ref_put(&ca->io_ref); 1899 bch2_write_ref_put(c, BCH_WRITE_REF_discard); 1900 } 1901 1902 void bch2_dev_do_discards(struct bch_dev *ca) 1903 { 1904 struct bch_fs *c = ca->fs; 1905 1906 if (!bch2_write_ref_tryget(c, BCH_WRITE_REF_discard)) 1907 return; 1908 1909 if (!bch2_dev_get_ioref(c, ca->dev_idx, WRITE)) 1910 goto put_write_ref; 1911 1912 if (queue_work(c->write_ref_wq, &ca->discard_work)) 1913 return; 1914 1915 percpu_ref_put(&ca->io_ref); 1916 put_write_ref: 1917 bch2_write_ref_put(c, BCH_WRITE_REF_discard); 1918 } 1919 1920 void bch2_do_discards(struct bch_fs *c) 1921 { 1922 for_each_member_device(c, ca) 1923 bch2_dev_do_discards(ca); 1924 } 1925 1926 static int bch2_clear_bucket_needs_discard(struct btree_trans *trans, struct bpos bucket) 1927 { 1928 struct btree_iter iter; 1929 bch2_trans_iter_init(trans, &iter, BTREE_ID_alloc, bucket, BTREE_ITER_intent); 1930 struct bkey_s_c k = bch2_btree_iter_peek_slot(&iter); 1931 int ret = bkey_err(k); 1932 if (ret) 1933 goto err; 1934 1935 struct bkey_i_alloc_v4 *a = bch2_alloc_to_v4_mut(trans, k); 1936 ret = PTR_ERR_OR_ZERO(a); 1937 if (ret) 1938 goto err; 1939 1940 BUG_ON(a->v.dirty_sectors); 1941 SET_BCH_ALLOC_V4_NEED_DISCARD(&a->v, false); 1942 alloc_data_type_set(&a->v, a->v.data_type); 1943 1944 ret = bch2_trans_update(trans, &iter, &a->k_i, 0); 1945 err: 1946 bch2_trans_iter_exit(trans, &iter); 1947 return ret; 1948 } 1949 1950 static void bch2_do_discards_fast_work(struct work_struct *work) 1951 { 1952 struct bch_dev *ca = container_of(work, struct bch_dev, discard_fast_work); 1953 struct bch_fs *c = ca->fs; 1954 1955 while (1) { 1956 bool got_bucket = false; 1957 u64 bucket; 1958 1959 mutex_lock(&ca->discard_buckets_in_flight_lock); 1960 darray_for_each(ca->discard_buckets_in_flight, i) { 1961 if (i->in_progress) 1962 continue; 1963 1964 got_bucket = true; 1965 bucket = i->bucket; 1966 i->in_progress = true; 1967 break; 1968 } 1969 mutex_unlock(&ca->discard_buckets_in_flight_lock); 1970 1971 if (!got_bucket) 1972 break; 1973 1974 if (ca->mi.discard && !c->opts.nochanges) 1975 blkdev_issue_discard(ca->disk_sb.bdev, 1976 bucket_to_sector(ca, bucket), 1977 ca->mi.bucket_size, 1978 GFP_KERNEL); 1979 1980 int ret = bch2_trans_commit_do(c, NULL, NULL, 1981 BCH_WATERMARK_btree| 1982 BCH_TRANS_COMMIT_no_enospc, 1983 bch2_clear_bucket_needs_discard(trans, POS(ca->dev_idx, bucket))); 1984 bch_err_fn(c, ret); 1985 1986 discard_in_flight_remove(ca, bucket); 1987 1988 if (ret) 1989 break; 1990 } 1991 1992 percpu_ref_put(&ca->io_ref); 1993 bch2_write_ref_put(c, BCH_WRITE_REF_discard_fast); 1994 } 1995 1996 static void bch2_discard_one_bucket_fast(struct bch_dev *ca, u64 bucket) 1997 { 1998 struct bch_fs *c = ca->fs; 1999 2000 if (discard_in_flight_add(ca, bucket, false)) 2001 return; 2002 2003 if (!bch2_write_ref_tryget(c, BCH_WRITE_REF_discard_fast)) 2004 return; 2005 2006 if (!bch2_dev_get_ioref(c, ca->dev_idx, WRITE)) 2007 goto put_ref; 2008 2009 if (queue_work(c->write_ref_wq, &ca->discard_fast_work)) 2010 return; 2011 2012 percpu_ref_put(&ca->io_ref); 2013 put_ref: 2014 bch2_write_ref_put(c, BCH_WRITE_REF_discard_fast); 2015 } 2016 2017 static int invalidate_one_bucket(struct btree_trans *trans, 2018 struct btree_iter *lru_iter, 2019 struct bkey_s_c lru_k, 2020 s64 *nr_to_invalidate) 2021 { 2022 struct bch_fs *c = trans->c; 2023 struct bkey_i_alloc_v4 *a = NULL; 2024 struct printbuf buf = PRINTBUF; 2025 struct bpos bucket = u64_to_bucket(lru_k.k->p.offset); 2026 unsigned cached_sectors; 2027 int ret = 0; 2028 2029 if (*nr_to_invalidate <= 0) 2030 return 1; 2031 2032 if (!bch2_dev_bucket_exists(c, bucket)) { 2033 prt_str(&buf, "lru entry points to invalid bucket"); 2034 goto err; 2035 } 2036 2037 if (bch2_bucket_is_open_safe(c, bucket.inode, bucket.offset)) 2038 return 0; 2039 2040 a = bch2_trans_start_alloc_update(trans, bucket, BTREE_TRIGGER_bucket_invalidate); 2041 ret = PTR_ERR_OR_ZERO(a); 2042 if (ret) 2043 goto out; 2044 2045 /* We expect harmless races here due to the btree write buffer: */ 2046 if (lru_pos_time(lru_iter->pos) != alloc_lru_idx_read(a->v)) 2047 goto out; 2048 2049 BUG_ON(a->v.data_type != BCH_DATA_cached); 2050 BUG_ON(a->v.dirty_sectors); 2051 2052 if (!a->v.cached_sectors) 2053 bch_err(c, "invalidating empty bucket, confused"); 2054 2055 cached_sectors = a->v.cached_sectors; 2056 2057 SET_BCH_ALLOC_V4_NEED_INC_GEN(&a->v, false); 2058 a->v.gen++; 2059 a->v.data_type = 0; 2060 a->v.dirty_sectors = 0; 2061 a->v.stripe_sectors = 0; 2062 a->v.cached_sectors = 0; 2063 a->v.io_time[READ] = bch2_current_io_time(c, READ); 2064 a->v.io_time[WRITE] = bch2_current_io_time(c, WRITE); 2065 2066 ret = bch2_trans_commit(trans, NULL, NULL, 2067 BCH_WATERMARK_btree| 2068 BCH_TRANS_COMMIT_no_enospc); 2069 if (ret) 2070 goto out; 2071 2072 trace_and_count(c, bucket_invalidate, c, bucket.inode, bucket.offset, cached_sectors); 2073 --*nr_to_invalidate; 2074 out: 2075 printbuf_exit(&buf); 2076 return ret; 2077 err: 2078 prt_str(&buf, "\n lru key: "); 2079 bch2_bkey_val_to_text(&buf, c, lru_k); 2080 2081 prt_str(&buf, "\n lru entry: "); 2082 bch2_lru_pos_to_text(&buf, lru_iter->pos); 2083 2084 prt_str(&buf, "\n alloc key: "); 2085 if (!a) 2086 bch2_bpos_to_text(&buf, bucket); 2087 else 2088 bch2_bkey_val_to_text(&buf, c, bkey_i_to_s_c(&a->k_i)); 2089 2090 bch_err(c, "%s", buf.buf); 2091 if (c->curr_recovery_pass > BCH_RECOVERY_PASS_check_lrus) { 2092 bch2_inconsistent_error(c); 2093 ret = -EINVAL; 2094 } 2095 2096 goto out; 2097 } 2098 2099 static struct bkey_s_c next_lru_key(struct btree_trans *trans, struct btree_iter *iter, 2100 struct bch_dev *ca, bool *wrapped) 2101 { 2102 struct bkey_s_c k; 2103 again: 2104 k = bch2_btree_iter_peek_upto(iter, lru_pos(ca->dev_idx, U64_MAX, LRU_TIME_MAX)); 2105 if (!k.k && !*wrapped) { 2106 bch2_btree_iter_set_pos(iter, lru_pos(ca->dev_idx, 0, 0)); 2107 *wrapped = true; 2108 goto again; 2109 } 2110 2111 return k; 2112 } 2113 2114 static void bch2_do_invalidates_work(struct work_struct *work) 2115 { 2116 struct bch_dev *ca = container_of(work, struct bch_dev, invalidate_work); 2117 struct bch_fs *c = ca->fs; 2118 struct btree_trans *trans = bch2_trans_get(c); 2119 int ret = 0; 2120 2121 ret = bch2_btree_write_buffer_tryflush(trans); 2122 if (ret) 2123 goto err; 2124 2125 s64 nr_to_invalidate = 2126 should_invalidate_buckets(ca, bch2_dev_usage_read(ca)); 2127 struct btree_iter iter; 2128 bool wrapped = false; 2129 2130 bch2_trans_iter_init(trans, &iter, BTREE_ID_lru, 2131 lru_pos(ca->dev_idx, 0, 2132 ((bch2_current_io_time(c, READ) + U32_MAX) & 2133 LRU_TIME_MAX)), 0); 2134 2135 while (true) { 2136 bch2_trans_begin(trans); 2137 2138 struct bkey_s_c k = next_lru_key(trans, &iter, ca, &wrapped); 2139 ret = bkey_err(k); 2140 if (ret) 2141 goto restart_err; 2142 if (!k.k) 2143 break; 2144 2145 ret = invalidate_one_bucket(trans, &iter, k, &nr_to_invalidate); 2146 restart_err: 2147 if (bch2_err_matches(ret, BCH_ERR_transaction_restart)) 2148 continue; 2149 if (ret) 2150 break; 2151 2152 bch2_btree_iter_advance(&iter); 2153 } 2154 bch2_trans_iter_exit(trans, &iter); 2155 err: 2156 bch2_trans_put(trans); 2157 percpu_ref_put(&ca->io_ref); 2158 bch2_write_ref_put(c, BCH_WRITE_REF_invalidate); 2159 } 2160 2161 void bch2_dev_do_invalidates(struct bch_dev *ca) 2162 { 2163 struct bch_fs *c = ca->fs; 2164 2165 if (!bch2_write_ref_tryget(c, BCH_WRITE_REF_invalidate)) 2166 return; 2167 2168 if (!bch2_dev_get_ioref(c, ca->dev_idx, WRITE)) 2169 goto put_ref; 2170 2171 if (queue_work(c->write_ref_wq, &ca->invalidate_work)) 2172 return; 2173 2174 percpu_ref_put(&ca->io_ref); 2175 put_ref: 2176 bch2_write_ref_put(c, BCH_WRITE_REF_invalidate); 2177 } 2178 2179 void bch2_do_invalidates(struct bch_fs *c) 2180 { 2181 for_each_member_device(c, ca) 2182 bch2_dev_do_invalidates(ca); 2183 } 2184 2185 int bch2_dev_freespace_init(struct bch_fs *c, struct bch_dev *ca, 2186 u64 bucket_start, u64 bucket_end) 2187 { 2188 struct btree_trans *trans = bch2_trans_get(c); 2189 struct btree_iter iter; 2190 struct bkey_s_c k; 2191 struct bkey hole; 2192 struct bpos end = POS(ca->dev_idx, bucket_end); 2193 struct bch_member *m; 2194 unsigned long last_updated = jiffies; 2195 int ret; 2196 2197 BUG_ON(bucket_start > bucket_end); 2198 BUG_ON(bucket_end > ca->mi.nbuckets); 2199 2200 bch2_trans_iter_init(trans, &iter, BTREE_ID_alloc, 2201 POS(ca->dev_idx, max_t(u64, ca->mi.first_bucket, bucket_start)), 2202 BTREE_ITER_prefetch); 2203 /* 2204 * Scan the alloc btree for every bucket on @ca, and add buckets to the 2205 * freespace/need_discard/need_gc_gens btrees as needed: 2206 */ 2207 while (1) { 2208 if (time_after(jiffies, last_updated + HZ * 10)) { 2209 bch_info(ca, "%s: currently at %llu/%llu", 2210 __func__, iter.pos.offset, ca->mi.nbuckets); 2211 last_updated = jiffies; 2212 } 2213 2214 bch2_trans_begin(trans); 2215 2216 if (bkey_ge(iter.pos, end)) { 2217 ret = 0; 2218 break; 2219 } 2220 2221 k = bch2_get_key_or_hole(&iter, end, &hole); 2222 ret = bkey_err(k); 2223 if (ret) 2224 goto bkey_err; 2225 2226 if (k.k->type) { 2227 /* 2228 * We process live keys in the alloc btree one at a 2229 * time: 2230 */ 2231 struct bch_alloc_v4 a_convert; 2232 const struct bch_alloc_v4 *a = bch2_alloc_to_v4(k, &a_convert); 2233 2234 ret = bch2_bucket_do_index(trans, ca, k, a, true) ?: 2235 bch2_trans_commit(trans, NULL, NULL, 2236 BCH_TRANS_COMMIT_no_enospc); 2237 if (ret) 2238 goto bkey_err; 2239 2240 bch2_btree_iter_advance(&iter); 2241 } else { 2242 struct bkey_i *freespace; 2243 2244 freespace = bch2_trans_kmalloc(trans, sizeof(*freespace)); 2245 ret = PTR_ERR_OR_ZERO(freespace); 2246 if (ret) 2247 goto bkey_err; 2248 2249 bkey_init(&freespace->k); 2250 freespace->k.type = KEY_TYPE_set; 2251 freespace->k.p = k.k->p; 2252 freespace->k.size = k.k->size; 2253 2254 ret = bch2_btree_insert_trans(trans, BTREE_ID_freespace, freespace, 0) ?: 2255 bch2_trans_commit(trans, NULL, NULL, 2256 BCH_TRANS_COMMIT_no_enospc); 2257 if (ret) 2258 goto bkey_err; 2259 2260 bch2_btree_iter_set_pos(&iter, k.k->p); 2261 } 2262 bkey_err: 2263 if (bch2_err_matches(ret, BCH_ERR_transaction_restart)) 2264 continue; 2265 if (ret) 2266 break; 2267 } 2268 2269 bch2_trans_iter_exit(trans, &iter); 2270 bch2_trans_put(trans); 2271 2272 if (ret < 0) { 2273 bch_err_msg(ca, ret, "initializing free space"); 2274 return ret; 2275 } 2276 2277 mutex_lock(&c->sb_lock); 2278 m = bch2_members_v2_get_mut(c->disk_sb.sb, ca->dev_idx); 2279 SET_BCH_MEMBER_FREESPACE_INITIALIZED(m, true); 2280 mutex_unlock(&c->sb_lock); 2281 2282 return 0; 2283 } 2284 2285 int bch2_fs_freespace_init(struct bch_fs *c) 2286 { 2287 int ret = 0; 2288 bool doing_init = false; 2289 2290 /* 2291 * We can crash during the device add path, so we need to check this on 2292 * every mount: 2293 */ 2294 2295 for_each_member_device(c, ca) { 2296 if (ca->mi.freespace_initialized) 2297 continue; 2298 2299 if (!doing_init) { 2300 bch_info(c, "initializing freespace"); 2301 doing_init = true; 2302 } 2303 2304 ret = bch2_dev_freespace_init(c, ca, 0, ca->mi.nbuckets); 2305 if (ret) { 2306 bch2_dev_put(ca); 2307 bch_err_fn(c, ret); 2308 return ret; 2309 } 2310 } 2311 2312 if (doing_init) { 2313 mutex_lock(&c->sb_lock); 2314 bch2_write_super(c); 2315 mutex_unlock(&c->sb_lock); 2316 bch_verbose(c, "done initializing freespace"); 2317 } 2318 2319 return 0; 2320 } 2321 2322 /* device removal */ 2323 2324 int bch2_dev_remove_alloc(struct bch_fs *c, struct bch_dev *ca) 2325 { 2326 struct bpos start = POS(ca->dev_idx, 0); 2327 struct bpos end = POS(ca->dev_idx, U64_MAX); 2328 int ret; 2329 2330 /* 2331 * We clear the LRU and need_discard btrees first so that we don't race 2332 * with bch2_do_invalidates() and bch2_do_discards() 2333 */ 2334 ret = bch2_dev_remove_stripes(c, ca->dev_idx) ?: 2335 bch2_btree_delete_range(c, BTREE_ID_lru, start, end, 2336 BTREE_TRIGGER_norun, NULL) ?: 2337 bch2_btree_delete_range(c, BTREE_ID_need_discard, start, end, 2338 BTREE_TRIGGER_norun, NULL) ?: 2339 bch2_btree_delete_range(c, BTREE_ID_freespace, start, end, 2340 BTREE_TRIGGER_norun, NULL) ?: 2341 bch2_btree_delete_range(c, BTREE_ID_backpointers, start, end, 2342 BTREE_TRIGGER_norun, NULL) ?: 2343 bch2_btree_delete_range(c, BTREE_ID_bucket_gens, start, end, 2344 BTREE_TRIGGER_norun, NULL) ?: 2345 bch2_btree_delete_range(c, BTREE_ID_alloc, start, end, 2346 BTREE_TRIGGER_norun, NULL) ?: 2347 bch2_dev_usage_remove(c, ca->dev_idx); 2348 bch_err_msg(ca, ret, "removing dev alloc info"); 2349 return ret; 2350 } 2351 2352 /* Bucket IO clocks: */ 2353 2354 static int __bch2_bucket_io_time_reset(struct btree_trans *trans, unsigned dev, 2355 size_t bucket_nr, int rw) 2356 { 2357 struct bch_fs *c = trans->c; 2358 2359 struct btree_iter iter; 2360 struct bkey_i_alloc_v4 *a = 2361 bch2_trans_start_alloc_update_noupdate(trans, &iter, POS(dev, bucket_nr)); 2362 int ret = PTR_ERR_OR_ZERO(a); 2363 if (ret) 2364 return ret; 2365 2366 u64 now = bch2_current_io_time(c, rw); 2367 if (a->v.io_time[rw] == now) 2368 goto out; 2369 2370 a->v.io_time[rw] = now; 2371 2372 ret = bch2_trans_update(trans, &iter, &a->k_i, 0) ?: 2373 bch2_trans_commit(trans, NULL, NULL, 0); 2374 out: 2375 bch2_trans_iter_exit(trans, &iter); 2376 return ret; 2377 } 2378 2379 int bch2_bucket_io_time_reset(struct btree_trans *trans, unsigned dev, 2380 size_t bucket_nr, int rw) 2381 { 2382 if (bch2_trans_relock(trans)) 2383 bch2_trans_begin(trans); 2384 2385 return nested_lockrestart_do(trans, __bch2_bucket_io_time_reset(trans, dev, bucket_nr, rw)); 2386 } 2387 2388 /* Startup/shutdown (ro/rw): */ 2389 2390 void bch2_recalc_capacity(struct bch_fs *c) 2391 { 2392 u64 capacity = 0, reserved_sectors = 0, gc_reserve; 2393 unsigned bucket_size_max = 0; 2394 unsigned long ra_pages = 0; 2395 2396 lockdep_assert_held(&c->state_lock); 2397 2398 for_each_online_member(c, ca) { 2399 struct backing_dev_info *bdi = ca->disk_sb.bdev->bd_disk->bdi; 2400 2401 ra_pages += bdi->ra_pages; 2402 } 2403 2404 bch2_set_ra_pages(c, ra_pages); 2405 2406 for_each_rw_member(c, ca) { 2407 u64 dev_reserve = 0; 2408 2409 /* 2410 * We need to reserve buckets (from the number 2411 * of currently available buckets) against 2412 * foreground writes so that mainly copygc can 2413 * make forward progress. 2414 * 2415 * We need enough to refill the various reserves 2416 * from scratch - copygc will use its entire 2417 * reserve all at once, then run against when 2418 * its reserve is refilled (from the formerly 2419 * available buckets). 2420 * 2421 * This reserve is just used when considering if 2422 * allocations for foreground writes must wait - 2423 * not -ENOSPC calculations. 2424 */ 2425 2426 dev_reserve += ca->nr_btree_reserve * 2; 2427 dev_reserve += ca->mi.nbuckets >> 6; /* copygc reserve */ 2428 2429 dev_reserve += 1; /* btree write point */ 2430 dev_reserve += 1; /* copygc write point */ 2431 dev_reserve += 1; /* rebalance write point */ 2432 2433 dev_reserve *= ca->mi.bucket_size; 2434 2435 capacity += bucket_to_sector(ca, ca->mi.nbuckets - 2436 ca->mi.first_bucket); 2437 2438 reserved_sectors += dev_reserve * 2; 2439 2440 bucket_size_max = max_t(unsigned, bucket_size_max, 2441 ca->mi.bucket_size); 2442 } 2443 2444 gc_reserve = c->opts.gc_reserve_bytes 2445 ? c->opts.gc_reserve_bytes >> 9 2446 : div64_u64(capacity * c->opts.gc_reserve_percent, 100); 2447 2448 reserved_sectors = max(gc_reserve, reserved_sectors); 2449 2450 reserved_sectors = min(reserved_sectors, capacity); 2451 2452 c->reserved = reserved_sectors; 2453 c->capacity = capacity - reserved_sectors; 2454 2455 c->bucket_size_max = bucket_size_max; 2456 2457 /* Wake up case someone was waiting for buckets */ 2458 closure_wake_up(&c->freelist_wait); 2459 } 2460 2461 u64 bch2_min_rw_member_capacity(struct bch_fs *c) 2462 { 2463 u64 ret = U64_MAX; 2464 2465 for_each_rw_member(c, ca) 2466 ret = min(ret, ca->mi.nbuckets * ca->mi.bucket_size); 2467 return ret; 2468 } 2469 2470 static bool bch2_dev_has_open_write_point(struct bch_fs *c, struct bch_dev *ca) 2471 { 2472 struct open_bucket *ob; 2473 bool ret = false; 2474 2475 for (ob = c->open_buckets; 2476 ob < c->open_buckets + ARRAY_SIZE(c->open_buckets); 2477 ob++) { 2478 spin_lock(&ob->lock); 2479 if (ob->valid && !ob->on_partial_list && 2480 ob->dev == ca->dev_idx) 2481 ret = true; 2482 spin_unlock(&ob->lock); 2483 } 2484 2485 return ret; 2486 } 2487 2488 /* device goes ro: */ 2489 void bch2_dev_allocator_remove(struct bch_fs *c, struct bch_dev *ca) 2490 { 2491 lockdep_assert_held(&c->state_lock); 2492 2493 /* First, remove device from allocation groups: */ 2494 2495 for (unsigned i = 0; i < ARRAY_SIZE(c->rw_devs); i++) 2496 clear_bit(ca->dev_idx, c->rw_devs[i].d); 2497 2498 c->rw_devs_change_count++; 2499 2500 /* 2501 * Capacity is calculated based off of devices in allocation groups: 2502 */ 2503 bch2_recalc_capacity(c); 2504 2505 bch2_open_buckets_stop(c, ca, false); 2506 2507 /* 2508 * Wake up threads that were blocked on allocation, so they can notice 2509 * the device can no longer be removed and the capacity has changed: 2510 */ 2511 closure_wake_up(&c->freelist_wait); 2512 2513 /* 2514 * journal_res_get() can block waiting for free space in the journal - 2515 * it needs to notice there may not be devices to allocate from anymore: 2516 */ 2517 wake_up(&c->journal.wait); 2518 2519 /* Now wait for any in flight writes: */ 2520 2521 closure_wait_event(&c->open_buckets_wait, 2522 !bch2_dev_has_open_write_point(c, ca)); 2523 } 2524 2525 /* device goes rw: */ 2526 void bch2_dev_allocator_add(struct bch_fs *c, struct bch_dev *ca) 2527 { 2528 lockdep_assert_held(&c->state_lock); 2529 2530 for (unsigned i = 0; i < ARRAY_SIZE(c->rw_devs); i++) 2531 if (ca->mi.data_allowed & (1 << i)) 2532 set_bit(ca->dev_idx, c->rw_devs[i].d); 2533 2534 c->rw_devs_change_count++; 2535 } 2536 2537 void bch2_dev_allocator_background_exit(struct bch_dev *ca) 2538 { 2539 darray_exit(&ca->discard_buckets_in_flight); 2540 } 2541 2542 void bch2_dev_allocator_background_init(struct bch_dev *ca) 2543 { 2544 mutex_init(&ca->discard_buckets_in_flight_lock); 2545 INIT_WORK(&ca->discard_work, bch2_do_discards_work); 2546 INIT_WORK(&ca->discard_fast_work, bch2_do_discards_fast_work); 2547 INIT_WORK(&ca->invalidate_work, bch2_do_invalidates_work); 2548 } 2549 2550 void bch2_fs_allocator_background_init(struct bch_fs *c) 2551 { 2552 spin_lock_init(&c->freelist_lock); 2553 } 2554