1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _BCACHEFS_ALLOC_FOREGROUND_H 3 #define _BCACHEFS_ALLOC_FOREGROUND_H 4 5 #include "bcachefs.h" 6 #include "alloc_types.h" 7 #include "extents.h" 8 #include "sb-members.h" 9 10 #include <linux/hash.h> 11 12 struct bkey; 13 struct bch_dev; 14 struct bch_fs; 15 struct bch_devs_List; 16 17 extern const char * const bch2_watermarks[]; 18 19 void bch2_reset_alloc_cursors(struct bch_fs *); 20 21 struct dev_alloc_list { 22 unsigned nr; 23 u8 devs[BCH_SB_MEMBERS_MAX]; 24 }; 25 26 struct dev_alloc_list bch2_dev_alloc_list(struct bch_fs *, 27 struct dev_stripe_state *, 28 struct bch_devs_mask *); 29 void bch2_dev_stripe_increment(struct bch_dev *, struct dev_stripe_state *); 30 31 long bch2_bucket_alloc_new_fs(struct bch_dev *); 32 33 static inline struct bch_dev *ob_dev(struct bch_fs *c, struct open_bucket *ob) 34 { 35 return bch2_dev_have_ref(c, ob->dev); 36 } 37 38 struct open_bucket *bch2_bucket_alloc(struct bch_fs *, struct bch_dev *, 39 enum bch_watermark, enum bch_data_type, 40 struct closure *); 41 42 static inline void ob_push(struct bch_fs *c, struct open_buckets *obs, 43 struct open_bucket *ob) 44 { 45 BUG_ON(obs->nr >= ARRAY_SIZE(obs->v)); 46 47 obs->v[obs->nr++] = ob - c->open_buckets; 48 } 49 50 #define open_bucket_for_each(_c, _obs, _ob, _i) \ 51 for ((_i) = 0; \ 52 (_i) < (_obs)->nr && \ 53 ((_ob) = (_c)->open_buckets + (_obs)->v[_i], true); \ 54 (_i)++) 55 56 static inline struct open_bucket *ec_open_bucket(struct bch_fs *c, 57 struct open_buckets *obs) 58 { 59 struct open_bucket *ob; 60 unsigned i; 61 62 open_bucket_for_each(c, obs, ob, i) 63 if (ob->ec) 64 return ob; 65 66 return NULL; 67 } 68 69 void bch2_open_bucket_write_error(struct bch_fs *, 70 struct open_buckets *, unsigned); 71 72 void __bch2_open_bucket_put(struct bch_fs *, struct open_bucket *); 73 74 static inline void bch2_open_bucket_put(struct bch_fs *c, struct open_bucket *ob) 75 { 76 if (atomic_dec_and_test(&ob->pin)) 77 __bch2_open_bucket_put(c, ob); 78 } 79 80 static inline void bch2_open_buckets_put(struct bch_fs *c, 81 struct open_buckets *ptrs) 82 { 83 struct open_bucket *ob; 84 unsigned i; 85 86 open_bucket_for_each(c, ptrs, ob, i) 87 bch2_open_bucket_put(c, ob); 88 ptrs->nr = 0; 89 } 90 91 static inline void bch2_alloc_sectors_done_inlined(struct bch_fs *c, struct write_point *wp) 92 { 93 struct open_buckets ptrs = { .nr = 0 }, keep = { .nr = 0 }; 94 struct open_bucket *ob; 95 unsigned i; 96 97 open_bucket_for_each(c, &wp->ptrs, ob, i) 98 ob_push(c, !ob->sectors_free ? &ptrs : &keep, ob); 99 wp->ptrs = keep; 100 101 mutex_unlock(&wp->lock); 102 103 bch2_open_buckets_put(c, &ptrs); 104 } 105 106 static inline void bch2_open_bucket_get(struct bch_fs *c, 107 struct write_point *wp, 108 struct open_buckets *ptrs) 109 { 110 struct open_bucket *ob; 111 unsigned i; 112 113 open_bucket_for_each(c, &wp->ptrs, ob, i) { 114 ob->data_type = wp->data_type; 115 atomic_inc(&ob->pin); 116 ob_push(c, ptrs, ob); 117 } 118 } 119 120 static inline open_bucket_idx_t *open_bucket_hashslot(struct bch_fs *c, 121 unsigned dev, u64 bucket) 122 { 123 return c->open_buckets_hash + 124 (jhash_3words(dev, bucket, bucket >> 32, 0) & 125 (OPEN_BUCKETS_COUNT - 1)); 126 } 127 128 static inline bool bch2_bucket_is_open(struct bch_fs *c, unsigned dev, u64 bucket) 129 { 130 open_bucket_idx_t slot = *open_bucket_hashslot(c, dev, bucket); 131 132 while (slot) { 133 struct open_bucket *ob = &c->open_buckets[slot]; 134 135 if (ob->dev == dev && ob->bucket == bucket) 136 return true; 137 138 slot = ob->hash; 139 } 140 141 return false; 142 } 143 144 static inline bool bch2_bucket_is_open_safe(struct bch_fs *c, unsigned dev, u64 bucket) 145 { 146 bool ret; 147 148 if (bch2_bucket_is_open(c, dev, bucket)) 149 return true; 150 151 spin_lock(&c->freelist_lock); 152 ret = bch2_bucket_is_open(c, dev, bucket); 153 spin_unlock(&c->freelist_lock); 154 155 return ret; 156 } 157 158 int bch2_bucket_alloc_set_trans(struct btree_trans *, struct open_buckets *, 159 struct dev_stripe_state *, struct bch_devs_mask *, 160 unsigned, unsigned *, bool *, unsigned, 161 enum bch_data_type, enum bch_watermark, 162 struct closure *); 163 164 int bch2_alloc_sectors_start_trans(struct btree_trans *, 165 unsigned, unsigned, 166 struct write_point_specifier, 167 struct bch_devs_list *, 168 unsigned, unsigned, 169 enum bch_watermark, 170 unsigned, 171 struct closure *, 172 struct write_point **); 173 174 struct bch_extent_ptr bch2_ob_ptr(struct bch_fs *, struct open_bucket *); 175 176 /* 177 * Append pointers to the space we just allocated to @k, and mark @sectors space 178 * as allocated out of @ob 179 */ 180 static inline void 181 bch2_alloc_sectors_append_ptrs_inlined(struct bch_fs *c, struct write_point *wp, 182 struct bkey_i *k, unsigned sectors, 183 bool cached) 184 { 185 struct open_bucket *ob; 186 unsigned i; 187 188 BUG_ON(sectors > wp->sectors_free); 189 wp->sectors_free -= sectors; 190 wp->sectors_allocated += sectors; 191 192 open_bucket_for_each(c, &wp->ptrs, ob, i) { 193 struct bch_dev *ca = ob_dev(c, ob); 194 struct bch_extent_ptr ptr = bch2_ob_ptr(c, ob); 195 196 ptr.cached = cached || 197 (!ca->mi.durability && 198 wp->data_type == BCH_DATA_user); 199 200 bch2_bkey_append_ptr(k, ptr); 201 202 BUG_ON(sectors > ob->sectors_free); 203 ob->sectors_free -= sectors; 204 } 205 } 206 207 void bch2_alloc_sectors_append_ptrs(struct bch_fs *, struct write_point *, 208 struct bkey_i *, unsigned, bool); 209 void bch2_alloc_sectors_done(struct bch_fs *, struct write_point *); 210 211 void bch2_open_buckets_stop(struct bch_fs *c, struct bch_dev *, bool); 212 213 static inline struct write_point_specifier writepoint_hashed(unsigned long v) 214 { 215 return (struct write_point_specifier) { .v = v | 1 }; 216 } 217 218 static inline struct write_point_specifier writepoint_ptr(struct write_point *wp) 219 { 220 return (struct write_point_specifier) { .v = (unsigned long) wp }; 221 } 222 223 void bch2_fs_allocator_foreground_init(struct bch_fs *); 224 225 void bch2_open_buckets_to_text(struct printbuf *, struct bch_fs *); 226 void bch2_open_buckets_partial_to_text(struct printbuf *, struct bch_fs *); 227 228 void bch2_write_points_to_text(struct printbuf *, struct bch_fs *); 229 230 void bch2_fs_alloc_debug_to_text(struct printbuf *, struct bch_fs *); 231 void bch2_dev_alloc_debug_to_text(struct printbuf *, struct bch_dev *); 232 233 void bch2_print_allocator_stuck(struct bch_fs *); 234 235 #endif /* _BCACHEFS_ALLOC_FOREGROUND_H */ 236