1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _BCACHEFS_ALLOC_FOREGROUND_H 3 #define _BCACHEFS_ALLOC_FOREGROUND_H 4 5 #include "bcachefs.h" 6 #include "alloc_types.h" 7 #include "extents.h" 8 #include "sb-members.h" 9 10 #include <linux/hash.h> 11 12 struct bkey; 13 struct bch_dev; 14 struct bch_fs; 15 struct bch_devs_List; 16 17 extern const char * const bch2_watermarks[]; 18 19 void bch2_reset_alloc_cursors(struct bch_fs *); 20 21 struct dev_alloc_list { 22 unsigned nr; 23 u8 data[BCH_SB_MEMBERS_MAX]; 24 }; 25 26 struct dev_alloc_list bch2_dev_alloc_list(struct bch_fs *, 27 struct dev_stripe_state *, 28 struct bch_devs_mask *); 29 void bch2_dev_stripe_increment(struct bch_dev *, struct dev_stripe_state *); 30 31 static inline struct bch_dev *ob_dev(struct bch_fs *c, struct open_bucket *ob) 32 { 33 return bch2_dev_have_ref(c, ob->dev); 34 } 35 36 struct open_bucket *bch2_bucket_alloc(struct bch_fs *, struct bch_dev *, 37 enum bch_watermark, enum bch_data_type, 38 struct closure *); 39 40 static inline void ob_push(struct bch_fs *c, struct open_buckets *obs, 41 struct open_bucket *ob) 42 { 43 BUG_ON(obs->nr >= ARRAY_SIZE(obs->v)); 44 45 obs->v[obs->nr++] = ob - c->open_buckets; 46 } 47 48 #define open_bucket_for_each(_c, _obs, _ob, _i) \ 49 for ((_i) = 0; \ 50 (_i) < (_obs)->nr && \ 51 ((_ob) = (_c)->open_buckets + (_obs)->v[_i], true); \ 52 (_i)++) 53 54 static inline struct open_bucket *ec_open_bucket(struct bch_fs *c, 55 struct open_buckets *obs) 56 { 57 struct open_bucket *ob; 58 unsigned i; 59 60 open_bucket_for_each(c, obs, ob, i) 61 if (ob->ec) 62 return ob; 63 64 return NULL; 65 } 66 67 void bch2_open_bucket_write_error(struct bch_fs *, 68 struct open_buckets *, unsigned); 69 70 void __bch2_open_bucket_put(struct bch_fs *, struct open_bucket *); 71 72 static inline void bch2_open_bucket_put(struct bch_fs *c, struct open_bucket *ob) 73 { 74 if (atomic_dec_and_test(&ob->pin)) 75 __bch2_open_bucket_put(c, ob); 76 } 77 78 static inline void bch2_open_buckets_put(struct bch_fs *c, 79 struct open_buckets *ptrs) 80 { 81 struct open_bucket *ob; 82 unsigned i; 83 84 open_bucket_for_each(c, ptrs, ob, i) 85 bch2_open_bucket_put(c, ob); 86 ptrs->nr = 0; 87 } 88 89 static inline void bch2_alloc_sectors_done_inlined(struct bch_fs *c, struct write_point *wp) 90 { 91 struct open_buckets ptrs = { .nr = 0 }, keep = { .nr = 0 }; 92 struct open_bucket *ob; 93 unsigned i; 94 95 open_bucket_for_each(c, &wp->ptrs, ob, i) 96 ob_push(c, !ob->sectors_free ? &ptrs : &keep, ob); 97 wp->ptrs = keep; 98 99 mutex_unlock(&wp->lock); 100 101 bch2_open_buckets_put(c, &ptrs); 102 } 103 104 static inline void bch2_open_bucket_get(struct bch_fs *c, 105 struct write_point *wp, 106 struct open_buckets *ptrs) 107 { 108 struct open_bucket *ob; 109 unsigned i; 110 111 open_bucket_for_each(c, &wp->ptrs, ob, i) { 112 ob->data_type = wp->data_type; 113 atomic_inc(&ob->pin); 114 ob_push(c, ptrs, ob); 115 } 116 } 117 118 static inline open_bucket_idx_t *open_bucket_hashslot(struct bch_fs *c, 119 unsigned dev, u64 bucket) 120 { 121 return c->open_buckets_hash + 122 (jhash_3words(dev, bucket, bucket >> 32, 0) & 123 (OPEN_BUCKETS_COUNT - 1)); 124 } 125 126 static inline bool bch2_bucket_is_open(struct bch_fs *c, unsigned dev, u64 bucket) 127 { 128 open_bucket_idx_t slot = *open_bucket_hashslot(c, dev, bucket); 129 130 while (slot) { 131 struct open_bucket *ob = &c->open_buckets[slot]; 132 133 if (ob->dev == dev && ob->bucket == bucket) 134 return true; 135 136 slot = ob->hash; 137 } 138 139 return false; 140 } 141 142 static inline bool bch2_bucket_is_open_safe(struct bch_fs *c, unsigned dev, u64 bucket) 143 { 144 bool ret; 145 146 if (bch2_bucket_is_open(c, dev, bucket)) 147 return true; 148 149 spin_lock(&c->freelist_lock); 150 ret = bch2_bucket_is_open(c, dev, bucket); 151 spin_unlock(&c->freelist_lock); 152 153 return ret; 154 } 155 156 enum bch_write_flags; 157 int bch2_bucket_alloc_set_trans(struct btree_trans *, struct open_buckets *, 158 struct dev_stripe_state *, struct bch_devs_mask *, 159 unsigned, unsigned *, bool *, enum bch_write_flags, 160 enum bch_data_type, enum bch_watermark, 161 struct closure *); 162 163 int bch2_alloc_sectors_start_trans(struct btree_trans *, 164 unsigned, unsigned, 165 struct write_point_specifier, 166 struct bch_devs_list *, 167 unsigned, unsigned, 168 enum bch_watermark, 169 enum bch_write_flags, 170 struct closure *, 171 struct write_point **); 172 173 struct bch_extent_ptr bch2_ob_ptr(struct bch_fs *, struct open_bucket *); 174 175 /* 176 * Append pointers to the space we just allocated to @k, and mark @sectors space 177 * as allocated out of @ob 178 */ 179 static inline void 180 bch2_alloc_sectors_append_ptrs_inlined(struct bch_fs *c, struct write_point *wp, 181 struct bkey_i *k, unsigned sectors, 182 bool cached) 183 { 184 struct open_bucket *ob; 185 unsigned i; 186 187 BUG_ON(sectors > wp->sectors_free); 188 wp->sectors_free -= sectors; 189 wp->sectors_allocated += sectors; 190 191 open_bucket_for_each(c, &wp->ptrs, ob, i) { 192 struct bch_dev *ca = ob_dev(c, ob); 193 struct bch_extent_ptr ptr = bch2_ob_ptr(c, ob); 194 195 ptr.cached = cached || 196 (!ca->mi.durability && 197 wp->data_type == BCH_DATA_user); 198 199 bch2_bkey_append_ptr(k, ptr); 200 201 BUG_ON(sectors > ob->sectors_free); 202 ob->sectors_free -= sectors; 203 } 204 } 205 206 void bch2_alloc_sectors_append_ptrs(struct bch_fs *, struct write_point *, 207 struct bkey_i *, unsigned, bool); 208 void bch2_alloc_sectors_done(struct bch_fs *, struct write_point *); 209 210 void bch2_open_buckets_stop(struct bch_fs *c, struct bch_dev *, bool); 211 212 static inline struct write_point_specifier writepoint_hashed(unsigned long v) 213 { 214 return (struct write_point_specifier) { .v = v | 1 }; 215 } 216 217 static inline struct write_point_specifier writepoint_ptr(struct write_point *wp) 218 { 219 return (struct write_point_specifier) { .v = (unsigned long) wp }; 220 } 221 222 void bch2_fs_allocator_foreground_init(struct bch_fs *); 223 224 void bch2_open_bucket_to_text(struct printbuf *, struct bch_fs *, struct open_bucket *); 225 void bch2_open_buckets_to_text(struct printbuf *, struct bch_fs *, struct bch_dev *); 226 void bch2_open_buckets_partial_to_text(struct printbuf *, struct bch_fs *); 227 228 void bch2_write_points_to_text(struct printbuf *, struct bch_fs *); 229 230 void bch2_fs_alloc_debug_to_text(struct printbuf *, struct bch_fs *); 231 void bch2_dev_alloc_debug_to_text(struct printbuf *, struct bch_dev *); 232 233 void __bch2_wait_on_allocator(struct bch_fs *, struct closure *); 234 static inline void bch2_wait_on_allocator(struct bch_fs *c, struct closure *cl) 235 { 236 if (cl->closure_get_happened) 237 __bch2_wait_on_allocator(c, cl); 238 } 239 240 #endif /* _BCACHEFS_ALLOC_FOREGROUND_H */ 241