1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _BCACHEFS_ALLOC_FOREGROUND_H
3 #define _BCACHEFS_ALLOC_FOREGROUND_H
4
5 #include "bcachefs.h"
6 #include "alloc_types.h"
7 #include "extents.h"
8 #include "sb-members.h"
9
10 #include <linux/hash.h>
11
12 struct bkey;
13 struct bch_dev;
14 struct bch_fs;
15 struct bch_devs_List;
16
17 extern const char * const bch2_watermarks[];
18
19 void bch2_reset_alloc_cursors(struct bch_fs *);
20
21 struct dev_alloc_list {
22 unsigned nr;
23 u8 data[BCH_SB_MEMBERS_MAX];
24 };
25
26 struct dev_alloc_list bch2_dev_alloc_list(struct bch_fs *,
27 struct dev_stripe_state *,
28 struct bch_devs_mask *);
29 void bch2_dev_stripe_increment(struct bch_dev *, struct dev_stripe_state *);
30
ob_dev(struct bch_fs * c,struct open_bucket * ob)31 static inline struct bch_dev *ob_dev(struct bch_fs *c, struct open_bucket *ob)
32 {
33 return bch2_dev_have_ref(c, ob->dev);
34 }
35
36 struct open_bucket *bch2_bucket_alloc(struct bch_fs *, struct bch_dev *,
37 enum bch_watermark, enum bch_data_type,
38 struct closure *);
39
ob_push(struct bch_fs * c,struct open_buckets * obs,struct open_bucket * ob)40 static inline void ob_push(struct bch_fs *c, struct open_buckets *obs,
41 struct open_bucket *ob)
42 {
43 BUG_ON(obs->nr >= ARRAY_SIZE(obs->v));
44
45 obs->v[obs->nr++] = ob - c->open_buckets;
46 }
47
48 #define open_bucket_for_each(_c, _obs, _ob, _i) \
49 for ((_i) = 0; \
50 (_i) < (_obs)->nr && \
51 ((_ob) = (_c)->open_buckets + (_obs)->v[_i], true); \
52 (_i)++)
53
ec_open_bucket(struct bch_fs * c,struct open_buckets * obs)54 static inline struct open_bucket *ec_open_bucket(struct bch_fs *c,
55 struct open_buckets *obs)
56 {
57 struct open_bucket *ob;
58 unsigned i;
59
60 open_bucket_for_each(c, obs, ob, i)
61 if (ob->ec)
62 return ob;
63
64 return NULL;
65 }
66
67 void bch2_open_bucket_write_error(struct bch_fs *,
68 struct open_buckets *, unsigned);
69
70 void __bch2_open_bucket_put(struct bch_fs *, struct open_bucket *);
71
bch2_open_bucket_put(struct bch_fs * c,struct open_bucket * ob)72 static inline void bch2_open_bucket_put(struct bch_fs *c, struct open_bucket *ob)
73 {
74 if (atomic_dec_and_test(&ob->pin))
75 __bch2_open_bucket_put(c, ob);
76 }
77
bch2_open_buckets_put(struct bch_fs * c,struct open_buckets * ptrs)78 static inline void bch2_open_buckets_put(struct bch_fs *c,
79 struct open_buckets *ptrs)
80 {
81 struct open_bucket *ob;
82 unsigned i;
83
84 open_bucket_for_each(c, ptrs, ob, i)
85 bch2_open_bucket_put(c, ob);
86 ptrs->nr = 0;
87 }
88
bch2_alloc_sectors_done_inlined(struct bch_fs * c,struct write_point * wp)89 static inline void bch2_alloc_sectors_done_inlined(struct bch_fs *c, struct write_point *wp)
90 {
91 struct open_buckets ptrs = { .nr = 0 }, keep = { .nr = 0 };
92 struct open_bucket *ob;
93 unsigned i;
94
95 open_bucket_for_each(c, &wp->ptrs, ob, i)
96 ob_push(c, !ob->sectors_free ? &ptrs : &keep, ob);
97 wp->ptrs = keep;
98
99 mutex_unlock(&wp->lock);
100
101 bch2_open_buckets_put(c, &ptrs);
102 }
103
bch2_open_bucket_get(struct bch_fs * c,struct write_point * wp,struct open_buckets * ptrs)104 static inline void bch2_open_bucket_get(struct bch_fs *c,
105 struct write_point *wp,
106 struct open_buckets *ptrs)
107 {
108 struct open_bucket *ob;
109 unsigned i;
110
111 open_bucket_for_each(c, &wp->ptrs, ob, i) {
112 ob->data_type = wp->data_type;
113 atomic_inc(&ob->pin);
114 ob_push(c, ptrs, ob);
115 }
116 }
117
open_bucket_hashslot(struct bch_fs * c,unsigned dev,u64 bucket)118 static inline open_bucket_idx_t *open_bucket_hashslot(struct bch_fs *c,
119 unsigned dev, u64 bucket)
120 {
121 return c->open_buckets_hash +
122 (jhash_3words(dev, bucket, bucket >> 32, 0) &
123 (OPEN_BUCKETS_COUNT - 1));
124 }
125
bch2_bucket_is_open(struct bch_fs * c,unsigned dev,u64 bucket)126 static inline bool bch2_bucket_is_open(struct bch_fs *c, unsigned dev, u64 bucket)
127 {
128 open_bucket_idx_t slot = *open_bucket_hashslot(c, dev, bucket);
129
130 while (slot) {
131 struct open_bucket *ob = &c->open_buckets[slot];
132
133 if (ob->dev == dev && ob->bucket == bucket)
134 return true;
135
136 slot = ob->hash;
137 }
138
139 return false;
140 }
141
bch2_bucket_is_open_safe(struct bch_fs * c,unsigned dev,u64 bucket)142 static inline bool bch2_bucket_is_open_safe(struct bch_fs *c, unsigned dev, u64 bucket)
143 {
144 bool ret;
145
146 if (bch2_bucket_is_open(c, dev, bucket))
147 return true;
148
149 spin_lock(&c->freelist_lock);
150 ret = bch2_bucket_is_open(c, dev, bucket);
151 spin_unlock(&c->freelist_lock);
152
153 return ret;
154 }
155
156 enum bch_write_flags;
157 int bch2_bucket_alloc_set_trans(struct btree_trans *, struct open_buckets *,
158 struct dev_stripe_state *, struct bch_devs_mask *,
159 unsigned, unsigned *, bool *, enum bch_write_flags,
160 enum bch_data_type, enum bch_watermark,
161 struct closure *);
162
163 int bch2_alloc_sectors_start_trans(struct btree_trans *,
164 unsigned, unsigned,
165 struct write_point_specifier,
166 struct bch_devs_list *,
167 unsigned, unsigned,
168 enum bch_watermark,
169 enum bch_write_flags,
170 struct closure *,
171 struct write_point **);
172
173 struct bch_extent_ptr bch2_ob_ptr(struct bch_fs *, struct open_bucket *);
174
175 /*
176 * Append pointers to the space we just allocated to @k, and mark @sectors space
177 * as allocated out of @ob
178 */
179 static inline void
bch2_alloc_sectors_append_ptrs_inlined(struct bch_fs * c,struct write_point * wp,struct bkey_i * k,unsigned sectors,bool cached)180 bch2_alloc_sectors_append_ptrs_inlined(struct bch_fs *c, struct write_point *wp,
181 struct bkey_i *k, unsigned sectors,
182 bool cached)
183 {
184 struct open_bucket *ob;
185 unsigned i;
186
187 BUG_ON(sectors > wp->sectors_free);
188 wp->sectors_free -= sectors;
189 wp->sectors_allocated += sectors;
190
191 open_bucket_for_each(c, &wp->ptrs, ob, i) {
192 struct bch_dev *ca = ob_dev(c, ob);
193 struct bch_extent_ptr ptr = bch2_ob_ptr(c, ob);
194
195 ptr.cached = cached ||
196 (!ca->mi.durability &&
197 wp->data_type == BCH_DATA_user);
198
199 bch2_bkey_append_ptr(k, ptr);
200
201 BUG_ON(sectors > ob->sectors_free);
202 ob->sectors_free -= sectors;
203 }
204 }
205
206 void bch2_alloc_sectors_append_ptrs(struct bch_fs *, struct write_point *,
207 struct bkey_i *, unsigned, bool);
208 void bch2_alloc_sectors_done(struct bch_fs *, struct write_point *);
209
210 void bch2_open_buckets_stop(struct bch_fs *c, struct bch_dev *, bool);
211
writepoint_hashed(unsigned long v)212 static inline struct write_point_specifier writepoint_hashed(unsigned long v)
213 {
214 return (struct write_point_specifier) { .v = v | 1 };
215 }
216
writepoint_ptr(struct write_point * wp)217 static inline struct write_point_specifier writepoint_ptr(struct write_point *wp)
218 {
219 return (struct write_point_specifier) { .v = (unsigned long) wp };
220 }
221
222 void bch2_fs_allocator_foreground_init(struct bch_fs *);
223
224 void bch2_open_bucket_to_text(struct printbuf *, struct bch_fs *, struct open_bucket *);
225 void bch2_open_buckets_to_text(struct printbuf *, struct bch_fs *, struct bch_dev *);
226 void bch2_open_buckets_partial_to_text(struct printbuf *, struct bch_fs *);
227
228 void bch2_write_points_to_text(struct printbuf *, struct bch_fs *);
229
230 void bch2_fs_alloc_debug_to_text(struct printbuf *, struct bch_fs *);
231 void bch2_dev_alloc_debug_to_text(struct printbuf *, struct bch_dev *);
232
233 void __bch2_wait_on_allocator(struct bch_fs *, struct closure *);
bch2_wait_on_allocator(struct bch_fs * c,struct closure * cl)234 static inline void bch2_wait_on_allocator(struct bch_fs *c, struct closure *cl)
235 {
236 if (cl->closure_get_happened)
237 __bch2_wait_on_allocator(c, cl);
238 }
239
240 #endif /* _BCACHEFS_ALLOC_FOREGROUND_H */
241