1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _BCACHEFS_DISK_ACCOUNTING_H 3 #define _BCACHEFS_DISK_ACCOUNTING_H 4 5 #include "btree_update.h" 6 #include "eytzinger.h" 7 #include "sb-members.h" 8 9 static inline void bch2_u64s_neg(u64 *v, unsigned nr) 10 { 11 for (unsigned i = 0; i < nr; i++) 12 v[i] = -v[i]; 13 } 14 15 static inline unsigned bch2_accounting_counters(const struct bkey *k) 16 { 17 return bkey_val_u64s(k) - offsetof(struct bch_accounting, d) / sizeof(u64); 18 } 19 20 static inline void bch2_accounting_neg(struct bkey_s_accounting a) 21 { 22 bch2_u64s_neg(a.v->d, bch2_accounting_counters(a.k)); 23 } 24 25 static inline bool bch2_accounting_key_is_zero(struct bkey_s_c_accounting a) 26 { 27 for (unsigned i = 0; i < bch2_accounting_counters(a.k); i++) 28 if (a.v->d[i]) 29 return false; 30 return true; 31 } 32 33 static inline void bch2_accounting_accumulate(struct bkey_i_accounting *dst, 34 struct bkey_s_c_accounting src) 35 { 36 EBUG_ON(dst->k.u64s != src.k->u64s); 37 38 for (unsigned i = 0; i < bch2_accounting_counters(&dst->k); i++) 39 dst->v.d[i] += src.v->d[i]; 40 if (bversion_cmp(dst->k.bversion, src.k->bversion) < 0) 41 dst->k.bversion = src.k->bversion; 42 } 43 44 static inline void fs_usage_data_type_to_base(struct bch_fs_usage_base *fs_usage, 45 enum bch_data_type data_type, 46 s64 sectors) 47 { 48 switch (data_type) { 49 case BCH_DATA_btree: 50 fs_usage->btree += sectors; 51 break; 52 case BCH_DATA_user: 53 case BCH_DATA_parity: 54 fs_usage->data += sectors; 55 break; 56 case BCH_DATA_cached: 57 fs_usage->cached += sectors; 58 break; 59 default: 60 break; 61 } 62 } 63 64 static inline void bpos_to_disk_accounting_pos(struct disk_accounting_pos *acc, struct bpos p) 65 { 66 BUILD_BUG_ON(sizeof(*acc) != sizeof(p)); 67 68 #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ 69 acc->_pad = p; 70 #else 71 memcpy_swab(acc, &p, sizeof(p)); 72 #endif 73 } 74 75 static inline struct bpos disk_accounting_pos_to_bpos(struct disk_accounting_pos *acc) 76 { 77 struct bpos p; 78 #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ 79 p = acc->_pad; 80 #else 81 memcpy_swab(&p, acc, sizeof(p)); 82 #endif 83 return p; 84 } 85 86 int bch2_disk_accounting_mod(struct btree_trans *, struct disk_accounting_pos *, 87 s64 *, unsigned, bool); 88 int bch2_mod_dev_cached_sectors(struct btree_trans *, unsigned, s64, bool); 89 90 int bch2_accounting_validate(struct bch_fs *, struct bkey_s_c, 91 struct bkey_validate_context); 92 void bch2_accounting_key_to_text(struct printbuf *, struct disk_accounting_pos *); 93 void bch2_accounting_to_text(struct printbuf *, struct bch_fs *, struct bkey_s_c); 94 void bch2_accounting_swab(struct bkey_s); 95 96 #define bch2_bkey_ops_accounting ((struct bkey_ops) { \ 97 .key_validate = bch2_accounting_validate, \ 98 .val_to_text = bch2_accounting_to_text, \ 99 .swab = bch2_accounting_swab, \ 100 .min_val_size = 8, \ 101 }) 102 103 int bch2_accounting_update_sb(struct btree_trans *); 104 105 static inline int accounting_pos_cmp(const void *_l, const void *_r) 106 { 107 const struct bpos *l = _l, *r = _r; 108 109 return bpos_cmp(*l, *r); 110 } 111 112 enum bch_accounting_mode { 113 BCH_ACCOUNTING_normal, 114 BCH_ACCOUNTING_gc, 115 BCH_ACCOUNTING_read, 116 }; 117 118 int bch2_accounting_mem_insert(struct bch_fs *, struct bkey_s_c_accounting, enum bch_accounting_mode); 119 void bch2_accounting_mem_gc(struct bch_fs *); 120 121 static inline bool bch2_accounting_is_mem(struct disk_accounting_pos acc) 122 { 123 return acc.type < BCH_DISK_ACCOUNTING_TYPE_NR && 124 acc.type != BCH_DISK_ACCOUNTING_inum; 125 } 126 127 /* 128 * Update in memory counters so they match the btree update we're doing; called 129 * from transaction commit path 130 */ 131 static inline int bch2_accounting_mem_mod_locked(struct btree_trans *trans, 132 struct bkey_s_c_accounting a, 133 enum bch_accounting_mode mode) 134 { 135 struct bch_fs *c = trans->c; 136 struct bch_accounting_mem *acc = &c->accounting; 137 struct disk_accounting_pos acc_k; 138 bpos_to_disk_accounting_pos(&acc_k, a.k->p); 139 bool gc = mode == BCH_ACCOUNTING_gc; 140 141 if (gc && !acc->gc_running) 142 return 0; 143 144 if (!bch2_accounting_is_mem(acc_k)) 145 return 0; 146 147 if (mode == BCH_ACCOUNTING_normal) { 148 switch (acc_k.type) { 149 case BCH_DISK_ACCOUNTING_persistent_reserved: 150 trans->fs_usage_delta.reserved += acc_k.persistent_reserved.nr_replicas * a.v->d[0]; 151 break; 152 case BCH_DISK_ACCOUNTING_replicas: 153 fs_usage_data_type_to_base(&trans->fs_usage_delta, acc_k.replicas.data_type, a.v->d[0]); 154 break; 155 case BCH_DISK_ACCOUNTING_dev_data_type: 156 rcu_read_lock(); 157 struct bch_dev *ca = bch2_dev_rcu_noerror(c, acc_k.dev_data_type.dev); 158 if (ca) { 159 this_cpu_add(ca->usage->d[acc_k.dev_data_type.data_type].buckets, a.v->d[0]); 160 this_cpu_add(ca->usage->d[acc_k.dev_data_type.data_type].sectors, a.v->d[1]); 161 this_cpu_add(ca->usage->d[acc_k.dev_data_type.data_type].fragmented, a.v->d[2]); 162 } 163 rcu_read_unlock(); 164 break; 165 } 166 } 167 168 unsigned idx; 169 170 while ((idx = eytzinger0_find(acc->k.data, acc->k.nr, sizeof(acc->k.data[0]), 171 accounting_pos_cmp, &a.k->p)) >= acc->k.nr) { 172 int ret = bch2_accounting_mem_insert(c, a, mode); 173 if (ret) 174 return ret; 175 } 176 177 struct accounting_mem_entry *e = &acc->k.data[idx]; 178 179 EBUG_ON(bch2_accounting_counters(a.k) != e->nr_counters); 180 181 for (unsigned i = 0; i < bch2_accounting_counters(a.k); i++) 182 this_cpu_add(e->v[gc][i], a.v->d[i]); 183 return 0; 184 } 185 186 static inline int bch2_accounting_mem_add(struct btree_trans *trans, struct bkey_s_c_accounting a, bool gc) 187 { 188 percpu_down_read(&trans->c->mark_lock); 189 int ret = bch2_accounting_mem_mod_locked(trans, a, gc ? BCH_ACCOUNTING_gc : BCH_ACCOUNTING_normal); 190 percpu_up_read(&trans->c->mark_lock); 191 return ret; 192 } 193 194 static inline void bch2_accounting_mem_read_counters(struct bch_accounting_mem *acc, 195 unsigned idx, u64 *v, unsigned nr, bool gc) 196 { 197 memset(v, 0, sizeof(*v) * nr); 198 199 if (unlikely(idx >= acc->k.nr)) 200 return; 201 202 struct accounting_mem_entry *e = &acc->k.data[idx]; 203 204 nr = min_t(unsigned, nr, e->nr_counters); 205 206 for (unsigned i = 0; i < nr; i++) 207 v[i] = percpu_u64_get(e->v[gc] + i); 208 } 209 210 static inline void bch2_accounting_mem_read(struct bch_fs *c, struct bpos p, 211 u64 *v, unsigned nr) 212 { 213 struct bch_accounting_mem *acc = &c->accounting; 214 unsigned idx = eytzinger0_find(acc->k.data, acc->k.nr, sizeof(acc->k.data[0]), 215 accounting_pos_cmp, &p); 216 217 bch2_accounting_mem_read_counters(acc, idx, v, nr, false); 218 } 219 220 static inline struct bversion journal_pos_to_bversion(struct journal_res *res, unsigned offset) 221 { 222 EBUG_ON(!res->ref); 223 224 return (struct bversion) { 225 .hi = res->seq >> 32, 226 .lo = (res->seq << 32) | (res->offset + offset), 227 }; 228 } 229 230 static inline int bch2_accounting_trans_commit_hook(struct btree_trans *trans, 231 struct bkey_i_accounting *a, 232 unsigned commit_flags) 233 { 234 a->k.bversion = journal_pos_to_bversion(&trans->journal_res, 235 (u64 *) a - (u64 *) trans->journal_entries); 236 237 EBUG_ON(bversion_zero(a->k.bversion)); 238 239 return likely(!(commit_flags & BCH_TRANS_COMMIT_skip_accounting_apply)) 240 ? bch2_accounting_mem_mod_locked(trans, accounting_i_to_s_c(a), BCH_ACCOUNTING_normal) 241 : 0; 242 } 243 244 static inline void bch2_accounting_trans_commit_revert(struct btree_trans *trans, 245 struct bkey_i_accounting *a_i, 246 unsigned commit_flags) 247 { 248 if (likely(!(commit_flags & BCH_TRANS_COMMIT_skip_accounting_apply))) { 249 struct bkey_s_accounting a = accounting_i_to_s(a_i); 250 251 bch2_accounting_neg(a); 252 bch2_accounting_mem_mod_locked(trans, a.c, BCH_ACCOUNTING_normal); 253 bch2_accounting_neg(a); 254 } 255 } 256 257 int bch2_fs_replicas_usage_read(struct bch_fs *, darray_char *); 258 int bch2_fs_accounting_read(struct bch_fs *, darray_char *, unsigned); 259 260 int bch2_gc_accounting_start(struct bch_fs *); 261 int bch2_gc_accounting_done(struct bch_fs *); 262 263 int bch2_accounting_read(struct bch_fs *); 264 265 int bch2_dev_usage_remove(struct bch_fs *, unsigned); 266 int bch2_dev_usage_init(struct bch_dev *, bool); 267 268 void bch2_verify_accounting_clean(struct bch_fs *c); 269 270 void bch2_accounting_gc_free(struct bch_fs *); 271 void bch2_fs_accounting_exit(struct bch_fs *); 272 273 #endif /* _BCACHEFS_DISK_ACCOUNTING_H */ 274