1 /* SPDX-License-Identifier: GPL-2.0 2 * 3 * Legacy blkg rwstat helpers enabled by CONFIG_BLK_CGROUP_RWSTAT. 4 * Do not use in new code. 5 */ 6 #ifndef _BLK_CGROUP_RWSTAT_H 7 #define _BLK_CGROUP_RWSTAT_H 8 9 #include "blk-cgroup.h" 10 11 enum blkg_rwstat_type { 12 BLKG_RWSTAT_READ, 13 BLKG_RWSTAT_WRITE, 14 BLKG_RWSTAT_SYNC, 15 BLKG_RWSTAT_ASYNC, 16 BLKG_RWSTAT_DISCARD, 17 18 BLKG_RWSTAT_NR, 19 BLKG_RWSTAT_TOTAL = BLKG_RWSTAT_NR, 20 }; 21 22 /* 23 * blkg_[rw]stat->aux_cnt is excluded for local stats but included for 24 * recursive. Used to carry stats of dead children. 25 */ 26 struct blkg_rwstat { 27 struct percpu_counter cpu_cnt[BLKG_RWSTAT_NR]; 28 atomic64_t aux_cnt[BLKG_RWSTAT_NR]; 29 }; 30 31 struct blkg_rwstat_sample { 32 u64 cnt[BLKG_RWSTAT_NR]; 33 }; 34 35 static inline u64 blkg_rwstat_read_counter(struct blkg_rwstat *rwstat, 36 unsigned int idx) 37 { 38 return atomic64_read(&rwstat->aux_cnt[idx]) + 39 percpu_counter_sum_positive(&rwstat->cpu_cnt[idx]); 40 } 41 42 int blkg_rwstat_init(struct blkg_rwstat *rwstat, gfp_t gfp); 43 void blkg_rwstat_exit(struct blkg_rwstat *rwstat); 44 u64 __blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd, 45 const struct blkg_rwstat_sample *rwstat); 46 u64 blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd, 47 int off); 48 void blkg_rwstat_recursive_sum(struct blkcg_gq *blkg, struct blkcg_policy *pol, 49 int off, struct blkg_rwstat_sample *sum); 50 51 52 /** 53 * blkg_rwstat_add - add a value to a blkg_rwstat 54 * @rwstat: target blkg_rwstat 55 * @opf: REQ_OP and flags 56 * @val: value to add 57 * 58 * Add @val to @rwstat. The counters are chosen according to @rw. The 59 * caller is responsible for synchronizing calls to this function. 60 */ 61 static inline void blkg_rwstat_add(struct blkg_rwstat *rwstat, 62 blk_opf_t opf, uint64_t val) 63 { 64 struct percpu_counter *cnt; 65 66 if (op_is_discard(opf)) 67 cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_DISCARD]; 68 else if (op_is_write(opf)) 69 cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_WRITE]; 70 else 71 cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_READ]; 72 73 percpu_counter_add_batch(cnt, val, BLKG_STAT_CPU_BATCH); 74 75 if (op_is_sync(opf)) 76 cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_SYNC]; 77 else 78 cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_ASYNC]; 79 80 percpu_counter_add_batch(cnt, val, BLKG_STAT_CPU_BATCH); 81 } 82 83 /** 84 * blkg_rwstat_read - read the current values of a blkg_rwstat 85 * @rwstat: blkg_rwstat to read 86 * @result: where to put the current values 87 * 88 * Read the current snapshot of @rwstat and return it in the @result counts. 89 */ 90 static inline void blkg_rwstat_read(struct blkg_rwstat *rwstat, 91 struct blkg_rwstat_sample *result) 92 { 93 int i; 94 95 for (i = 0; i < BLKG_RWSTAT_NR; i++) 96 result->cnt[i] = 97 percpu_counter_sum_positive(&rwstat->cpu_cnt[i]); 98 } 99 100 /** 101 * blkg_rwstat_total - read the total count of a blkg_rwstat 102 * @rwstat: blkg_rwstat to read 103 * 104 * Return the total count of @rwstat regardless of the IO direction. This 105 * function can be called without synchronization and takes care of u64 106 * atomicity. 107 */ 108 static inline uint64_t blkg_rwstat_total(struct blkg_rwstat *rwstat) 109 { 110 struct blkg_rwstat_sample tmp = { }; 111 112 blkg_rwstat_read(rwstat, &tmp); 113 return tmp.cnt[BLKG_RWSTAT_READ] + tmp.cnt[BLKG_RWSTAT_WRITE]; 114 } 115 116 /** 117 * blkg_rwstat_reset - reset a blkg_rwstat 118 * @rwstat: blkg_rwstat to reset 119 */ 120 static inline void blkg_rwstat_reset(struct blkg_rwstat *rwstat) 121 { 122 int i; 123 124 for (i = 0; i < BLKG_RWSTAT_NR; i++) { 125 percpu_counter_set(&rwstat->cpu_cnt[i], 0); 126 atomic64_set(&rwstat->aux_cnt[i], 0); 127 } 128 } 129 130 /** 131 * blkg_rwstat_add_aux - add a blkg_rwstat into another's aux count 132 * @to: the destination blkg_rwstat 133 * @from: the source 134 * 135 * Add @from's count including the aux one to @to's aux count. 136 */ 137 static inline void blkg_rwstat_add_aux(struct blkg_rwstat *to, 138 struct blkg_rwstat *from) 139 { 140 u64 sum[BLKG_RWSTAT_NR]; 141 int i; 142 143 for (i = 0; i < BLKG_RWSTAT_NR; i++) 144 sum[i] = percpu_counter_sum_positive(&from->cpu_cnt[i]); 145 146 for (i = 0; i < BLKG_RWSTAT_NR; i++) 147 atomic64_add(sum[i] + atomic64_read(&from->aux_cnt[i]), 148 &to->aux_cnt[i]); 149 } 150 #endif /* _BLK_CGROUP_RWSTAT_H */ 151