1 /* 2 * Block stat tracking code 3 * 4 * Copyright (C) 2016 Jens Axboe 5 */ 6 #include <linux/kernel.h> 7 #include <linux/blk-mq.h> 8 9 #include "blk-stat.h" 10 #include "blk-mq.h" 11 12 static void blk_stat_flush_batch(struct blk_rq_stat *stat) 13 { 14 const s32 nr_batch = READ_ONCE(stat->nr_batch); 15 const s32 nr_samples = READ_ONCE(stat->nr_samples); 16 17 if (!nr_batch) 18 return; 19 if (!nr_samples) 20 stat->mean = div64_s64(stat->batch, nr_batch); 21 else { 22 stat->mean = div64_s64((stat->mean * nr_samples) + 23 stat->batch, 24 nr_batch + nr_samples); 25 } 26 27 stat->nr_samples += nr_batch; 28 stat->nr_batch = stat->batch = 0; 29 } 30 31 static void blk_stat_sum(struct blk_rq_stat *dst, struct blk_rq_stat *src) 32 { 33 if (!src->nr_samples) 34 return; 35 36 blk_stat_flush_batch(src); 37 38 dst->min = min(dst->min, src->min); 39 dst->max = max(dst->max, src->max); 40 41 if (!dst->nr_samples) 42 dst->mean = src->mean; 43 else { 44 dst->mean = div64_s64((src->mean * src->nr_samples) + 45 (dst->mean * dst->nr_samples), 46 dst->nr_samples + src->nr_samples); 47 } 48 dst->nr_samples += src->nr_samples; 49 } 50 51 static void blk_mq_stat_get(struct request_queue *q, struct blk_rq_stat *dst) 52 { 53 struct blk_mq_hw_ctx *hctx; 54 struct blk_mq_ctx *ctx; 55 uint64_t latest = 0; 56 int i, j, nr; 57 58 blk_stat_init(&dst[BLK_STAT_READ]); 59 blk_stat_init(&dst[BLK_STAT_WRITE]); 60 61 nr = 0; 62 do { 63 uint64_t newest = 0; 64 65 queue_for_each_hw_ctx(q, hctx, i) { 66 hctx_for_each_ctx(hctx, ctx, j) { 67 if (!ctx->stat[BLK_STAT_READ].nr_samples && 68 !ctx->stat[BLK_STAT_WRITE].nr_samples) 69 continue; 70 if (ctx->stat[BLK_STAT_READ].time > newest) 71 newest = ctx->stat[BLK_STAT_READ].time; 72 if (ctx->stat[BLK_STAT_WRITE].time > newest) 73 newest = ctx->stat[BLK_STAT_WRITE].time; 74 } 75 } 76 77 /* 78 * No samples 79 */ 80 if (!newest) 81 break; 82 83 if (newest > latest) 84 latest = newest; 85 86 queue_for_each_hw_ctx(q, hctx, i) { 87 hctx_for_each_ctx(hctx, ctx, j) { 88 if (ctx->stat[BLK_STAT_READ].time == newest) { 89 blk_stat_sum(&dst[BLK_STAT_READ], 90 &ctx->stat[BLK_STAT_READ]); 91 nr++; 92 } 93 if (ctx->stat[BLK_STAT_WRITE].time == newest) { 94 blk_stat_sum(&dst[BLK_STAT_WRITE], 95 &ctx->stat[BLK_STAT_WRITE]); 96 nr++; 97 } 98 } 99 } 100 /* 101 * If we race on finding an entry, just loop back again. 102 * Should be very rare. 103 */ 104 } while (!nr); 105 106 dst[BLK_STAT_READ].time = dst[BLK_STAT_WRITE].time = latest; 107 } 108 109 void blk_queue_stat_get(struct request_queue *q, struct blk_rq_stat *dst) 110 { 111 if (q->mq_ops) 112 blk_mq_stat_get(q, dst); 113 else { 114 memcpy(&dst[BLK_STAT_READ], &q->rq_stats[BLK_STAT_READ], 115 sizeof(struct blk_rq_stat)); 116 memcpy(&dst[BLK_STAT_WRITE], &q->rq_stats[BLK_STAT_WRITE], 117 sizeof(struct blk_rq_stat)); 118 } 119 } 120 121 void blk_hctx_stat_get(struct blk_mq_hw_ctx *hctx, struct blk_rq_stat *dst) 122 { 123 struct blk_mq_ctx *ctx; 124 unsigned int i, nr; 125 126 nr = 0; 127 do { 128 uint64_t newest = 0; 129 130 hctx_for_each_ctx(hctx, ctx, i) { 131 if (!ctx->stat[BLK_STAT_READ].nr_samples && 132 !ctx->stat[BLK_STAT_WRITE].nr_samples) 133 continue; 134 135 if (ctx->stat[BLK_STAT_READ].time > newest) 136 newest = ctx->stat[BLK_STAT_READ].time; 137 if (ctx->stat[BLK_STAT_WRITE].time > newest) 138 newest = ctx->stat[BLK_STAT_WRITE].time; 139 } 140 141 if (!newest) 142 break; 143 144 hctx_for_each_ctx(hctx, ctx, i) { 145 if (ctx->stat[BLK_STAT_READ].time == newest) { 146 blk_stat_sum(&dst[BLK_STAT_READ], 147 &ctx->stat[BLK_STAT_READ]); 148 nr++; 149 } 150 if (ctx->stat[BLK_STAT_WRITE].time == newest) { 151 blk_stat_sum(&dst[BLK_STAT_WRITE], 152 &ctx->stat[BLK_STAT_WRITE]); 153 nr++; 154 } 155 } 156 /* 157 * If we race on finding an entry, just loop back again. 158 * Should be very rare, as the window is only updated 159 * occasionally 160 */ 161 } while (!nr); 162 } 163 164 static void __blk_stat_init(struct blk_rq_stat *stat, s64 time_now) 165 { 166 stat->min = -1ULL; 167 stat->max = stat->nr_samples = stat->mean = 0; 168 stat->batch = stat->nr_batch = 0; 169 stat->time = time_now & BLK_STAT_NSEC_MASK; 170 } 171 172 void blk_stat_init(struct blk_rq_stat *stat) 173 { 174 __blk_stat_init(stat, ktime_to_ns(ktime_get())); 175 } 176 177 static bool __blk_stat_is_current(struct blk_rq_stat *stat, s64 now) 178 { 179 return (now & BLK_STAT_NSEC_MASK) == (stat->time & BLK_STAT_NSEC_MASK); 180 } 181 182 bool blk_stat_is_current(struct blk_rq_stat *stat) 183 { 184 return __blk_stat_is_current(stat, ktime_to_ns(ktime_get())); 185 } 186 187 void blk_stat_add(struct blk_rq_stat *stat, struct request *rq) 188 { 189 s64 now, value; 190 191 now = __blk_stat_time(ktime_to_ns(ktime_get())); 192 if (now < blk_stat_time(&rq->issue_stat)) 193 return; 194 195 if (!__blk_stat_is_current(stat, now)) 196 __blk_stat_init(stat, now); 197 198 value = now - blk_stat_time(&rq->issue_stat); 199 if (value > stat->max) 200 stat->max = value; 201 if (value < stat->min) 202 stat->min = value; 203 204 if (stat->batch + value < stat->batch || 205 stat->nr_batch + 1 == BLK_RQ_STAT_BATCH) 206 blk_stat_flush_batch(stat); 207 208 stat->batch += value; 209 stat->nr_batch++; 210 } 211 212 void blk_stat_clear(struct request_queue *q) 213 { 214 if (q->mq_ops) { 215 struct blk_mq_hw_ctx *hctx; 216 struct blk_mq_ctx *ctx; 217 int i, j; 218 219 queue_for_each_hw_ctx(q, hctx, i) { 220 hctx_for_each_ctx(hctx, ctx, j) { 221 blk_stat_init(&ctx->stat[BLK_STAT_READ]); 222 blk_stat_init(&ctx->stat[BLK_STAT_WRITE]); 223 } 224 } 225 } else { 226 blk_stat_init(&q->rq_stats[BLK_STAT_READ]); 227 blk_stat_init(&q->rq_stats[BLK_STAT_WRITE]); 228 } 229 } 230 231 void blk_stat_set_issue_time(struct blk_issue_stat *stat) 232 { 233 stat->time = (stat->time & BLK_STAT_MASK) | 234 (ktime_to_ns(ktime_get()) & BLK_STAT_TIME_MASK); 235 } 236 237 /* 238 * Enable stat tracking, return whether it was enabled 239 */ 240 bool blk_stat_enable(struct request_queue *q) 241 { 242 if (!test_bit(QUEUE_FLAG_STATS, &q->queue_flags)) { 243 set_bit(QUEUE_FLAG_STATS, &q->queue_flags); 244 return false; 245 } 246 247 return true; 248 } 249