1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * f2fs iostat support
4 *
5 * Copyright 2021 Google LLC
6 * Author: Daeho Jeong <daehojeong@google.com>
7 */
8
9 #include <linux/fs.h>
10 #include <linux/f2fs_fs.h>
11 #include <linux/seq_file.h>
12
13 #include "f2fs.h"
14 #include "iostat.h"
15 #include <trace/events/f2fs.h>
16
17 static struct kmem_cache *bio_iostat_ctx_cache;
18 static mempool_t *bio_iostat_ctx_pool;
19
iostat_get_avg_bytes(struct f2fs_sb_info * sbi,enum iostat_type type)20 static inline unsigned long long iostat_get_avg_bytes(struct f2fs_sb_info *sbi,
21 enum iostat_type type)
22 {
23 return sbi->iostat_count[type] ? div64_u64(sbi->iostat_bytes[type],
24 sbi->iostat_count[type]) : 0;
25 }
26
27 #define IOSTAT_INFO_SHOW(name, type) \
28 seq_printf(seq, "%-23s %-16llu %-16llu %-16llu\n", \
29 name":", sbi->iostat_bytes[type], \
30 sbi->iostat_count[type], \
31 iostat_get_avg_bytes(sbi, type))
32
iostat_info_seq_show(struct seq_file * seq,void * offset)33 int __maybe_unused iostat_info_seq_show(struct seq_file *seq, void *offset)
34 {
35 struct super_block *sb = seq->private;
36 struct f2fs_sb_info *sbi = F2FS_SB(sb);
37 int i;
38
39 if (!sbi->iostat_enable)
40 return 0;
41
42 seq_printf(seq, "time: %-16llu\n", ktime_get_real_seconds());
43 seq_printf(seq, "\t\t\t%-16s %-16s %-16s\n",
44 "io_bytes", "count", "avg_bytes");
45
46 /* print app write IOs */
47 seq_puts(seq, "[WRITE]\n");
48 IOSTAT_INFO_SHOW("app buffered data", APP_BUFFERED_IO);
49 IOSTAT_INFO_SHOW("app direct data", APP_DIRECT_IO);
50 IOSTAT_INFO_SHOW("app mapped data", APP_MAPPED_IO);
51 IOSTAT_INFO_SHOW("app buffered cdata", APP_BUFFERED_CDATA_IO);
52 IOSTAT_INFO_SHOW("app mapped cdata", APP_MAPPED_CDATA_IO);
53
54 /* print fs write IOs */
55 IOSTAT_INFO_SHOW("fs data", FS_DATA_IO);
56 IOSTAT_INFO_SHOW("fs cdata", FS_CDATA_IO);
57 IOSTAT_INFO_SHOW("fs node", FS_NODE_IO);
58 IOSTAT_INFO_SHOW("fs meta", FS_META_IO);
59 IOSTAT_INFO_SHOW("fs gc data", FS_GC_DATA_IO);
60 IOSTAT_INFO_SHOW("fs gc node", FS_GC_NODE_IO);
61 IOSTAT_INFO_SHOW("fs cp data", FS_CP_DATA_IO);
62 IOSTAT_INFO_SHOW("fs cp node", FS_CP_NODE_IO);
63 IOSTAT_INFO_SHOW("fs cp meta", FS_CP_META_IO);
64
65 /* print app read IOs */
66 seq_puts(seq, "[READ]\n");
67 IOSTAT_INFO_SHOW("app buffered data", APP_BUFFERED_READ_IO);
68 IOSTAT_INFO_SHOW("app direct data", APP_DIRECT_READ_IO);
69 IOSTAT_INFO_SHOW("app mapped data", APP_MAPPED_READ_IO);
70 IOSTAT_INFO_SHOW("app buffered cdata", APP_BUFFERED_CDATA_READ_IO);
71 IOSTAT_INFO_SHOW("app mapped cdata", APP_MAPPED_CDATA_READ_IO);
72
73 /* print fs read IOs */
74 IOSTAT_INFO_SHOW("fs data", FS_DATA_READ_IO);
75 IOSTAT_INFO_SHOW("fs gc data", FS_GDATA_READ_IO);
76 IOSTAT_INFO_SHOW("fs cdata", FS_CDATA_READ_IO);
77 IOSTAT_INFO_SHOW("fs node", FS_NODE_READ_IO);
78 IOSTAT_INFO_SHOW("fs meta", FS_META_READ_IO);
79
80 /* print read folio order stats */
81 seq_printf(seq, "%-23s", "fs read folio order:");
82 for (i = 0; i < NR_PAGE_ORDERS; i++)
83 seq_printf(seq, " %llu", sbi->iostat_read_folio_count[i]);
84 seq_putc(seq, '\n');
85
86 /* print other IOs */
87 seq_puts(seq, "[OTHER]\n");
88 IOSTAT_INFO_SHOW("fs discard", FS_DISCARD_IO);
89 IOSTAT_INFO_SHOW("fs flush", FS_FLUSH_IO);
90 IOSTAT_INFO_SHOW("fs zone reset", FS_ZONE_RESET_IO);
91
92 return 0;
93 }
94
__record_iostat_latency(struct f2fs_sb_info * sbi)95 static inline void __record_iostat_latency(struct f2fs_sb_info *sbi)
96 {
97 int io, idx;
98 struct f2fs_iostat_latency iostat_lat[MAX_IO_TYPE][NR_PAGE_TYPE];
99 struct iostat_lat_info *io_lat = sbi->iostat_io_lat;
100 unsigned long flags;
101
102 spin_lock_irqsave(&sbi->iostat_lat_lock, flags);
103 for (idx = 0; idx < MAX_IO_TYPE; idx++) {
104 for (io = 0; io < NR_PAGE_TYPE; io++) {
105 iostat_lat[idx][io].peak_lat =
106 jiffies_to_msecs(io_lat->peak_lat[idx][io]);
107 iostat_lat[idx][io].cnt = io_lat->bio_cnt[idx][io];
108 iostat_lat[idx][io].avg_lat = iostat_lat[idx][io].cnt ?
109 jiffies_to_msecs(io_lat->sum_lat[idx][io]) / iostat_lat[idx][io].cnt : 0;
110 io_lat->sum_lat[idx][io] = 0;
111 io_lat->peak_lat[idx][io] = 0;
112 io_lat->bio_cnt[idx][io] = 0;
113 }
114 }
115 spin_unlock_irqrestore(&sbi->iostat_lat_lock, flags);
116
117 trace_f2fs_iostat_latency(sbi, iostat_lat);
118 }
119
f2fs_record_iostat(struct f2fs_sb_info * sbi)120 static inline void f2fs_record_iostat(struct f2fs_sb_info *sbi)
121 {
122 unsigned long long iostat_diff[NR_IO_TYPE];
123 unsigned long long read_folio_count_diff[NR_PAGE_ORDERS];
124 int i;
125 unsigned long flags;
126
127 if (time_is_after_jiffies(sbi->iostat_next_period))
128 return;
129
130 /* Need double check under the lock */
131 spin_lock_irqsave(&sbi->iostat_lock, flags);
132 if (time_is_after_jiffies(sbi->iostat_next_period)) {
133 spin_unlock_irqrestore(&sbi->iostat_lock, flags);
134 return;
135 }
136 sbi->iostat_next_period = jiffies +
137 msecs_to_jiffies(sbi->iostat_period_ms);
138
139 for (i = 0; i < NR_IO_TYPE; i++) {
140 iostat_diff[i] = sbi->iostat_bytes[i] -
141 sbi->prev_iostat_bytes[i];
142 sbi->prev_iostat_bytes[i] = sbi->iostat_bytes[i];
143 }
144
145 for (i = 0; i < NR_PAGE_ORDERS; i++) {
146 read_folio_count_diff[i] = sbi->iostat_read_folio_count[i] -
147 sbi->prev_iostat_read_folio_count[i];
148 sbi->prev_iostat_read_folio_count[i] = sbi->iostat_read_folio_count[i];
149 }
150 spin_unlock_irqrestore(&sbi->iostat_lock, flags);
151
152 trace_f2fs_iostat(sbi, iostat_diff, read_folio_count_diff);
153
154 __record_iostat_latency(sbi);
155 }
156
f2fs_reset_iostat(struct f2fs_sb_info * sbi)157 void f2fs_reset_iostat(struct f2fs_sb_info *sbi)
158 {
159 struct iostat_lat_info *io_lat = sbi->iostat_io_lat;
160 int i;
161
162 spin_lock_irq(&sbi->iostat_lock);
163 for (i = 0; i < NR_IO_TYPE; i++) {
164 sbi->iostat_count[i] = 0;
165 sbi->iostat_bytes[i] = 0;
166 sbi->prev_iostat_bytes[i] = 0;
167 }
168 for (i = 0; i < NR_PAGE_ORDERS; i++) {
169 sbi->iostat_read_folio_count[i] = 0;
170 sbi->prev_iostat_read_folio_count[i] = 0;
171 }
172 spin_unlock_irq(&sbi->iostat_lock);
173
174 spin_lock_irq(&sbi->iostat_lat_lock);
175 memset(io_lat, 0, sizeof(struct iostat_lat_info));
176 spin_unlock_irq(&sbi->iostat_lat_lock);
177 }
178
__f2fs_update_iostat(struct f2fs_sb_info * sbi,enum iostat_type type,unsigned long long io_bytes)179 static inline void __f2fs_update_iostat(struct f2fs_sb_info *sbi,
180 enum iostat_type type, unsigned long long io_bytes)
181 {
182 sbi->iostat_bytes[type] += io_bytes;
183 sbi->iostat_count[type]++;
184 }
185
f2fs_update_read_folio_count(struct f2fs_sb_info * sbi,struct folio * folio)186 void f2fs_update_read_folio_count(struct f2fs_sb_info *sbi, struct folio *folio)
187 {
188 unsigned int order = folio_order(folio);
189 unsigned long flags;
190
191 if (!sbi->iostat_enable)
192 return;
193
194 if (order >= NR_PAGE_ORDERS)
195 order = NR_PAGE_ORDERS - 1;
196
197 spin_lock_irqsave(&sbi->iostat_lock, flags);
198 sbi->iostat_read_folio_count[order]++;
199 spin_unlock_irqrestore(&sbi->iostat_lock, flags);
200
201 f2fs_record_iostat(sbi);
202 }
203
f2fs_update_iostat(struct f2fs_sb_info * sbi,struct inode * inode,enum iostat_type type,unsigned long long io_bytes)204 void f2fs_update_iostat(struct f2fs_sb_info *sbi, struct inode *inode,
205 enum iostat_type type, unsigned long long io_bytes)
206 {
207 unsigned long flags;
208
209 if (!sbi->iostat_enable)
210 return;
211
212 spin_lock_irqsave(&sbi->iostat_lock, flags);
213 __f2fs_update_iostat(sbi, type, io_bytes);
214
215 if (type == APP_BUFFERED_IO || type == APP_DIRECT_IO)
216 __f2fs_update_iostat(sbi, APP_WRITE_IO, io_bytes);
217
218 if (type == APP_BUFFERED_READ_IO || type == APP_DIRECT_READ_IO)
219 __f2fs_update_iostat(sbi, APP_READ_IO, io_bytes);
220
221 #ifdef CONFIG_F2FS_FS_COMPRESSION
222 if (inode && f2fs_compressed_file(inode)) {
223 if (type == APP_BUFFERED_IO)
224 __f2fs_update_iostat(sbi, APP_BUFFERED_CDATA_IO, io_bytes);
225
226 if (type == APP_BUFFERED_READ_IO)
227 __f2fs_update_iostat(sbi, APP_BUFFERED_CDATA_READ_IO, io_bytes);
228
229 if (type == APP_MAPPED_READ_IO)
230 __f2fs_update_iostat(sbi, APP_MAPPED_CDATA_READ_IO, io_bytes);
231
232 if (type == APP_MAPPED_IO)
233 __f2fs_update_iostat(sbi, APP_MAPPED_CDATA_IO, io_bytes);
234
235 if (type == FS_DATA_READ_IO)
236 __f2fs_update_iostat(sbi, FS_CDATA_READ_IO, io_bytes);
237
238 if (type == FS_DATA_IO)
239 __f2fs_update_iostat(sbi, FS_CDATA_IO, io_bytes);
240 }
241 #endif
242
243 spin_unlock_irqrestore(&sbi->iostat_lock, flags);
244
245 f2fs_record_iostat(sbi);
246 }
247
__update_iostat_latency(struct bio_iostat_ctx * iostat_ctx,enum iostat_lat_type lat_type)248 static inline void __update_iostat_latency(struct bio_iostat_ctx *iostat_ctx,
249 enum iostat_lat_type lat_type)
250 {
251 unsigned long ts_diff;
252 unsigned int page_type = iostat_ctx->type;
253 struct f2fs_sb_info *sbi = iostat_ctx->sbi;
254 struct iostat_lat_info *io_lat = sbi->iostat_io_lat;
255 unsigned long flags;
256
257 if (!sbi->iostat_enable)
258 return;
259
260 ts_diff = jiffies - iostat_ctx->submit_ts;
261 if (page_type == META_FLUSH) {
262 page_type = META;
263 } else if (page_type >= NR_PAGE_TYPE) {
264 f2fs_warn(sbi, "%s: %d over NR_PAGE_TYPE", __func__, page_type);
265 return;
266 }
267
268 spin_lock_irqsave(&sbi->iostat_lat_lock, flags);
269 io_lat->sum_lat[lat_type][page_type] += ts_diff;
270 io_lat->bio_cnt[lat_type][page_type]++;
271 if (ts_diff > io_lat->peak_lat[lat_type][page_type])
272 io_lat->peak_lat[lat_type][page_type] = ts_diff;
273 spin_unlock_irqrestore(&sbi->iostat_lat_lock, flags);
274 }
275
iostat_update_and_unbind_ctx(struct bio * bio)276 void iostat_update_and_unbind_ctx(struct bio *bio)
277 {
278 struct bio_iostat_ctx *iostat_ctx = bio->bi_private;
279 enum iostat_lat_type lat_type;
280
281 if (op_is_write(bio_op(bio))) {
282 lat_type = bio->bi_opf & REQ_SYNC ?
283 WRITE_SYNC_IO : WRITE_ASYNC_IO;
284 bio->bi_private = iostat_ctx->sbi;
285 } else {
286 lat_type = READ_IO;
287 bio->bi_private = iostat_ctx->post_read_ctx;
288 }
289
290 __update_iostat_latency(iostat_ctx, lat_type);
291 mempool_free(iostat_ctx, bio_iostat_ctx_pool);
292 }
293
iostat_alloc_and_bind_ctx(struct f2fs_sb_info * sbi,struct bio * bio,struct bio_post_read_ctx * ctx)294 void iostat_alloc_and_bind_ctx(struct f2fs_sb_info *sbi,
295 struct bio *bio, struct bio_post_read_ctx *ctx)
296 {
297 struct bio_iostat_ctx *iostat_ctx;
298 /* Due to the mempool, this never fails. */
299 iostat_ctx = mempool_alloc(bio_iostat_ctx_pool, GFP_NOFS);
300 iostat_ctx->sbi = sbi;
301 iostat_ctx->submit_ts = 0;
302 iostat_ctx->type = 0;
303 iostat_ctx->post_read_ctx = ctx;
304 bio->bi_private = iostat_ctx;
305 }
306
f2fs_init_iostat_processing(void)307 int __init f2fs_init_iostat_processing(void)
308 {
309 bio_iostat_ctx_cache =
310 kmem_cache_create("f2fs_bio_iostat_ctx",
311 sizeof(struct bio_iostat_ctx), 0, 0, NULL);
312 if (!bio_iostat_ctx_cache)
313 goto fail;
314 bio_iostat_ctx_pool =
315 mempool_create_slab_pool(NUM_PREALLOC_IOSTAT_CTXS,
316 bio_iostat_ctx_cache);
317 if (!bio_iostat_ctx_pool)
318 goto fail_free_cache;
319 return 0;
320
321 fail_free_cache:
322 kmem_cache_destroy(bio_iostat_ctx_cache);
323 fail:
324 return -ENOMEM;
325 }
326
f2fs_destroy_iostat_processing(void)327 void f2fs_destroy_iostat_processing(void)
328 {
329 mempool_destroy(bio_iostat_ctx_pool);
330 kmem_cache_destroy(bio_iostat_ctx_cache);
331 }
332
f2fs_init_iostat(struct f2fs_sb_info * sbi)333 int f2fs_init_iostat(struct f2fs_sb_info *sbi)
334 {
335 /*
336 * The f2fs_iostat tracepoint emits a fixed number of read folio order
337 * buckets; make sure every order fits so none is silently dropped.
338 */
339 BUILD_BUG_ON(NR_PAGE_ORDERS > F2FS_IOSTAT_RD_FOLIO_ORDERS);
340
341 /* init iostat info */
342 spin_lock_init(&sbi->iostat_lock);
343 spin_lock_init(&sbi->iostat_lat_lock);
344 sbi->iostat_enable = false;
345 sbi->iostat_period_ms = DEFAULT_IOSTAT_PERIOD_MS;
346 sbi->iostat_io_lat = f2fs_kzalloc(sbi, sizeof(struct iostat_lat_info),
347 GFP_KERNEL);
348 if (!sbi->iostat_io_lat)
349 return -ENOMEM;
350
351 return 0;
352 }
353
f2fs_destroy_iostat(struct f2fs_sb_info * sbi)354 void f2fs_destroy_iostat(struct f2fs_sb_info *sbi)
355 {
356 kfree(sbi->iostat_io_lat);
357 }
358