152118743SDaeho Jeong // SPDX-License-Identifier: GPL-2.0
252118743SDaeho Jeong /*
352118743SDaeho Jeong * f2fs iostat support
452118743SDaeho Jeong *
552118743SDaeho Jeong * Copyright 2021 Google LLC
652118743SDaeho Jeong * Author: Daeho Jeong <daehojeong@google.com>
752118743SDaeho Jeong */
852118743SDaeho Jeong
952118743SDaeho Jeong #include <linux/fs.h>
1052118743SDaeho Jeong #include <linux/f2fs_fs.h>
1152118743SDaeho Jeong #include <linux/seq_file.h>
1252118743SDaeho Jeong
1352118743SDaeho Jeong #include "f2fs.h"
1452118743SDaeho Jeong #include "iostat.h"
1552118743SDaeho Jeong #include <trace/events/f2fs.h>
1652118743SDaeho Jeong
17a4b68176SDaeho Jeong static struct kmem_cache *bio_iostat_ctx_cache;
18a4b68176SDaeho Jeong static mempool_t *bio_iostat_ctx_pool;
19a4b68176SDaeho Jeong
iostat_get_avg_bytes(struct f2fs_sb_info * sbi,enum iostat_type type)207a2b15cfSYangtao Li static inline unsigned long long iostat_get_avg_bytes(struct f2fs_sb_info *sbi,
217a2b15cfSYangtao Li enum iostat_type type)
227a2b15cfSYangtao Li {
237a2b15cfSYangtao Li return sbi->iostat_count[type] ? div64_u64(sbi->iostat_bytes[type],
247a2b15cfSYangtao Li sbi->iostat_count[type]) : 0;
257a2b15cfSYangtao Li }
267a2b15cfSYangtao Li
277a2b15cfSYangtao Li #define IOSTAT_INFO_SHOW(name, type) \
287a2b15cfSYangtao Li seq_printf(seq, "%-23s %-16llu %-16llu %-16llu\n", \
297a2b15cfSYangtao Li name":", sbi->iostat_bytes[type], \
307a2b15cfSYangtao Li sbi->iostat_count[type], \
317a2b15cfSYangtao Li iostat_get_avg_bytes(sbi, type))
327a2b15cfSYangtao Li
iostat_info_seq_show(struct seq_file * seq,void * offset)3352118743SDaeho Jeong int __maybe_unused iostat_info_seq_show(struct seq_file *seq, void *offset)
3452118743SDaeho Jeong {
3552118743SDaeho Jeong struct super_block *sb = seq->private;
3652118743SDaeho Jeong struct f2fs_sb_info *sbi = F2FS_SB(sb);
3752118743SDaeho Jeong
3852118743SDaeho Jeong if (!sbi->iostat_enable)
3952118743SDaeho Jeong return 0;
4052118743SDaeho Jeong
417a2b15cfSYangtao Li seq_printf(seq, "time: %-16llu\n", ktime_get_real_seconds());
427a2b15cfSYangtao Li seq_printf(seq, "\t\t\t%-16s %-16s %-16s\n",
437a2b15cfSYangtao Li "io_bytes", "count", "avg_bytes");
4452118743SDaeho Jeong
4552118743SDaeho Jeong /* print app write IOs */
4652118743SDaeho Jeong seq_puts(seq, "[WRITE]\n");
477a2b15cfSYangtao Li IOSTAT_INFO_SHOW("app buffered data", APP_BUFFERED_IO);
487a2b15cfSYangtao Li IOSTAT_INFO_SHOW("app direct data", APP_DIRECT_IO);
497a2b15cfSYangtao Li IOSTAT_INFO_SHOW("app mapped data", APP_MAPPED_IO);
507a2b15cfSYangtao Li IOSTAT_INFO_SHOW("app buffered cdata", APP_BUFFERED_CDATA_IO);
517a2b15cfSYangtao Li IOSTAT_INFO_SHOW("app mapped cdata", APP_MAPPED_CDATA_IO);
5252118743SDaeho Jeong
5352118743SDaeho Jeong /* print fs write IOs */
547a2b15cfSYangtao Li IOSTAT_INFO_SHOW("fs data", FS_DATA_IO);
557a2b15cfSYangtao Li IOSTAT_INFO_SHOW("fs cdata", FS_CDATA_IO);
567a2b15cfSYangtao Li IOSTAT_INFO_SHOW("fs node", FS_NODE_IO);
577a2b15cfSYangtao Li IOSTAT_INFO_SHOW("fs meta", FS_META_IO);
587a2b15cfSYangtao Li IOSTAT_INFO_SHOW("fs gc data", FS_GC_DATA_IO);
597a2b15cfSYangtao Li IOSTAT_INFO_SHOW("fs gc node", FS_GC_NODE_IO);
607a2b15cfSYangtao Li IOSTAT_INFO_SHOW("fs cp data", FS_CP_DATA_IO);
617a2b15cfSYangtao Li IOSTAT_INFO_SHOW("fs cp node", FS_CP_NODE_IO);
627a2b15cfSYangtao Li IOSTAT_INFO_SHOW("fs cp meta", FS_CP_META_IO);
6352118743SDaeho Jeong
6452118743SDaeho Jeong /* print app read IOs */
6552118743SDaeho Jeong seq_puts(seq, "[READ]\n");
667a2b15cfSYangtao Li IOSTAT_INFO_SHOW("app buffered data", APP_BUFFERED_READ_IO);
677a2b15cfSYangtao Li IOSTAT_INFO_SHOW("app direct data", APP_DIRECT_READ_IO);
687a2b15cfSYangtao Li IOSTAT_INFO_SHOW("app mapped data", APP_MAPPED_READ_IO);
697a2b15cfSYangtao Li IOSTAT_INFO_SHOW("app buffered cdata", APP_BUFFERED_CDATA_READ_IO);
707a2b15cfSYangtao Li IOSTAT_INFO_SHOW("app mapped cdata", APP_MAPPED_CDATA_READ_IO);
7152118743SDaeho Jeong
7252118743SDaeho Jeong /* print fs read IOs */
737a2b15cfSYangtao Li IOSTAT_INFO_SHOW("fs data", FS_DATA_READ_IO);
747a2b15cfSYangtao Li IOSTAT_INFO_SHOW("fs gc data", FS_GDATA_READ_IO);
757a2b15cfSYangtao Li IOSTAT_INFO_SHOW("fs cdata", FS_CDATA_READ_IO);
767a2b15cfSYangtao Li IOSTAT_INFO_SHOW("fs node", FS_NODE_READ_IO);
777a2b15cfSYangtao Li IOSTAT_INFO_SHOW("fs meta", FS_META_READ_IO);
7852118743SDaeho Jeong
7952118743SDaeho Jeong /* print other IOs */
8052118743SDaeho Jeong seq_puts(seq, "[OTHER]\n");
817a2b15cfSYangtao Li IOSTAT_INFO_SHOW("fs discard", FS_DISCARD_IO);
82193a639fSYangtao Li IOSTAT_INFO_SHOW("fs flush", FS_FLUSH_IO);
83*25f90805SDaejun Park IOSTAT_INFO_SHOW("fs zone reset", FS_ZONE_RESET_IO);
8452118743SDaeho Jeong
8552118743SDaeho Jeong return 0;
8652118743SDaeho Jeong }
8752118743SDaeho Jeong
__record_iostat_latency(struct f2fs_sb_info * sbi)88a4b68176SDaeho Jeong static inline void __record_iostat_latency(struct f2fs_sb_info *sbi)
89a4b68176SDaeho Jeong {
90c5f9db25SYangtao Li int io, idx;
91a4b68176SDaeho Jeong struct f2fs_iostat_latency iostat_lat[MAX_IO_TYPE][NR_PAGE_TYPE];
92a4b68176SDaeho Jeong struct iostat_lat_info *io_lat = sbi->iostat_io_lat;
9361803e98SDaeho Jeong unsigned long flags;
94a4b68176SDaeho Jeong
9561803e98SDaeho Jeong spin_lock_irqsave(&sbi->iostat_lat_lock, flags);
96a4b68176SDaeho Jeong for (idx = 0; idx < MAX_IO_TYPE; idx++) {
97a4b68176SDaeho Jeong for (io = 0; io < NR_PAGE_TYPE; io++) {
98a4b68176SDaeho Jeong iostat_lat[idx][io].peak_lat =
99a4b68176SDaeho Jeong jiffies_to_msecs(io_lat->peak_lat[idx][io]);
100c5f9db25SYangtao Li iostat_lat[idx][io].cnt = io_lat->bio_cnt[idx][io];
101c5f9db25SYangtao Li iostat_lat[idx][io].avg_lat = iostat_lat[idx][io].cnt ?
102c5f9db25SYangtao Li jiffies_to_msecs(io_lat->sum_lat[idx][io]) / iostat_lat[idx][io].cnt : 0;
103a4b68176SDaeho Jeong io_lat->sum_lat[idx][io] = 0;
104a4b68176SDaeho Jeong io_lat->peak_lat[idx][io] = 0;
105a4b68176SDaeho Jeong io_lat->bio_cnt[idx][io] = 0;
106a4b68176SDaeho Jeong }
107a4b68176SDaeho Jeong }
10861803e98SDaeho Jeong spin_unlock_irqrestore(&sbi->iostat_lat_lock, flags);
109a4b68176SDaeho Jeong
110a4b68176SDaeho Jeong trace_f2fs_iostat_latency(sbi, iostat_lat);
111a4b68176SDaeho Jeong }
112a4b68176SDaeho Jeong
f2fs_record_iostat(struct f2fs_sb_info * sbi)11352118743SDaeho Jeong static inline void f2fs_record_iostat(struct f2fs_sb_info *sbi)
11452118743SDaeho Jeong {
11552118743SDaeho Jeong unsigned long long iostat_diff[NR_IO_TYPE];
11652118743SDaeho Jeong int i;
11761803e98SDaeho Jeong unsigned long flags;
11852118743SDaeho Jeong
11952118743SDaeho Jeong if (time_is_after_jiffies(sbi->iostat_next_period))
12052118743SDaeho Jeong return;
12152118743SDaeho Jeong
12252118743SDaeho Jeong /* Need double check under the lock */
12361803e98SDaeho Jeong spin_lock_irqsave(&sbi->iostat_lock, flags);
12452118743SDaeho Jeong if (time_is_after_jiffies(sbi->iostat_next_period)) {
12561803e98SDaeho Jeong spin_unlock_irqrestore(&sbi->iostat_lock, flags);
12652118743SDaeho Jeong return;
12752118743SDaeho Jeong }
12852118743SDaeho Jeong sbi->iostat_next_period = jiffies +
12952118743SDaeho Jeong msecs_to_jiffies(sbi->iostat_period_ms);
13052118743SDaeho Jeong
13152118743SDaeho Jeong for (i = 0; i < NR_IO_TYPE; i++) {
1327a2b15cfSYangtao Li iostat_diff[i] = sbi->iostat_bytes[i] -
1337a2b15cfSYangtao Li sbi->prev_iostat_bytes[i];
1347a2b15cfSYangtao Li sbi->prev_iostat_bytes[i] = sbi->iostat_bytes[i];
13552118743SDaeho Jeong }
13661803e98SDaeho Jeong spin_unlock_irqrestore(&sbi->iostat_lock, flags);
13752118743SDaeho Jeong
13852118743SDaeho Jeong trace_f2fs_iostat(sbi, iostat_diff);
139a4b68176SDaeho Jeong
140a4b68176SDaeho Jeong __record_iostat_latency(sbi);
14152118743SDaeho Jeong }
14252118743SDaeho Jeong
f2fs_reset_iostat(struct f2fs_sb_info * sbi)14352118743SDaeho Jeong void f2fs_reset_iostat(struct f2fs_sb_info *sbi)
14452118743SDaeho Jeong {
145a4b68176SDaeho Jeong struct iostat_lat_info *io_lat = sbi->iostat_io_lat;
14652118743SDaeho Jeong int i;
14752118743SDaeho Jeong
14861803e98SDaeho Jeong spin_lock_irq(&sbi->iostat_lock);
14952118743SDaeho Jeong for (i = 0; i < NR_IO_TYPE; i++) {
1507a2b15cfSYangtao Li sbi->iostat_count[i] = 0;
1517a2b15cfSYangtao Li sbi->iostat_bytes[i] = 0;
1527a2b15cfSYangtao Li sbi->prev_iostat_bytes[i] = 0;
15352118743SDaeho Jeong }
15461803e98SDaeho Jeong spin_unlock_irq(&sbi->iostat_lock);
155a4b68176SDaeho Jeong
15661803e98SDaeho Jeong spin_lock_irq(&sbi->iostat_lat_lock);
157a4b68176SDaeho Jeong memset(io_lat, 0, sizeof(struct iostat_lat_info));
15861803e98SDaeho Jeong spin_unlock_irq(&sbi->iostat_lat_lock);
15952118743SDaeho Jeong }
16052118743SDaeho Jeong
__f2fs_update_iostat(struct f2fs_sb_info * sbi,enum iostat_type type,unsigned long long io_bytes)1617a2b15cfSYangtao Li static inline void __f2fs_update_iostat(struct f2fs_sb_info *sbi,
1627a2b15cfSYangtao Li enum iostat_type type, unsigned long long io_bytes)
1637a2b15cfSYangtao Li {
1647a2b15cfSYangtao Li sbi->iostat_bytes[type] += io_bytes;
1657a2b15cfSYangtao Li sbi->iostat_count[type]++;
1667a2b15cfSYangtao Li }
1677a2b15cfSYangtao Li
f2fs_update_iostat(struct f2fs_sb_info * sbi,struct inode * inode,enum iostat_type type,unsigned long long io_bytes)16834a23525SChao Yu void f2fs_update_iostat(struct f2fs_sb_info *sbi, struct inode *inode,
16952118743SDaeho Jeong enum iostat_type type, unsigned long long io_bytes)
17052118743SDaeho Jeong {
17161803e98SDaeho Jeong unsigned long flags;
17261803e98SDaeho Jeong
17352118743SDaeho Jeong if (!sbi->iostat_enable)
17452118743SDaeho Jeong return;
17552118743SDaeho Jeong
17661803e98SDaeho Jeong spin_lock_irqsave(&sbi->iostat_lock, flags);
1777a2b15cfSYangtao Li __f2fs_update_iostat(sbi, type, io_bytes);
17852118743SDaeho Jeong
179a1e09b03SEric Biggers if (type == APP_BUFFERED_IO || type == APP_DIRECT_IO)
1807a2b15cfSYangtao Li __f2fs_update_iostat(sbi, APP_WRITE_IO, io_bytes);
18152118743SDaeho Jeong
182a1e09b03SEric Biggers if (type == APP_BUFFERED_READ_IO || type == APP_DIRECT_READ_IO)
1837a2b15cfSYangtao Li __f2fs_update_iostat(sbi, APP_READ_IO, io_bytes);
184a1e09b03SEric Biggers
18534a23525SChao Yu #ifdef CONFIG_F2FS_FS_COMPRESSION
18634a23525SChao Yu if (inode && f2fs_compressed_file(inode)) {
18734a23525SChao Yu if (type == APP_BUFFERED_IO)
1887a2b15cfSYangtao Li __f2fs_update_iostat(sbi, APP_BUFFERED_CDATA_IO, io_bytes);
18934a23525SChao Yu
19034a23525SChao Yu if (type == APP_BUFFERED_READ_IO)
1917a2b15cfSYangtao Li __f2fs_update_iostat(sbi, APP_BUFFERED_CDATA_READ_IO, io_bytes);
19234a23525SChao Yu
19334a23525SChao Yu if (type == APP_MAPPED_READ_IO)
1947a2b15cfSYangtao Li __f2fs_update_iostat(sbi, APP_MAPPED_CDATA_READ_IO, io_bytes);
19534a23525SChao Yu
19634a23525SChao Yu if (type == APP_MAPPED_IO)
1977a2b15cfSYangtao Li __f2fs_update_iostat(sbi, APP_MAPPED_CDATA_IO, io_bytes);
19834a23525SChao Yu
19934a23525SChao Yu if (type == FS_DATA_READ_IO)
2007a2b15cfSYangtao Li __f2fs_update_iostat(sbi, FS_CDATA_READ_IO, io_bytes);
20134a23525SChao Yu
20234a23525SChao Yu if (type == FS_DATA_IO)
2037a2b15cfSYangtao Li __f2fs_update_iostat(sbi, FS_CDATA_IO, io_bytes);
20434a23525SChao Yu }
20534a23525SChao Yu #endif
20634a23525SChao Yu
20761803e98SDaeho Jeong spin_unlock_irqrestore(&sbi->iostat_lock, flags);
20852118743SDaeho Jeong
20952118743SDaeho Jeong f2fs_record_iostat(sbi);
21052118743SDaeho Jeong }
21152118743SDaeho Jeong
__update_iostat_latency(struct bio_iostat_ctx * iostat_ctx,enum iostat_lat_type lat_type)212a4b68176SDaeho Jeong static inline void __update_iostat_latency(struct bio_iostat_ctx *iostat_ctx,
213d9bac032SYangtao Li enum iostat_lat_type lat_type)
214a4b68176SDaeho Jeong {
215a4b68176SDaeho Jeong unsigned long ts_diff;
216d9bac032SYangtao Li unsigned int page_type = iostat_ctx->type;
217a4b68176SDaeho Jeong struct f2fs_sb_info *sbi = iostat_ctx->sbi;
218a4b68176SDaeho Jeong struct iostat_lat_info *io_lat = sbi->iostat_io_lat;
21961803e98SDaeho Jeong unsigned long flags;
220a4b68176SDaeho Jeong
221a4b68176SDaeho Jeong if (!sbi->iostat_enable)
222a4b68176SDaeho Jeong return;
223a4b68176SDaeho Jeong
224a4b68176SDaeho Jeong ts_diff = jiffies - iostat_ctx->submit_ts;
225d9bac032SYangtao Li if (page_type == META_FLUSH) {
226d9bac032SYangtao Li page_type = META;
227d9bac032SYangtao Li } else if (page_type >= NR_PAGE_TYPE) {
228d9bac032SYangtao Li f2fs_warn(sbi, "%s: %d over NR_PAGE_TYPE", __func__, page_type);
2290dbbf0fbSYangtao Li return;
2300dbbf0fbSYangtao Li }
231a4b68176SDaeho Jeong
23261803e98SDaeho Jeong spin_lock_irqsave(&sbi->iostat_lat_lock, flags);
233d9bac032SYangtao Li io_lat->sum_lat[lat_type][page_type] += ts_diff;
234d9bac032SYangtao Li io_lat->bio_cnt[lat_type][page_type]++;
235d9bac032SYangtao Li if (ts_diff > io_lat->peak_lat[lat_type][page_type])
236d9bac032SYangtao Li io_lat->peak_lat[lat_type][page_type] = ts_diff;
23761803e98SDaeho Jeong spin_unlock_irqrestore(&sbi->iostat_lat_lock, flags);
238a4b68176SDaeho Jeong }
239a4b68176SDaeho Jeong
iostat_update_and_unbind_ctx(struct bio * bio)240d9bac032SYangtao Li void iostat_update_and_unbind_ctx(struct bio *bio)
241a4b68176SDaeho Jeong {
242a4b68176SDaeho Jeong struct bio_iostat_ctx *iostat_ctx = bio->bi_private;
243d9bac032SYangtao Li enum iostat_lat_type lat_type;
244a4b68176SDaeho Jeong
245d9bac032SYangtao Li if (op_is_write(bio_op(bio))) {
246d9bac032SYangtao Li lat_type = bio->bi_opf & REQ_SYNC ?
247d9bac032SYangtao Li WRITE_SYNC_IO : WRITE_ASYNC_IO;
248a4b68176SDaeho Jeong bio->bi_private = iostat_ctx->sbi;
249d9bac032SYangtao Li } else {
250d9bac032SYangtao Li lat_type = READ_IO;
251d9bac032SYangtao Li bio->bi_private = iostat_ctx->post_read_ctx;
252d9bac032SYangtao Li }
253d9bac032SYangtao Li
254d9bac032SYangtao Li __update_iostat_latency(iostat_ctx, lat_type);
255a4b68176SDaeho Jeong mempool_free(iostat_ctx, bio_iostat_ctx_pool);
256a4b68176SDaeho Jeong }
257a4b68176SDaeho Jeong
iostat_alloc_and_bind_ctx(struct f2fs_sb_info * sbi,struct bio * bio,struct bio_post_read_ctx * ctx)258a4b68176SDaeho Jeong void iostat_alloc_and_bind_ctx(struct f2fs_sb_info *sbi,
259a4b68176SDaeho Jeong struct bio *bio, struct bio_post_read_ctx *ctx)
260a4b68176SDaeho Jeong {
261a4b68176SDaeho Jeong struct bio_iostat_ctx *iostat_ctx;
262a4b68176SDaeho Jeong /* Due to the mempool, this never fails. */
263a4b68176SDaeho Jeong iostat_ctx = mempool_alloc(bio_iostat_ctx_pool, GFP_NOFS);
264a4b68176SDaeho Jeong iostat_ctx->sbi = sbi;
265a4b68176SDaeho Jeong iostat_ctx->submit_ts = 0;
266a4b68176SDaeho Jeong iostat_ctx->type = 0;
267a4b68176SDaeho Jeong iostat_ctx->post_read_ctx = ctx;
268a4b68176SDaeho Jeong bio->bi_private = iostat_ctx;
269a4b68176SDaeho Jeong }
270a4b68176SDaeho Jeong
f2fs_init_iostat_processing(void)271a4b68176SDaeho Jeong int __init f2fs_init_iostat_processing(void)
272a4b68176SDaeho Jeong {
273a4b68176SDaeho Jeong bio_iostat_ctx_cache =
274a4b68176SDaeho Jeong kmem_cache_create("f2fs_bio_iostat_ctx",
275a4b68176SDaeho Jeong sizeof(struct bio_iostat_ctx), 0, 0, NULL);
276a4b68176SDaeho Jeong if (!bio_iostat_ctx_cache)
277a4b68176SDaeho Jeong goto fail;
278a4b68176SDaeho Jeong bio_iostat_ctx_pool =
279a4b68176SDaeho Jeong mempool_create_slab_pool(NUM_PREALLOC_IOSTAT_CTXS,
280a4b68176SDaeho Jeong bio_iostat_ctx_cache);
281a4b68176SDaeho Jeong if (!bio_iostat_ctx_pool)
282a4b68176SDaeho Jeong goto fail_free_cache;
283a4b68176SDaeho Jeong return 0;
284a4b68176SDaeho Jeong
285a4b68176SDaeho Jeong fail_free_cache:
286a4b68176SDaeho Jeong kmem_cache_destroy(bio_iostat_ctx_cache);
287a4b68176SDaeho Jeong fail:
288a4b68176SDaeho Jeong return -ENOMEM;
289a4b68176SDaeho Jeong }
290a4b68176SDaeho Jeong
f2fs_destroy_iostat_processing(void)291a4b68176SDaeho Jeong void f2fs_destroy_iostat_processing(void)
292a4b68176SDaeho Jeong {
293a4b68176SDaeho Jeong mempool_destroy(bio_iostat_ctx_pool);
294a4b68176SDaeho Jeong kmem_cache_destroy(bio_iostat_ctx_cache);
295a4b68176SDaeho Jeong }
296a4b68176SDaeho Jeong
f2fs_init_iostat(struct f2fs_sb_info * sbi)29752118743SDaeho Jeong int f2fs_init_iostat(struct f2fs_sb_info *sbi)
29852118743SDaeho Jeong {
29952118743SDaeho Jeong /* init iostat info */
30052118743SDaeho Jeong spin_lock_init(&sbi->iostat_lock);
301a4b68176SDaeho Jeong spin_lock_init(&sbi->iostat_lat_lock);
30252118743SDaeho Jeong sbi->iostat_enable = false;
30352118743SDaeho Jeong sbi->iostat_period_ms = DEFAULT_IOSTAT_PERIOD_MS;
304a4b68176SDaeho Jeong sbi->iostat_io_lat = f2fs_kzalloc(sbi, sizeof(struct iostat_lat_info),
305a4b68176SDaeho Jeong GFP_KERNEL);
306a4b68176SDaeho Jeong if (!sbi->iostat_io_lat)
307a4b68176SDaeho Jeong return -ENOMEM;
30852118743SDaeho Jeong
30952118743SDaeho Jeong return 0;
31052118743SDaeho Jeong }
311a4b68176SDaeho Jeong
f2fs_destroy_iostat(struct f2fs_sb_info * sbi)312a4b68176SDaeho Jeong void f2fs_destroy_iostat(struct f2fs_sb_info *sbi)
313a4b68176SDaeho Jeong {
314a4b68176SDaeho Jeong kfree(sbi->iostat_io_lat);
315a4b68176SDaeho Jeong }
316