1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * f2fs iostat support 4 * 5 * Copyright 2021 Google LLC 6 * Author: Daeho Jeong <daehojeong@google.com> 7 */ 8 9 #include <linux/fs.h> 10 #include <linux/f2fs_fs.h> 11 #include <linux/seq_file.h> 12 13 #include "f2fs.h" 14 #include "iostat.h" 15 #include <trace/events/f2fs.h> 16 17 static struct kmem_cache *bio_iostat_ctx_cache; 18 static mempool_t *bio_iostat_ctx_pool; 19 20 static inline unsigned long long iostat_get_avg_bytes(struct f2fs_sb_info *sbi, 21 enum iostat_type type) 22 { 23 return sbi->iostat_count[type] ? div64_u64(sbi->iostat_bytes[type], 24 sbi->iostat_count[type]) : 0; 25 } 26 27 #define IOSTAT_INFO_SHOW(name, type) \ 28 seq_printf(seq, "%-23s %-16llu %-16llu %-16llu\n", \ 29 name":", sbi->iostat_bytes[type], \ 30 sbi->iostat_count[type], \ 31 iostat_get_avg_bytes(sbi, type)) 32 33 int __maybe_unused iostat_info_seq_show(struct seq_file *seq, void *offset) 34 { 35 struct super_block *sb = seq->private; 36 struct f2fs_sb_info *sbi = F2FS_SB(sb); 37 int i; 38 39 if (!sbi->iostat_enable) 40 return 0; 41 42 seq_printf(seq, "time: %-16llu\n", ktime_get_real_seconds()); 43 seq_printf(seq, "\t\t\t%-16s %-16s %-16s\n", 44 "io_bytes", "count", "avg_bytes"); 45 46 /* print app write IOs */ 47 seq_puts(seq, "[WRITE]\n"); 48 IOSTAT_INFO_SHOW("app buffered data", APP_BUFFERED_IO); 49 IOSTAT_INFO_SHOW("app direct data", APP_DIRECT_IO); 50 IOSTAT_INFO_SHOW("app mapped data", APP_MAPPED_IO); 51 IOSTAT_INFO_SHOW("app buffered cdata", APP_BUFFERED_CDATA_IO); 52 IOSTAT_INFO_SHOW("app mapped cdata", APP_MAPPED_CDATA_IO); 53 54 /* print fs write IOs */ 55 IOSTAT_INFO_SHOW("fs data", FS_DATA_IO); 56 IOSTAT_INFO_SHOW("fs cdata", FS_CDATA_IO); 57 IOSTAT_INFO_SHOW("fs node", FS_NODE_IO); 58 IOSTAT_INFO_SHOW("fs meta", FS_META_IO); 59 IOSTAT_INFO_SHOW("fs gc data", FS_GC_DATA_IO); 60 IOSTAT_INFO_SHOW("fs gc node", FS_GC_NODE_IO); 61 IOSTAT_INFO_SHOW("fs cp data", FS_CP_DATA_IO); 62 IOSTAT_INFO_SHOW("fs cp node", FS_CP_NODE_IO); 63 IOSTAT_INFO_SHOW("fs cp meta", FS_CP_META_IO); 64 65 /* print app read IOs */ 66 seq_puts(seq, "[READ]\n"); 67 IOSTAT_INFO_SHOW("app buffered data", APP_BUFFERED_READ_IO); 68 IOSTAT_INFO_SHOW("app direct data", APP_DIRECT_READ_IO); 69 IOSTAT_INFO_SHOW("app mapped data", APP_MAPPED_READ_IO); 70 IOSTAT_INFO_SHOW("app buffered cdata", APP_BUFFERED_CDATA_READ_IO); 71 IOSTAT_INFO_SHOW("app mapped cdata", APP_MAPPED_CDATA_READ_IO); 72 73 /* print fs read IOs */ 74 IOSTAT_INFO_SHOW("fs data", FS_DATA_READ_IO); 75 IOSTAT_INFO_SHOW("fs gc data", FS_GDATA_READ_IO); 76 IOSTAT_INFO_SHOW("fs cdata", FS_CDATA_READ_IO); 77 IOSTAT_INFO_SHOW("fs node", FS_NODE_READ_IO); 78 IOSTAT_INFO_SHOW("fs meta", FS_META_READ_IO); 79 80 /* print read folio order stats */ 81 seq_printf(seq, "%-23s", "fs read folio order:"); 82 for (i = 0; i < NR_PAGE_ORDERS; i++) 83 seq_printf(seq, " %llu", sbi->iostat_read_folio_count[i]); 84 seq_putc(seq, '\n'); 85 86 /* print other IOs */ 87 seq_puts(seq, "[OTHER]\n"); 88 IOSTAT_INFO_SHOW("fs discard", FS_DISCARD_IO); 89 IOSTAT_INFO_SHOW("fs flush", FS_FLUSH_IO); 90 IOSTAT_INFO_SHOW("fs zone reset", FS_ZONE_RESET_IO); 91 92 return 0; 93 } 94 95 static inline void __record_iostat_latency(struct f2fs_sb_info *sbi) 96 { 97 int io, idx; 98 struct f2fs_iostat_latency iostat_lat[MAX_IO_TYPE][NR_PAGE_TYPE]; 99 struct iostat_lat_info *io_lat = sbi->iostat_io_lat; 100 unsigned long flags; 101 102 spin_lock_irqsave(&sbi->iostat_lat_lock, flags); 103 for (idx = 0; idx < MAX_IO_TYPE; idx++) { 104 for (io = 0; io < NR_PAGE_TYPE; io++) { 105 iostat_lat[idx][io].peak_lat = 106 jiffies_to_msecs(io_lat->peak_lat[idx][io]); 107 iostat_lat[idx][io].cnt = io_lat->bio_cnt[idx][io]; 108 iostat_lat[idx][io].avg_lat = iostat_lat[idx][io].cnt ? 109 jiffies_to_msecs(io_lat->sum_lat[idx][io]) / iostat_lat[idx][io].cnt : 0; 110 io_lat->sum_lat[idx][io] = 0; 111 io_lat->peak_lat[idx][io] = 0; 112 io_lat->bio_cnt[idx][io] = 0; 113 } 114 } 115 spin_unlock_irqrestore(&sbi->iostat_lat_lock, flags); 116 117 trace_f2fs_iostat_latency(sbi, iostat_lat); 118 } 119 120 static inline void f2fs_record_iostat(struct f2fs_sb_info *sbi) 121 { 122 unsigned long long iostat_diff[NR_IO_TYPE]; 123 unsigned long long read_folio_count_diff[NR_PAGE_ORDERS]; 124 int i; 125 unsigned long flags; 126 127 if (time_is_after_jiffies(sbi->iostat_next_period)) 128 return; 129 130 /* Need double check under the lock */ 131 spin_lock_irqsave(&sbi->iostat_lock, flags); 132 if (time_is_after_jiffies(sbi->iostat_next_period)) { 133 spin_unlock_irqrestore(&sbi->iostat_lock, flags); 134 return; 135 } 136 sbi->iostat_next_period = jiffies + 137 msecs_to_jiffies(sbi->iostat_period_ms); 138 139 for (i = 0; i < NR_IO_TYPE; i++) { 140 iostat_diff[i] = sbi->iostat_bytes[i] - 141 sbi->prev_iostat_bytes[i]; 142 sbi->prev_iostat_bytes[i] = sbi->iostat_bytes[i]; 143 } 144 145 for (i = 0; i < NR_PAGE_ORDERS; i++) { 146 read_folio_count_diff[i] = sbi->iostat_read_folio_count[i] - 147 sbi->prev_iostat_read_folio_count[i]; 148 sbi->prev_iostat_read_folio_count[i] = sbi->iostat_read_folio_count[i]; 149 } 150 spin_unlock_irqrestore(&sbi->iostat_lock, flags); 151 152 trace_f2fs_iostat(sbi, iostat_diff, read_folio_count_diff); 153 154 __record_iostat_latency(sbi); 155 } 156 157 void f2fs_reset_iostat(struct f2fs_sb_info *sbi) 158 { 159 struct iostat_lat_info *io_lat = sbi->iostat_io_lat; 160 int i; 161 162 spin_lock_irq(&sbi->iostat_lock); 163 for (i = 0; i < NR_IO_TYPE; i++) { 164 sbi->iostat_count[i] = 0; 165 sbi->iostat_bytes[i] = 0; 166 sbi->prev_iostat_bytes[i] = 0; 167 } 168 for (i = 0; i < NR_PAGE_ORDERS; i++) { 169 sbi->iostat_read_folio_count[i] = 0; 170 sbi->prev_iostat_read_folio_count[i] = 0; 171 } 172 spin_unlock_irq(&sbi->iostat_lock); 173 174 spin_lock_irq(&sbi->iostat_lat_lock); 175 memset(io_lat, 0, sizeof(struct iostat_lat_info)); 176 spin_unlock_irq(&sbi->iostat_lat_lock); 177 } 178 179 static inline void __f2fs_update_iostat(struct f2fs_sb_info *sbi, 180 enum iostat_type type, unsigned long long io_bytes) 181 { 182 sbi->iostat_bytes[type] += io_bytes; 183 sbi->iostat_count[type]++; 184 } 185 186 void f2fs_update_read_folio_count(struct f2fs_sb_info *sbi, struct folio *folio) 187 { 188 unsigned int order = folio_order(folio); 189 unsigned long flags; 190 191 if (!sbi->iostat_enable) 192 return; 193 194 if (order >= NR_PAGE_ORDERS) 195 order = NR_PAGE_ORDERS - 1; 196 197 spin_lock_irqsave(&sbi->iostat_lock, flags); 198 sbi->iostat_read_folio_count[order]++; 199 spin_unlock_irqrestore(&sbi->iostat_lock, flags); 200 201 f2fs_record_iostat(sbi); 202 } 203 204 void f2fs_update_iostat(struct f2fs_sb_info *sbi, struct inode *inode, 205 enum iostat_type type, unsigned long long io_bytes) 206 { 207 unsigned long flags; 208 209 if (!sbi->iostat_enable) 210 return; 211 212 spin_lock_irqsave(&sbi->iostat_lock, flags); 213 __f2fs_update_iostat(sbi, type, io_bytes); 214 215 if (type == APP_BUFFERED_IO || type == APP_DIRECT_IO) 216 __f2fs_update_iostat(sbi, APP_WRITE_IO, io_bytes); 217 218 if (type == APP_BUFFERED_READ_IO || type == APP_DIRECT_READ_IO) 219 __f2fs_update_iostat(sbi, APP_READ_IO, io_bytes); 220 221 #ifdef CONFIG_F2FS_FS_COMPRESSION 222 if (inode && f2fs_compressed_file(inode)) { 223 if (type == APP_BUFFERED_IO) 224 __f2fs_update_iostat(sbi, APP_BUFFERED_CDATA_IO, io_bytes); 225 226 if (type == APP_BUFFERED_READ_IO) 227 __f2fs_update_iostat(sbi, APP_BUFFERED_CDATA_READ_IO, io_bytes); 228 229 if (type == APP_MAPPED_READ_IO) 230 __f2fs_update_iostat(sbi, APP_MAPPED_CDATA_READ_IO, io_bytes); 231 232 if (type == APP_MAPPED_IO) 233 __f2fs_update_iostat(sbi, APP_MAPPED_CDATA_IO, io_bytes); 234 235 if (type == FS_DATA_READ_IO) 236 __f2fs_update_iostat(sbi, FS_CDATA_READ_IO, io_bytes); 237 238 if (type == FS_DATA_IO) 239 __f2fs_update_iostat(sbi, FS_CDATA_IO, io_bytes); 240 } 241 #endif 242 243 spin_unlock_irqrestore(&sbi->iostat_lock, flags); 244 245 f2fs_record_iostat(sbi); 246 } 247 248 static inline void __update_iostat_latency(struct bio_iostat_ctx *iostat_ctx, 249 enum iostat_lat_type lat_type) 250 { 251 unsigned long ts_diff; 252 unsigned int page_type = iostat_ctx->type; 253 struct f2fs_sb_info *sbi = iostat_ctx->sbi; 254 struct iostat_lat_info *io_lat = sbi->iostat_io_lat; 255 unsigned long flags; 256 257 if (!sbi->iostat_enable) 258 return; 259 260 ts_diff = jiffies - iostat_ctx->submit_ts; 261 if (page_type == META_FLUSH) { 262 page_type = META; 263 } else if (page_type >= NR_PAGE_TYPE) { 264 f2fs_warn(sbi, "%s: %d over NR_PAGE_TYPE", __func__, page_type); 265 return; 266 } 267 268 spin_lock_irqsave(&sbi->iostat_lat_lock, flags); 269 io_lat->sum_lat[lat_type][page_type] += ts_diff; 270 io_lat->bio_cnt[lat_type][page_type]++; 271 if (ts_diff > io_lat->peak_lat[lat_type][page_type]) 272 io_lat->peak_lat[lat_type][page_type] = ts_diff; 273 spin_unlock_irqrestore(&sbi->iostat_lat_lock, flags); 274 } 275 276 void iostat_update_and_unbind_ctx(struct bio *bio) 277 { 278 struct bio_iostat_ctx *iostat_ctx = bio->bi_private; 279 enum iostat_lat_type lat_type; 280 281 if (op_is_write(bio_op(bio))) { 282 lat_type = bio->bi_opf & REQ_SYNC ? 283 WRITE_SYNC_IO : WRITE_ASYNC_IO; 284 bio->bi_private = iostat_ctx->sbi; 285 } else { 286 lat_type = READ_IO; 287 bio->bi_private = iostat_ctx->post_read_ctx; 288 } 289 290 __update_iostat_latency(iostat_ctx, lat_type); 291 mempool_free(iostat_ctx, bio_iostat_ctx_pool); 292 } 293 294 void iostat_alloc_and_bind_ctx(struct f2fs_sb_info *sbi, 295 struct bio *bio, struct bio_post_read_ctx *ctx) 296 { 297 struct bio_iostat_ctx *iostat_ctx; 298 /* Due to the mempool, this never fails. */ 299 iostat_ctx = mempool_alloc(bio_iostat_ctx_pool, GFP_NOFS); 300 iostat_ctx->sbi = sbi; 301 iostat_ctx->submit_ts = 0; 302 iostat_ctx->type = 0; 303 iostat_ctx->post_read_ctx = ctx; 304 bio->bi_private = iostat_ctx; 305 } 306 307 int __init f2fs_init_iostat_processing(void) 308 { 309 bio_iostat_ctx_cache = 310 kmem_cache_create("f2fs_bio_iostat_ctx", 311 sizeof(struct bio_iostat_ctx), 0, 0, NULL); 312 if (!bio_iostat_ctx_cache) 313 goto fail; 314 bio_iostat_ctx_pool = 315 mempool_create_slab_pool(NUM_PREALLOC_IOSTAT_CTXS, 316 bio_iostat_ctx_cache); 317 if (!bio_iostat_ctx_pool) 318 goto fail_free_cache; 319 return 0; 320 321 fail_free_cache: 322 kmem_cache_destroy(bio_iostat_ctx_cache); 323 fail: 324 return -ENOMEM; 325 } 326 327 void f2fs_destroy_iostat_processing(void) 328 { 329 mempool_destroy(bio_iostat_ctx_pool); 330 kmem_cache_destroy(bio_iostat_ctx_cache); 331 } 332 333 int f2fs_init_iostat(struct f2fs_sb_info *sbi) 334 { 335 /* init iostat info */ 336 spin_lock_init(&sbi->iostat_lock); 337 spin_lock_init(&sbi->iostat_lat_lock); 338 sbi->iostat_enable = false; 339 sbi->iostat_period_ms = DEFAULT_IOSTAT_PERIOD_MS; 340 sbi->iostat_io_lat = f2fs_kzalloc(sbi, sizeof(struct iostat_lat_info), 341 GFP_KERNEL); 342 if (!sbi->iostat_io_lat) 343 return -ENOMEM; 344 345 return 0; 346 } 347 348 void f2fs_destroy_iostat(struct f2fs_sb_info *sbi) 349 { 350 kfree(sbi->iostat_io_lat); 351 } 352