1 /* 2 * Copyright (C) 2017 Facebook 3 * 4 * This program is free software; you can redistribute it and/or 5 * modify it under the terms of the GNU General Public 6 * License v2 as published by the Free Software Foundation. 7 * 8 * This program is distributed in the hope that it will be useful, 9 * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 * General Public License for more details. 12 * 13 * You should have received a copy of the GNU General Public License 14 * along with this program. If not, see <https://www.gnu.org/licenses/>. 15 */ 16 17 #include <linux/kernel.h> 18 #include <linux/blkdev.h> 19 #include <linux/debugfs.h> 20 21 #include <linux/blk-mq.h> 22 #include "blk.h" 23 #include "blk-mq.h" 24 #include "blk-mq-debugfs.h" 25 #include "blk-mq-tag.h" 26 27 static void print_stat(struct seq_file *m, struct blk_rq_stat *stat) 28 { 29 if (stat->nr_samples) { 30 seq_printf(m, "samples=%d, mean=%lld, min=%llu, max=%llu", 31 stat->nr_samples, stat->mean, stat->min, stat->max); 32 } else { 33 seq_puts(m, "samples=0"); 34 } 35 } 36 37 static int queue_poll_stat_show(void *data, struct seq_file *m) 38 { 39 struct request_queue *q = data; 40 int bucket; 41 42 for (bucket = 0; bucket < BLK_MQ_POLL_STATS_BKTS/2; bucket++) { 43 seq_printf(m, "read (%d Bytes): ", 1 << (9+bucket)); 44 print_stat(m, &q->poll_stat[2*bucket]); 45 seq_puts(m, "\n"); 46 47 seq_printf(m, "write (%d Bytes): ", 1 << (9+bucket)); 48 print_stat(m, &q->poll_stat[2*bucket+1]); 49 seq_puts(m, "\n"); 50 } 51 return 0; 52 } 53 54 static void *queue_requeue_list_start(struct seq_file *m, loff_t *pos) 55 __acquires(&q->requeue_lock) 56 { 57 struct request_queue *q = m->private; 58 59 spin_lock_irq(&q->requeue_lock); 60 return seq_list_start(&q->requeue_list, *pos); 61 } 62 63 static void *queue_requeue_list_next(struct seq_file *m, void *v, loff_t *pos) 64 { 65 struct request_queue *q = m->private; 66 67 return seq_list_next(v, &q->requeue_list, pos); 68 } 69 70 static void queue_requeue_list_stop(struct seq_file *m, void *v) 71 __releases(&q->requeue_lock) 72 { 73 struct request_queue *q = m->private; 74 75 spin_unlock_irq(&q->requeue_lock); 76 } 77 78 static const struct seq_operations queue_requeue_list_seq_ops = { 79 .start = queue_requeue_list_start, 80 .next = queue_requeue_list_next, 81 .stop = queue_requeue_list_stop, 82 .show = blk_mq_debugfs_rq_show, 83 }; 84 85 static int blk_flags_show(struct seq_file *m, const unsigned long flags, 86 const char *const *flag_name, int flag_name_count) 87 { 88 bool sep = false; 89 int i; 90 91 for (i = 0; i < sizeof(flags) * BITS_PER_BYTE; i++) { 92 if (!(flags & BIT(i))) 93 continue; 94 if (sep) 95 seq_puts(m, "|"); 96 sep = true; 97 if (i < flag_name_count && flag_name[i]) 98 seq_puts(m, flag_name[i]); 99 else 100 seq_printf(m, "%d", i); 101 } 102 return 0; 103 } 104 105 #define QUEUE_FLAG_NAME(name) [QUEUE_FLAG_##name] = #name 106 static const char *const blk_queue_flag_name[] = { 107 QUEUE_FLAG_NAME(QUEUED), 108 QUEUE_FLAG_NAME(STOPPED), 109 QUEUE_FLAG_NAME(DYING), 110 QUEUE_FLAG_NAME(BYPASS), 111 QUEUE_FLAG_NAME(BIDI), 112 QUEUE_FLAG_NAME(NOMERGES), 113 QUEUE_FLAG_NAME(SAME_COMP), 114 QUEUE_FLAG_NAME(FAIL_IO), 115 QUEUE_FLAG_NAME(NONROT), 116 QUEUE_FLAG_NAME(IO_STAT), 117 QUEUE_FLAG_NAME(DISCARD), 118 QUEUE_FLAG_NAME(NOXMERGES), 119 QUEUE_FLAG_NAME(ADD_RANDOM), 120 QUEUE_FLAG_NAME(SECERASE), 121 QUEUE_FLAG_NAME(SAME_FORCE), 122 QUEUE_FLAG_NAME(DEAD), 123 QUEUE_FLAG_NAME(INIT_DONE), 124 QUEUE_FLAG_NAME(NO_SG_MERGE), 125 QUEUE_FLAG_NAME(POLL), 126 QUEUE_FLAG_NAME(WC), 127 QUEUE_FLAG_NAME(FUA), 128 QUEUE_FLAG_NAME(FLUSH_NQ), 129 QUEUE_FLAG_NAME(DAX), 130 QUEUE_FLAG_NAME(STATS), 131 QUEUE_FLAG_NAME(POLL_STATS), 132 QUEUE_FLAG_NAME(REGISTERED), 133 QUEUE_FLAG_NAME(SCSI_PASSTHROUGH), 134 QUEUE_FLAG_NAME(QUIESCED), 135 QUEUE_FLAG_NAME(PREEMPT_ONLY), 136 }; 137 #undef QUEUE_FLAG_NAME 138 139 static int queue_state_show(void *data, struct seq_file *m) 140 { 141 struct request_queue *q = data; 142 143 blk_flags_show(m, q->queue_flags, blk_queue_flag_name, 144 ARRAY_SIZE(blk_queue_flag_name)); 145 seq_puts(m, "\n"); 146 return 0; 147 } 148 149 static ssize_t queue_state_write(void *data, const char __user *buf, 150 size_t count, loff_t *ppos) 151 { 152 struct request_queue *q = data; 153 char opbuf[16] = { }, *op; 154 155 /* 156 * The "state" attribute is removed after blk_cleanup_queue() has called 157 * blk_mq_free_queue(). Return if QUEUE_FLAG_DEAD has been set to avoid 158 * triggering a use-after-free. 159 */ 160 if (blk_queue_dead(q)) 161 return -ENOENT; 162 163 if (count >= sizeof(opbuf)) { 164 pr_err("%s: operation too long\n", __func__); 165 goto inval; 166 } 167 168 if (copy_from_user(opbuf, buf, count)) 169 return -EFAULT; 170 op = strstrip(opbuf); 171 if (strcmp(op, "run") == 0) { 172 blk_mq_run_hw_queues(q, true); 173 } else if (strcmp(op, "start") == 0) { 174 blk_mq_start_stopped_hw_queues(q, true); 175 } else if (strcmp(op, "kick") == 0) { 176 blk_mq_kick_requeue_list(q); 177 } else { 178 pr_err("%s: unsupported operation '%s'\n", __func__, op); 179 inval: 180 pr_err("%s: use 'run', 'start' or 'kick'\n", __func__); 181 return -EINVAL; 182 } 183 return count; 184 } 185 186 static int queue_write_hint_show(void *data, struct seq_file *m) 187 { 188 struct request_queue *q = data; 189 int i; 190 191 for (i = 0; i < BLK_MAX_WRITE_HINTS; i++) 192 seq_printf(m, "hint%d: %llu\n", i, q->write_hints[i]); 193 194 return 0; 195 } 196 197 static ssize_t queue_write_hint_store(void *data, const char __user *buf, 198 size_t count, loff_t *ppos) 199 { 200 struct request_queue *q = data; 201 int i; 202 203 for (i = 0; i < BLK_MAX_WRITE_HINTS; i++) 204 q->write_hints[i] = 0; 205 206 return count; 207 } 208 209 static int queue_zone_wlock_show(void *data, struct seq_file *m) 210 { 211 struct request_queue *q = data; 212 unsigned int i; 213 214 if (!q->seq_zones_wlock) 215 return 0; 216 217 for (i = 0; i < blk_queue_nr_zones(q); i++) 218 if (test_bit(i, q->seq_zones_wlock)) 219 seq_printf(m, "%u\n", i); 220 221 return 0; 222 } 223 224 static const struct blk_mq_debugfs_attr blk_mq_debugfs_queue_attrs[] = { 225 { "poll_stat", 0400, queue_poll_stat_show }, 226 { "requeue_list", 0400, .seq_ops = &queue_requeue_list_seq_ops }, 227 { "state", 0600, queue_state_show, queue_state_write }, 228 { "write_hints", 0600, queue_write_hint_show, queue_write_hint_store }, 229 { "zone_wlock", 0400, queue_zone_wlock_show, NULL }, 230 { }, 231 }; 232 233 #define HCTX_STATE_NAME(name) [BLK_MQ_S_##name] = #name 234 static const char *const hctx_state_name[] = { 235 HCTX_STATE_NAME(STOPPED), 236 HCTX_STATE_NAME(TAG_ACTIVE), 237 HCTX_STATE_NAME(SCHED_RESTART), 238 }; 239 #undef HCTX_STATE_NAME 240 241 static int hctx_state_show(void *data, struct seq_file *m) 242 { 243 struct blk_mq_hw_ctx *hctx = data; 244 245 blk_flags_show(m, hctx->state, hctx_state_name, 246 ARRAY_SIZE(hctx_state_name)); 247 seq_puts(m, "\n"); 248 return 0; 249 } 250 251 #define BLK_TAG_ALLOC_NAME(name) [BLK_TAG_ALLOC_##name] = #name 252 static const char *const alloc_policy_name[] = { 253 BLK_TAG_ALLOC_NAME(FIFO), 254 BLK_TAG_ALLOC_NAME(RR), 255 }; 256 #undef BLK_TAG_ALLOC_NAME 257 258 #define HCTX_FLAG_NAME(name) [ilog2(BLK_MQ_F_##name)] = #name 259 static const char *const hctx_flag_name[] = { 260 HCTX_FLAG_NAME(SHOULD_MERGE), 261 HCTX_FLAG_NAME(TAG_SHARED), 262 HCTX_FLAG_NAME(SG_MERGE), 263 HCTX_FLAG_NAME(BLOCKING), 264 HCTX_FLAG_NAME(NO_SCHED), 265 }; 266 #undef HCTX_FLAG_NAME 267 268 static int hctx_flags_show(void *data, struct seq_file *m) 269 { 270 struct blk_mq_hw_ctx *hctx = data; 271 const int alloc_policy = BLK_MQ_FLAG_TO_ALLOC_POLICY(hctx->flags); 272 273 seq_puts(m, "alloc_policy="); 274 if (alloc_policy < ARRAY_SIZE(alloc_policy_name) && 275 alloc_policy_name[alloc_policy]) 276 seq_puts(m, alloc_policy_name[alloc_policy]); 277 else 278 seq_printf(m, "%d", alloc_policy); 279 seq_puts(m, " "); 280 blk_flags_show(m, 281 hctx->flags ^ BLK_ALLOC_POLICY_TO_MQ_FLAG(alloc_policy), 282 hctx_flag_name, ARRAY_SIZE(hctx_flag_name)); 283 seq_puts(m, "\n"); 284 return 0; 285 } 286 287 #define REQ_OP_NAME(name) [REQ_OP_##name] = #name 288 static const char *const op_name[] = { 289 REQ_OP_NAME(READ), 290 REQ_OP_NAME(WRITE), 291 REQ_OP_NAME(FLUSH), 292 REQ_OP_NAME(DISCARD), 293 REQ_OP_NAME(ZONE_REPORT), 294 REQ_OP_NAME(SECURE_ERASE), 295 REQ_OP_NAME(ZONE_RESET), 296 REQ_OP_NAME(WRITE_SAME), 297 REQ_OP_NAME(WRITE_ZEROES), 298 REQ_OP_NAME(SCSI_IN), 299 REQ_OP_NAME(SCSI_OUT), 300 REQ_OP_NAME(DRV_IN), 301 REQ_OP_NAME(DRV_OUT), 302 }; 303 #undef REQ_OP_NAME 304 305 #define CMD_FLAG_NAME(name) [__REQ_##name] = #name 306 static const char *const cmd_flag_name[] = { 307 CMD_FLAG_NAME(FAILFAST_DEV), 308 CMD_FLAG_NAME(FAILFAST_TRANSPORT), 309 CMD_FLAG_NAME(FAILFAST_DRIVER), 310 CMD_FLAG_NAME(SYNC), 311 CMD_FLAG_NAME(META), 312 CMD_FLAG_NAME(PRIO), 313 CMD_FLAG_NAME(NOMERGE), 314 CMD_FLAG_NAME(IDLE), 315 CMD_FLAG_NAME(INTEGRITY), 316 CMD_FLAG_NAME(FUA), 317 CMD_FLAG_NAME(PREFLUSH), 318 CMD_FLAG_NAME(RAHEAD), 319 CMD_FLAG_NAME(BACKGROUND), 320 CMD_FLAG_NAME(NOUNMAP), 321 CMD_FLAG_NAME(NOWAIT), 322 }; 323 #undef CMD_FLAG_NAME 324 325 #define RQF_NAME(name) [ilog2((__force u32)RQF_##name)] = #name 326 static const char *const rqf_name[] = { 327 RQF_NAME(SORTED), 328 RQF_NAME(STARTED), 329 RQF_NAME(QUEUED), 330 RQF_NAME(SOFTBARRIER), 331 RQF_NAME(FLUSH_SEQ), 332 RQF_NAME(MIXED_MERGE), 333 RQF_NAME(MQ_INFLIGHT), 334 RQF_NAME(DONTPREP), 335 RQF_NAME(PREEMPT), 336 RQF_NAME(COPY_USER), 337 RQF_NAME(FAILED), 338 RQF_NAME(QUIET), 339 RQF_NAME(ELVPRIV), 340 RQF_NAME(IO_STAT), 341 RQF_NAME(ALLOCED), 342 RQF_NAME(PM), 343 RQF_NAME(HASHED), 344 RQF_NAME(STATS), 345 RQF_NAME(SPECIAL_PAYLOAD), 346 RQF_NAME(ZONE_WRITE_LOCKED), 347 RQF_NAME(MQ_POLL_SLEPT), 348 }; 349 #undef RQF_NAME 350 351 static const char *const blk_mq_rq_state_name_array[] = { 352 [MQ_RQ_IDLE] = "idle", 353 [MQ_RQ_IN_FLIGHT] = "in_flight", 354 [MQ_RQ_COMPLETE] = "complete", 355 }; 356 357 static const char *blk_mq_rq_state_name(enum mq_rq_state rq_state) 358 { 359 if (WARN_ON_ONCE((unsigned int)rq_state > 360 ARRAY_SIZE(blk_mq_rq_state_name_array))) 361 return "(?)"; 362 return blk_mq_rq_state_name_array[rq_state]; 363 } 364 365 int __blk_mq_debugfs_rq_show(struct seq_file *m, struct request *rq) 366 { 367 const struct blk_mq_ops *const mq_ops = rq->q->mq_ops; 368 const unsigned int op = rq->cmd_flags & REQ_OP_MASK; 369 370 seq_printf(m, "%p {.op=", rq); 371 if (op < ARRAY_SIZE(op_name) && op_name[op]) 372 seq_printf(m, "%s", op_name[op]); 373 else 374 seq_printf(m, "%d", op); 375 seq_puts(m, ", .cmd_flags="); 376 blk_flags_show(m, rq->cmd_flags & ~REQ_OP_MASK, cmd_flag_name, 377 ARRAY_SIZE(cmd_flag_name)); 378 seq_puts(m, ", .rq_flags="); 379 blk_flags_show(m, (__force unsigned int)rq->rq_flags, rqf_name, 380 ARRAY_SIZE(rqf_name)); 381 seq_printf(m, ", .state=%s", blk_mq_rq_state_name(blk_mq_rq_state(rq))); 382 seq_printf(m, ", .tag=%d, .internal_tag=%d", rq->tag, 383 rq->internal_tag); 384 if (mq_ops->show_rq) 385 mq_ops->show_rq(m, rq); 386 seq_puts(m, "}\n"); 387 return 0; 388 } 389 EXPORT_SYMBOL_GPL(__blk_mq_debugfs_rq_show); 390 391 int blk_mq_debugfs_rq_show(struct seq_file *m, void *v) 392 { 393 return __blk_mq_debugfs_rq_show(m, list_entry_rq(v)); 394 } 395 EXPORT_SYMBOL_GPL(blk_mq_debugfs_rq_show); 396 397 static void *hctx_dispatch_start(struct seq_file *m, loff_t *pos) 398 __acquires(&hctx->lock) 399 { 400 struct blk_mq_hw_ctx *hctx = m->private; 401 402 spin_lock(&hctx->lock); 403 return seq_list_start(&hctx->dispatch, *pos); 404 } 405 406 static void *hctx_dispatch_next(struct seq_file *m, void *v, loff_t *pos) 407 { 408 struct blk_mq_hw_ctx *hctx = m->private; 409 410 return seq_list_next(v, &hctx->dispatch, pos); 411 } 412 413 static void hctx_dispatch_stop(struct seq_file *m, void *v) 414 __releases(&hctx->lock) 415 { 416 struct blk_mq_hw_ctx *hctx = m->private; 417 418 spin_unlock(&hctx->lock); 419 } 420 421 static const struct seq_operations hctx_dispatch_seq_ops = { 422 .start = hctx_dispatch_start, 423 .next = hctx_dispatch_next, 424 .stop = hctx_dispatch_stop, 425 .show = blk_mq_debugfs_rq_show, 426 }; 427 428 struct show_busy_params { 429 struct seq_file *m; 430 struct blk_mq_hw_ctx *hctx; 431 }; 432 433 /* 434 * Note: the state of a request may change while this function is in progress, 435 * e.g. due to a concurrent blk_mq_finish_request() call. 436 */ 437 static void hctx_show_busy_rq(struct request *rq, void *data, bool reserved) 438 { 439 const struct show_busy_params *params = data; 440 441 if (blk_mq_map_queue(rq->q, rq->mq_ctx->cpu) == params->hctx && 442 blk_mq_rq_state(rq) != MQ_RQ_IDLE) 443 __blk_mq_debugfs_rq_show(params->m, 444 list_entry_rq(&rq->queuelist)); 445 } 446 447 static int hctx_busy_show(void *data, struct seq_file *m) 448 { 449 struct blk_mq_hw_ctx *hctx = data; 450 struct show_busy_params params = { .m = m, .hctx = hctx }; 451 452 blk_mq_tagset_busy_iter(hctx->queue->tag_set, hctx_show_busy_rq, 453 ¶ms); 454 455 return 0; 456 } 457 458 static int hctx_ctx_map_show(void *data, struct seq_file *m) 459 { 460 struct blk_mq_hw_ctx *hctx = data; 461 462 sbitmap_bitmap_show(&hctx->ctx_map, m); 463 return 0; 464 } 465 466 static void blk_mq_debugfs_tags_show(struct seq_file *m, 467 struct blk_mq_tags *tags) 468 { 469 seq_printf(m, "nr_tags=%u\n", tags->nr_tags); 470 seq_printf(m, "nr_reserved_tags=%u\n", tags->nr_reserved_tags); 471 seq_printf(m, "active_queues=%d\n", 472 atomic_read(&tags->active_queues)); 473 474 seq_puts(m, "\nbitmap_tags:\n"); 475 sbitmap_queue_show(&tags->bitmap_tags, m); 476 477 if (tags->nr_reserved_tags) { 478 seq_puts(m, "\nbreserved_tags:\n"); 479 sbitmap_queue_show(&tags->breserved_tags, m); 480 } 481 } 482 483 static int hctx_tags_show(void *data, struct seq_file *m) 484 { 485 struct blk_mq_hw_ctx *hctx = data; 486 struct request_queue *q = hctx->queue; 487 int res; 488 489 res = mutex_lock_interruptible(&q->sysfs_lock); 490 if (res) 491 goto out; 492 if (hctx->tags) 493 blk_mq_debugfs_tags_show(m, hctx->tags); 494 mutex_unlock(&q->sysfs_lock); 495 496 out: 497 return res; 498 } 499 500 static int hctx_tags_bitmap_show(void *data, struct seq_file *m) 501 { 502 struct blk_mq_hw_ctx *hctx = data; 503 struct request_queue *q = hctx->queue; 504 int res; 505 506 res = mutex_lock_interruptible(&q->sysfs_lock); 507 if (res) 508 goto out; 509 if (hctx->tags) 510 sbitmap_bitmap_show(&hctx->tags->bitmap_tags.sb, m); 511 mutex_unlock(&q->sysfs_lock); 512 513 out: 514 return res; 515 } 516 517 static int hctx_sched_tags_show(void *data, struct seq_file *m) 518 { 519 struct blk_mq_hw_ctx *hctx = data; 520 struct request_queue *q = hctx->queue; 521 int res; 522 523 res = mutex_lock_interruptible(&q->sysfs_lock); 524 if (res) 525 goto out; 526 if (hctx->sched_tags) 527 blk_mq_debugfs_tags_show(m, hctx->sched_tags); 528 mutex_unlock(&q->sysfs_lock); 529 530 out: 531 return res; 532 } 533 534 static int hctx_sched_tags_bitmap_show(void *data, struct seq_file *m) 535 { 536 struct blk_mq_hw_ctx *hctx = data; 537 struct request_queue *q = hctx->queue; 538 int res; 539 540 res = mutex_lock_interruptible(&q->sysfs_lock); 541 if (res) 542 goto out; 543 if (hctx->sched_tags) 544 sbitmap_bitmap_show(&hctx->sched_tags->bitmap_tags.sb, m); 545 mutex_unlock(&q->sysfs_lock); 546 547 out: 548 return res; 549 } 550 551 static int hctx_io_poll_show(void *data, struct seq_file *m) 552 { 553 struct blk_mq_hw_ctx *hctx = data; 554 555 seq_printf(m, "considered=%lu\n", hctx->poll_considered); 556 seq_printf(m, "invoked=%lu\n", hctx->poll_invoked); 557 seq_printf(m, "success=%lu\n", hctx->poll_success); 558 return 0; 559 } 560 561 static ssize_t hctx_io_poll_write(void *data, const char __user *buf, 562 size_t count, loff_t *ppos) 563 { 564 struct blk_mq_hw_ctx *hctx = data; 565 566 hctx->poll_considered = hctx->poll_invoked = hctx->poll_success = 0; 567 return count; 568 } 569 570 static int hctx_dispatched_show(void *data, struct seq_file *m) 571 { 572 struct blk_mq_hw_ctx *hctx = data; 573 int i; 574 575 seq_printf(m, "%8u\t%lu\n", 0U, hctx->dispatched[0]); 576 577 for (i = 1; i < BLK_MQ_MAX_DISPATCH_ORDER - 1; i++) { 578 unsigned int d = 1U << (i - 1); 579 580 seq_printf(m, "%8u\t%lu\n", d, hctx->dispatched[i]); 581 } 582 583 seq_printf(m, "%8u+\t%lu\n", 1U << (i - 1), hctx->dispatched[i]); 584 return 0; 585 } 586 587 static ssize_t hctx_dispatched_write(void *data, const char __user *buf, 588 size_t count, loff_t *ppos) 589 { 590 struct blk_mq_hw_ctx *hctx = data; 591 int i; 592 593 for (i = 0; i < BLK_MQ_MAX_DISPATCH_ORDER; i++) 594 hctx->dispatched[i] = 0; 595 return count; 596 } 597 598 static int hctx_queued_show(void *data, struct seq_file *m) 599 { 600 struct blk_mq_hw_ctx *hctx = data; 601 602 seq_printf(m, "%lu\n", hctx->queued); 603 return 0; 604 } 605 606 static ssize_t hctx_queued_write(void *data, const char __user *buf, 607 size_t count, loff_t *ppos) 608 { 609 struct blk_mq_hw_ctx *hctx = data; 610 611 hctx->queued = 0; 612 return count; 613 } 614 615 static int hctx_run_show(void *data, struct seq_file *m) 616 { 617 struct blk_mq_hw_ctx *hctx = data; 618 619 seq_printf(m, "%lu\n", hctx->run); 620 return 0; 621 } 622 623 static ssize_t hctx_run_write(void *data, const char __user *buf, size_t count, 624 loff_t *ppos) 625 { 626 struct blk_mq_hw_ctx *hctx = data; 627 628 hctx->run = 0; 629 return count; 630 } 631 632 static int hctx_active_show(void *data, struct seq_file *m) 633 { 634 struct blk_mq_hw_ctx *hctx = data; 635 636 seq_printf(m, "%d\n", atomic_read(&hctx->nr_active)); 637 return 0; 638 } 639 640 static void *ctx_rq_list_start(struct seq_file *m, loff_t *pos) 641 __acquires(&ctx->lock) 642 { 643 struct blk_mq_ctx *ctx = m->private; 644 645 spin_lock(&ctx->lock); 646 return seq_list_start(&ctx->rq_list, *pos); 647 } 648 649 static void *ctx_rq_list_next(struct seq_file *m, void *v, loff_t *pos) 650 { 651 struct blk_mq_ctx *ctx = m->private; 652 653 return seq_list_next(v, &ctx->rq_list, pos); 654 } 655 656 static void ctx_rq_list_stop(struct seq_file *m, void *v) 657 __releases(&ctx->lock) 658 { 659 struct blk_mq_ctx *ctx = m->private; 660 661 spin_unlock(&ctx->lock); 662 } 663 664 static const struct seq_operations ctx_rq_list_seq_ops = { 665 .start = ctx_rq_list_start, 666 .next = ctx_rq_list_next, 667 .stop = ctx_rq_list_stop, 668 .show = blk_mq_debugfs_rq_show, 669 }; 670 static int ctx_dispatched_show(void *data, struct seq_file *m) 671 { 672 struct blk_mq_ctx *ctx = data; 673 674 seq_printf(m, "%lu %lu\n", ctx->rq_dispatched[1], ctx->rq_dispatched[0]); 675 return 0; 676 } 677 678 static ssize_t ctx_dispatched_write(void *data, const char __user *buf, 679 size_t count, loff_t *ppos) 680 { 681 struct blk_mq_ctx *ctx = data; 682 683 ctx->rq_dispatched[0] = ctx->rq_dispatched[1] = 0; 684 return count; 685 } 686 687 static int ctx_merged_show(void *data, struct seq_file *m) 688 { 689 struct blk_mq_ctx *ctx = data; 690 691 seq_printf(m, "%lu\n", ctx->rq_merged); 692 return 0; 693 } 694 695 static ssize_t ctx_merged_write(void *data, const char __user *buf, 696 size_t count, loff_t *ppos) 697 { 698 struct blk_mq_ctx *ctx = data; 699 700 ctx->rq_merged = 0; 701 return count; 702 } 703 704 static int ctx_completed_show(void *data, struct seq_file *m) 705 { 706 struct blk_mq_ctx *ctx = data; 707 708 seq_printf(m, "%lu %lu\n", ctx->rq_completed[1], ctx->rq_completed[0]); 709 return 0; 710 } 711 712 static ssize_t ctx_completed_write(void *data, const char __user *buf, 713 size_t count, loff_t *ppos) 714 { 715 struct blk_mq_ctx *ctx = data; 716 717 ctx->rq_completed[0] = ctx->rq_completed[1] = 0; 718 return count; 719 } 720 721 static int blk_mq_debugfs_show(struct seq_file *m, void *v) 722 { 723 const struct blk_mq_debugfs_attr *attr = m->private; 724 void *data = d_inode(m->file->f_path.dentry->d_parent)->i_private; 725 726 return attr->show(data, m); 727 } 728 729 static ssize_t blk_mq_debugfs_write(struct file *file, const char __user *buf, 730 size_t count, loff_t *ppos) 731 { 732 struct seq_file *m = file->private_data; 733 const struct blk_mq_debugfs_attr *attr = m->private; 734 void *data = d_inode(file->f_path.dentry->d_parent)->i_private; 735 736 /* 737 * Attributes that only implement .seq_ops are read-only and 'attr' is 738 * the same with 'data' in this case. 739 */ 740 if (attr == data || !attr->write) 741 return -EPERM; 742 743 return attr->write(data, buf, count, ppos); 744 } 745 746 static int blk_mq_debugfs_open(struct inode *inode, struct file *file) 747 { 748 const struct blk_mq_debugfs_attr *attr = inode->i_private; 749 void *data = d_inode(file->f_path.dentry->d_parent)->i_private; 750 struct seq_file *m; 751 int ret; 752 753 if (attr->seq_ops) { 754 ret = seq_open(file, attr->seq_ops); 755 if (!ret) { 756 m = file->private_data; 757 m->private = data; 758 } 759 return ret; 760 } 761 762 if (WARN_ON_ONCE(!attr->show)) 763 return -EPERM; 764 765 return single_open(file, blk_mq_debugfs_show, inode->i_private); 766 } 767 768 static int blk_mq_debugfs_release(struct inode *inode, struct file *file) 769 { 770 const struct blk_mq_debugfs_attr *attr = inode->i_private; 771 772 if (attr->show) 773 return single_release(inode, file); 774 else 775 return seq_release(inode, file); 776 } 777 778 static const struct file_operations blk_mq_debugfs_fops = { 779 .open = blk_mq_debugfs_open, 780 .read = seq_read, 781 .write = blk_mq_debugfs_write, 782 .llseek = seq_lseek, 783 .release = blk_mq_debugfs_release, 784 }; 785 786 static const struct blk_mq_debugfs_attr blk_mq_debugfs_hctx_attrs[] = { 787 {"state", 0400, hctx_state_show}, 788 {"flags", 0400, hctx_flags_show}, 789 {"dispatch", 0400, .seq_ops = &hctx_dispatch_seq_ops}, 790 {"busy", 0400, hctx_busy_show}, 791 {"ctx_map", 0400, hctx_ctx_map_show}, 792 {"tags", 0400, hctx_tags_show}, 793 {"tags_bitmap", 0400, hctx_tags_bitmap_show}, 794 {"sched_tags", 0400, hctx_sched_tags_show}, 795 {"sched_tags_bitmap", 0400, hctx_sched_tags_bitmap_show}, 796 {"io_poll", 0600, hctx_io_poll_show, hctx_io_poll_write}, 797 {"dispatched", 0600, hctx_dispatched_show, hctx_dispatched_write}, 798 {"queued", 0600, hctx_queued_show, hctx_queued_write}, 799 {"run", 0600, hctx_run_show, hctx_run_write}, 800 {"active", 0400, hctx_active_show}, 801 {}, 802 }; 803 804 static const struct blk_mq_debugfs_attr blk_mq_debugfs_ctx_attrs[] = { 805 {"rq_list", 0400, .seq_ops = &ctx_rq_list_seq_ops}, 806 {"dispatched", 0600, ctx_dispatched_show, ctx_dispatched_write}, 807 {"merged", 0600, ctx_merged_show, ctx_merged_write}, 808 {"completed", 0600, ctx_completed_show, ctx_completed_write}, 809 {}, 810 }; 811 812 static bool debugfs_create_files(struct dentry *parent, void *data, 813 const struct blk_mq_debugfs_attr *attr) 814 { 815 d_inode(parent)->i_private = data; 816 817 for (; attr->name; attr++) { 818 if (!debugfs_create_file(attr->name, attr->mode, parent, 819 (void *)attr, &blk_mq_debugfs_fops)) 820 return false; 821 } 822 return true; 823 } 824 825 int blk_mq_debugfs_register(struct request_queue *q) 826 { 827 struct blk_mq_hw_ctx *hctx; 828 int i; 829 830 if (!blk_debugfs_root) 831 return -ENOENT; 832 833 q->debugfs_dir = debugfs_create_dir(kobject_name(q->kobj.parent), 834 blk_debugfs_root); 835 if (!q->debugfs_dir) 836 return -ENOMEM; 837 838 if (!debugfs_create_files(q->debugfs_dir, q, 839 blk_mq_debugfs_queue_attrs)) 840 goto err; 841 842 /* 843 * blk_mq_init_sched() attempted to do this already, but q->debugfs_dir 844 * didn't exist yet (because we don't know what to name the directory 845 * until the queue is registered to a gendisk). 846 */ 847 if (q->elevator && !q->sched_debugfs_dir) 848 blk_mq_debugfs_register_sched(q); 849 850 /* Similarly, blk_mq_init_hctx() couldn't do this previously. */ 851 queue_for_each_hw_ctx(q, hctx, i) { 852 if (!hctx->debugfs_dir && blk_mq_debugfs_register_hctx(q, hctx)) 853 goto err; 854 if (q->elevator && !hctx->sched_debugfs_dir && 855 blk_mq_debugfs_register_sched_hctx(q, hctx)) 856 goto err; 857 } 858 859 return 0; 860 861 err: 862 blk_mq_debugfs_unregister(q); 863 return -ENOMEM; 864 } 865 866 void blk_mq_debugfs_unregister(struct request_queue *q) 867 { 868 debugfs_remove_recursive(q->debugfs_dir); 869 q->sched_debugfs_dir = NULL; 870 q->debugfs_dir = NULL; 871 } 872 873 static int blk_mq_debugfs_register_ctx(struct blk_mq_hw_ctx *hctx, 874 struct blk_mq_ctx *ctx) 875 { 876 struct dentry *ctx_dir; 877 char name[20]; 878 879 snprintf(name, sizeof(name), "cpu%u", ctx->cpu); 880 ctx_dir = debugfs_create_dir(name, hctx->debugfs_dir); 881 if (!ctx_dir) 882 return -ENOMEM; 883 884 if (!debugfs_create_files(ctx_dir, ctx, blk_mq_debugfs_ctx_attrs)) 885 return -ENOMEM; 886 887 return 0; 888 } 889 890 int blk_mq_debugfs_register_hctx(struct request_queue *q, 891 struct blk_mq_hw_ctx *hctx) 892 { 893 struct blk_mq_ctx *ctx; 894 char name[20]; 895 int i; 896 897 if (!q->debugfs_dir) 898 return -ENOENT; 899 900 snprintf(name, sizeof(name), "hctx%u", hctx->queue_num); 901 hctx->debugfs_dir = debugfs_create_dir(name, q->debugfs_dir); 902 if (!hctx->debugfs_dir) 903 return -ENOMEM; 904 905 if (!debugfs_create_files(hctx->debugfs_dir, hctx, 906 blk_mq_debugfs_hctx_attrs)) 907 goto err; 908 909 hctx_for_each_ctx(hctx, ctx, i) { 910 if (blk_mq_debugfs_register_ctx(hctx, ctx)) 911 goto err; 912 } 913 914 return 0; 915 916 err: 917 blk_mq_debugfs_unregister_hctx(hctx); 918 return -ENOMEM; 919 } 920 921 void blk_mq_debugfs_unregister_hctx(struct blk_mq_hw_ctx *hctx) 922 { 923 debugfs_remove_recursive(hctx->debugfs_dir); 924 hctx->sched_debugfs_dir = NULL; 925 hctx->debugfs_dir = NULL; 926 } 927 928 int blk_mq_debugfs_register_hctxs(struct request_queue *q) 929 { 930 struct blk_mq_hw_ctx *hctx; 931 int i; 932 933 queue_for_each_hw_ctx(q, hctx, i) { 934 if (blk_mq_debugfs_register_hctx(q, hctx)) 935 return -ENOMEM; 936 } 937 938 return 0; 939 } 940 941 void blk_mq_debugfs_unregister_hctxs(struct request_queue *q) 942 { 943 struct blk_mq_hw_ctx *hctx; 944 int i; 945 946 queue_for_each_hw_ctx(q, hctx, i) 947 blk_mq_debugfs_unregister_hctx(hctx); 948 } 949 950 int blk_mq_debugfs_register_sched(struct request_queue *q) 951 { 952 struct elevator_type *e = q->elevator->type; 953 954 if (!q->debugfs_dir) 955 return -ENOENT; 956 957 if (!e->queue_debugfs_attrs) 958 return 0; 959 960 q->sched_debugfs_dir = debugfs_create_dir("sched", q->debugfs_dir); 961 if (!q->sched_debugfs_dir) 962 return -ENOMEM; 963 964 if (!debugfs_create_files(q->sched_debugfs_dir, q, 965 e->queue_debugfs_attrs)) 966 goto err; 967 968 return 0; 969 970 err: 971 blk_mq_debugfs_unregister_sched(q); 972 return -ENOMEM; 973 } 974 975 void blk_mq_debugfs_unregister_sched(struct request_queue *q) 976 { 977 debugfs_remove_recursive(q->sched_debugfs_dir); 978 q->sched_debugfs_dir = NULL; 979 } 980 981 int blk_mq_debugfs_register_sched_hctx(struct request_queue *q, 982 struct blk_mq_hw_ctx *hctx) 983 { 984 struct elevator_type *e = q->elevator->type; 985 986 if (!hctx->debugfs_dir) 987 return -ENOENT; 988 989 if (!e->hctx_debugfs_attrs) 990 return 0; 991 992 hctx->sched_debugfs_dir = debugfs_create_dir("sched", 993 hctx->debugfs_dir); 994 if (!hctx->sched_debugfs_dir) 995 return -ENOMEM; 996 997 if (!debugfs_create_files(hctx->sched_debugfs_dir, hctx, 998 e->hctx_debugfs_attrs)) 999 return -ENOMEM; 1000 1001 return 0; 1002 } 1003 1004 void blk_mq_debugfs_unregister_sched_hctx(struct blk_mq_hw_ctx *hctx) 1005 { 1006 debugfs_remove_recursive(hctx->sched_debugfs_dir); 1007 hctx->sched_debugfs_dir = NULL; 1008 } 1009