1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * kernel/sched/debug.c 4 * 5 * Print the CFS rbtree and other debugging details 6 * 7 * Copyright(C) 2007, Red Hat, Inc., Ingo Molnar 8 */ 9 #include <linux/debugfs.h> 10 #include <linux/nmi.h> 11 #include <linux/log2.h> 12 #include "sched.h" 13 14 /* 15 * This allows printing both to /sys/kernel/debug/sched/debug and 16 * to the console 17 */ 18 #define SEQ_printf(m, x...) \ 19 do { \ 20 if (m) \ 21 seq_printf(m, x); \ 22 else \ 23 pr_cont(x); \ 24 } while (0) 25 26 /* 27 * Ease the printing of nsec fields: 28 */ 29 static long long nsec_high(unsigned long long nsec) 30 { 31 if ((long long)nsec < 0) { 32 nsec = -nsec; 33 do_div(nsec, 1000000); 34 return -nsec; 35 } 36 do_div(nsec, 1000000); 37 38 return nsec; 39 } 40 41 static unsigned long nsec_low(unsigned long long nsec) 42 { 43 if ((long long)nsec < 0) 44 nsec = -nsec; 45 46 return do_div(nsec, 1000000); 47 } 48 49 #define SPLIT_NS(x) nsec_high(x), nsec_low(x) 50 51 #define SCHED_FEAT(name, enabled) \ 52 #name , 53 54 static const char * const sched_feat_names[] = { 55 #include "features.h" 56 }; 57 58 #undef SCHED_FEAT 59 60 static int sched_feat_show(struct seq_file *m, void *v) 61 { 62 int i; 63 64 for (i = 0; i < __SCHED_FEAT_NR; i++) { 65 if (!(sysctl_sched_features & (1UL << i))) 66 seq_puts(m, "NO_"); 67 seq_printf(m, "%s ", sched_feat_names[i]); 68 } 69 seq_puts(m, "\n"); 70 71 return 0; 72 } 73 74 #ifdef CONFIG_JUMP_LABEL 75 76 #define jump_label_key__true STATIC_KEY_INIT_TRUE 77 #define jump_label_key__false STATIC_KEY_INIT_FALSE 78 79 #define SCHED_FEAT(name, enabled) \ 80 jump_label_key__##enabled , 81 82 struct static_key sched_feat_keys[__SCHED_FEAT_NR] = { 83 #include "features.h" 84 }; 85 86 #undef SCHED_FEAT 87 88 static void sched_feat_disable(int i) 89 { 90 static_key_disable_cpuslocked(&sched_feat_keys[i]); 91 } 92 93 static void sched_feat_enable(int i) 94 { 95 static_key_enable_cpuslocked(&sched_feat_keys[i]); 96 } 97 #else /* !CONFIG_JUMP_LABEL: */ 98 static void sched_feat_disable(int i) { }; 99 static void sched_feat_enable(int i) { }; 100 #endif /* !CONFIG_JUMP_LABEL */ 101 102 static int sched_feat_set(char *cmp) 103 { 104 int i; 105 int neg = 0; 106 107 if (strncmp(cmp, "NO_", 3) == 0) { 108 neg = 1; 109 cmp += 3; 110 } 111 112 i = match_string(sched_feat_names, __SCHED_FEAT_NR, cmp); 113 if (i < 0) 114 return i; 115 116 if (neg) { 117 sysctl_sched_features &= ~(1UL << i); 118 sched_feat_disable(i); 119 } else { 120 sysctl_sched_features |= (1UL << i); 121 sched_feat_enable(i); 122 } 123 124 return 0; 125 } 126 127 static ssize_t 128 sched_feat_write(struct file *filp, const char __user *ubuf, 129 size_t cnt, loff_t *ppos) 130 { 131 char buf[64]; 132 char *cmp; 133 int ret; 134 struct inode *inode; 135 136 if (cnt > 63) 137 cnt = 63; 138 139 if (copy_from_user(buf, ubuf, cnt)) 140 return -EFAULT; 141 142 buf[cnt] = 0; 143 cmp = strstrip(buf); 144 145 /* Ensure the static_key remains in a consistent state */ 146 inode = file_inode(filp); 147 cpus_read_lock(); 148 inode_lock(inode); 149 ret = sched_feat_set(cmp); 150 inode_unlock(inode); 151 cpus_read_unlock(); 152 if (ret < 0) 153 return ret; 154 155 *ppos += cnt; 156 157 return cnt; 158 } 159 160 static int sched_feat_open(struct inode *inode, struct file *filp) 161 { 162 return single_open(filp, sched_feat_show, NULL); 163 } 164 165 static const struct file_operations sched_feat_fops = { 166 .open = sched_feat_open, 167 .write = sched_feat_write, 168 .read = seq_read, 169 .llseek = seq_lseek, 170 .release = single_release, 171 }; 172 173 static ssize_t sched_scaling_write(struct file *filp, const char __user *ubuf, 174 size_t cnt, loff_t *ppos) 175 { 176 unsigned int scaling; 177 int ret; 178 179 ret = kstrtouint_from_user(ubuf, cnt, 10, &scaling); 180 if (ret) 181 return ret; 182 183 if (scaling >= SCHED_TUNABLESCALING_END) 184 return -EINVAL; 185 186 sysctl_sched_tunable_scaling = scaling; 187 if (sched_update_scaling()) 188 return -EINVAL; 189 190 *ppos += cnt; 191 return cnt; 192 } 193 194 static int sched_scaling_show(struct seq_file *m, void *v) 195 { 196 seq_printf(m, "%d\n", sysctl_sched_tunable_scaling); 197 return 0; 198 } 199 200 static int sched_scaling_open(struct inode *inode, struct file *filp) 201 { 202 return single_open(filp, sched_scaling_show, NULL); 203 } 204 205 static const struct file_operations sched_scaling_fops = { 206 .open = sched_scaling_open, 207 .write = sched_scaling_write, 208 .read = seq_read, 209 .llseek = seq_lseek, 210 .release = single_release, 211 }; 212 213 #ifdef CONFIG_SCHED_CACHE 214 static ssize_t 215 sched_cache_enable_write(struct file *filp, const char __user *ubuf, 216 size_t cnt, loff_t *ppos) 217 { 218 bool val; 219 int ret; 220 221 ret = kstrtobool_from_user(ubuf, cnt, &val); 222 if (ret) 223 return ret; 224 225 sysctl_sched_cache_user = val; 226 227 sched_cache_active_set(); 228 229 *ppos += cnt; 230 231 return cnt; 232 } 233 234 static int sched_cache_enable_show(struct seq_file *m, void *v) 235 { 236 seq_printf(m, "%d\n", sysctl_sched_cache_user); 237 return 0; 238 } 239 240 static int sched_cache_enable_open(struct inode *inode, 241 struct file *filp) 242 { 243 return single_open(filp, sched_cache_enable_show, NULL); 244 } 245 246 static const struct file_operations sched_cache_enable_fops = { 247 .open = sched_cache_enable_open, 248 .write = sched_cache_enable_write, 249 .read = seq_read, 250 .llseek = seq_lseek, 251 .release = single_release, 252 }; 253 #endif 254 255 #ifdef CONFIG_PREEMPT_DYNAMIC 256 257 static ssize_t sched_dynamic_write(struct file *filp, const char __user *ubuf, 258 size_t cnt, loff_t *ppos) 259 { 260 char buf[16]; 261 int mode; 262 263 if (cnt > 15) 264 cnt = 15; 265 266 if (copy_from_user(buf, ubuf, cnt)) 267 return -EFAULT; 268 269 buf[cnt] = 0; 270 mode = sched_dynamic_mode(strstrip(buf)); 271 if (mode < 0) 272 return mode; 273 274 sched_dynamic_update(mode); 275 276 *ppos += cnt; 277 278 return cnt; 279 } 280 281 static int sched_dynamic_show(struct seq_file *m, void *v) 282 { 283 int i = (IS_ENABLED(CONFIG_PREEMPT_RT) || IS_ENABLED(CONFIG_ARCH_HAS_PREEMPT_LAZY)) * 2; 284 int mode = READ_ONCE(preempt_dynamic_mode); 285 int j; 286 287 /* Count entries in NULL terminated preempt_modes */ 288 for (j = 0; preempt_modes[j]; j++) 289 ; 290 j -= !IS_ENABLED(CONFIG_ARCH_HAS_PREEMPT_LAZY); 291 292 for (; i < j; i++) { 293 if (mode == i) 294 seq_puts(m, "("); 295 seq_puts(m, preempt_modes[i]); 296 if (mode == i) 297 seq_puts(m, ")"); 298 299 seq_puts(m, " "); 300 } 301 302 seq_puts(m, "\n"); 303 return 0; 304 } 305 306 static int sched_dynamic_open(struct inode *inode, struct file *filp) 307 { 308 return single_open(filp, sched_dynamic_show, NULL); 309 } 310 311 static const struct file_operations sched_dynamic_fops = { 312 .open = sched_dynamic_open, 313 .write = sched_dynamic_write, 314 .read = seq_read, 315 .llseek = seq_lseek, 316 .release = single_release, 317 }; 318 319 #endif /* CONFIG_PREEMPT_DYNAMIC */ 320 321 __read_mostly bool sched_debug_verbose; 322 323 static struct dentry *sd_dentry; 324 325 326 static ssize_t sched_verbose_write(struct file *filp, const char __user *ubuf, 327 size_t cnt, loff_t *ppos) 328 { 329 ssize_t result; 330 bool orig; 331 332 cpus_read_lock(); 333 sched_domains_mutex_lock(); 334 335 orig = sched_debug_verbose; 336 result = debugfs_write_file_bool(filp, ubuf, cnt, ppos); 337 338 if (sched_debug_verbose && !orig) 339 update_sched_domain_debugfs(); 340 else if (!sched_debug_verbose && orig) { 341 debugfs_remove(sd_dentry); 342 sd_dentry = NULL; 343 } 344 345 sched_domains_mutex_unlock(); 346 cpus_read_unlock(); 347 348 return result; 349 } 350 351 static const struct file_operations sched_verbose_fops = { 352 .read = debugfs_read_file_bool, 353 .write = sched_verbose_write, 354 .open = simple_open, 355 .llseek = default_llseek, 356 }; 357 358 static const struct seq_operations sched_debug_sops; 359 360 static int sched_debug_open(struct inode *inode, struct file *filp) 361 { 362 return seq_open(filp, &sched_debug_sops); 363 } 364 365 static const struct file_operations sched_debug_fops = { 366 .open = sched_debug_open, 367 .read = seq_read, 368 .llseek = seq_lseek, 369 .release = seq_release, 370 }; 371 372 enum dl_param { 373 DL_RUNTIME = 0, 374 DL_PERIOD, 375 }; 376 377 static unsigned long dl_server_period_max = (1UL << 22) * NSEC_PER_USEC; /* ~4 seconds */ 378 static unsigned long dl_server_period_min = (100) * NSEC_PER_USEC; /* 100 us */ 379 380 static ssize_t sched_server_write_common(struct file *filp, const char __user *ubuf, 381 size_t cnt, loff_t *ppos, enum dl_param param, 382 void *server) 383 { 384 long cpu = (long) ((struct seq_file *) filp->private_data)->private; 385 struct sched_dl_entity *dl_se = (struct sched_dl_entity *)server; 386 u64 old_runtime, runtime, period; 387 struct rq *rq = cpu_rq(cpu); 388 int retval = 0; 389 size_t err; 390 u64 value; 391 392 err = kstrtoull_from_user(ubuf, cnt, 10, &value); 393 if (err) 394 return err; 395 396 scoped_guard (rq_lock_irqsave, rq) { 397 old_runtime = runtime = dl_se->dl_runtime; 398 period = dl_se->dl_period; 399 400 switch (param) { 401 case DL_RUNTIME: 402 if (runtime == value) 403 break; 404 runtime = value; 405 break; 406 case DL_PERIOD: 407 if (value == period) 408 break; 409 period = value; 410 break; 411 } 412 413 if (runtime > period || 414 period > dl_server_period_max || 415 period < dl_server_period_min) { 416 return -EINVAL; 417 } 418 419 if (!cpu_online(cpu_of(rq))) 420 return -EBUSY; 421 422 update_rq_clock(rq); 423 dl_server_stop(dl_se); 424 retval = dl_server_apply_params(dl_se, runtime, period, 0); 425 dl_server_start(dl_se); 426 427 if (retval < 0) 428 return retval; 429 } 430 431 if (!!old_runtime ^ !!runtime) { 432 pr_info("%s server %sabled on CPU %d%s.\n", 433 server == &rq->fair_server ? "Fair" : "Ext", 434 runtime ? "en" : "dis", 435 cpu_of(rq), 436 runtime ? "" : ", system may malfunction due to starvation"); 437 } 438 439 *ppos += cnt; 440 return cnt; 441 } 442 443 static size_t sched_server_show_common(struct seq_file *m, void *v, enum dl_param param, 444 void *server) 445 { 446 struct sched_dl_entity *dl_se = (struct sched_dl_entity *)server; 447 u64 value; 448 449 switch (param) { 450 case DL_RUNTIME: 451 value = dl_se->dl_runtime; 452 break; 453 case DL_PERIOD: 454 value = dl_se->dl_period; 455 break; 456 } 457 458 seq_printf(m, "%llu\n", value); 459 return 0; 460 } 461 462 static ssize_t 463 sched_fair_server_runtime_write(struct file *filp, const char __user *ubuf, 464 size_t cnt, loff_t *ppos) 465 { 466 long cpu = (long) ((struct seq_file *) filp->private_data)->private; 467 struct rq *rq = cpu_rq(cpu); 468 469 return sched_server_write_common(filp, ubuf, cnt, ppos, DL_RUNTIME, 470 &rq->fair_server); 471 } 472 473 static int sched_fair_server_runtime_show(struct seq_file *m, void *v) 474 { 475 unsigned long cpu = (unsigned long) m->private; 476 struct rq *rq = cpu_rq(cpu); 477 478 return sched_server_show_common(m, v, DL_RUNTIME, &rq->fair_server); 479 } 480 481 static int sched_fair_server_runtime_open(struct inode *inode, struct file *filp) 482 { 483 return single_open(filp, sched_fair_server_runtime_show, inode->i_private); 484 } 485 486 static const struct file_operations fair_server_runtime_fops = { 487 .open = sched_fair_server_runtime_open, 488 .write = sched_fair_server_runtime_write, 489 .read = seq_read, 490 .llseek = seq_lseek, 491 .release = single_release, 492 }; 493 494 static struct dentry *debugfs_sched; 495 496 #ifdef CONFIG_SCHED_CLASS_EXT 497 static ssize_t 498 sched_ext_server_runtime_write(struct file *filp, const char __user *ubuf, 499 size_t cnt, loff_t *ppos) 500 { 501 long cpu = (long) ((struct seq_file *) filp->private_data)->private; 502 struct rq *rq = cpu_rq(cpu); 503 504 return sched_server_write_common(filp, ubuf, cnt, ppos, DL_RUNTIME, 505 &rq->ext_server); 506 } 507 508 static int sched_ext_server_runtime_show(struct seq_file *m, void *v) 509 { 510 unsigned long cpu = (unsigned long) m->private; 511 struct rq *rq = cpu_rq(cpu); 512 513 return sched_server_show_common(m, v, DL_RUNTIME, &rq->ext_server); 514 } 515 516 static int sched_ext_server_runtime_open(struct inode *inode, struct file *filp) 517 { 518 return single_open(filp, sched_ext_server_runtime_show, inode->i_private); 519 } 520 521 static const struct file_operations ext_server_runtime_fops = { 522 .open = sched_ext_server_runtime_open, 523 .write = sched_ext_server_runtime_write, 524 .read = seq_read, 525 .llseek = seq_lseek, 526 .release = single_release, 527 }; 528 529 static ssize_t 530 sched_ext_server_period_write(struct file *filp, const char __user *ubuf, 531 size_t cnt, loff_t *ppos) 532 { 533 long cpu = (long) ((struct seq_file *) filp->private_data)->private; 534 struct rq *rq = cpu_rq(cpu); 535 536 return sched_server_write_common(filp, ubuf, cnt, ppos, DL_PERIOD, 537 &rq->ext_server); 538 } 539 540 static int sched_ext_server_period_show(struct seq_file *m, void *v) 541 { 542 unsigned long cpu = (unsigned long) m->private; 543 struct rq *rq = cpu_rq(cpu); 544 545 return sched_server_show_common(m, v, DL_PERIOD, &rq->ext_server); 546 } 547 548 static int sched_ext_server_period_open(struct inode *inode, struct file *filp) 549 { 550 return single_open(filp, sched_ext_server_period_show, inode->i_private); 551 } 552 553 static const struct file_operations ext_server_period_fops = { 554 .open = sched_ext_server_period_open, 555 .write = sched_ext_server_period_write, 556 .read = seq_read, 557 .llseek = seq_lseek, 558 .release = single_release, 559 }; 560 561 static void debugfs_ext_server_init(void) 562 { 563 struct dentry *d_ext; 564 unsigned long cpu; 565 566 d_ext = debugfs_create_dir("ext_server", debugfs_sched); 567 if (!d_ext) 568 return; 569 570 for_each_possible_cpu(cpu) { 571 struct dentry *d_cpu; 572 char buf[32]; 573 574 snprintf(buf, sizeof(buf), "cpu%lu", cpu); 575 d_cpu = debugfs_create_dir(buf, d_ext); 576 577 debugfs_create_file("runtime", 0644, d_cpu, (void *) cpu, &ext_server_runtime_fops); 578 debugfs_create_file("period", 0644, d_cpu, (void *) cpu, &ext_server_period_fops); 579 } 580 } 581 #endif /* CONFIG_SCHED_CLASS_EXT */ 582 583 static ssize_t 584 sched_fair_server_period_write(struct file *filp, const char __user *ubuf, 585 size_t cnt, loff_t *ppos) 586 { 587 long cpu = (long) ((struct seq_file *) filp->private_data)->private; 588 struct rq *rq = cpu_rq(cpu); 589 590 return sched_server_write_common(filp, ubuf, cnt, ppos, DL_PERIOD, 591 &rq->fair_server); 592 } 593 594 static int sched_fair_server_period_show(struct seq_file *m, void *v) 595 { 596 unsigned long cpu = (unsigned long) m->private; 597 struct rq *rq = cpu_rq(cpu); 598 599 return sched_server_show_common(m, v, DL_PERIOD, &rq->fair_server); 600 } 601 602 static int sched_fair_server_period_open(struct inode *inode, struct file *filp) 603 { 604 return single_open(filp, sched_fair_server_period_show, inode->i_private); 605 } 606 607 static const struct file_operations fair_server_period_fops = { 608 .open = sched_fair_server_period_open, 609 .write = sched_fair_server_period_write, 610 .read = seq_read, 611 .llseek = seq_lseek, 612 .release = single_release, 613 }; 614 615 static void debugfs_fair_server_init(void) 616 { 617 struct dentry *d_fair; 618 unsigned long cpu; 619 620 d_fair = debugfs_create_dir("fair_server", debugfs_sched); 621 if (!d_fair) 622 return; 623 624 for_each_possible_cpu(cpu) { 625 struct dentry *d_cpu; 626 char buf[32]; 627 628 snprintf(buf, sizeof(buf), "cpu%lu", cpu); 629 d_cpu = debugfs_create_dir(buf, d_fair); 630 631 debugfs_create_file("runtime", 0644, d_cpu, (void *) cpu, &fair_server_runtime_fops); 632 debugfs_create_file("period", 0644, d_cpu, (void *) cpu, &fair_server_period_fops); 633 } 634 } 635 636 #ifdef CONFIG_FAIR_GROUP_SCHED 637 static int cgroup_mode = 2; 638 639 /* See __sched_cgroup_mode_update(). */ 640 static const char *cgroup_mode_str[] = { 641 "up", 642 "smp", 643 "concur", 644 "max", 645 "tasks", 646 }; 647 648 static int sched_cgroup_mode(const char *str) 649 { 650 for (int i = 0; i < ARRAY_SIZE(cgroup_mode_str); i++) { 651 if (!strcmp(str, cgroup_mode_str[i])) 652 return i; 653 } 654 return -EINVAL; 655 } 656 657 static ssize_t sched_cgroup_write(struct file *filp, const char __user *ubuf, 658 size_t cnt, loff_t *ppos) 659 { 660 char buf[16]; 661 int mode; 662 663 if (cnt > 15) 664 cnt = 15; 665 666 if (copy_from_user(buf, ubuf, cnt)) 667 return -EFAULT; 668 669 buf[cnt] = 0; 670 mode = sched_cgroup_mode(strstrip(buf)); 671 if (mode < 0) 672 return mode; 673 674 __sched_cgroup_mode_update(mode); 675 WRITE_ONCE(cgroup_mode, mode); 676 677 *ppos += cnt; 678 return cnt; 679 } 680 681 static int sched_cgroup_show(struct seq_file *m, void *v) 682 { 683 int mode = READ_ONCE(cgroup_mode); 684 685 for (int i = 0; i < ARRAY_SIZE(cgroup_mode_str); i++) { 686 if (mode == i) 687 seq_puts(m, "("); 688 seq_puts(m, cgroup_mode_str[i]); 689 if (mode == i) 690 seq_puts(m, ")"); 691 692 seq_puts(m, " "); 693 } 694 seq_puts(m, "\n"); 695 return 0; 696 } 697 698 static int sched_cgroup_open(struct inode *inode, struct file *filp) 699 { 700 return single_open(filp, sched_cgroup_show, NULL); 701 } 702 703 static const struct file_operations sched_cgroup_fops = { 704 .open = sched_cgroup_open, 705 .write = sched_cgroup_write, 706 .read = seq_read, 707 .llseek = seq_lseek, 708 .release = single_release, 709 }; 710 #endif 711 712 static __init int sched_init_debug(void) 713 { 714 struct dentry __maybe_unused *numa, *llc; 715 716 debugfs_sched = debugfs_create_dir("sched", NULL); 717 718 debugfs_create_file("features", 0644, debugfs_sched, NULL, &sched_feat_fops); 719 debugfs_create_file_unsafe("verbose", 0644, debugfs_sched, &sched_debug_verbose, &sched_verbose_fops); 720 #ifdef CONFIG_PREEMPT_DYNAMIC 721 debugfs_create_file("preempt", 0644, debugfs_sched, NULL, &sched_dynamic_fops); 722 #endif 723 724 debugfs_create_u32("base_slice_ns", 0644, debugfs_sched, &sysctl_sched_base_slice); 725 726 debugfs_create_u32("latency_warn_ms", 0644, debugfs_sched, &sysctl_resched_latency_warn_ms); 727 debugfs_create_u32("latency_warn_once", 0644, debugfs_sched, &sysctl_resched_latency_warn_once); 728 729 debugfs_create_file("tunable_scaling", 0644, debugfs_sched, NULL, &sched_scaling_fops); 730 debugfs_create_u32("migration_cost_ns", 0644, debugfs_sched, &sysctl_sched_migration_cost); 731 debugfs_create_u32("nr_migrate", 0644, debugfs_sched, &sysctl_sched_nr_migrate); 732 733 sched_domains_mutex_lock(); 734 update_sched_domain_debugfs(); 735 sched_domains_mutex_unlock(); 736 737 #ifdef CONFIG_NUMA_BALANCING 738 numa = debugfs_create_dir("numa_balancing", debugfs_sched); 739 740 debugfs_create_u32("scan_delay_ms", 0644, numa, &sysctl_numa_balancing_scan_delay); 741 debugfs_create_u32("scan_period_min_ms", 0644, numa, &sysctl_numa_balancing_scan_period_min); 742 debugfs_create_u32("scan_period_max_ms", 0644, numa, &sysctl_numa_balancing_scan_period_max); 743 debugfs_create_u32("scan_size_mb", 0644, numa, &sysctl_numa_balancing_scan_size); 744 debugfs_create_u32("hot_threshold_ms", 0644, numa, &sysctl_numa_balancing_hot_threshold); 745 #endif /* CONFIG_NUMA_BALANCING */ 746 747 #ifdef CONFIG_SCHED_CACHE 748 llc = debugfs_create_dir("llc_balancing", debugfs_sched); 749 debugfs_create_file("enabled", 0644, llc, NULL, 750 &sched_cache_enable_fops); 751 debugfs_create_u32("aggr_tolerance", 0644, llc, 752 &llc_aggr_tolerance); 753 debugfs_create_u32("epoch_period", 0644, llc, 754 &llc_epoch_period); 755 debugfs_create_u32("epoch_affinity_timeout", 0644, llc, 756 &llc_epoch_affinity_timeout); 757 debugfs_create_u32("overaggr_pct", 0644, llc, 758 &llc_overaggr_pct); 759 debugfs_create_u32("imb_pct", 0644, llc, 760 &llc_imb_pct); 761 #endif 762 763 debugfs_create_file("debug", 0444, debugfs_sched, NULL, &sched_debug_fops); 764 765 #ifdef CONFIG_FAIR_GROUP_SCHED 766 debugfs_create_file("cgroup_mode", 0644, debugfs_sched, NULL, &sched_cgroup_fops); 767 #endif 768 769 debugfs_fair_server_init(); 770 #ifdef CONFIG_SCHED_CLASS_EXT 771 debugfs_ext_server_init(); 772 #endif 773 774 return 0; 775 } 776 late_initcall(sched_init_debug); 777 778 static cpumask_var_t sd_sysctl_cpus; 779 780 static int sd_flags_show(struct seq_file *m, void *v) 781 { 782 unsigned long flags = *(unsigned int *)m->private; 783 int idx; 784 785 for_each_set_bit(idx, &flags, __SD_FLAG_CNT) { 786 seq_puts(m, sd_flag_debug[idx].name); 787 seq_puts(m, " "); 788 } 789 seq_puts(m, "\n"); 790 791 return 0; 792 } 793 794 static int sd_flags_open(struct inode *inode, struct file *file) 795 { 796 return single_open(file, sd_flags_show, inode->i_private); 797 } 798 799 static const struct file_operations sd_flags_fops = { 800 .open = sd_flags_open, 801 .read = seq_read, 802 .llseek = seq_lseek, 803 .release = single_release, 804 }; 805 806 static void register_sd(struct sched_domain *sd, struct dentry *parent) 807 { 808 #define SDM(type, mode, member) \ 809 debugfs_create_##type(#member, mode, parent, &sd->member) 810 811 SDM(ulong, 0644, min_interval); 812 SDM(ulong, 0644, max_interval); 813 SDM(u64, 0644, max_newidle_lb_cost); 814 SDM(u32, 0644, busy_factor); 815 SDM(u32, 0644, imbalance_pct); 816 SDM(u32, 0644, cache_nice_tries); 817 SDM(str, 0444, name); 818 819 #undef SDM 820 821 debugfs_create_file("flags", 0444, parent, &sd->flags, &sd_flags_fops); 822 debugfs_create_file("groups_flags", 0444, parent, &sd->groups->flags, &sd_flags_fops); 823 debugfs_create_u32("level", 0444, parent, (u32 *)&sd->level); 824 825 if (sd->flags & SD_ASYM_PACKING) 826 debugfs_create_u32("group_asym_prefer_cpu", 0444, parent, 827 (u32 *)&sd->groups->asym_prefer_cpu); 828 } 829 830 void update_sched_domain_debugfs(void) 831 { 832 int cpu, i; 833 834 /* 835 * This can unfortunately be invoked before sched_debug_init() creates 836 * the debug directory. Don't touch sd_sysctl_cpus until then. 837 */ 838 if (!debugfs_sched) 839 return; 840 841 if (!sched_debug_verbose) 842 return; 843 844 if (!cpumask_available(sd_sysctl_cpus)) { 845 if (!alloc_cpumask_var(&sd_sysctl_cpus, GFP_KERNEL)) 846 return; 847 cpumask_copy(sd_sysctl_cpus, cpu_possible_mask); 848 } 849 850 if (!sd_dentry) { 851 sd_dentry = debugfs_create_dir("domains", debugfs_sched); 852 853 /* rebuild sd_sysctl_cpus if empty since it gets cleared below */ 854 if (cpumask_empty(sd_sysctl_cpus)) 855 cpumask_copy(sd_sysctl_cpus, cpu_online_mask); 856 } 857 858 for_each_cpu(cpu, sd_sysctl_cpus) { 859 struct sched_domain *sd; 860 struct dentry *d_cpu; 861 char buf[32]; 862 863 snprintf(buf, sizeof(buf), "cpu%d", cpu); 864 debugfs_lookup_and_remove(buf, sd_dentry); 865 d_cpu = debugfs_create_dir(buf, sd_dentry); 866 867 i = 0; 868 for_each_domain(cpu, sd) { 869 struct dentry *d_sd; 870 871 snprintf(buf, sizeof(buf), "domain%d", i); 872 d_sd = debugfs_create_dir(buf, d_cpu); 873 874 register_sd(sd, d_sd); 875 i++; 876 } 877 878 __cpumask_clear_cpu(cpu, sd_sysctl_cpus); 879 } 880 } 881 882 void dirty_sched_domain_sysctl(int cpu) 883 { 884 if (cpumask_available(sd_sysctl_cpus)) 885 __cpumask_set_cpu(cpu, sd_sysctl_cpus); 886 } 887 888 #ifdef CONFIG_FAIR_GROUP_SCHED 889 static void print_cfs_group_stats(struct seq_file *m, int cpu, struct task_group *tg) 890 { 891 struct sched_entity *se = tg_se(tg, cpu); 892 893 #define P(F) SEQ_printf(m, " .%-30s: %lld\n", #F, (long long)F) 894 #define P_SCHEDSTAT(F) SEQ_printf(m, " .%-30s: %lld\n", \ 895 #F, (long long)schedstat_val(stats->F)) 896 #define PN(F) SEQ_printf(m, " .%-30s: %lld.%06ld\n", #F, SPLIT_NS((long long)F)) 897 #define PN_SCHEDSTAT(F) SEQ_printf(m, " .%-30s: %lld.%06ld\n", \ 898 #F, SPLIT_NS((long long)schedstat_val(stats->F))) 899 900 if (!se) 901 return; 902 903 PN(se->exec_start); 904 PN(se->vruntime); 905 PN(se->sum_exec_runtime); 906 907 if (schedstat_enabled()) { 908 struct sched_statistics *stats; 909 stats = __schedstats_from_se(se); 910 911 PN_SCHEDSTAT(wait_start); 912 PN_SCHEDSTAT(sleep_start); 913 PN_SCHEDSTAT(block_start); 914 PN_SCHEDSTAT(sleep_max); 915 PN_SCHEDSTAT(block_max); 916 PN_SCHEDSTAT(exec_max); 917 PN_SCHEDSTAT(slice_max); 918 PN_SCHEDSTAT(wait_max); 919 PN_SCHEDSTAT(wait_sum); 920 P_SCHEDSTAT(wait_count); 921 } 922 923 P(se->load.weight); 924 P(se->avg.load_avg); 925 P(se->avg.util_avg); 926 P(se->avg.runnable_avg); 927 928 #undef PN_SCHEDSTAT 929 #undef PN 930 #undef P_SCHEDSTAT 931 #undef P 932 } 933 #endif /* CONFIG_FAIR_GROUP_SCHED */ 934 935 #ifdef CONFIG_CGROUP_SCHED 936 static DEFINE_SPINLOCK(sched_debug_lock); 937 static char group_path[PATH_MAX]; 938 939 static void task_group_path(struct task_group *tg, char *path, int plen) 940 { 941 if (autogroup_path(tg, path, plen)) 942 return; 943 944 cgroup_path(tg->css.cgroup, path, plen); 945 } 946 947 /* 948 * Only 1 SEQ_printf_task_group_path() caller can use the full length 949 * group_path[] for cgroup path. Other simultaneous callers will have 950 * to use a shorter stack buffer. A "..." suffix is appended at the end 951 * of the stack buffer so that it will show up in case the output length 952 * matches the given buffer size to indicate possible path name truncation. 953 */ 954 #define SEQ_printf_task_group_path(m, tg, fmt...) \ 955 { \ 956 if (spin_trylock(&sched_debug_lock)) { \ 957 task_group_path(tg, group_path, sizeof(group_path)); \ 958 SEQ_printf(m, fmt, group_path); \ 959 spin_unlock(&sched_debug_lock); \ 960 } else { \ 961 char buf[128]; \ 962 char *bufend = buf + sizeof(buf) - 3; \ 963 task_group_path(tg, buf, bufend - buf); \ 964 strcpy(bufend - 1, "..."); \ 965 SEQ_printf(m, fmt, buf); \ 966 } \ 967 } 968 #endif 969 970 static void 971 print_task(struct seq_file *m, struct rq *rq, struct task_struct *p) 972 { 973 if (task_current(rq, p)) 974 SEQ_printf(m, ">R"); 975 else 976 SEQ_printf(m, " %c", task_state_to_char(p)); 977 978 SEQ_printf(m, " %15s %5d %10ld %9Ld.%06ld %c %9Ld.%06ld %c %9Ld.%06ld %9Ld.%06ld %9Ld %5d ", 979 p->comm, task_pid_nr(p), 980 p->se.h_load.weight, 981 SPLIT_NS(p->se.vruntime), 982 entity_eligible(&rq->cfs, &p->se) ? 'E' : 'N', 983 SPLIT_NS(p->se.deadline), 984 p->se.custom_slice ? 'S' : ' ', 985 SPLIT_NS(p->se.slice), 986 SPLIT_NS(p->se.sum_exec_runtime), 987 (long long)(p->nvcsw + p->nivcsw), 988 p->prio); 989 990 SEQ_printf(m, "%9lld.%06ld %9lld.%06ld %9lld.%06ld", 991 SPLIT_NS(schedstat_val_or_zero(p->stats.wait_sum)), 992 SPLIT_NS(schedstat_val_or_zero(p->stats.sum_sleep_runtime)), 993 SPLIT_NS(schedstat_val_or_zero(p->stats.sum_block_runtime))); 994 995 #ifdef CONFIG_NUMA_BALANCING 996 SEQ_printf(m, " %d %d", task_node(p), task_numa_group_id(p)); 997 #endif 998 #ifdef CONFIG_CGROUP_SCHED 999 SEQ_printf_task_group_path(m, task_group(p), " %s") 1000 #endif 1001 1002 SEQ_printf(m, "\n"); 1003 } 1004 1005 static void print_rq(struct seq_file *m, struct rq *rq, int rq_cpu) 1006 { 1007 struct task_struct *g, *p; 1008 1009 SEQ_printf(m, "\n"); 1010 SEQ_printf(m, "runnable tasks:\n"); 1011 SEQ_printf(m, " S task PID weight vruntime eligible " 1012 "deadline slice sum-exec switches " 1013 "prio wait-time sum-sleep sum-block" 1014 #ifdef CONFIG_NUMA_BALANCING 1015 " node group-id" 1016 #endif 1017 #ifdef CONFIG_CGROUP_SCHED 1018 " group-path" 1019 #endif 1020 "\n"); 1021 SEQ_printf(m, "-------------------------------------------------------" 1022 "------------------------------------------------------" 1023 "------------------------------------------------------" 1024 #ifdef CONFIG_NUMA_BALANCING 1025 "--------------" 1026 #endif 1027 #ifdef CONFIG_CGROUP_SCHED 1028 "--------------" 1029 #endif 1030 "\n"); 1031 1032 rcu_read_lock(); 1033 for_each_process_thread(g, p) { 1034 if (task_cpu(p) != rq_cpu) 1035 continue; 1036 1037 print_task(m, rq, p); 1038 } 1039 rcu_read_unlock(); 1040 } 1041 1042 void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq) 1043 { 1044 s64 left_vruntime = -1, right_vruntime = -1, left_deadline = -1, spread; 1045 s64 zero_vruntime = -1, sum_w_vruntime = -1; 1046 u64 avruntime; 1047 struct sched_entity *last, *first, *root; 1048 struct rq *rq = cpu_rq(cpu); 1049 unsigned int sum_shift; 1050 unsigned long flags; 1051 u64 sum_weight; 1052 1053 #ifdef CONFIG_FAIR_GROUP_SCHED 1054 SEQ_printf(m, "\n"); 1055 SEQ_printf_task_group_path(m, cfs_rq->tg, "cfs_rq[%d]:%s\n", cpu); 1056 #else 1057 SEQ_printf(m, "\n"); 1058 SEQ_printf(m, "cfs_rq[%d]:\n", cpu); 1059 #endif 1060 1061 raw_spin_rq_lock_irqsave(rq, flags); 1062 root = __pick_root_entity(cfs_rq); 1063 if (root) 1064 left_vruntime = root->min_vruntime; 1065 first = __pick_first_entity(cfs_rq); 1066 if (first) 1067 left_deadline = first->deadline; 1068 last = __pick_last_entity(cfs_rq); 1069 if (last) 1070 right_vruntime = last->vruntime; 1071 zero_vruntime = cfs_rq->zero_vruntime; 1072 sum_w_vruntime = cfs_rq->sum_w_vruntime; 1073 sum_weight = cfs_rq->sum_weight; 1074 sum_shift = cfs_rq->sum_shift; 1075 avruntime = avg_vruntime(cfs_rq); 1076 raw_spin_rq_unlock_irqrestore(rq, flags); 1077 1078 SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "left_deadline", 1079 SPLIT_NS(left_deadline)); 1080 SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "left_vruntime", 1081 SPLIT_NS(left_vruntime)); 1082 SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "zero_vruntime", 1083 SPLIT_NS(zero_vruntime)); 1084 SEQ_printf(m, " .%-30s: %Ld (%d bits)\n", "sum_w_vruntime", 1085 sum_w_vruntime, ilog2(abs(sum_w_vruntime))); 1086 SEQ_printf(m, " .%-30s: %Lu\n", "sum_weight", 1087 sum_weight); 1088 SEQ_printf(m, " .%-30s: %u\n", "sum_shift", sum_shift); 1089 SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "avg_vruntime", 1090 SPLIT_NS(avruntime)); 1091 SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "right_vruntime", 1092 SPLIT_NS(right_vruntime)); 1093 spread = right_vruntime - left_vruntime; 1094 SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "spread", SPLIT_NS(spread)); 1095 SEQ_printf(m, " .%-30s: %d\n", "nr_queued", cfs_rq->nr_queued); 1096 SEQ_printf(m, " .%-30s: %d\n", "h_nr_runnable", cfs_rq->h_nr_runnable); 1097 SEQ_printf(m, " .%-30s: %d\n", "h_nr_queued", cfs_rq->h_nr_queued); 1098 SEQ_printf(m, " .%-30s: %d\n", "h_nr_idle", cfs_rq->h_nr_idle); 1099 SEQ_printf(m, " .%-30s: %ld\n", "load", cfs_rq->load.weight); 1100 SEQ_printf(m, " .%-30s: %lu\n", "load_avg", 1101 cfs_rq->avg.load_avg); 1102 SEQ_printf(m, " .%-30s: %lu\n", "runnable_avg", 1103 cfs_rq->avg.runnable_avg); 1104 SEQ_printf(m, " .%-30s: %lu\n", "util_avg", 1105 cfs_rq->avg.util_avg); 1106 SEQ_printf(m, " .%-30s: %u\n", "util_est", 1107 cfs_rq->avg.util_est); 1108 SEQ_printf(m, " .%-30s: %ld\n", "removed.load_avg", 1109 cfs_rq->removed.load_avg); 1110 SEQ_printf(m, " .%-30s: %ld\n", "removed.util_avg", 1111 cfs_rq->removed.util_avg); 1112 SEQ_printf(m, " .%-30s: %ld\n", "removed.runnable_avg", 1113 cfs_rq->removed.runnable_avg); 1114 #ifdef CONFIG_FAIR_GROUP_SCHED 1115 SEQ_printf(m, " .%-30s: %lu\n", "tg_load_avg_contrib", 1116 cfs_rq->tg_load_avg_contrib); 1117 SEQ_printf(m, " .%-30s: %ld\n", "tg_load_avg", 1118 atomic_long_read(&cfs_rq->tg->load_avg)); 1119 SEQ_printf(m, " .%-30s: %lu\n", "h_load", 1120 cfs_rq->h_load); 1121 #endif /* CONFIG_FAIR_GROUP_SCHED */ 1122 #ifdef CONFIG_CFS_BANDWIDTH 1123 SEQ_printf(m, " .%-30s: %d\n", "throttled", 1124 cfs_rq->throttled); 1125 SEQ_printf(m, " .%-30s: %d\n", "throttle_count", 1126 cfs_rq->throttle_count); 1127 #endif 1128 1129 #ifdef CONFIG_FAIR_GROUP_SCHED 1130 print_cfs_group_stats(m, cpu, cfs_rq->tg); 1131 #endif 1132 } 1133 1134 void print_rt_rq(struct seq_file *m, int cpu, struct rt_rq *rt_rq) 1135 { 1136 #ifdef CONFIG_RT_GROUP_SCHED 1137 SEQ_printf(m, "\n"); 1138 SEQ_printf_task_group_path(m, rt_rq->tg, "rt_rq[%d]:%s\n", cpu); 1139 #else 1140 SEQ_printf(m, "\n"); 1141 SEQ_printf(m, "rt_rq[%d]:\n", cpu); 1142 #endif 1143 1144 #define P(x) \ 1145 SEQ_printf(m, " .%-30s: %Ld\n", #x, (long long)(rt_rq->x)) 1146 #define PU(x) \ 1147 SEQ_printf(m, " .%-30s: %lu\n", #x, (unsigned long)(rt_rq->x)) 1148 #define PN(x) \ 1149 SEQ_printf(m, " .%-30s: %Ld.%06ld\n", #x, SPLIT_NS(rt_rq->x)) 1150 1151 PU(rt_nr_running); 1152 1153 #ifdef CONFIG_RT_GROUP_SCHED 1154 P(rt_throttled); 1155 PN(rt_time); 1156 PN(rt_runtime); 1157 #endif 1158 1159 #undef PN 1160 #undef PU 1161 #undef P 1162 } 1163 1164 void print_dl_rq(struct seq_file *m, int cpu, struct dl_rq *dl_rq) 1165 { 1166 struct dl_bw *dl_bw; 1167 1168 SEQ_printf(m, "\n"); 1169 SEQ_printf(m, "dl_rq[%d]:\n", cpu); 1170 1171 #define PU(x) \ 1172 SEQ_printf(m, " .%-30s: %lu\n", #x, (unsigned long)(dl_rq->x)) 1173 1174 PU(dl_nr_running); 1175 dl_bw = &cpu_rq(cpu)->rd->dl_bw; 1176 SEQ_printf(m, " .%-30s: %lld\n", "dl_bw->bw", dl_bw->bw); 1177 SEQ_printf(m, " .%-30s: %lld\n", "dl_bw->total_bw", dl_bw->total_bw); 1178 1179 #undef PU 1180 } 1181 1182 static void print_cpu(struct seq_file *m, int cpu) 1183 { 1184 struct rq *rq = cpu_rq(cpu); 1185 1186 #ifdef CONFIG_X86 1187 { 1188 unsigned int freq = cpu_khz ? : 1; 1189 1190 SEQ_printf(m, "cpu#%d, %u.%03u MHz\n", 1191 cpu, freq / 1000, (freq % 1000)); 1192 } 1193 #else /* !CONFIG_X86: */ 1194 SEQ_printf(m, "cpu#%d\n", cpu); 1195 #endif /* !CONFIG_X86 */ 1196 1197 #define P(x) \ 1198 do { \ 1199 if (sizeof(rq->x) == 4) \ 1200 SEQ_printf(m, " .%-30s: %d\n", #x, (int)(rq->x)); \ 1201 else \ 1202 SEQ_printf(m, " .%-30s: %Ld\n", #x, (long long)(rq->x));\ 1203 } while (0) 1204 1205 #define PN(x) \ 1206 SEQ_printf(m, " .%-30s: %Ld.%06ld\n", #x, SPLIT_NS(rq->x)) 1207 1208 P(nr_running); 1209 P(nr_switches); 1210 P(nr_uninterruptible); 1211 PN(next_balance); 1212 SEQ_printf(m, " .%-30s: %ld\n", "curr->pid", (long)(task_pid_nr(rq->curr))); 1213 PN(clock); 1214 PN(clock_task); 1215 #undef P 1216 #undef PN 1217 1218 #define P64(n) SEQ_printf(m, " .%-30s: %Ld\n", #n, rq->n); 1219 P64(avg_idle); 1220 P64(max_idle_balance_cost); 1221 #undef P64 1222 1223 #define P(n) SEQ_printf(m, " .%-30s: %d\n", #n, schedstat_val(rq->n)); 1224 if (schedstat_enabled()) { 1225 P(yld_count); 1226 P(sched_count); 1227 P(sched_goidle); 1228 P(ttwu_count); 1229 P(ttwu_local); 1230 } 1231 #undef P 1232 1233 print_cfs_stats(m, cpu); 1234 print_rt_stats(m, cpu); 1235 print_dl_stats(m, cpu); 1236 1237 print_rq(m, rq, cpu); 1238 SEQ_printf(m, "\n"); 1239 } 1240 1241 static const char *sched_tunable_scaling_names[] = { 1242 "none", 1243 "logarithmic", 1244 "linear" 1245 }; 1246 1247 static void sched_debug_header(struct seq_file *m) 1248 { 1249 u64 ktime, sched_clk, cpu_clk; 1250 unsigned long flags; 1251 1252 local_irq_save(flags); 1253 ktime = ktime_to_ns(ktime_get()); 1254 sched_clk = sched_clock(); 1255 cpu_clk = local_clock(); 1256 local_irq_restore(flags); 1257 1258 SEQ_printf(m, "Sched Debug Version: v0.11, %s %.*s\n", 1259 init_utsname()->release, 1260 (int)strcspn(init_utsname()->version, " "), 1261 init_utsname()->version); 1262 1263 #define P(x) \ 1264 SEQ_printf(m, "%-40s: %Ld\n", #x, (long long)(x)) 1265 #define PN(x) \ 1266 SEQ_printf(m, "%-40s: %Ld.%06ld\n", #x, SPLIT_NS(x)) 1267 PN(ktime); 1268 PN(sched_clk); 1269 PN(cpu_clk); 1270 P(jiffies); 1271 #ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK 1272 P(sched_clock_stable()); 1273 #endif 1274 #undef PN 1275 #undef P 1276 1277 SEQ_printf(m, "\n"); 1278 SEQ_printf(m, "sysctl_sched\n"); 1279 1280 #define P(x) \ 1281 SEQ_printf(m, " .%-40s: %Ld\n", #x, (long long)(x)) 1282 #define PN(x) \ 1283 SEQ_printf(m, " .%-40s: %Ld.%06ld\n", #x, SPLIT_NS(x)) 1284 PN(sysctl_sched_base_slice); 1285 P(sysctl_sched_features); 1286 #undef PN 1287 #undef P 1288 1289 SEQ_printf(m, " .%-40s: %d (%s)\n", 1290 "sysctl_sched_tunable_scaling", 1291 sysctl_sched_tunable_scaling, 1292 sched_tunable_scaling_names[sysctl_sched_tunable_scaling]); 1293 SEQ_printf(m, "\n"); 1294 } 1295 1296 static int sched_debug_show(struct seq_file *m, void *v) 1297 { 1298 int cpu = (unsigned long)(v - 2); 1299 1300 if (cpu != -1) 1301 print_cpu(m, cpu); 1302 else 1303 sched_debug_header(m); 1304 1305 return 0; 1306 } 1307 1308 void sysrq_sched_debug_show(void) 1309 { 1310 int cpu; 1311 1312 sched_debug_header(NULL); 1313 for_each_online_cpu(cpu) { 1314 /* 1315 * Need to reset softlockup watchdogs on all CPUs, because 1316 * another CPU might be blocked waiting for us to process 1317 * an IPI or stop_machine. 1318 */ 1319 touch_nmi_watchdog(); 1320 touch_all_softlockup_watchdogs(); 1321 print_cpu(NULL, cpu); 1322 } 1323 } 1324 1325 /* 1326 * This iterator needs some explanation. 1327 * It returns 1 for the header position. 1328 * This means 2 is CPU 0. 1329 * In a hotplugged system some CPUs, including CPU 0, may be missing so we have 1330 * to use cpumask_* to iterate over the CPUs. 1331 */ 1332 static void *sched_debug_start(struct seq_file *file, loff_t *offset) 1333 { 1334 unsigned long n = *offset; 1335 1336 if (n == 0) 1337 return (void *) 1; 1338 1339 n--; 1340 1341 if (n > 0) 1342 n = cpumask_next(n - 1, cpu_online_mask); 1343 else 1344 n = cpumask_first(cpu_online_mask); 1345 1346 *offset = n + 1; 1347 1348 if (n < nr_cpu_ids) 1349 return (void *)(unsigned long)(n + 2); 1350 1351 return NULL; 1352 } 1353 1354 static void *sched_debug_next(struct seq_file *file, void *data, loff_t *offset) 1355 { 1356 (*offset)++; 1357 return sched_debug_start(file, offset); 1358 } 1359 1360 static void sched_debug_stop(struct seq_file *file, void *data) 1361 { 1362 } 1363 1364 static const struct seq_operations sched_debug_sops = { 1365 .start = sched_debug_start, 1366 .next = sched_debug_next, 1367 .stop = sched_debug_stop, 1368 .show = sched_debug_show, 1369 }; 1370 1371 #define __PS(S, F) SEQ_printf(m, "%-45s:%21Ld\n", S, (long long)(F)) 1372 #define __P(F) __PS(#F, F) 1373 #define P(F) __PS(#F, p->F) 1374 #define PM(F, M) __PS(#F, p->F & (M)) 1375 #define __PSN(S, F) SEQ_printf(m, "%-45s:%14Ld.%06ld\n", S, SPLIT_NS((long long)(F))) 1376 #define __PN(F) __PSN(#F, F) 1377 #define PN(F) __PSN(#F, p->F) 1378 1379 1380 #ifdef CONFIG_NUMA_BALANCING 1381 void print_numa_stats(struct seq_file *m, int node, unsigned long tsf, 1382 unsigned long tpf, unsigned long gsf, unsigned long gpf) 1383 { 1384 SEQ_printf(m, "numa_faults node=%d ", node); 1385 SEQ_printf(m, "task_private=%lu task_shared=%lu ", tpf, tsf); 1386 SEQ_printf(m, "group_private=%lu group_shared=%lu\n", gpf, gsf); 1387 } 1388 #endif 1389 1390 1391 static void sched_show_numa(struct task_struct *p, struct seq_file *m) 1392 { 1393 #ifdef CONFIG_NUMA_BALANCING 1394 if (p->mm) 1395 P(mm->numa_scan_seq); 1396 1397 P(numa_pages_migrated); 1398 P(numa_preferred_nid); 1399 P(total_numa_faults); 1400 SEQ_printf(m, "current_node=%d, numa_group_id=%d\n", 1401 task_node(p), task_numa_group_id(p)); 1402 show_numa_stats(p, m); 1403 #endif /* CONFIG_NUMA_BALANCING */ 1404 } 1405 1406 void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, 1407 struct seq_file *m) 1408 { 1409 unsigned long nr_switches; 1410 1411 SEQ_printf(m, "%s (%d, #threads: %d)\n", p->comm, task_pid_nr_ns(p, ns), 1412 get_nr_threads(p)); 1413 SEQ_printf(m, 1414 "---------------------------------------------------------" 1415 "----------\n"); 1416 1417 #define P_SCHEDSTAT(F) __PS(#F, schedstat_val(p->stats.F)) 1418 #define PN_SCHEDSTAT(F) __PSN(#F, schedstat_val(p->stats.F)) 1419 1420 PN(se.exec_start); 1421 PN(se.vruntime); 1422 PN(se.sum_exec_runtime); 1423 1424 nr_switches = p->nvcsw + p->nivcsw; 1425 1426 P(se.nr_migrations); 1427 1428 if (schedstat_enabled()) { 1429 u64 avg_atom, avg_per_cpu; 1430 1431 PN_SCHEDSTAT(sum_sleep_runtime); 1432 PN_SCHEDSTAT(sum_block_runtime); 1433 PN_SCHEDSTAT(wait_start); 1434 PN_SCHEDSTAT(sleep_start); 1435 PN_SCHEDSTAT(block_start); 1436 PN_SCHEDSTAT(sleep_max); 1437 PN_SCHEDSTAT(block_max); 1438 PN_SCHEDSTAT(exec_max); 1439 PN_SCHEDSTAT(slice_max); 1440 PN_SCHEDSTAT(wait_max); 1441 PN_SCHEDSTAT(wait_sum); 1442 P_SCHEDSTAT(wait_count); 1443 PN_SCHEDSTAT(iowait_sum); 1444 P_SCHEDSTAT(iowait_count); 1445 P_SCHEDSTAT(nr_failed_migrations_affine); 1446 P_SCHEDSTAT(nr_failed_migrations_running); 1447 P_SCHEDSTAT(nr_failed_migrations_hot); 1448 P_SCHEDSTAT(nr_forced_migrations); 1449 P_SCHEDSTAT(nr_wakeups); 1450 P_SCHEDSTAT(nr_wakeups_sync); 1451 P_SCHEDSTAT(nr_wakeups_migrate); 1452 P_SCHEDSTAT(nr_wakeups_local); 1453 P_SCHEDSTAT(nr_wakeups_remote); 1454 P_SCHEDSTAT(nr_wakeups_affine); 1455 P_SCHEDSTAT(nr_wakeups_affine_attempts); 1456 1457 avg_atom = p->se.sum_exec_runtime; 1458 if (nr_switches) 1459 avg_atom = div64_ul(avg_atom, nr_switches); 1460 else 1461 avg_atom = -1LL; 1462 1463 avg_per_cpu = p->se.sum_exec_runtime; 1464 if (p->se.nr_migrations) { 1465 avg_per_cpu = div64_u64(avg_per_cpu, 1466 p->se.nr_migrations); 1467 } else { 1468 avg_per_cpu = -1LL; 1469 } 1470 1471 __PN(avg_atom); 1472 __PN(avg_per_cpu); 1473 1474 #ifdef CONFIG_SCHED_CORE 1475 PN_SCHEDSTAT(core_forceidle_sum); 1476 #endif 1477 } 1478 1479 __P(nr_switches); 1480 __PS("nr_voluntary_switches", p->nvcsw); 1481 __PS("nr_involuntary_switches", p->nivcsw); 1482 1483 P(se.load.weight); 1484 P(se.avg.load_sum); 1485 P(se.avg.runnable_sum); 1486 P(se.avg.util_sum); 1487 P(se.avg.load_avg); 1488 P(se.avg.runnable_avg); 1489 P(se.avg.util_avg); 1490 P(se.avg.last_update_time); 1491 PM(se.avg.util_est, ~UTIL_AVG_UNCHANGED); 1492 #ifdef CONFIG_UCLAMP_TASK 1493 __PS("uclamp.min", p->uclamp_req[UCLAMP_MIN].value); 1494 __PS("uclamp.max", p->uclamp_req[UCLAMP_MAX].value); 1495 __PS("effective uclamp.min", uclamp_eff_value(p, UCLAMP_MIN)); 1496 __PS("effective uclamp.max", uclamp_eff_value(p, UCLAMP_MAX)); 1497 #endif /* CONFIG_UCLAMP_TASK */ 1498 P(policy); 1499 P(prio); 1500 if (task_has_dl_policy(p)) { 1501 P(dl.runtime); 1502 P(dl.deadline); 1503 } else if (fair_policy(p->policy)) { 1504 P(se.slice); 1505 } 1506 #ifdef CONFIG_SCHED_CLASS_EXT 1507 __PS("ext.enabled", task_on_scx(p)); 1508 #endif 1509 #undef PN_SCHEDSTAT 1510 #undef P_SCHEDSTAT 1511 1512 { 1513 unsigned int this_cpu = raw_smp_processor_id(); 1514 u64 t0, t1; 1515 1516 t0 = cpu_clock(this_cpu); 1517 t1 = cpu_clock(this_cpu); 1518 __PS("clock-delta", t1-t0); 1519 } 1520 1521 sched_show_numa(p, m); 1522 } 1523 1524 void proc_sched_set_task(struct task_struct *p) 1525 { 1526 #ifdef CONFIG_SCHEDSTATS 1527 memset(&p->stats, 0, sizeof(p->stats)); 1528 #endif 1529 } 1530 1531 void resched_latency_warn(int cpu, u64 latency) 1532 { 1533 static DEFINE_RATELIMIT_STATE(latency_check_ratelimit, 60 * 60 * HZ, 1); 1534 1535 if (likely(!__ratelimit(&latency_check_ratelimit))) 1536 return; 1537 1538 pr_err("sched: CPU %d need_resched set for > %llu ns (%d ticks) without schedule\n", 1539 cpu, latency, cpu_rq(cpu)->ticks_without_resched); 1540 dump_stack(); 1541 } 1542