1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Detect hard and soft lockups on a system 4 * 5 * started by Don Zickus, Copyright (C) 2010 Red Hat, Inc. 6 * 7 * Note: Most of this code is borrowed heavily from the original softlockup 8 * detector, so thanks to Ingo for the initial implementation. 9 * Some chunks also taken from the old x86-specific nmi watchdog code, thanks 10 * to those contributors as well. 11 */ 12 13 #define pr_fmt(fmt) "watchdog: " fmt 14 15 #include <linux/mm.h> 16 #include <linux/cpu.h> 17 #include <linux/nmi.h> 18 #include <linux/init.h> 19 #include <linux/module.h> 20 #include <linux/sysctl.h> 21 #include <linux/tick.h> 22 #include <linux/sched/clock.h> 23 #include <linux/sched/debug.h> 24 #include <linux/sched/isolation.h> 25 #include <linux/stop_machine.h> 26 27 #include <asm/irq_regs.h> 28 #include <linux/kvm_para.h> 29 30 static DEFINE_MUTEX(watchdog_mutex); 31 32 #if defined(CONFIG_HARDLOCKUP_DETECTOR) || defined(CONFIG_HARDLOCKUP_DETECTOR_SPARC64) 33 # define WATCHDOG_HARDLOCKUP_DEFAULT 1 34 #else 35 # define WATCHDOG_HARDLOCKUP_DEFAULT 0 36 #endif 37 38 unsigned long __read_mostly watchdog_enabled; 39 int __read_mostly watchdog_user_enabled = 1; 40 static int __read_mostly watchdog_hardlockup_user_enabled = WATCHDOG_HARDLOCKUP_DEFAULT; 41 static int __read_mostly watchdog_softlockup_user_enabled = 1; 42 int __read_mostly watchdog_thresh = 10; 43 static int __read_mostly watchdog_hardlockup_available; 44 45 struct cpumask watchdog_cpumask __read_mostly; 46 unsigned long *watchdog_cpumask_bits = cpumask_bits(&watchdog_cpumask); 47 48 #ifdef CONFIG_HARDLOCKUP_DETECTOR 49 50 # ifdef CONFIG_SMP 51 int __read_mostly sysctl_hardlockup_all_cpu_backtrace; 52 # endif /* CONFIG_SMP */ 53 54 /* 55 * Should we panic when a soft-lockup or hard-lockup occurs: 56 */ 57 unsigned int __read_mostly hardlockup_panic = 58 IS_ENABLED(CONFIG_BOOTPARAM_HARDLOCKUP_PANIC); 59 /* 60 * We may not want to enable hard lockup detection by default in all cases, 61 * for example when running the kernel as a guest on a hypervisor. In these 62 * cases this function can be called to disable hard lockup detection. This 63 * function should only be executed once by the boot processor before the 64 * kernel command line parameters are parsed, because otherwise it is not 65 * possible to override this in hardlockup_panic_setup(). 66 */ 67 void __init hardlockup_detector_disable(void) 68 { 69 watchdog_hardlockup_user_enabled = 0; 70 } 71 72 static int __init hardlockup_panic_setup(char *str) 73 { 74 next: 75 if (!strncmp(str, "panic", 5)) 76 hardlockup_panic = 1; 77 else if (!strncmp(str, "nopanic", 7)) 78 hardlockup_panic = 0; 79 else if (!strncmp(str, "0", 1)) 80 watchdog_hardlockup_user_enabled = 0; 81 else if (!strncmp(str, "1", 1)) 82 watchdog_hardlockup_user_enabled = 1; 83 else if (!strncmp(str, "r", 1)) 84 hardlockup_config_perf_event(str + 1); 85 while (*(str++)) { 86 if (*str == ',') { 87 str++; 88 goto next; 89 } 90 } 91 return 1; 92 } 93 __setup("nmi_watchdog=", hardlockup_panic_setup); 94 95 #endif /* CONFIG_HARDLOCKUP_DETECTOR */ 96 97 #if defined(CONFIG_HARDLOCKUP_DETECTOR_COUNTS_HRTIMER) 98 99 static DEFINE_PER_CPU(atomic_t, hrtimer_interrupts); 100 static DEFINE_PER_CPU(int, hrtimer_interrupts_saved); 101 static DEFINE_PER_CPU(bool, watchdog_hardlockup_warned); 102 static DEFINE_PER_CPU(bool, watchdog_hardlockup_touched); 103 static unsigned long hard_lockup_nmi_warn; 104 105 notrace void arch_touch_nmi_watchdog(void) 106 { 107 /* 108 * Using __raw here because some code paths have 109 * preemption enabled. If preemption is enabled 110 * then interrupts should be enabled too, in which 111 * case we shouldn't have to worry about the watchdog 112 * going off. 113 */ 114 raw_cpu_write(watchdog_hardlockup_touched, true); 115 } 116 EXPORT_SYMBOL(arch_touch_nmi_watchdog); 117 118 void watchdog_hardlockup_touch_cpu(unsigned int cpu) 119 { 120 per_cpu(watchdog_hardlockup_touched, cpu) = true; 121 } 122 123 static bool is_hardlockup(unsigned int cpu) 124 { 125 int hrint = atomic_read(&per_cpu(hrtimer_interrupts, cpu)); 126 127 if (per_cpu(hrtimer_interrupts_saved, cpu) == hrint) 128 return true; 129 130 /* 131 * NOTE: we don't need any fancy atomic_t or READ_ONCE/WRITE_ONCE 132 * for hrtimer_interrupts_saved. hrtimer_interrupts_saved is 133 * written/read by a single CPU. 134 */ 135 per_cpu(hrtimer_interrupts_saved, cpu) = hrint; 136 137 return false; 138 } 139 140 static void watchdog_hardlockup_kick(void) 141 { 142 int new_interrupts; 143 144 new_interrupts = atomic_inc_return(this_cpu_ptr(&hrtimer_interrupts)); 145 watchdog_buddy_check_hardlockup(new_interrupts); 146 } 147 148 void watchdog_hardlockup_check(unsigned int cpu, struct pt_regs *regs) 149 { 150 if (per_cpu(watchdog_hardlockup_touched, cpu)) { 151 per_cpu(watchdog_hardlockup_touched, cpu) = false; 152 return; 153 } 154 155 /* 156 * Check for a hardlockup by making sure the CPU's timer 157 * interrupt is incrementing. The timer interrupt should have 158 * fired multiple times before we overflow'd. If it hasn't 159 * then this is a good indication the cpu is stuck 160 */ 161 if (is_hardlockup(cpu)) { 162 unsigned int this_cpu = smp_processor_id(); 163 unsigned long flags; 164 165 /* Only print hardlockups once. */ 166 if (per_cpu(watchdog_hardlockup_warned, cpu)) 167 return; 168 169 /* 170 * Prevent multiple hard-lockup reports if one cpu is already 171 * engaged in dumping all cpu back traces. 172 */ 173 if (sysctl_hardlockup_all_cpu_backtrace) { 174 if (test_and_set_bit_lock(0, &hard_lockup_nmi_warn)) 175 return; 176 } 177 178 /* 179 * NOTE: we call printk_cpu_sync_get_irqsave() after printing 180 * the lockup message. While it would be nice to serialize 181 * that printout, we really want to make sure that if some 182 * other CPU somehow locked up while holding the lock associated 183 * with printk_cpu_sync_get_irqsave() that we can still at least 184 * get the message about the lockup out. 185 */ 186 pr_emerg("Watchdog detected hard LOCKUP on cpu %d\n", cpu); 187 printk_cpu_sync_get_irqsave(flags); 188 189 print_modules(); 190 print_irqtrace_events(current); 191 if (cpu == this_cpu) { 192 if (regs) 193 show_regs(regs); 194 else 195 dump_stack(); 196 printk_cpu_sync_put_irqrestore(flags); 197 } else { 198 printk_cpu_sync_put_irqrestore(flags); 199 trigger_single_cpu_backtrace(cpu); 200 } 201 202 if (sysctl_hardlockup_all_cpu_backtrace) { 203 trigger_allbutcpu_cpu_backtrace(cpu); 204 if (!hardlockup_panic) 205 clear_bit_unlock(0, &hard_lockup_nmi_warn); 206 } 207 208 if (hardlockup_panic) 209 nmi_panic(regs, "Hard LOCKUP"); 210 211 per_cpu(watchdog_hardlockup_warned, cpu) = true; 212 } else { 213 per_cpu(watchdog_hardlockup_warned, cpu) = false; 214 } 215 } 216 217 #else /* CONFIG_HARDLOCKUP_DETECTOR_COUNTS_HRTIMER */ 218 219 static inline void watchdog_hardlockup_kick(void) { } 220 221 #endif /* !CONFIG_HARDLOCKUP_DETECTOR_COUNTS_HRTIMER */ 222 223 /* 224 * These functions can be overridden based on the configured hardlockdup detector. 225 * 226 * watchdog_hardlockup_enable/disable can be implemented to start and stop when 227 * softlockup watchdog start and stop. The detector must select the 228 * SOFTLOCKUP_DETECTOR Kconfig. 229 */ 230 void __weak watchdog_hardlockup_enable(unsigned int cpu) { } 231 232 void __weak watchdog_hardlockup_disable(unsigned int cpu) { } 233 234 /* 235 * Watchdog-detector specific API. 236 * 237 * Return 0 when hardlockup watchdog is available, negative value otherwise. 238 * Note that the negative value means that a delayed probe might 239 * succeed later. 240 */ 241 int __weak __init watchdog_hardlockup_probe(void) 242 { 243 return -ENODEV; 244 } 245 246 /** 247 * watchdog_hardlockup_stop - Stop the watchdog for reconfiguration 248 * 249 * The reconfiguration steps are: 250 * watchdog_hardlockup_stop(); 251 * update_variables(); 252 * watchdog_hardlockup_start(); 253 */ 254 void __weak watchdog_hardlockup_stop(void) { } 255 256 /** 257 * watchdog_hardlockup_start - Start the watchdog after reconfiguration 258 * 259 * Counterpart to watchdog_hardlockup_stop(). 260 * 261 * The following variables have been updated in update_variables() and 262 * contain the currently valid configuration: 263 * - watchdog_enabled 264 * - watchdog_thresh 265 * - watchdog_cpumask 266 */ 267 void __weak watchdog_hardlockup_start(void) { } 268 269 /** 270 * lockup_detector_update_enable - Update the sysctl enable bit 271 * 272 * Caller needs to make sure that the hard watchdogs are off, so this 273 * can't race with watchdog_hardlockup_disable(). 274 */ 275 static void lockup_detector_update_enable(void) 276 { 277 watchdog_enabled = 0; 278 if (!watchdog_user_enabled) 279 return; 280 if (watchdog_hardlockup_available && watchdog_hardlockup_user_enabled) 281 watchdog_enabled |= WATCHDOG_HARDLOCKUP_ENABLED; 282 if (watchdog_softlockup_user_enabled) 283 watchdog_enabled |= WATCHDOG_SOFTOCKUP_ENABLED; 284 } 285 286 #ifdef CONFIG_SOFTLOCKUP_DETECTOR 287 288 /* 289 * Delay the soflockup report when running a known slow code. 290 * It does _not_ affect the timestamp of the last successdul reschedule. 291 */ 292 #define SOFTLOCKUP_DELAY_REPORT ULONG_MAX 293 294 #ifdef CONFIG_SMP 295 int __read_mostly sysctl_softlockup_all_cpu_backtrace; 296 #endif 297 298 static struct cpumask watchdog_allowed_mask __read_mostly; 299 300 /* Global variables, exported for sysctl */ 301 unsigned int __read_mostly softlockup_panic = 302 IS_ENABLED(CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC); 303 304 static bool softlockup_initialized __read_mostly; 305 static u64 __read_mostly sample_period; 306 307 /* Timestamp taken after the last successful reschedule. */ 308 static DEFINE_PER_CPU(unsigned long, watchdog_touch_ts); 309 /* Timestamp of the last softlockup report. */ 310 static DEFINE_PER_CPU(unsigned long, watchdog_report_ts); 311 static DEFINE_PER_CPU(struct hrtimer, watchdog_hrtimer); 312 static DEFINE_PER_CPU(bool, softlockup_touch_sync); 313 static unsigned long soft_lockup_nmi_warn; 314 315 static int __init softlockup_panic_setup(char *str) 316 { 317 softlockup_panic = simple_strtoul(str, NULL, 0); 318 return 1; 319 } 320 __setup("softlockup_panic=", softlockup_panic_setup); 321 322 static int __init nowatchdog_setup(char *str) 323 { 324 watchdog_user_enabled = 0; 325 return 1; 326 } 327 __setup("nowatchdog", nowatchdog_setup); 328 329 static int __init nosoftlockup_setup(char *str) 330 { 331 watchdog_softlockup_user_enabled = 0; 332 return 1; 333 } 334 __setup("nosoftlockup", nosoftlockup_setup); 335 336 static int __init watchdog_thresh_setup(char *str) 337 { 338 get_option(&str, &watchdog_thresh); 339 return 1; 340 } 341 __setup("watchdog_thresh=", watchdog_thresh_setup); 342 343 static void __lockup_detector_cleanup(void); 344 345 /* 346 * Hard-lockup warnings should be triggered after just a few seconds. Soft- 347 * lockups can have false positives under extreme conditions. So we generally 348 * want a higher threshold for soft lockups than for hard lockups. So we couple 349 * the thresholds with a factor: we make the soft threshold twice the amount of 350 * time the hard threshold is. 351 */ 352 static int get_softlockup_thresh(void) 353 { 354 return watchdog_thresh * 2; 355 } 356 357 /* 358 * Returns seconds, approximately. We don't need nanosecond 359 * resolution, and we don't need to waste time with a big divide when 360 * 2^30ns == 1.074s. 361 */ 362 static unsigned long get_timestamp(void) 363 { 364 return running_clock() >> 30LL; /* 2^30 ~= 10^9 */ 365 } 366 367 static void set_sample_period(void) 368 { 369 /* 370 * convert watchdog_thresh from seconds to ns 371 * the divide by 5 is to give hrtimer several chances (two 372 * or three with the current relation between the soft 373 * and hard thresholds) to increment before the 374 * hardlockup detector generates a warning 375 */ 376 sample_period = get_softlockup_thresh() * ((u64)NSEC_PER_SEC / 5); 377 watchdog_update_hrtimer_threshold(sample_period); 378 } 379 380 static void update_report_ts(void) 381 { 382 __this_cpu_write(watchdog_report_ts, get_timestamp()); 383 } 384 385 /* Commands for resetting the watchdog */ 386 static void update_touch_ts(void) 387 { 388 __this_cpu_write(watchdog_touch_ts, get_timestamp()); 389 update_report_ts(); 390 } 391 392 /** 393 * touch_softlockup_watchdog_sched - touch watchdog on scheduler stalls 394 * 395 * Call when the scheduler may have stalled for legitimate reasons 396 * preventing the watchdog task from executing - e.g. the scheduler 397 * entering idle state. This should only be used for scheduler events. 398 * Use touch_softlockup_watchdog() for everything else. 399 */ 400 notrace void touch_softlockup_watchdog_sched(void) 401 { 402 /* 403 * Preemption can be enabled. It doesn't matter which CPU's watchdog 404 * report period gets restarted here, so use the raw_ operation. 405 */ 406 raw_cpu_write(watchdog_report_ts, SOFTLOCKUP_DELAY_REPORT); 407 } 408 409 notrace void touch_softlockup_watchdog(void) 410 { 411 touch_softlockup_watchdog_sched(); 412 wq_watchdog_touch(raw_smp_processor_id()); 413 } 414 EXPORT_SYMBOL(touch_softlockup_watchdog); 415 416 void touch_all_softlockup_watchdogs(void) 417 { 418 int cpu; 419 420 /* 421 * watchdog_mutex cannpt be taken here, as this might be called 422 * from (soft)interrupt context, so the access to 423 * watchdog_allowed_cpumask might race with a concurrent update. 424 * 425 * The watchdog time stamp can race against a concurrent real 426 * update as well, the only side effect might be a cycle delay for 427 * the softlockup check. 428 */ 429 for_each_cpu(cpu, &watchdog_allowed_mask) { 430 per_cpu(watchdog_report_ts, cpu) = SOFTLOCKUP_DELAY_REPORT; 431 wq_watchdog_touch(cpu); 432 } 433 } 434 435 void touch_softlockup_watchdog_sync(void) 436 { 437 __this_cpu_write(softlockup_touch_sync, true); 438 __this_cpu_write(watchdog_report_ts, SOFTLOCKUP_DELAY_REPORT); 439 } 440 441 static int is_softlockup(unsigned long touch_ts, 442 unsigned long period_ts, 443 unsigned long now) 444 { 445 if ((watchdog_enabled & WATCHDOG_SOFTOCKUP_ENABLED) && watchdog_thresh) { 446 /* Warn about unreasonable delays. */ 447 if (time_after(now, period_ts + get_softlockup_thresh())) 448 return now - touch_ts; 449 } 450 return 0; 451 } 452 453 /* watchdog detector functions */ 454 static DEFINE_PER_CPU(struct completion, softlockup_completion); 455 static DEFINE_PER_CPU(struct cpu_stop_work, softlockup_stop_work); 456 457 /* 458 * The watchdog feed function - touches the timestamp. 459 * 460 * It only runs once every sample_period seconds (4 seconds by 461 * default) to reset the softlockup timestamp. If this gets delayed 462 * for more than 2*watchdog_thresh seconds then the debug-printout 463 * triggers in watchdog_timer_fn(). 464 */ 465 static int softlockup_fn(void *data) 466 { 467 update_touch_ts(); 468 complete(this_cpu_ptr(&softlockup_completion)); 469 470 return 0; 471 } 472 473 /* watchdog kicker functions */ 474 static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer) 475 { 476 unsigned long touch_ts, period_ts, now; 477 struct pt_regs *regs = get_irq_regs(); 478 int duration; 479 int softlockup_all_cpu_backtrace = sysctl_softlockup_all_cpu_backtrace; 480 unsigned long flags; 481 482 if (!watchdog_enabled) 483 return HRTIMER_NORESTART; 484 485 watchdog_hardlockup_kick(); 486 487 /* kick the softlockup detector */ 488 if (completion_done(this_cpu_ptr(&softlockup_completion))) { 489 reinit_completion(this_cpu_ptr(&softlockup_completion)); 490 stop_one_cpu_nowait(smp_processor_id(), 491 softlockup_fn, NULL, 492 this_cpu_ptr(&softlockup_stop_work)); 493 } 494 495 /* .. and repeat */ 496 hrtimer_forward_now(hrtimer, ns_to_ktime(sample_period)); 497 498 /* 499 * Read the current timestamp first. It might become invalid anytime 500 * when a virtual machine is stopped by the host or when the watchog 501 * is touched from NMI. 502 */ 503 now = get_timestamp(); 504 /* 505 * If a virtual machine is stopped by the host it can look to 506 * the watchdog like a soft lockup. This function touches the watchdog. 507 */ 508 kvm_check_and_clear_guest_paused(); 509 /* 510 * The stored timestamp is comparable with @now only when not touched. 511 * It might get touched anytime from NMI. Make sure that is_softlockup() 512 * uses the same (valid) value. 513 */ 514 period_ts = READ_ONCE(*this_cpu_ptr(&watchdog_report_ts)); 515 516 /* Reset the interval when touched by known problematic code. */ 517 if (period_ts == SOFTLOCKUP_DELAY_REPORT) { 518 if (unlikely(__this_cpu_read(softlockup_touch_sync))) { 519 /* 520 * If the time stamp was touched atomically 521 * make sure the scheduler tick is up to date. 522 */ 523 __this_cpu_write(softlockup_touch_sync, false); 524 sched_clock_tick(); 525 } 526 527 update_report_ts(); 528 return HRTIMER_RESTART; 529 } 530 531 /* Check for a softlockup. */ 532 touch_ts = __this_cpu_read(watchdog_touch_ts); 533 duration = is_softlockup(touch_ts, period_ts, now); 534 if (unlikely(duration)) { 535 /* 536 * Prevent multiple soft-lockup reports if one cpu is already 537 * engaged in dumping all cpu back traces. 538 */ 539 if (softlockup_all_cpu_backtrace) { 540 if (test_and_set_bit_lock(0, &soft_lockup_nmi_warn)) 541 return HRTIMER_RESTART; 542 } 543 544 /* Start period for the next softlockup warning. */ 545 update_report_ts(); 546 547 printk_cpu_sync_get_irqsave(flags); 548 pr_emerg("BUG: soft lockup - CPU#%d stuck for %us! [%s:%d]\n", 549 smp_processor_id(), duration, 550 current->comm, task_pid_nr(current)); 551 print_modules(); 552 print_irqtrace_events(current); 553 if (regs) 554 show_regs(regs); 555 else 556 dump_stack(); 557 printk_cpu_sync_put_irqrestore(flags); 558 559 if (softlockup_all_cpu_backtrace) { 560 trigger_allbutcpu_cpu_backtrace(smp_processor_id()); 561 if (!softlockup_panic) 562 clear_bit_unlock(0, &soft_lockup_nmi_warn); 563 } 564 565 add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK); 566 if (softlockup_panic) 567 panic("softlockup: hung tasks"); 568 } 569 570 return HRTIMER_RESTART; 571 } 572 573 static void watchdog_enable(unsigned int cpu) 574 { 575 struct hrtimer *hrtimer = this_cpu_ptr(&watchdog_hrtimer); 576 struct completion *done = this_cpu_ptr(&softlockup_completion); 577 578 WARN_ON_ONCE(cpu != smp_processor_id()); 579 580 init_completion(done); 581 complete(done); 582 583 /* 584 * Start the timer first to prevent the hardlockup watchdog triggering 585 * before the timer has a chance to fire. 586 */ 587 hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD); 588 hrtimer->function = watchdog_timer_fn; 589 hrtimer_start(hrtimer, ns_to_ktime(sample_period), 590 HRTIMER_MODE_REL_PINNED_HARD); 591 592 /* Initialize timestamp */ 593 update_touch_ts(); 594 /* Enable the hardlockup detector */ 595 if (watchdog_enabled & WATCHDOG_HARDLOCKUP_ENABLED) 596 watchdog_hardlockup_enable(cpu); 597 } 598 599 static void watchdog_disable(unsigned int cpu) 600 { 601 struct hrtimer *hrtimer = this_cpu_ptr(&watchdog_hrtimer); 602 603 WARN_ON_ONCE(cpu != smp_processor_id()); 604 605 /* 606 * Disable the hardlockup detector first. That prevents that a large 607 * delay between disabling the timer and disabling the hardlockup 608 * detector causes a false positive. 609 */ 610 watchdog_hardlockup_disable(cpu); 611 hrtimer_cancel(hrtimer); 612 wait_for_completion(this_cpu_ptr(&softlockup_completion)); 613 } 614 615 static int softlockup_stop_fn(void *data) 616 { 617 watchdog_disable(smp_processor_id()); 618 return 0; 619 } 620 621 static void softlockup_stop_all(void) 622 { 623 int cpu; 624 625 if (!softlockup_initialized) 626 return; 627 628 for_each_cpu(cpu, &watchdog_allowed_mask) 629 smp_call_on_cpu(cpu, softlockup_stop_fn, NULL, false); 630 631 cpumask_clear(&watchdog_allowed_mask); 632 } 633 634 static int softlockup_start_fn(void *data) 635 { 636 watchdog_enable(smp_processor_id()); 637 return 0; 638 } 639 640 static void softlockup_start_all(void) 641 { 642 int cpu; 643 644 cpumask_copy(&watchdog_allowed_mask, &watchdog_cpumask); 645 for_each_cpu(cpu, &watchdog_allowed_mask) 646 smp_call_on_cpu(cpu, softlockup_start_fn, NULL, false); 647 } 648 649 int lockup_detector_online_cpu(unsigned int cpu) 650 { 651 if (cpumask_test_cpu(cpu, &watchdog_allowed_mask)) 652 watchdog_enable(cpu); 653 return 0; 654 } 655 656 int lockup_detector_offline_cpu(unsigned int cpu) 657 { 658 if (cpumask_test_cpu(cpu, &watchdog_allowed_mask)) 659 watchdog_disable(cpu); 660 return 0; 661 } 662 663 static void __lockup_detector_reconfigure(void) 664 { 665 cpus_read_lock(); 666 watchdog_hardlockup_stop(); 667 668 softlockup_stop_all(); 669 set_sample_period(); 670 lockup_detector_update_enable(); 671 if (watchdog_enabled && watchdog_thresh) 672 softlockup_start_all(); 673 674 watchdog_hardlockup_start(); 675 cpus_read_unlock(); 676 /* 677 * Must be called outside the cpus locked section to prevent 678 * recursive locking in the perf code. 679 */ 680 __lockup_detector_cleanup(); 681 } 682 683 void lockup_detector_reconfigure(void) 684 { 685 mutex_lock(&watchdog_mutex); 686 __lockup_detector_reconfigure(); 687 mutex_unlock(&watchdog_mutex); 688 } 689 690 /* 691 * Create the watchdog infrastructure and configure the detector(s). 692 */ 693 static __init void lockup_detector_setup(void) 694 { 695 /* 696 * If sysctl is off and watchdog got disabled on the command line, 697 * nothing to do here. 698 */ 699 lockup_detector_update_enable(); 700 701 if (!IS_ENABLED(CONFIG_SYSCTL) && 702 !(watchdog_enabled && watchdog_thresh)) 703 return; 704 705 mutex_lock(&watchdog_mutex); 706 __lockup_detector_reconfigure(); 707 softlockup_initialized = true; 708 mutex_unlock(&watchdog_mutex); 709 } 710 711 #else /* CONFIG_SOFTLOCKUP_DETECTOR */ 712 static void __lockup_detector_reconfigure(void) 713 { 714 cpus_read_lock(); 715 watchdog_hardlockup_stop(); 716 lockup_detector_update_enable(); 717 watchdog_hardlockup_start(); 718 cpus_read_unlock(); 719 } 720 void lockup_detector_reconfigure(void) 721 { 722 __lockup_detector_reconfigure(); 723 } 724 static inline void lockup_detector_setup(void) 725 { 726 __lockup_detector_reconfigure(); 727 } 728 #endif /* !CONFIG_SOFTLOCKUP_DETECTOR */ 729 730 static void __lockup_detector_cleanup(void) 731 { 732 lockdep_assert_held(&watchdog_mutex); 733 hardlockup_detector_perf_cleanup(); 734 } 735 736 /** 737 * lockup_detector_cleanup - Cleanup after cpu hotplug or sysctl changes 738 * 739 * Caller must not hold the cpu hotplug rwsem. 740 */ 741 void lockup_detector_cleanup(void) 742 { 743 mutex_lock(&watchdog_mutex); 744 __lockup_detector_cleanup(); 745 mutex_unlock(&watchdog_mutex); 746 } 747 748 /** 749 * lockup_detector_soft_poweroff - Interface to stop lockup detector(s) 750 * 751 * Special interface for parisc. It prevents lockup detector warnings from 752 * the default pm_poweroff() function which busy loops forever. 753 */ 754 void lockup_detector_soft_poweroff(void) 755 { 756 watchdog_enabled = 0; 757 } 758 759 #ifdef CONFIG_SYSCTL 760 761 /* Propagate any changes to the watchdog infrastructure */ 762 static void proc_watchdog_update(void) 763 { 764 /* Remove impossible cpus to keep sysctl output clean. */ 765 cpumask_and(&watchdog_cpumask, &watchdog_cpumask, cpu_possible_mask); 766 __lockup_detector_reconfigure(); 767 } 768 769 /* 770 * common function for watchdog, nmi_watchdog and soft_watchdog parameter 771 * 772 * caller | table->data points to | 'which' 773 * -------------------|----------------------------------|------------------------------- 774 * proc_watchdog | watchdog_user_enabled | WATCHDOG_HARDLOCKUP_ENABLED | 775 * | | WATCHDOG_SOFTOCKUP_ENABLED 776 * -------------------|----------------------------------|------------------------------- 777 * proc_nmi_watchdog | watchdog_hardlockup_user_enabled | WATCHDOG_HARDLOCKUP_ENABLED 778 * -------------------|----------------------------------|------------------------------- 779 * proc_soft_watchdog | watchdog_softlockup_user_enabled | WATCHDOG_SOFTOCKUP_ENABLED 780 */ 781 static int proc_watchdog_common(int which, struct ctl_table *table, int write, 782 void *buffer, size_t *lenp, loff_t *ppos) 783 { 784 int err, old, *param = table->data; 785 786 mutex_lock(&watchdog_mutex); 787 788 if (!write) { 789 /* 790 * On read synchronize the userspace interface. This is a 791 * racy snapshot. 792 */ 793 *param = (watchdog_enabled & which) != 0; 794 err = proc_dointvec_minmax(table, write, buffer, lenp, ppos); 795 } else { 796 old = READ_ONCE(*param); 797 err = proc_dointvec_minmax(table, write, buffer, lenp, ppos); 798 if (!err && old != READ_ONCE(*param)) 799 proc_watchdog_update(); 800 } 801 mutex_unlock(&watchdog_mutex); 802 return err; 803 } 804 805 /* 806 * /proc/sys/kernel/watchdog 807 */ 808 static int proc_watchdog(struct ctl_table *table, int write, 809 void *buffer, size_t *lenp, loff_t *ppos) 810 { 811 return proc_watchdog_common(WATCHDOG_HARDLOCKUP_ENABLED | 812 WATCHDOG_SOFTOCKUP_ENABLED, 813 table, write, buffer, lenp, ppos); 814 } 815 816 /* 817 * /proc/sys/kernel/nmi_watchdog 818 */ 819 static int proc_nmi_watchdog(struct ctl_table *table, int write, 820 void *buffer, size_t *lenp, loff_t *ppos) 821 { 822 if (!watchdog_hardlockup_available && write) 823 return -ENOTSUPP; 824 return proc_watchdog_common(WATCHDOG_HARDLOCKUP_ENABLED, 825 table, write, buffer, lenp, ppos); 826 } 827 828 #ifdef CONFIG_SOFTLOCKUP_DETECTOR 829 /* 830 * /proc/sys/kernel/soft_watchdog 831 */ 832 static int proc_soft_watchdog(struct ctl_table *table, int write, 833 void *buffer, size_t *lenp, loff_t *ppos) 834 { 835 return proc_watchdog_common(WATCHDOG_SOFTOCKUP_ENABLED, 836 table, write, buffer, lenp, ppos); 837 } 838 #endif 839 840 /* 841 * /proc/sys/kernel/watchdog_thresh 842 */ 843 static int proc_watchdog_thresh(struct ctl_table *table, int write, 844 void *buffer, size_t *lenp, loff_t *ppos) 845 { 846 int err, old; 847 848 mutex_lock(&watchdog_mutex); 849 850 old = READ_ONCE(watchdog_thresh); 851 err = proc_dointvec_minmax(table, write, buffer, lenp, ppos); 852 853 if (!err && write && old != READ_ONCE(watchdog_thresh)) 854 proc_watchdog_update(); 855 856 mutex_unlock(&watchdog_mutex); 857 return err; 858 } 859 860 /* 861 * The cpumask is the mask of possible cpus that the watchdog can run 862 * on, not the mask of cpus it is actually running on. This allows the 863 * user to specify a mask that will include cpus that have not yet 864 * been brought online, if desired. 865 */ 866 static int proc_watchdog_cpumask(struct ctl_table *table, int write, 867 void *buffer, size_t *lenp, loff_t *ppos) 868 { 869 int err; 870 871 mutex_lock(&watchdog_mutex); 872 873 err = proc_do_large_bitmap(table, write, buffer, lenp, ppos); 874 if (!err && write) 875 proc_watchdog_update(); 876 877 mutex_unlock(&watchdog_mutex); 878 return err; 879 } 880 881 static const int sixty = 60; 882 883 static struct ctl_table watchdog_sysctls[] = { 884 { 885 .procname = "watchdog", 886 .data = &watchdog_user_enabled, 887 .maxlen = sizeof(int), 888 .mode = 0644, 889 .proc_handler = proc_watchdog, 890 .extra1 = SYSCTL_ZERO, 891 .extra2 = SYSCTL_ONE, 892 }, 893 { 894 .procname = "watchdog_thresh", 895 .data = &watchdog_thresh, 896 .maxlen = sizeof(int), 897 .mode = 0644, 898 .proc_handler = proc_watchdog_thresh, 899 .extra1 = SYSCTL_ZERO, 900 .extra2 = (void *)&sixty, 901 }, 902 { 903 .procname = "watchdog_cpumask", 904 .data = &watchdog_cpumask_bits, 905 .maxlen = NR_CPUS, 906 .mode = 0644, 907 .proc_handler = proc_watchdog_cpumask, 908 }, 909 #ifdef CONFIG_SOFTLOCKUP_DETECTOR 910 { 911 .procname = "soft_watchdog", 912 .data = &watchdog_softlockup_user_enabled, 913 .maxlen = sizeof(int), 914 .mode = 0644, 915 .proc_handler = proc_soft_watchdog, 916 .extra1 = SYSCTL_ZERO, 917 .extra2 = SYSCTL_ONE, 918 }, 919 { 920 .procname = "softlockup_panic", 921 .data = &softlockup_panic, 922 .maxlen = sizeof(int), 923 .mode = 0644, 924 .proc_handler = proc_dointvec_minmax, 925 .extra1 = SYSCTL_ZERO, 926 .extra2 = SYSCTL_ONE, 927 }, 928 #ifdef CONFIG_SMP 929 { 930 .procname = "softlockup_all_cpu_backtrace", 931 .data = &sysctl_softlockup_all_cpu_backtrace, 932 .maxlen = sizeof(int), 933 .mode = 0644, 934 .proc_handler = proc_dointvec_minmax, 935 .extra1 = SYSCTL_ZERO, 936 .extra2 = SYSCTL_ONE, 937 }, 938 #endif /* CONFIG_SMP */ 939 #endif 940 #ifdef CONFIG_HARDLOCKUP_DETECTOR 941 { 942 .procname = "hardlockup_panic", 943 .data = &hardlockup_panic, 944 .maxlen = sizeof(int), 945 .mode = 0644, 946 .proc_handler = proc_dointvec_minmax, 947 .extra1 = SYSCTL_ZERO, 948 .extra2 = SYSCTL_ONE, 949 }, 950 #ifdef CONFIG_SMP 951 { 952 .procname = "hardlockup_all_cpu_backtrace", 953 .data = &sysctl_hardlockup_all_cpu_backtrace, 954 .maxlen = sizeof(int), 955 .mode = 0644, 956 .proc_handler = proc_dointvec_minmax, 957 .extra1 = SYSCTL_ZERO, 958 .extra2 = SYSCTL_ONE, 959 }, 960 #endif /* CONFIG_SMP */ 961 #endif 962 {} 963 }; 964 965 static struct ctl_table watchdog_hardlockup_sysctl[] = { 966 { 967 .procname = "nmi_watchdog", 968 .data = &watchdog_hardlockup_user_enabled, 969 .maxlen = sizeof(int), 970 .mode = 0444, 971 .proc_handler = proc_nmi_watchdog, 972 .extra1 = SYSCTL_ZERO, 973 .extra2 = SYSCTL_ONE, 974 }, 975 {} 976 }; 977 978 static void __init watchdog_sysctl_init(void) 979 { 980 register_sysctl_init("kernel", watchdog_sysctls); 981 982 if (watchdog_hardlockup_available) 983 watchdog_hardlockup_sysctl[0].mode = 0644; 984 register_sysctl_init("kernel", watchdog_hardlockup_sysctl); 985 } 986 987 #else 988 #define watchdog_sysctl_init() do { } while (0) 989 #endif /* CONFIG_SYSCTL */ 990 991 static void __init lockup_detector_delay_init(struct work_struct *work); 992 static bool allow_lockup_detector_init_retry __initdata; 993 994 static struct work_struct detector_work __initdata = 995 __WORK_INITIALIZER(detector_work, lockup_detector_delay_init); 996 997 static void __init lockup_detector_delay_init(struct work_struct *work) 998 { 999 int ret; 1000 1001 ret = watchdog_hardlockup_probe(); 1002 if (ret) { 1003 pr_info("Delayed init of the lockup detector failed: %d\n", ret); 1004 pr_info("Hard watchdog permanently disabled\n"); 1005 return; 1006 } 1007 1008 allow_lockup_detector_init_retry = false; 1009 1010 watchdog_hardlockup_available = true; 1011 lockup_detector_setup(); 1012 } 1013 1014 /* 1015 * lockup_detector_retry_init - retry init lockup detector if possible. 1016 * 1017 * Retry hardlockup detector init. It is useful when it requires some 1018 * functionality that has to be initialized later on a particular 1019 * platform. 1020 */ 1021 void __init lockup_detector_retry_init(void) 1022 { 1023 /* Must be called before late init calls */ 1024 if (!allow_lockup_detector_init_retry) 1025 return; 1026 1027 schedule_work(&detector_work); 1028 } 1029 1030 /* 1031 * Ensure that optional delayed hardlockup init is proceed before 1032 * the init code and memory is freed. 1033 */ 1034 static int __init lockup_detector_check(void) 1035 { 1036 /* Prevent any later retry. */ 1037 allow_lockup_detector_init_retry = false; 1038 1039 /* Make sure no work is pending. */ 1040 flush_work(&detector_work); 1041 1042 watchdog_sysctl_init(); 1043 1044 return 0; 1045 1046 } 1047 late_initcall_sync(lockup_detector_check); 1048 1049 void __init lockup_detector_init(void) 1050 { 1051 if (tick_nohz_full_enabled()) 1052 pr_info("Disabling watchdog on nohz_full cores by default\n"); 1053 1054 cpumask_copy(&watchdog_cpumask, 1055 housekeeping_cpumask(HK_TYPE_TIMER)); 1056 1057 if (!watchdog_hardlockup_probe()) 1058 watchdog_hardlockup_available = true; 1059 else 1060 allow_lockup_detector_init_retry = true; 1061 1062 lockup_detector_setup(); 1063 } 1064