1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Detect hard lockups on a system using perf 4 * 5 * started by Don Zickus, Copyright (C) 2010 Red Hat, Inc. 6 * 7 * Note: Most of this code is borrowed heavily from the original softlockup 8 * detector, so thanks to Ingo for the initial implementation. 9 * Some chunks also taken from the old x86-specific nmi watchdog code, thanks 10 * to those contributors as well. 11 */ 12 13 #define pr_fmt(fmt) "NMI watchdog: " fmt 14 15 #include <linux/panic.h> 16 #include <linux/nmi.h> 17 #include <linux/atomic.h> 18 #include <linux/module.h> 19 #include <linux/sched/debug.h> 20 21 #include <asm/irq_regs.h> 22 #include <linux/perf_event.h> 23 24 static DEFINE_PER_CPU(struct perf_event *, watchdog_ev); 25 26 static atomic_t watchdog_cpus = ATOMIC_INIT(0); 27 28 #ifdef CONFIG_HARDLOCKUP_CHECK_TIMESTAMP 29 static DEFINE_PER_CPU(ktime_t, last_timestamp); 30 static DEFINE_PER_CPU(unsigned int, nmi_rearmed); 31 static ktime_t watchdog_hrtimer_sample_threshold __read_mostly; 32 33 void watchdog_update_hrtimer_threshold(u64 period) 34 { 35 /* 36 * The hrtimer runs with a period of (watchdog_threshold * 2) / 5 37 * 38 * So it runs effectively with 2.5 times the rate of the NMI 39 * watchdog. That means the hrtimer should fire 2-3 times before 40 * the NMI watchdog expires. The NMI watchdog on x86 is based on 41 * unhalted CPU cycles, so if Turbo-Mode is enabled the CPU cycles 42 * might run way faster than expected and the NMI fires in a 43 * smaller period than the one deduced from the nominal CPU 44 * frequency. Depending on the Turbo-Mode factor this might be fast 45 * enough to get the NMI period smaller than the hrtimer watchdog 46 * period and trigger false positives. 47 * 48 * The sample threshold is used to check in the NMI handler whether 49 * the minimum time between two NMI samples has elapsed. That 50 * prevents false positives. 51 * 52 * Set this to 4/5 of the actual watchdog threshold period so the 53 * hrtimer is guaranteed to fire at least once within the real 54 * watchdog threshold. 55 */ 56 watchdog_hrtimer_sample_threshold = period * 2; 57 } 58 59 static bool watchdog_check_timestamp(void) 60 { 61 ktime_t delta, now = ktime_get_mono_fast_ns(); 62 63 delta = now - __this_cpu_read(last_timestamp); 64 if (delta < watchdog_hrtimer_sample_threshold) { 65 /* 66 * If ktime is jiffies based, a stalled timer would prevent 67 * jiffies from being incremented and the filter would look 68 * at a stale timestamp and never trigger. 69 */ 70 if (__this_cpu_inc_return(nmi_rearmed) < 10) 71 return false; 72 } 73 __this_cpu_write(nmi_rearmed, 0); 74 __this_cpu_write(last_timestamp, now); 75 return true; 76 } 77 78 static void watchdog_init_timestamp(void) 79 { 80 __this_cpu_write(nmi_rearmed, 0); 81 __this_cpu_write(last_timestamp, ktime_get_mono_fast_ns()); 82 } 83 #else 84 static inline bool watchdog_check_timestamp(void) { return true; } 85 static inline void watchdog_init_timestamp(void) { } 86 #endif 87 88 static struct perf_event_attr wd_hw_attr = { 89 .type = PERF_TYPE_HARDWARE, 90 .config = PERF_COUNT_HW_CPU_CYCLES, 91 .size = sizeof(struct perf_event_attr), 92 .pinned = 1, 93 .disabled = 1, 94 }; 95 96 static struct perf_event_attr fallback_wd_hw_attr = { 97 .type = PERF_TYPE_HARDWARE, 98 .config = PERF_COUNT_HW_CPU_CYCLES, 99 .size = sizeof(struct perf_event_attr), 100 .pinned = 1, 101 .disabled = 1, 102 }; 103 104 /* Callback function for perf event subsystem */ 105 static void watchdog_overflow_callback(struct perf_event *event, 106 struct perf_sample_data *data, 107 struct pt_regs *regs) 108 { 109 /* Ensure the watchdog never gets throttled */ 110 event->hw.interrupts = 0; 111 112 if (panic_in_progress()) 113 return; 114 115 if (!watchdog_check_timestamp()) 116 return; 117 118 watchdog_hardlockup_check(smp_processor_id(), regs); 119 } 120 121 static int hardlockup_detector_event_create(void) 122 { 123 unsigned int cpu; 124 struct perf_event_attr *wd_attr; 125 struct perf_event *evt; 126 127 /* 128 * Preemption is not disabled because memory will be allocated. 129 * Ensure CPU-locality by calling this in per-CPU kthread. 130 */ 131 WARN_ON(!is_percpu_thread()); 132 cpu = raw_smp_processor_id(); 133 wd_attr = &wd_hw_attr; 134 wd_attr->sample_period = hw_nmi_get_sample_period(watchdog_thresh); 135 136 /* Try to register using hardware perf events */ 137 evt = perf_event_create_kernel_counter(wd_attr, cpu, NULL, 138 watchdog_overflow_callback, NULL); 139 if (IS_ERR(evt)) { 140 wd_attr = &fallback_wd_hw_attr; 141 wd_attr->sample_period = hw_nmi_get_sample_period(watchdog_thresh); 142 evt = perf_event_create_kernel_counter(wd_attr, cpu, NULL, 143 watchdog_overflow_callback, NULL); 144 } 145 146 if (IS_ERR(evt)) { 147 pr_debug("Perf event create on CPU %d failed with %ld\n", cpu, 148 PTR_ERR(evt)); 149 return PTR_ERR(evt); 150 } 151 WARN_ONCE(this_cpu_read(watchdog_ev), "unexpected watchdog_ev leak"); 152 this_cpu_write(watchdog_ev, evt); 153 return 0; 154 } 155 156 /** 157 * watchdog_hardlockup_enable - Enable the local event 158 * @cpu: The CPU to enable hard lockup on. 159 */ 160 void watchdog_hardlockup_enable(unsigned int cpu) 161 { 162 WARN_ON_ONCE(cpu != smp_processor_id()); 163 164 if (hardlockup_detector_event_create()) 165 return; 166 167 /* use original value for check */ 168 if (!atomic_fetch_inc(&watchdog_cpus)) 169 pr_info("Enabled. Permanently consumes one hw-PMU counter.\n"); 170 171 watchdog_init_timestamp(); 172 perf_event_enable(this_cpu_read(watchdog_ev)); 173 } 174 175 /** 176 * watchdog_hardlockup_disable - Disable the local event 177 * @cpu: The CPU to enable hard lockup on. 178 */ 179 void watchdog_hardlockup_disable(unsigned int cpu) 180 { 181 struct perf_event *event = this_cpu_read(watchdog_ev); 182 183 WARN_ON_ONCE(cpu != smp_processor_id()); 184 185 if (event) { 186 perf_event_disable(event); 187 perf_event_release_kernel(event); 188 this_cpu_write(watchdog_ev, NULL); 189 atomic_dec(&watchdog_cpus); 190 } 191 } 192 193 /** 194 * hardlockup_detector_perf_adjust_period - Adjust the event period due 195 * to current cpu frequency change 196 * @period: The target period to be set 197 */ 198 void hardlockup_detector_perf_adjust_period(u64 period) 199 { 200 struct perf_event *event = this_cpu_read(watchdog_ev); 201 202 if (!(watchdog_enabled & WATCHDOG_HARDLOCKUP_ENABLED)) 203 return; 204 205 if (!event) 206 return; 207 208 if (event->attr.sample_period == period) 209 return; 210 211 if (perf_event_period(event, period)) 212 pr_err("failed to change period to %llu\n", period); 213 } 214 215 /** 216 * hardlockup_detector_perf_stop - Globally stop watchdog events 217 * 218 * Special interface for x86 to handle the perf HT bug. 219 */ 220 void __init hardlockup_detector_perf_stop(void) 221 { 222 int cpu; 223 224 lockdep_assert_cpus_held(); 225 226 for_each_online_cpu(cpu) { 227 struct perf_event *event = per_cpu(watchdog_ev, cpu); 228 229 if (event) 230 perf_event_disable(event); 231 } 232 } 233 234 /** 235 * hardlockup_detector_perf_restart - Globally restart watchdog events 236 * 237 * Special interface for x86 to handle the perf HT bug. 238 */ 239 void __init hardlockup_detector_perf_restart(void) 240 { 241 int cpu; 242 243 lockdep_assert_cpus_held(); 244 245 if (!(watchdog_enabled & WATCHDOG_HARDLOCKUP_ENABLED)) 246 return; 247 248 for_each_online_cpu(cpu) { 249 struct perf_event *event = per_cpu(watchdog_ev, cpu); 250 251 if (event) 252 perf_event_enable(event); 253 } 254 } 255 256 bool __weak __init arch_perf_nmi_is_available(void) 257 { 258 return true; 259 } 260 261 /** 262 * watchdog_hardlockup_probe - Probe whether NMI event is available at all 263 */ 264 int __init watchdog_hardlockup_probe(void) 265 { 266 int ret; 267 268 if (!arch_perf_nmi_is_available()) 269 return -ENODEV; 270 271 ret = hardlockup_detector_event_create(); 272 273 if (ret) { 274 pr_info("Perf NMI watchdog permanently disabled\n"); 275 } else { 276 perf_event_release_kernel(this_cpu_read(watchdog_ev)); 277 this_cpu_write(watchdog_ev, NULL); 278 } 279 return ret; 280 } 281 282 /** 283 * hardlockup_config_perf_event - Overwrite config of wd_hw_attr. 284 * @str: number which identifies the raw perf event to use 285 */ 286 void __init hardlockup_config_perf_event(const char *str) 287 { 288 u64 config; 289 char buf[24]; 290 char *comma = strchr(str, ','); 291 292 if (!comma) { 293 if (kstrtoull(str, 16, &config)) 294 return; 295 } else { 296 unsigned int len = comma - str; 297 298 if (len > sizeof(buf)) 299 return; 300 301 strscpy(buf, str, len); 302 if (kstrtoull(buf, 16, &config)) 303 return; 304 } 305 306 wd_hw_attr.type = PERF_TYPE_RAW; 307 wd_hw_attr.config = config; 308 } 309