1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Detect hard lockups on a system using perf 4 * 5 * started by Don Zickus, Copyright (C) 2010 Red Hat, Inc. 6 * 7 * Note: Most of this code is borrowed heavily from the original softlockup 8 * detector, so thanks to Ingo for the initial implementation. 9 * Some chunks also taken from the old x86-specific nmi watchdog code, thanks 10 * to those contributors as well. 11 */ 12 13 #define pr_fmt(fmt) "NMI watchdog: " fmt 14 15 #include <linux/panic.h> 16 #include <linux/nmi.h> 17 #include <linux/atomic.h> 18 #include <linux/module.h> 19 #include <linux/sched/debug.h> 20 21 #include <asm/irq_regs.h> 22 #include <linux/perf_event.h> 23 24 static DEFINE_PER_CPU(struct perf_event *, watchdog_ev); 25 26 static atomic_t watchdog_cpus = ATOMIC_INIT(0); 27 28 #ifdef CONFIG_HARDLOCKUP_CHECK_TIMESTAMP 29 static DEFINE_PER_CPU(ktime_t, last_timestamp); 30 static DEFINE_PER_CPU(unsigned int, nmi_rearmed); 31 static ktime_t watchdog_hrtimer_sample_threshold __read_mostly; 32 33 void watchdog_update_hrtimer_threshold(u64 period) 34 { 35 /* 36 * The hrtimer runs with a period of (watchdog_threshold * 2) / 5 37 * 38 * So it runs effectively with 2.5 times the rate of the NMI 39 * watchdog. That means the hrtimer should fire 2-3 times before 40 * the NMI watchdog expires. The NMI watchdog on x86 is based on 41 * unhalted CPU cycles, so if Turbo-Mode is enabled the CPU cycles 42 * might run way faster than expected and the NMI fires in a 43 * smaller period than the one deduced from the nominal CPU 44 * frequency. Depending on the Turbo-Mode factor this might be fast 45 * enough to get the NMI period smaller than the hrtimer watchdog 46 * period and trigger false positives. 47 * 48 * The sample threshold is used to check in the NMI handler whether 49 * the minimum time between two NMI samples has elapsed. That 50 * prevents false positives. 51 * 52 * Set this to 4/5 of the actual watchdog threshold period so the 53 * hrtimer is guaranteed to fire at least once within the real 54 * watchdog threshold. 55 */ 56 watchdog_hrtimer_sample_threshold = period * 2; 57 } 58 59 static bool watchdog_check_timestamp(void) 60 { 61 ktime_t delta, now = ktime_get_mono_fast_ns(); 62 63 delta = now - __this_cpu_read(last_timestamp); 64 if (delta < watchdog_hrtimer_sample_threshold) { 65 /* 66 * If ktime is jiffies based, a stalled timer would prevent 67 * jiffies from being incremented and the filter would look 68 * at a stale timestamp and never trigger. 69 */ 70 if (__this_cpu_inc_return(nmi_rearmed) < 10) 71 return false; 72 } 73 __this_cpu_write(nmi_rearmed, 0); 74 __this_cpu_write(last_timestamp, now); 75 return true; 76 } 77 78 static void watchdog_init_timestamp(void) 79 { 80 __this_cpu_write(nmi_rearmed, 0); 81 __this_cpu_write(last_timestamp, ktime_get_mono_fast_ns()); 82 } 83 #else 84 static inline bool watchdog_check_timestamp(void) { return true; } 85 static inline void watchdog_init_timestamp(void) { } 86 #endif 87 88 static struct perf_event_attr wd_hw_attr = { 89 .type = PERF_TYPE_HARDWARE, 90 .config = PERF_COUNT_HW_CPU_CYCLES, 91 .size = sizeof(struct perf_event_attr), 92 .pinned = 1, 93 .disabled = 1, 94 }; 95 96 static struct perf_event_attr fallback_wd_hw_attr = { 97 .type = PERF_TYPE_HARDWARE, 98 .config = PERF_COUNT_HW_CPU_CYCLES, 99 .size = sizeof(struct perf_event_attr), 100 .pinned = 1, 101 .disabled = 1, 102 }; 103 104 /* Callback function for perf event subsystem */ 105 static void watchdog_overflow_callback(struct perf_event *event, 106 struct perf_sample_data *data, 107 struct pt_regs *regs) 108 { 109 /* Ensure the watchdog never gets throttled */ 110 event->hw.interrupts = 0; 111 112 if (panic_in_progress()) 113 return; 114 115 if (!watchdog_check_timestamp()) 116 return; 117 118 watchdog_hardlockup_check(smp_processor_id(), regs); 119 } 120 121 static struct perf_event *hardlockup_detector_event_create(unsigned int cpu) 122 { 123 struct perf_event_attr *wd_attr; 124 struct perf_event *evt; 125 126 wd_attr = &wd_hw_attr; 127 wd_attr->sample_period = hw_nmi_get_sample_period(watchdog_thresh); 128 129 /* Try to register using hardware perf events */ 130 evt = perf_event_create_kernel_counter(wd_attr, cpu, NULL, 131 watchdog_overflow_callback, NULL); 132 if (IS_ERR(evt)) { 133 wd_attr = &fallback_wd_hw_attr; 134 wd_attr->sample_period = hw_nmi_get_sample_period(watchdog_thresh); 135 evt = perf_event_create_kernel_counter(wd_attr, cpu, NULL, 136 watchdog_overflow_callback, NULL); 137 } 138 139 return evt; 140 } 141 142 /** 143 * watchdog_hardlockup_enable - Enable the local event 144 * @cpu: The CPU to enable hard lockup on. 145 */ 146 void watchdog_hardlockup_enable(unsigned int cpu) 147 { 148 struct perf_event *evt; 149 150 WARN_ON_ONCE(cpu != smp_processor_id()); 151 152 evt = hardlockup_detector_event_create(cpu); 153 if (IS_ERR(evt)) { 154 pr_debug("Perf event create on CPU %d failed with %ld\n", cpu, 155 PTR_ERR(evt)); 156 return; 157 } 158 159 /* use original value for check */ 160 if (!atomic_fetch_inc(&watchdog_cpus)) 161 pr_info("Enabled. Permanently consumes one hw-PMU counter.\n"); 162 163 WARN_ONCE(this_cpu_read(watchdog_ev), "unexpected watchdog_ev leak"); 164 this_cpu_write(watchdog_ev, evt); 165 166 watchdog_init_timestamp(); 167 perf_event_enable(evt); 168 } 169 170 /** 171 * watchdog_hardlockup_disable - Disable the local event 172 * @cpu: The CPU to enable hard lockup on. 173 */ 174 void watchdog_hardlockup_disable(unsigned int cpu) 175 { 176 struct perf_event *event = this_cpu_read(watchdog_ev); 177 178 WARN_ON_ONCE(cpu != smp_processor_id()); 179 180 if (event) { 181 perf_event_disable(event); 182 perf_event_release_kernel(event); 183 this_cpu_write(watchdog_ev, NULL); 184 atomic_dec(&watchdog_cpus); 185 } 186 } 187 188 /** 189 * hardlockup_detector_perf_adjust_period - Adjust the event period due 190 * to current cpu frequency change 191 * @period: The target period to be set 192 */ 193 void hardlockup_detector_perf_adjust_period(u64 period) 194 { 195 struct perf_event *event = this_cpu_read(watchdog_ev); 196 197 if (!(watchdog_enabled & WATCHDOG_HARDLOCKUP_ENABLED)) 198 return; 199 200 if (!event) 201 return; 202 203 if (event->attr.sample_period == period) 204 return; 205 206 if (perf_event_period(event, period)) 207 pr_err("failed to change period to %llu\n", period); 208 } 209 210 /** 211 * hardlockup_detector_perf_stop - Globally stop watchdog events 212 * 213 * Special interface for x86 to handle the perf HT bug. 214 */ 215 void __init hardlockup_detector_perf_stop(void) 216 { 217 int cpu; 218 219 lockdep_assert_cpus_held(); 220 221 for_each_online_cpu(cpu) { 222 struct perf_event *event = per_cpu(watchdog_ev, cpu); 223 224 if (event) 225 perf_event_disable(event); 226 } 227 } 228 229 /** 230 * hardlockup_detector_perf_restart - Globally restart watchdog events 231 * 232 * Special interface for x86 to handle the perf HT bug. 233 */ 234 void __init hardlockup_detector_perf_restart(void) 235 { 236 int cpu; 237 238 lockdep_assert_cpus_held(); 239 240 if (!(watchdog_enabled & WATCHDOG_HARDLOCKUP_ENABLED)) 241 return; 242 243 for_each_online_cpu(cpu) { 244 struct perf_event *event = per_cpu(watchdog_ev, cpu); 245 246 if (event) 247 perf_event_enable(event); 248 } 249 } 250 251 bool __weak __init arch_perf_nmi_is_available(void) 252 { 253 return true; 254 } 255 256 /** 257 * watchdog_hardlockup_probe - Probe whether NMI event is available at all 258 */ 259 int __init watchdog_hardlockup_probe(void) 260 { 261 struct perf_event *evt; 262 unsigned int cpu; 263 int ret; 264 265 if (!arch_perf_nmi_is_available()) 266 return -ENODEV; 267 268 if (!hw_nmi_get_sample_period(watchdog_thresh)) 269 return -EINVAL; 270 271 /* 272 * Test hardware PMU availability by creating a temporary perf event. 273 * The event is released immediately. 274 */ 275 cpu = raw_smp_processor_id(); 276 evt = hardlockup_detector_event_create(cpu); 277 if (IS_ERR(evt)) { 278 pr_info("Perf NMI watchdog permanently disabled\n"); 279 ret = PTR_ERR(evt); 280 } else { 281 perf_event_release_kernel(evt); 282 ret = 0; 283 } 284 285 return ret; 286 } 287 288 /** 289 * hardlockup_config_perf_event - Overwrite config of wd_hw_attr. 290 * @str: number which identifies the raw perf event to use 291 */ 292 void __init hardlockup_config_perf_event(const char *str) 293 { 294 u64 config; 295 char buf[24]; 296 char *comma = strchr(str, ','); 297 298 if (!comma) { 299 if (kstrtoull(str, 16, &config)) 300 return; 301 } else { 302 unsigned int len = comma - str; 303 304 if (len > sizeof(buf)) 305 return; 306 307 strscpy(buf, str, len); 308 if (kstrtoull(buf, 16, &config)) 309 return; 310 } 311 312 wd_hw_attr.type = PERF_TYPE_RAW; 313 wd_hw_attr.config = config; 314 } 315