1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * arm64 SDEI-based cross-CPU NMI service. 4 * 5 * Delivering an "NMI-shaped" event to an EL1 context that has locally 6 * masked interrupts, on silicon without FEAT_NMI, can be done two ways: 7 * 8 * - pseudo-NMI: mask "interrupts" via the GIC priority register 9 * (ICC_PMR_EL1) instead of PSTATE.DAIF, leaving a high-priority band 10 * deliverable. Functionally this works -- but it reimplements every 11 * local_irq_disable()/enable() and exception entry/exit as a PMR 12 * write plus synchronisation, a cost paid on that hot path forever, 13 * whether or not an NMI is ever delivered. 14 * 15 * - SDEI: leave interrupt masking as the cheap PSTATE.DAIF operation 16 * and have the firmware bounce an EL3-routed Group-0 SGI back to 17 * NS-EL1 as an event callback. The cost is a firmware round-trip, 18 * but only at the rare moment delivery is actually needed. 19 * 20 * This driver takes the second path: it keeps the IRQ-mask hot path 21 * free and pays only when it fires, which is what makes cross-CPU NMI 22 * affordable on hardware where the pseudo-NMI tax isn't, until FEAT_NMI 23 * makes NMI masking cheap in the architecture itself. 24 * 25 * Capabilities provided: 26 * 27 * - sdei_nmi_trigger_cpumask_backtrace() — override for arm64's 28 * arch_trigger_cpumask_backtrace(), so sysrq-l, RCU stall dumps, 29 * hardlockup_all_cpu_backtrace, soft-lockup/hung-task secondary 30 * dumps all reach interrupt-masked CPUs. 31 * 32 * - sdei_nmi_stop_cpus() — the last rung of smp_send_stop()'s 33 * escalation (reboot/halt and the panic/kdump crash stop alike), 34 * reaching CPUs that ignored the stop IPIs; on the kdump path the 35 * wedged context is captured into the vmcore before the CPU parks. 36 * 37 * Delivery uses the standard SDEI software-signalled event (event 0) and 38 * SDEI_EVENT_SIGNAL. We register a handler for event 0, enable it, and 39 * poke a target CPU with sdei_event_signal(0, mpidr): firmware makes 40 * event 0 pending on that PE and dispatches the handler NMI-like, 41 * regardless of the target's DAIF. 42 * Availability is simply whether event 0 registers and enables -- if SDEI 43 * and its software-signalled event are present we use it, otherwise the 44 * driver stays inert. 45 */ 46 47 #define pr_fmt(fmt) "sdei_nmi: " fmt 48 49 #include <linux/arm_sdei.h> 50 #include <linux/cpumask.h> 51 #include <linux/init.h> 52 #include <linux/kernel.h> 53 #include <linux/kprobes.h> 54 #include <linux/nmi.h> 55 #include <linux/printk.h> 56 #include <linux/ptrace.h> 57 #include <linux/smp.h> 58 #include <linux/types.h> 59 60 #include <asm/nmi.h> 61 #include <asm/smp_plat.h> 62 63 static bool sdei_nmi_available; 64 65 #define SDEI_NMI_EVENT 0 66 67 /* 68 * Backtrace and stop both ride SDEI event 0. That is not a chosen economy: 69 * event 0 is the only architecturally software-signalled event -- the sole 70 * event SDEI_EVENT_SIGNAL can target at an arbitrary PE. Every other event 71 * number is a firmware/platform interrupt-bound event, not something the 72 * kernel can raise cross-CPU, so a dedicated "stop" event would need 73 * firmware to define and bind it -- exactly the firmware dependency this 74 * driver sets out to avoid. 75 * 76 * Sharing one event means the handler must tell a stop apart from a 77 * backtrace. A stop is terminal and system-wide -- sdei_nmi_stop_cpus() is 78 * only reached from smp_send_stop() (reboot/halt/panic/kdump), which never 79 * returns -- so once a stop is requested, every later event-0 fire is a 80 * stop too. A single write-once flag therefore carries as much as a 81 * per-CPU mask would: sdei_nmi_stop_cpus() sets it before signalling, and 82 * the handler reads a set flag as "stop this CPU" and a clear flag as 83 * "backtrace" (handled by nmi_cpu_backtrace(), which self-gates on the 84 * framework's backtrace mask). A backtrace fire that races in after a stop 85 * has begun just stops that CPU instead -- harmless, it is going down. 86 */ 87 static bool sdei_nmi_stopping; 88 89 static int sdei_nmi_handler(u32 event, struct pt_regs *regs, void *arg) 90 { 91 /* 92 * No smp_rmb() pairing sdei_nmi_stop_cpus()'s dsb(ishst): the flag is 93 * the only shared value, and this handler runs only because firmware 94 * delivered the event -- a round-trip past that store -- so the read 95 * cannot be stale and there is no second load for a barrier to order. 96 */ 97 if (READ_ONCE(sdei_nmi_stopping)) { 98 /* 99 * Never returns, and deliberately never completes the SDEI 100 * event: SDEI_EVENT_COMPLETE has firmware restore the 101 * interrupted context, which would land the CPU back in 102 * the wedged loop (or in do_idle, which BUGs at 103 * cpuhp_report_idle_dead once it sees itself offline). 104 * Returning a modified pt_regs doesn't help -- 105 * arch/arm64/kernel/sdei.c::do_sdei_event only honours a PC 106 * override via its IRQ-state heuristic and otherwise hands 107 * EL3 its own saved-context slot back. 108 * 109 * Trade-off: EL3 retains ~one saved-context slot per parked 110 * CPU until the next hardware reset (~hundreds of bytes per 111 * CPU). Recoverability is unchanged versus an IPI-stopped 112 * CPU: neither comes back without a reset. 113 */ 114 arm64_nmi_cpu_stop(regs, false); 115 /* unreachable */ 116 } 117 118 /* 119 * nmi_cpu_backtrace() no-ops unless this CPU's bit is set in the 120 * global backtrace mask (driven by nmi_trigger_cpumask_backtrace()), 121 * so a fire that reaches a CPU not being backtraced is harmless. 122 */ 123 nmi_cpu_backtrace(regs); 124 return SDEI_EV_HANDLED; 125 } 126 NOKPROBE_SYMBOL(sdei_nmi_handler); 127 128 static void sdei_nmi_fire(unsigned int target_cpu) 129 { 130 int err = sdei_event_signal(SDEI_NMI_EVENT, cpu_logical_map(target_cpu)); 131 132 if (err) 133 pr_warn("SDEI_EVENT_SIGNAL to CPU %u failed: %d\n", 134 target_cpu, err); 135 } 136 137 /* 138 * Raise callback for nmi_trigger_cpumask_backtrace(): signal event 0 139 * at every CPU still pending in @mask. The framework excludes the local 140 * CPU from @mask before calling us. 141 */ 142 static void sdei_nmi_raise_backtrace(cpumask_t *mask) 143 { 144 unsigned int cpu; 145 146 /* 147 * Publish backtrace_mask (set by nmi_trigger_cpumask_backtrace()) 148 * before signalling. As in the stop path, the SMC is not a memory 149 * store, so dsb(ishst) is needed for the target to observe the mask. 150 */ 151 dsb(ishst); 152 153 for_each_cpu(cpu, mask) 154 sdei_nmi_fire(cpu); 155 } 156 157 /* 158 * Override hook for arch_trigger_cpumask_backtrace() (see 159 * arch/arm64/kernel/smp.c). Returns true when SDEI handled the request, 160 * which is the case whenever SDEI is active; on a false return the arch 161 * falls back to its regular-IRQ (or pseudo-NMI, if enabled) IPI. 162 * 163 * On a kernel built without paying the pseudo-NMI hot-path cost (the 164 * usual case for this driver's target), the IPI can't reach a CPU that 165 * has interrupts masked -- so the backtrace of the one CPU you care 166 * about comes back empty. SDEI is dispatched out of EL3 and lands 167 * regardless of the target's DAIF, without taxing the IRQ-mask path. 168 */ 169 bool sdei_nmi_trigger_cpumask_backtrace(const cpumask_t *mask, int exclude_cpu) 170 { 171 if (!sdei_nmi_available) 172 return false; 173 174 nmi_trigger_cpumask_backtrace(mask, exclude_cpu, 175 sdei_nmi_raise_backtrace); 176 return true; 177 } 178 179 bool sdei_nmi_active(void) 180 { 181 return sdei_nmi_available; 182 } 183 184 /* 185 * Last rung of the stop escalation in smp_send_stop() (see 186 * arch/arm64/kernel/smp.c). The caller runs the regular stop IPI (and 187 * the pseudo-NMI stop IPI, where available) first; @mask holds whatever 188 * stayed online through those -- typically CPUs wedged with interrupts 189 * masked, unreachable by an IPI. Mark the stop in progress and signal 190 * event 0 at each target; a target acks by marking itself offline, which 191 * the caller polls for. The caller has already confirmed sdei_nmi_active(). 192 */ 193 void sdei_nmi_stop_cpus(const cpumask_t *mask) 194 { 195 unsigned int cpu; 196 197 WRITE_ONCE(sdei_nmi_stopping, true); 198 199 /* 200 * Publish the flag before signalling. The signal goes out via an SMC 201 * to firmware, not a memory store, so smp_wmb() ordering is not 202 * enough: use dsb(ishst) to make the store globally visible before the 203 * SMC executes, as gic_ipi_send_mask() does for its SGI. The SDEI spec 204 * does not require the dispatch to order the caller's prior stores. 205 */ 206 dsb(ishst); 207 208 for_each_cpu(cpu, mask) 209 sdei_nmi_fire(cpu); 210 } 211 212 /* 213 * device_initcall (after arch_initcall(sdei_init), so the SDEI subsystem 214 * is up): probe the firmware, register the event, and turn on the 215 * cross-CPU service. If the probe fails the driver stays inert and the 216 * override hooks decline, leaving the arch's own paths in place. 217 */ 218 static int __init sdei_nmi_init(void) 219 { 220 int err; 221 222 if (!sdei_is_present()) 223 return 0; 224 225 err = sdei_event_register(SDEI_NMI_EVENT, sdei_nmi_handler, NULL); 226 if (err) { 227 pr_err("sdei_event_register(%u) failed: %d\n", 228 SDEI_NMI_EVENT, err); 229 return 0; 230 } 231 232 err = sdei_event_enable(SDEI_NMI_EVENT); 233 if (err) { 234 pr_err("sdei_event_enable(%u) failed: %d\n", 235 SDEI_NMI_EVENT, err); 236 sdei_event_unregister(SDEI_NMI_EVENT); 237 return 0; 238 } 239 240 sdei_nmi_available = true; 241 pr_info("using SDEI cross-CPU NMI (SDEI_EVENT_SIGNAL, event %u)\n", 242 SDEI_NMI_EVENT); 243 244 return 0; 245 } 246 device_initcall(sdei_nmi_init); 247