1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * The FRED specific kernel/user entry functions which are invoked from
4 * assembly code and dispatch to the associated handlers.
5 */
6 #include <linux/kernel.h>
7 #include <linux/kdebug.h>
8 #include <linux/nospec.h>
9
10 #include <asm/desc.h>
11 #include <asm/fred.h>
12 #include <asm/idtentry.h>
13 #include <asm/processor-flags.h>
14 #include <asm/syscall.h>
15 #include <asm/trapnr.h>
16 #include <asm/traps.h>
17
18 /* FRED EVENT_TYPE_OTHER vector numbers */
19 #define FRED_SYSCALL 1
20 #define FRED_SYSENTER 2
21
fred_bad_type(struct pt_regs * regs,unsigned long error_code)22 static noinstr void fred_bad_type(struct pt_regs *regs, unsigned long error_code)
23 {
24 irqentry_state_t irq_state = irqentry_nmi_enter(regs);
25
26 instrumentation_begin();
27
28 /* Panic on events from a high stack level */
29 if (regs->fred_cs.sl > 0) {
30 pr_emerg("PANIC: invalid or fatal FRED event; event type %u "
31 "vector %u error 0x%lx aux 0x%lx at %04x:%016lx\n",
32 regs->fred_ss.type, regs->fred_ss.vector, error_code,
33 fred_event_data(regs), regs->cs, regs->ip);
34 die("invalid or fatal FRED event", regs, error_code);
35 panic("invalid or fatal FRED event");
36 } else {
37 unsigned long flags = oops_begin();
38 int sig = SIGKILL;
39
40 pr_alert("BUG: invalid or fatal FRED event; event type %u "
41 "vector %u error 0x%lx aux 0x%lx at %04x:%016lx\n",
42 regs->fred_ss.type, regs->fred_ss.vector, error_code,
43 fred_event_data(regs), regs->cs, regs->ip);
44
45 if (__die("Invalid or fatal FRED event", regs, error_code))
46 sig = 0;
47
48 oops_end(flags, regs, sig);
49 }
50
51 instrumentation_end();
52 irqentry_nmi_exit(regs, irq_state);
53 }
54
fred_intx(struct pt_regs * regs)55 static noinstr void fred_intx(struct pt_regs *regs)
56 {
57 switch (regs->fred_ss.vector) {
58 /* Opcode 0xcd, 0x3, NOT INT3 (opcode 0xcc) */
59 case X86_TRAP_BP:
60 return exc_int3(regs);
61
62 /* Opcode 0xcd, 0x4, NOT INTO (opcode 0xce) */
63 case X86_TRAP_OF:
64 return exc_overflow(regs);
65
66 #ifdef CONFIG_IA32_EMULATION
67 /* INT80 */
68 case IA32_SYSCALL_VECTOR:
69 if (ia32_enabled())
70 return fred_int80_emulation(regs);
71 fallthrough;
72 #endif
73
74 default:
75 /*
76 * Reconstruct the #GP fault state that IDT delivery would produce.
77 * Clear the software event flag so ERETU with TF set does not trap
78 * before the resumed instruction. See prevent_single_step_upon_eretu().
79 */
80 regs->ip -= regs->fred_ss.insnlen;
81 regs->flags |= X86_EFLAGS_RF;
82 regs->fred_ss.swevent = 0;
83 return exc_general_protection(regs, (regs->fred_ss.vector << 3) | 2);
84 }
85 }
86
fred_other(struct pt_regs * regs)87 static __always_inline void fred_other(struct pt_regs *regs)
88 {
89 /* The compiler can fold these conditions into a single test */
90 if (likely(regs->fred_ss.vector == FRED_SYSCALL && regs->fred_ss.l)) {
91 regs->orig_ax = regs->ax;
92 regs->ax = -ENOSYS;
93 do_syscall_64(regs, regs->orig_ax);
94 return;
95 } else if (ia32_enabled() &&
96 likely(regs->fred_ss.vector == FRED_SYSENTER && !regs->fred_ss.l)) {
97 regs->orig_ax = regs->ax;
98 regs->ax = -ENOSYS;
99 do_fast_syscall_32(regs);
100 return;
101 } else {
102 exc_invalid_op(regs);
103 return;
104 }
105 }
106
107 #define SYSVEC(_vector, _function) [_vector - FIRST_SYSTEM_VECTOR] = fred_sysvec_##_function
108
109 static idtentry_t sysvec_table[NR_SYSTEM_VECTORS] __ro_after_init = {
110 SYSVEC(ERROR_APIC_VECTOR, error_interrupt),
111 SYSVEC(SPURIOUS_APIC_VECTOR, spurious_apic_interrupt),
112 SYSVEC(LOCAL_TIMER_VECTOR, apic_timer_interrupt),
113 SYSVEC(X86_PLATFORM_IPI_VECTOR, x86_platform_ipi),
114
115 SYSVEC(RESCHEDULE_VECTOR, reschedule_ipi),
116 SYSVEC(CALL_FUNCTION_SINGLE_VECTOR, call_function_single),
117 SYSVEC(CALL_FUNCTION_VECTOR, call_function),
118 SYSVEC(REBOOT_VECTOR, reboot),
119
120 SYSVEC(THRESHOLD_APIC_VECTOR, threshold),
121 SYSVEC(DEFERRED_ERROR_VECTOR, deferred_error),
122 SYSVEC(THERMAL_APIC_VECTOR, thermal),
123
124 SYSVEC(IRQ_WORK_VECTOR, irq_work),
125
126 SYSVEC(PERF_GUEST_MEDIATED_PMI_VECTOR, perf_guest_mediated_pmi_handler),
127 SYSVEC(POSTED_INTR_VECTOR, kvm_posted_intr_ipi),
128 SYSVEC(POSTED_INTR_WAKEUP_VECTOR, kvm_posted_intr_wakeup_ipi),
129 SYSVEC(POSTED_INTR_NESTED_VECTOR, kvm_posted_intr_nested_ipi),
130
131 SYSVEC(POSTED_MSI_NOTIFICATION_VECTOR, posted_msi_notification),
132 };
133
134 static bool fred_setup_done __initdata;
135
fred_install_sysvec(unsigned int sysvec,idtentry_t handler)136 void __init fred_install_sysvec(unsigned int sysvec, idtentry_t handler)
137 {
138 if (WARN_ON_ONCE(sysvec < FIRST_SYSTEM_VECTOR))
139 return;
140
141 if (WARN_ON_ONCE(fred_setup_done))
142 return;
143
144 if (!WARN_ON_ONCE(sysvec_table[sysvec - FIRST_SYSTEM_VECTOR]))
145 sysvec_table[sysvec - FIRST_SYSTEM_VECTOR] = handler;
146 }
147
fred_handle_spurious_interrupt(struct pt_regs * regs)148 static noinstr void fred_handle_spurious_interrupt(struct pt_regs *regs)
149 {
150 spurious_interrupt(regs, regs->fred_ss.vector);
151 }
152
fred_complete_exception_setup(void)153 void __init fred_complete_exception_setup(void)
154 {
155 unsigned int vector;
156
157 for (vector = 0; vector < FIRST_EXTERNAL_VECTOR; vector++)
158 set_bit(vector, system_vectors);
159
160 for (vector = 0; vector < NR_SYSTEM_VECTORS; vector++) {
161 if (sysvec_table[vector])
162 set_bit(vector + FIRST_SYSTEM_VECTOR, system_vectors);
163 else
164 sysvec_table[vector] = fred_handle_spurious_interrupt;
165 }
166 fred_setup_done = true;
167 }
168
fred_extint(struct pt_regs * regs)169 static noinstr void fred_extint(struct pt_regs *regs)
170 {
171 unsigned int vector = regs->fred_ss.vector;
172
173 if (WARN_ON_ONCE(vector < FIRST_EXTERNAL_VECTOR))
174 return;
175
176 if (likely(vector >= FIRST_SYSTEM_VECTOR)) {
177 irqentry_state_t state = irqentry_enter(regs);
178
179 instrumentation_begin();
180 sysvec_table[array_index_nospec(vector - FIRST_SYSTEM_VECTOR,
181 NR_SYSTEM_VECTORS)](regs);
182 instrumentation_end();
183 irqentry_exit(regs, state);
184 } else {
185 common_interrupt(regs, vector);
186 }
187 }
188
fred_hwexc(struct pt_regs * regs,unsigned long error_code)189 static noinstr void fred_hwexc(struct pt_regs *regs, unsigned long error_code)
190 {
191 /* Optimize for #PF. That's the only exception which matters performance wise */
192 if (likely(regs->fred_ss.vector == X86_TRAP_PF))
193 return exc_page_fault(regs, error_code);
194
195 switch (regs->fred_ss.vector) {
196 case X86_TRAP_DE: return exc_divide_error(regs);
197 case X86_TRAP_DB: return fred_exc_debug(regs);
198 case X86_TRAP_BR: return exc_bounds(regs);
199 case X86_TRAP_UD: return exc_invalid_op(regs);
200 case X86_TRAP_NM: return exc_device_not_available(regs);
201 case X86_TRAP_DF: return exc_double_fault(regs, error_code);
202 case X86_TRAP_TS: return exc_invalid_tss(regs, error_code);
203 case X86_TRAP_NP: return exc_segment_not_present(regs, error_code);
204 case X86_TRAP_SS: return exc_stack_segment(regs, error_code);
205 case X86_TRAP_GP: return exc_general_protection(regs, error_code);
206 case X86_TRAP_MF: return exc_coprocessor_error(regs);
207 case X86_TRAP_AC: return exc_alignment_check(regs, error_code);
208 case X86_TRAP_XF: return exc_simd_coprocessor_error(regs);
209
210 #ifdef CONFIG_X86_MCE
211 case X86_TRAP_MC: return fred_exc_machine_check(regs);
212 #endif
213 #ifdef CONFIG_INTEL_TDX_GUEST
214 case X86_TRAP_VE: return exc_virtualization_exception(regs);
215 #endif
216 #ifdef CONFIG_X86_CET
217 case X86_TRAP_CP: return exc_control_protection(regs, error_code);
218 #endif
219 #ifdef CONFIG_AMD_MEM_ENCRYPT
220 case X86_TRAP_VC: return exc_vmm_communication(regs, error_code);
221 #endif
222
223 default: return fred_bad_type(regs, error_code);
224 }
225
226 }
227
fred_swexc(struct pt_regs * regs,unsigned long error_code)228 static noinstr void fred_swexc(struct pt_regs *regs, unsigned long error_code)
229 {
230 switch (regs->fred_ss.vector) {
231 case X86_TRAP_BP: return exc_int3(regs);
232 case X86_TRAP_OF: return exc_overflow(regs);
233 default: return fred_bad_type(regs, error_code);
234 }
235 }
236
fred_entry_from_user(struct pt_regs * regs)237 __visible noinstr void fred_entry_from_user(struct pt_regs *regs)
238 {
239 unsigned long error_code = regs->orig_ax;
240
241 /* Invalidate orig_ax so that syscall_get_nr() works correctly */
242 regs->orig_ax = -1;
243
244 switch (regs->fred_ss.type) {
245 case EVENT_TYPE_EXTINT:
246 return fred_extint(regs);
247 case EVENT_TYPE_NMI:
248 if (likely(regs->fred_ss.vector == X86_TRAP_NMI))
249 return fred_exc_nmi(regs);
250 break;
251 case EVENT_TYPE_HWEXC:
252 return fred_hwexc(regs, error_code);
253 case EVENT_TYPE_SWINT:
254 return fred_intx(regs);
255 case EVENT_TYPE_PRIV_SWEXC:
256 if (likely(regs->fred_ss.vector == X86_TRAP_DB))
257 return fred_exc_debug(regs);
258 break;
259 case EVENT_TYPE_SWEXC:
260 return fred_swexc(regs, error_code);
261 case EVENT_TYPE_OTHER:
262 return fred_other(regs);
263 default: break;
264 }
265
266 return fred_bad_type(regs, error_code);
267 }
268
fred_entry_from_kernel(struct pt_regs * regs)269 __visible noinstr void fred_entry_from_kernel(struct pt_regs *regs)
270 {
271 unsigned long error_code = regs->orig_ax;
272
273 /* Invalidate orig_ax so that syscall_get_nr() works correctly */
274 regs->orig_ax = -1;
275
276 switch (regs->fred_ss.type) {
277 case EVENT_TYPE_EXTINT:
278 return fred_extint(regs);
279 case EVENT_TYPE_NMI:
280 if (likely(regs->fred_ss.vector == X86_TRAP_NMI))
281 return fred_exc_nmi(regs);
282 break;
283 case EVENT_TYPE_HWEXC:
284 return fred_hwexc(regs, error_code);
285 case EVENT_TYPE_PRIV_SWEXC:
286 if (likely(regs->fred_ss.vector == X86_TRAP_DB))
287 return fred_exc_debug(regs);
288 break;
289 case EVENT_TYPE_SWEXC:
290 return fred_swexc(regs, error_code);
291 default: break;
292 }
293
294 return fred_bad_type(regs, error_code);
295 }
296
297 #if IS_ENABLED(CONFIG_KVM_INTEL)
__fred_entry_from_kvm(struct pt_regs * regs)298 __visible noinstr void __fred_entry_from_kvm(struct pt_regs *regs)
299 {
300 switch (regs->fred_ss.type) {
301 case EVENT_TYPE_EXTINT:
302 return fred_extint(regs);
303 case EVENT_TYPE_NMI:
304 return fred_exc_nmi(regs);
305 default:
306 WARN_ON_ONCE(1);
307 }
308 }
309 #endif
310