1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * The FRED specific kernel/user entry functions which are invoked from
4 * assembly code and dispatch to the associated handlers.
5 */
6 #include <linux/kernel.h>
7 #include <linux/kdebug.h>
8 #include <linux/nospec.h>
9
10 #include <asm/desc.h>
11 #include <asm/fred.h>
12 #include <asm/idtentry.h>
13 #include <asm/syscall.h>
14 #include <asm/trapnr.h>
15 #include <asm/traps.h>
16
17 /* FRED EVENT_TYPE_OTHER vector numbers */
18 #define FRED_SYSCALL 1
19 #define FRED_SYSENTER 2
20
fred_bad_type(struct pt_regs * regs,unsigned long error_code)21 static noinstr void fred_bad_type(struct pt_regs *regs, unsigned long error_code)
22 {
23 irqentry_state_t irq_state = irqentry_nmi_enter(regs);
24
25 instrumentation_begin();
26
27 /* Panic on events from a high stack level */
28 if (regs->fred_cs.sl > 0) {
29 pr_emerg("PANIC: invalid or fatal FRED event; event type %u "
30 "vector %u error 0x%lx aux 0x%lx at %04x:%016lx\n",
31 regs->fred_ss.type, regs->fred_ss.vector, error_code,
32 fred_event_data(regs), regs->cs, regs->ip);
33 die("invalid or fatal FRED event", regs, error_code);
34 panic("invalid or fatal FRED event");
35 } else {
36 unsigned long flags = oops_begin();
37 int sig = SIGKILL;
38
39 pr_alert("BUG: invalid or fatal FRED event; event type %u "
40 "vector %u error 0x%lx aux 0x%lx at %04x:%016lx\n",
41 regs->fred_ss.type, regs->fred_ss.vector, error_code,
42 fred_event_data(regs), regs->cs, regs->ip);
43
44 if (__die("Invalid or fatal FRED event", regs, error_code))
45 sig = 0;
46
47 oops_end(flags, regs, sig);
48 }
49
50 instrumentation_end();
51 irqentry_nmi_exit(regs, irq_state);
52 }
53
fred_intx(struct pt_regs * regs)54 static noinstr void fred_intx(struct pt_regs *regs)
55 {
56 switch (regs->fred_ss.vector) {
57 /* Opcode 0xcd, 0x3, NOT INT3 (opcode 0xcc) */
58 case X86_TRAP_BP:
59 return exc_int3(regs);
60
61 /* Opcode 0xcd, 0x4, NOT INTO (opcode 0xce) */
62 case X86_TRAP_OF:
63 return exc_overflow(regs);
64
65 #ifdef CONFIG_IA32_EMULATION
66 /* INT80 */
67 case IA32_SYSCALL_VECTOR:
68 if (ia32_enabled())
69 return fred_int80_emulation(regs);
70 fallthrough;
71 #endif
72
73 default:
74 return exc_general_protection(regs, 0);
75 }
76 }
77
fred_other(struct pt_regs * regs)78 static __always_inline void fred_other(struct pt_regs *regs)
79 {
80 /* The compiler can fold these conditions into a single test */
81 if (likely(regs->fred_ss.vector == FRED_SYSCALL && regs->fred_ss.l)) {
82 regs->orig_ax = regs->ax;
83 regs->ax = -ENOSYS;
84 do_syscall_64(regs, regs->orig_ax);
85 return;
86 } else if (ia32_enabled() &&
87 likely(regs->fred_ss.vector == FRED_SYSENTER && !regs->fred_ss.l)) {
88 regs->orig_ax = regs->ax;
89 regs->ax = -ENOSYS;
90 do_fast_syscall_32(regs);
91 return;
92 } else {
93 exc_invalid_op(regs);
94 return;
95 }
96 }
97
98 #define SYSVEC(_vector, _function) [_vector - FIRST_SYSTEM_VECTOR] = fred_sysvec_##_function
99
100 static idtentry_t sysvec_table[NR_SYSTEM_VECTORS] __ro_after_init = {
101 SYSVEC(ERROR_APIC_VECTOR, error_interrupt),
102 SYSVEC(SPURIOUS_APIC_VECTOR, spurious_apic_interrupt),
103 SYSVEC(LOCAL_TIMER_VECTOR, apic_timer_interrupt),
104 SYSVEC(X86_PLATFORM_IPI_VECTOR, x86_platform_ipi),
105
106 SYSVEC(RESCHEDULE_VECTOR, reschedule_ipi),
107 SYSVEC(CALL_FUNCTION_SINGLE_VECTOR, call_function_single),
108 SYSVEC(CALL_FUNCTION_VECTOR, call_function),
109 SYSVEC(REBOOT_VECTOR, reboot),
110
111 SYSVEC(THRESHOLD_APIC_VECTOR, threshold),
112 SYSVEC(DEFERRED_ERROR_VECTOR, deferred_error),
113 SYSVEC(THERMAL_APIC_VECTOR, thermal),
114
115 SYSVEC(IRQ_WORK_VECTOR, irq_work),
116
117 SYSVEC(PERF_GUEST_MEDIATED_PMI_VECTOR, perf_guest_mediated_pmi_handler),
118 SYSVEC(POSTED_INTR_VECTOR, kvm_posted_intr_ipi),
119 SYSVEC(POSTED_INTR_WAKEUP_VECTOR, kvm_posted_intr_wakeup_ipi),
120 SYSVEC(POSTED_INTR_NESTED_VECTOR, kvm_posted_intr_nested_ipi),
121
122 SYSVEC(POSTED_MSI_NOTIFICATION_VECTOR, posted_msi_notification),
123 };
124
125 static bool fred_setup_done __initdata;
126
fred_install_sysvec(unsigned int sysvec,idtentry_t handler)127 void __init fred_install_sysvec(unsigned int sysvec, idtentry_t handler)
128 {
129 if (WARN_ON_ONCE(sysvec < FIRST_SYSTEM_VECTOR))
130 return;
131
132 if (WARN_ON_ONCE(fred_setup_done))
133 return;
134
135 if (!WARN_ON_ONCE(sysvec_table[sysvec - FIRST_SYSTEM_VECTOR]))
136 sysvec_table[sysvec - FIRST_SYSTEM_VECTOR] = handler;
137 }
138
fred_handle_spurious_interrupt(struct pt_regs * regs)139 static noinstr void fred_handle_spurious_interrupt(struct pt_regs *regs)
140 {
141 spurious_interrupt(regs, regs->fred_ss.vector);
142 }
143
fred_complete_exception_setup(void)144 void __init fred_complete_exception_setup(void)
145 {
146 unsigned int vector;
147
148 for (vector = 0; vector < FIRST_EXTERNAL_VECTOR; vector++)
149 set_bit(vector, system_vectors);
150
151 for (vector = 0; vector < NR_SYSTEM_VECTORS; vector++) {
152 if (sysvec_table[vector])
153 set_bit(vector + FIRST_SYSTEM_VECTOR, system_vectors);
154 else
155 sysvec_table[vector] = fred_handle_spurious_interrupt;
156 }
157 fred_setup_done = true;
158 }
159
fred_extint(struct pt_regs * regs)160 static noinstr void fred_extint(struct pt_regs *regs)
161 {
162 unsigned int vector = regs->fred_ss.vector;
163
164 if (WARN_ON_ONCE(vector < FIRST_EXTERNAL_VECTOR))
165 return;
166
167 if (likely(vector >= FIRST_SYSTEM_VECTOR)) {
168 irqentry_state_t state = irqentry_enter(regs);
169
170 instrumentation_begin();
171 sysvec_table[array_index_nospec(vector - FIRST_SYSTEM_VECTOR,
172 NR_SYSTEM_VECTORS)](regs);
173 instrumentation_end();
174 irqentry_exit(regs, state);
175 } else {
176 common_interrupt(regs, vector);
177 }
178 }
179
180 #ifdef CONFIG_AMD_MEM_ENCRYPT
exc_vmm_communication(struct pt_regs * regs,unsigned long error_code)181 noinstr void exc_vmm_communication(struct pt_regs *regs, unsigned long error_code)
182 {
183 if (user_mode(regs))
184 return user_exc_vmm_communication(regs, error_code);
185 else
186 return kernel_exc_vmm_communication(regs, error_code);
187 }
188 #endif
189
fred_hwexc(struct pt_regs * regs,unsigned long error_code)190 static noinstr void fred_hwexc(struct pt_regs *regs, unsigned long error_code)
191 {
192 /* Optimize for #PF. That's the only exception which matters performance wise */
193 if (likely(regs->fred_ss.vector == X86_TRAP_PF))
194 return exc_page_fault(regs, error_code);
195
196 switch (regs->fred_ss.vector) {
197 case X86_TRAP_DE: return exc_divide_error(regs);
198 case X86_TRAP_DB: return fred_exc_debug(regs);
199 case X86_TRAP_BR: return exc_bounds(regs);
200 case X86_TRAP_UD: return exc_invalid_op(regs);
201 case X86_TRAP_NM: return exc_device_not_available(regs);
202 case X86_TRAP_DF: return exc_double_fault(regs, error_code);
203 case X86_TRAP_TS: return exc_invalid_tss(regs, error_code);
204 case X86_TRAP_NP: return exc_segment_not_present(regs, error_code);
205 case X86_TRAP_SS: return exc_stack_segment(regs, error_code);
206 case X86_TRAP_GP: return exc_general_protection(regs, error_code);
207 case X86_TRAP_MF: return exc_coprocessor_error(regs);
208 case X86_TRAP_AC: return exc_alignment_check(regs, error_code);
209 case X86_TRAP_XF: return exc_simd_coprocessor_error(regs);
210
211 #ifdef CONFIG_X86_MCE
212 case X86_TRAP_MC: return fred_exc_machine_check(regs);
213 #endif
214 #ifdef CONFIG_INTEL_TDX_GUEST
215 case X86_TRAP_VE: return exc_virtualization_exception(regs);
216 #endif
217 #ifdef CONFIG_X86_CET
218 case X86_TRAP_CP: return exc_control_protection(regs, error_code);
219 #endif
220 #ifdef CONFIG_AMD_MEM_ENCRYPT
221 case X86_TRAP_VC: return exc_vmm_communication(regs, error_code);
222 #endif
223
224 default: return fred_bad_type(regs, error_code);
225 }
226
227 }
228
fred_swexc(struct pt_regs * regs,unsigned long error_code)229 static noinstr void fred_swexc(struct pt_regs *regs, unsigned long error_code)
230 {
231 switch (regs->fred_ss.vector) {
232 case X86_TRAP_BP: return exc_int3(regs);
233 case X86_TRAP_OF: return exc_overflow(regs);
234 default: return fred_bad_type(regs, error_code);
235 }
236 }
237
fred_entry_from_user(struct pt_regs * regs)238 __visible noinstr void fred_entry_from_user(struct pt_regs *regs)
239 {
240 unsigned long error_code = regs->orig_ax;
241
242 /* Invalidate orig_ax so that syscall_get_nr() works correctly */
243 regs->orig_ax = -1;
244
245 switch (regs->fred_ss.type) {
246 case EVENT_TYPE_EXTINT:
247 return fred_extint(regs);
248 case EVENT_TYPE_NMI:
249 if (likely(regs->fred_ss.vector == X86_TRAP_NMI))
250 return fred_exc_nmi(regs);
251 break;
252 case EVENT_TYPE_HWEXC:
253 return fred_hwexc(regs, error_code);
254 case EVENT_TYPE_SWINT:
255 return fred_intx(regs);
256 case EVENT_TYPE_PRIV_SWEXC:
257 if (likely(regs->fred_ss.vector == X86_TRAP_DB))
258 return fred_exc_debug(regs);
259 break;
260 case EVENT_TYPE_SWEXC:
261 return fred_swexc(regs, error_code);
262 case EVENT_TYPE_OTHER:
263 return fred_other(regs);
264 default: break;
265 }
266
267 return fred_bad_type(regs, error_code);
268 }
269
fred_entry_from_kernel(struct pt_regs * regs)270 __visible noinstr void fred_entry_from_kernel(struct pt_regs *regs)
271 {
272 unsigned long error_code = regs->orig_ax;
273
274 /* Invalidate orig_ax so that syscall_get_nr() works correctly */
275 regs->orig_ax = -1;
276
277 switch (regs->fred_ss.type) {
278 case EVENT_TYPE_EXTINT:
279 return fred_extint(regs);
280 case EVENT_TYPE_NMI:
281 if (likely(regs->fred_ss.vector == X86_TRAP_NMI))
282 return fred_exc_nmi(regs);
283 break;
284 case EVENT_TYPE_HWEXC:
285 return fred_hwexc(regs, error_code);
286 case EVENT_TYPE_PRIV_SWEXC:
287 if (likely(regs->fred_ss.vector == X86_TRAP_DB))
288 return fred_exc_debug(regs);
289 break;
290 case EVENT_TYPE_SWEXC:
291 return fred_swexc(regs, error_code);
292 default: break;
293 }
294
295 return fred_bad_type(regs, error_code);
296 }
297
298 #if IS_ENABLED(CONFIG_KVM_INTEL)
__fred_entry_from_kvm(struct pt_regs * regs)299 __visible noinstr void __fred_entry_from_kvm(struct pt_regs *regs)
300 {
301 switch (regs->fred_ss.type) {
302 case EVENT_TYPE_EXTINT:
303 return fred_extint(regs);
304 case EVENT_TYPE_NMI:
305 return fred_exc_nmi(regs);
306 default:
307 WARN_ON_ONCE(1);
308 }
309 }
310 #endif
311