1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * The FRED specific kernel/user entry functions which are invoked from 4 * assembly code and dispatch to the associated handlers. 5 */ 6 #include <linux/kernel.h> 7 #include <linux/kdebug.h> 8 #include <linux/nospec.h> 9 10 #include <asm/desc.h> 11 #include <asm/fred.h> 12 #include <asm/idtentry.h> 13 #include <asm/processor-flags.h> 14 #include <asm/syscall.h> 15 #include <asm/trapnr.h> 16 #include <asm/traps.h> 17 18 /* FRED EVENT_TYPE_OTHER vector numbers */ 19 #define FRED_SYSCALL 1 20 #define FRED_SYSENTER 2 21 22 static noinstr void fred_bad_type(struct pt_regs *regs, unsigned long error_code) 23 { 24 irqentry_state_t irq_state = irqentry_nmi_enter(regs); 25 26 instrumentation_begin(); 27 28 /* Panic on events from a high stack level */ 29 if (regs->fred_cs.sl > 0) { 30 pr_emerg("PANIC: invalid or fatal FRED event; event type %u " 31 "vector %u error 0x%lx aux 0x%lx at %04x:%016lx\n", 32 regs->fred_ss.type, regs->fred_ss.vector, error_code, 33 fred_event_data(regs), regs->cs, regs->ip); 34 die("invalid or fatal FRED event", regs, error_code); 35 panic("invalid or fatal FRED event"); 36 } else { 37 unsigned long flags = oops_begin(); 38 int sig = SIGKILL; 39 40 pr_alert("BUG: invalid or fatal FRED event; event type %u " 41 "vector %u error 0x%lx aux 0x%lx at %04x:%016lx\n", 42 regs->fred_ss.type, regs->fred_ss.vector, error_code, 43 fred_event_data(regs), regs->cs, regs->ip); 44 45 if (__die("Invalid or fatal FRED event", regs, error_code)) 46 sig = 0; 47 48 oops_end(flags, regs, sig); 49 } 50 51 instrumentation_end(); 52 irqentry_nmi_exit(regs, irq_state); 53 } 54 55 static noinstr void fred_intx(struct pt_regs *regs) 56 { 57 switch (regs->fred_ss.vector) { 58 /* Opcode 0xcd, 0x3, NOT INT3 (opcode 0xcc) */ 59 case X86_TRAP_BP: 60 return exc_int3(regs); 61 62 /* Opcode 0xcd, 0x4, NOT INTO (opcode 0xce) */ 63 case X86_TRAP_OF: 64 return exc_overflow(regs); 65 66 #ifdef CONFIG_IA32_EMULATION 67 /* INT80 */ 68 case IA32_SYSCALL_VECTOR: 69 if (ia32_enabled()) 70 return fred_int80_emulation(regs); 71 fallthrough; 72 #endif 73 74 default: 75 /* 76 * Reconstruct the #GP fault state that IDT delivery would produce. 77 * Clear the software event flag so ERETU with TF set does not trap 78 * before the resumed instruction. See prevent_single_step_upon_eretu(). 79 */ 80 regs->ip -= regs->fred_ss.insnlen; 81 regs->flags |= X86_EFLAGS_RF; 82 regs->fred_ss.swevent = 0; 83 return exc_general_protection(regs, (regs->fred_ss.vector << 3) | 2); 84 } 85 } 86 87 static __always_inline void fred_other(struct pt_regs *regs) 88 { 89 /* The compiler can fold these conditions into a single test */ 90 if (likely(regs->fred_ss.vector == FRED_SYSCALL && regs->fred_ss.l)) { 91 regs->orig_ax = regs->ax; 92 regs->ax = -ENOSYS; 93 do_syscall_64(regs, regs->orig_ax); 94 return; 95 } else if (ia32_enabled() && 96 likely(regs->fred_ss.vector == FRED_SYSENTER && !regs->fred_ss.l)) { 97 regs->orig_ax = regs->ax; 98 regs->ax = -ENOSYS; 99 do_fast_syscall_32(regs); 100 return; 101 } else { 102 exc_invalid_op(regs); 103 return; 104 } 105 } 106 107 #define SYSVEC(_vector, _function) [_vector - FIRST_SYSTEM_VECTOR] = fred_sysvec_##_function 108 109 static idtentry_t sysvec_table[NR_SYSTEM_VECTORS] __ro_after_init = { 110 SYSVEC(ERROR_APIC_VECTOR, error_interrupt), 111 SYSVEC(SPURIOUS_APIC_VECTOR, spurious_apic_interrupt), 112 SYSVEC(LOCAL_TIMER_VECTOR, apic_timer_interrupt), 113 SYSVEC(X86_PLATFORM_IPI_VECTOR, x86_platform_ipi), 114 115 SYSVEC(RESCHEDULE_VECTOR, reschedule_ipi), 116 SYSVEC(CALL_FUNCTION_SINGLE_VECTOR, call_function_single), 117 SYSVEC(CALL_FUNCTION_VECTOR, call_function), 118 SYSVEC(REBOOT_VECTOR, reboot), 119 120 SYSVEC(THRESHOLD_APIC_VECTOR, threshold), 121 SYSVEC(DEFERRED_ERROR_VECTOR, deferred_error), 122 SYSVEC(THERMAL_APIC_VECTOR, thermal), 123 124 SYSVEC(IRQ_WORK_VECTOR, irq_work), 125 126 SYSVEC(PERF_GUEST_MEDIATED_PMI_VECTOR, perf_guest_mediated_pmi_handler), 127 SYSVEC(POSTED_INTR_VECTOR, kvm_posted_intr_ipi), 128 SYSVEC(POSTED_INTR_WAKEUP_VECTOR, kvm_posted_intr_wakeup_ipi), 129 SYSVEC(POSTED_INTR_NESTED_VECTOR, kvm_posted_intr_nested_ipi), 130 131 SYSVEC(POSTED_MSI_NOTIFICATION_VECTOR, posted_msi_notification), 132 }; 133 134 static bool fred_setup_done __initdata; 135 136 void __init fred_install_sysvec(unsigned int sysvec, idtentry_t handler) 137 { 138 if (WARN_ON_ONCE(sysvec < FIRST_SYSTEM_VECTOR)) 139 return; 140 141 if (WARN_ON_ONCE(fred_setup_done)) 142 return; 143 144 if (!WARN_ON_ONCE(sysvec_table[sysvec - FIRST_SYSTEM_VECTOR])) 145 sysvec_table[sysvec - FIRST_SYSTEM_VECTOR] = handler; 146 } 147 148 static noinstr void fred_handle_spurious_interrupt(struct pt_regs *regs) 149 { 150 spurious_interrupt(regs, regs->fred_ss.vector); 151 } 152 153 void __init fred_complete_exception_setup(void) 154 { 155 unsigned int vector; 156 157 for (vector = 0; vector < FIRST_EXTERNAL_VECTOR; vector++) 158 set_bit(vector, system_vectors); 159 160 for (vector = 0; vector < NR_SYSTEM_VECTORS; vector++) { 161 if (sysvec_table[vector]) 162 set_bit(vector + FIRST_SYSTEM_VECTOR, system_vectors); 163 else 164 sysvec_table[vector] = fred_handle_spurious_interrupt; 165 } 166 fred_setup_done = true; 167 } 168 169 static noinstr void fred_extint(struct pt_regs *regs) 170 { 171 unsigned int vector = regs->fred_ss.vector; 172 173 if (WARN_ON_ONCE(vector < FIRST_EXTERNAL_VECTOR)) 174 return; 175 176 if (likely(vector >= FIRST_SYSTEM_VECTOR)) { 177 irqentry_state_t state = irqentry_enter(regs); 178 179 instrumentation_begin(); 180 sysvec_table[array_index_nospec(vector - FIRST_SYSTEM_VECTOR, 181 NR_SYSTEM_VECTORS)](regs); 182 instrumentation_end(); 183 irqentry_exit(regs, state); 184 } else { 185 common_interrupt(regs, vector); 186 } 187 } 188 189 static noinstr void fred_hwexc(struct pt_regs *regs, unsigned long error_code) 190 { 191 /* Optimize for #PF. That's the only exception which matters performance wise */ 192 if (likely(regs->fred_ss.vector == X86_TRAP_PF)) 193 return exc_page_fault(regs, error_code); 194 195 switch (regs->fred_ss.vector) { 196 case X86_TRAP_DE: return exc_divide_error(regs); 197 case X86_TRAP_DB: return fred_exc_debug(regs); 198 case X86_TRAP_BR: return exc_bounds(regs); 199 case X86_TRAP_UD: return exc_invalid_op(regs); 200 case X86_TRAP_NM: return exc_device_not_available(regs); 201 case X86_TRAP_DF: return exc_double_fault(regs, error_code); 202 case X86_TRAP_TS: return exc_invalid_tss(regs, error_code); 203 case X86_TRAP_NP: return exc_segment_not_present(regs, error_code); 204 case X86_TRAP_SS: return exc_stack_segment(regs, error_code); 205 case X86_TRAP_GP: return exc_general_protection(regs, error_code); 206 case X86_TRAP_MF: return exc_coprocessor_error(regs); 207 case X86_TRAP_AC: return exc_alignment_check(regs, error_code); 208 case X86_TRAP_XF: return exc_simd_coprocessor_error(regs); 209 210 #ifdef CONFIG_X86_MCE 211 case X86_TRAP_MC: return fred_exc_machine_check(regs); 212 #endif 213 #ifdef CONFIG_INTEL_TDX_GUEST 214 case X86_TRAP_VE: return exc_virtualization_exception(regs); 215 #endif 216 #ifdef CONFIG_X86_CET 217 case X86_TRAP_CP: return exc_control_protection(regs, error_code); 218 #endif 219 #ifdef CONFIG_AMD_MEM_ENCRYPT 220 case X86_TRAP_VC: return exc_vmm_communication(regs, error_code); 221 #endif 222 223 default: return fred_bad_type(regs, error_code); 224 } 225 226 } 227 228 static noinstr void fred_swexc(struct pt_regs *regs, unsigned long error_code) 229 { 230 switch (regs->fred_ss.vector) { 231 case X86_TRAP_BP: return exc_int3(regs); 232 case X86_TRAP_OF: return exc_overflow(regs); 233 default: return fred_bad_type(regs, error_code); 234 } 235 } 236 237 __visible noinstr void fred_entry_from_user(struct pt_regs *regs) 238 { 239 unsigned long error_code = regs->orig_ax; 240 241 /* Invalidate orig_ax so that syscall_get_nr() works correctly */ 242 regs->orig_ax = -1; 243 244 switch (regs->fred_ss.type) { 245 case EVENT_TYPE_EXTINT: 246 return fred_extint(regs); 247 case EVENT_TYPE_NMI: 248 if (likely(regs->fred_ss.vector == X86_TRAP_NMI)) 249 return fred_exc_nmi(regs); 250 break; 251 case EVENT_TYPE_HWEXC: 252 return fred_hwexc(regs, error_code); 253 case EVENT_TYPE_SWINT: 254 return fred_intx(regs); 255 case EVENT_TYPE_PRIV_SWEXC: 256 if (likely(regs->fred_ss.vector == X86_TRAP_DB)) 257 return fred_exc_debug(regs); 258 break; 259 case EVENT_TYPE_SWEXC: 260 return fred_swexc(regs, error_code); 261 case EVENT_TYPE_OTHER: 262 return fred_other(regs); 263 default: break; 264 } 265 266 return fred_bad_type(regs, error_code); 267 } 268 269 __visible noinstr void fred_entry_from_kernel(struct pt_regs *regs) 270 { 271 unsigned long error_code = regs->orig_ax; 272 273 /* Invalidate orig_ax so that syscall_get_nr() works correctly */ 274 regs->orig_ax = -1; 275 276 switch (regs->fred_ss.type) { 277 case EVENT_TYPE_EXTINT: 278 return fred_extint(regs); 279 case EVENT_TYPE_NMI: 280 if (likely(regs->fred_ss.vector == X86_TRAP_NMI)) 281 return fred_exc_nmi(regs); 282 break; 283 case EVENT_TYPE_HWEXC: 284 return fred_hwexc(regs, error_code); 285 case EVENT_TYPE_PRIV_SWEXC: 286 if (likely(regs->fred_ss.vector == X86_TRAP_DB)) 287 return fred_exc_debug(regs); 288 break; 289 case EVENT_TYPE_SWEXC: 290 return fred_swexc(regs, error_code); 291 default: break; 292 } 293 294 return fred_bad_type(regs, error_code); 295 } 296 297 #if IS_ENABLED(CONFIG_KVM_INTEL) 298 __visible noinstr void __fred_entry_from_kvm(struct pt_regs *regs) 299 { 300 switch (regs->fred_ss.type) { 301 case EVENT_TYPE_EXTINT: 302 return fred_extint(regs); 303 case EVENT_TYPE_NMI: 304 return fred_exc_nmi(regs); 305 default: 306 WARN_ON_ONCE(1); 307 } 308 } 309 #endif 310