1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * The FRED specific kernel/user entry functions which are invoked from 4 * assembly code and dispatch to the associated handlers. 5 */ 6 #include <linux/kernel.h> 7 #include <linux/kdebug.h> 8 #include <linux/nospec.h> 9 10 #include <asm/desc.h> 11 #include <asm/fred.h> 12 #include <asm/idtentry.h> 13 #include <asm/syscall.h> 14 #include <asm/trapnr.h> 15 #include <asm/traps.h> 16 17 /* FRED EVENT_TYPE_OTHER vector numbers */ 18 #define FRED_SYSCALL 1 19 #define FRED_SYSENTER 2 20 21 static noinstr void fred_bad_type(struct pt_regs *regs, unsigned long error_code) 22 { 23 irqentry_state_t irq_state = irqentry_nmi_enter(regs); 24 25 instrumentation_begin(); 26 27 /* Panic on events from a high stack level */ 28 if (regs->fred_cs.sl > 0) { 29 pr_emerg("PANIC: invalid or fatal FRED event; event type %u " 30 "vector %u error 0x%lx aux 0x%lx at %04x:%016lx\n", 31 regs->fred_ss.type, regs->fred_ss.vector, error_code, 32 fred_event_data(regs), regs->cs, regs->ip); 33 die("invalid or fatal FRED event", regs, error_code); 34 panic("invalid or fatal FRED event"); 35 } else { 36 unsigned long flags = oops_begin(); 37 int sig = SIGKILL; 38 39 pr_alert("BUG: invalid or fatal FRED event; event type %u " 40 "vector %u error 0x%lx aux 0x%lx at %04x:%016lx\n", 41 regs->fred_ss.type, regs->fred_ss.vector, error_code, 42 fred_event_data(regs), regs->cs, regs->ip); 43 44 if (__die("Invalid or fatal FRED event", regs, error_code)) 45 sig = 0; 46 47 oops_end(flags, regs, sig); 48 } 49 50 instrumentation_end(); 51 irqentry_nmi_exit(regs, irq_state); 52 } 53 54 static noinstr void fred_intx(struct pt_regs *regs) 55 { 56 switch (regs->fred_ss.vector) { 57 /* Opcode 0xcd, 0x3, NOT INT3 (opcode 0xcc) */ 58 case X86_TRAP_BP: 59 return exc_int3(regs); 60 61 /* Opcode 0xcd, 0x4, NOT INTO (opcode 0xce) */ 62 case X86_TRAP_OF: 63 return exc_overflow(regs); 64 65 #ifdef CONFIG_IA32_EMULATION 66 /* INT80 */ 67 case IA32_SYSCALL_VECTOR: 68 if (ia32_enabled()) 69 return fred_int80_emulation(regs); 70 fallthrough; 71 #endif 72 73 default: 74 return exc_general_protection(regs, 0); 75 } 76 } 77 78 static __always_inline void fred_other(struct pt_regs *regs) 79 { 80 /* The compiler can fold these conditions into a single test */ 81 if (likely(regs->fred_ss.vector == FRED_SYSCALL && regs->fred_ss.lm)) { 82 regs->orig_ax = regs->ax; 83 regs->ax = -ENOSYS; 84 do_syscall_64(regs, regs->orig_ax); 85 return; 86 } else if (ia32_enabled() && 87 likely(regs->fred_ss.vector == FRED_SYSENTER && !regs->fred_ss.lm)) { 88 regs->orig_ax = regs->ax; 89 regs->ax = -ENOSYS; 90 do_fast_syscall_32(regs); 91 return; 92 } else { 93 exc_invalid_op(regs); 94 return; 95 } 96 } 97 98 #define SYSVEC(_vector, _function) [_vector - FIRST_SYSTEM_VECTOR] = fred_sysvec_##_function 99 100 static idtentry_t sysvec_table[NR_SYSTEM_VECTORS] __ro_after_init = { 101 SYSVEC(ERROR_APIC_VECTOR, error_interrupt), 102 SYSVEC(SPURIOUS_APIC_VECTOR, spurious_apic_interrupt), 103 SYSVEC(LOCAL_TIMER_VECTOR, apic_timer_interrupt), 104 SYSVEC(X86_PLATFORM_IPI_VECTOR, x86_platform_ipi), 105 106 SYSVEC(RESCHEDULE_VECTOR, reschedule_ipi), 107 SYSVEC(CALL_FUNCTION_SINGLE_VECTOR, call_function_single), 108 SYSVEC(CALL_FUNCTION_VECTOR, call_function), 109 SYSVEC(REBOOT_VECTOR, reboot), 110 111 SYSVEC(THRESHOLD_APIC_VECTOR, threshold), 112 SYSVEC(DEFERRED_ERROR_VECTOR, deferred_error), 113 SYSVEC(THERMAL_APIC_VECTOR, thermal), 114 115 SYSVEC(IRQ_WORK_VECTOR, irq_work), 116 117 SYSVEC(POSTED_INTR_VECTOR, kvm_posted_intr_ipi), 118 SYSVEC(POSTED_INTR_WAKEUP_VECTOR, kvm_posted_intr_wakeup_ipi), 119 SYSVEC(POSTED_INTR_NESTED_VECTOR, kvm_posted_intr_nested_ipi), 120 121 SYSVEC(POSTED_MSI_NOTIFICATION_VECTOR, posted_msi_notification), 122 }; 123 124 static bool fred_setup_done __initdata; 125 126 void __init fred_install_sysvec(unsigned int sysvec, idtentry_t handler) 127 { 128 if (WARN_ON_ONCE(sysvec < FIRST_SYSTEM_VECTOR)) 129 return; 130 131 if (WARN_ON_ONCE(fred_setup_done)) 132 return; 133 134 if (!WARN_ON_ONCE(sysvec_table[sysvec - FIRST_SYSTEM_VECTOR])) 135 sysvec_table[sysvec - FIRST_SYSTEM_VECTOR] = handler; 136 } 137 138 static noinstr void fred_handle_spurious_interrupt(struct pt_regs *regs) 139 { 140 spurious_interrupt(regs, regs->fred_ss.vector); 141 } 142 143 void __init fred_complete_exception_setup(void) 144 { 145 unsigned int vector; 146 147 for (vector = 0; vector < FIRST_EXTERNAL_VECTOR; vector++) 148 set_bit(vector, system_vectors); 149 150 for (vector = 0; vector < NR_SYSTEM_VECTORS; vector++) { 151 if (sysvec_table[vector]) 152 set_bit(vector + FIRST_SYSTEM_VECTOR, system_vectors); 153 else 154 sysvec_table[vector] = fred_handle_spurious_interrupt; 155 } 156 fred_setup_done = true; 157 } 158 159 static noinstr void fred_extint(struct pt_regs *regs) 160 { 161 unsigned int vector = regs->fred_ss.vector; 162 unsigned int index = array_index_nospec(vector - FIRST_SYSTEM_VECTOR, 163 NR_SYSTEM_VECTORS); 164 165 if (WARN_ON_ONCE(vector < FIRST_EXTERNAL_VECTOR)) 166 return; 167 168 if (likely(vector >= FIRST_SYSTEM_VECTOR)) { 169 irqentry_state_t state = irqentry_enter(regs); 170 171 instrumentation_begin(); 172 sysvec_table[index](regs); 173 instrumentation_end(); 174 irqentry_exit(regs, state); 175 } else { 176 common_interrupt(regs, vector); 177 } 178 } 179 180 static noinstr void fred_hwexc(struct pt_regs *regs, unsigned long error_code) 181 { 182 /* Optimize for #PF. That's the only exception which matters performance wise */ 183 if (likely(regs->fred_ss.vector == X86_TRAP_PF)) 184 return exc_page_fault(regs, error_code); 185 186 switch (regs->fred_ss.vector) { 187 case X86_TRAP_DE: return exc_divide_error(regs); 188 case X86_TRAP_DB: return fred_exc_debug(regs); 189 case X86_TRAP_BR: return exc_bounds(regs); 190 case X86_TRAP_UD: return exc_invalid_op(regs); 191 case X86_TRAP_NM: return exc_device_not_available(regs); 192 case X86_TRAP_DF: return exc_double_fault(regs, error_code); 193 case X86_TRAP_TS: return exc_invalid_tss(regs, error_code); 194 case X86_TRAP_NP: return exc_segment_not_present(regs, error_code); 195 case X86_TRAP_SS: return exc_stack_segment(regs, error_code); 196 case X86_TRAP_GP: return exc_general_protection(regs, error_code); 197 case X86_TRAP_MF: return exc_coprocessor_error(regs); 198 case X86_TRAP_AC: return exc_alignment_check(regs, error_code); 199 case X86_TRAP_XF: return exc_simd_coprocessor_error(regs); 200 201 #ifdef CONFIG_X86_MCE 202 case X86_TRAP_MC: return fred_exc_machine_check(regs); 203 #endif 204 #ifdef CONFIG_INTEL_TDX_GUEST 205 case X86_TRAP_VE: return exc_virtualization_exception(regs); 206 #endif 207 #ifdef CONFIG_X86_CET 208 case X86_TRAP_CP: return exc_control_protection(regs, error_code); 209 #endif 210 default: return fred_bad_type(regs, error_code); 211 } 212 213 } 214 215 static noinstr void fred_swexc(struct pt_regs *regs, unsigned long error_code) 216 { 217 switch (regs->fred_ss.vector) { 218 case X86_TRAP_BP: return exc_int3(regs); 219 case X86_TRAP_OF: return exc_overflow(regs); 220 default: return fred_bad_type(regs, error_code); 221 } 222 } 223 224 __visible noinstr void fred_entry_from_user(struct pt_regs *regs) 225 { 226 unsigned long error_code = regs->orig_ax; 227 228 /* Invalidate orig_ax so that syscall_get_nr() works correctly */ 229 regs->orig_ax = -1; 230 231 switch (regs->fred_ss.type) { 232 case EVENT_TYPE_EXTINT: 233 return fred_extint(regs); 234 case EVENT_TYPE_NMI: 235 if (likely(regs->fred_ss.vector == X86_TRAP_NMI)) 236 return fred_exc_nmi(regs); 237 break; 238 case EVENT_TYPE_HWEXC: 239 return fred_hwexc(regs, error_code); 240 case EVENT_TYPE_SWINT: 241 return fred_intx(regs); 242 case EVENT_TYPE_PRIV_SWEXC: 243 if (likely(regs->fred_ss.vector == X86_TRAP_DB)) 244 return fred_exc_debug(regs); 245 break; 246 case EVENT_TYPE_SWEXC: 247 return fred_swexc(regs, error_code); 248 case EVENT_TYPE_OTHER: 249 return fred_other(regs); 250 default: break; 251 } 252 253 return fred_bad_type(regs, error_code); 254 } 255 256 __visible noinstr void fred_entry_from_kernel(struct pt_regs *regs) 257 { 258 unsigned long error_code = regs->orig_ax; 259 260 /* Invalidate orig_ax so that syscall_get_nr() works correctly */ 261 regs->orig_ax = -1; 262 263 switch (regs->fred_ss.type) { 264 case EVENT_TYPE_EXTINT: 265 return fred_extint(regs); 266 case EVENT_TYPE_NMI: 267 if (likely(regs->fred_ss.vector == X86_TRAP_NMI)) 268 return fred_exc_nmi(regs); 269 break; 270 case EVENT_TYPE_HWEXC: 271 return fred_hwexc(regs, error_code); 272 case EVENT_TYPE_PRIV_SWEXC: 273 if (likely(regs->fred_ss.vector == X86_TRAP_DB)) 274 return fred_exc_debug(regs); 275 break; 276 case EVENT_TYPE_SWEXC: 277 return fred_swexc(regs, error_code); 278 default: break; 279 } 280 281 return fred_bad_type(regs, error_code); 282 } 283 284 #if IS_ENABLED(CONFIG_KVM_INTEL) 285 __visible noinstr void __fred_entry_from_kvm(struct pt_regs *regs) 286 { 287 switch (regs->fred_ss.type) { 288 case EVENT_TYPE_EXTINT: 289 return fred_extint(regs); 290 case EVENT_TYPE_NMI: 291 return fred_exc_nmi(regs); 292 default: 293 WARN_ON_ONCE(1); 294 } 295 } 296 #endif 297