1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * The FRED specific kernel/user entry functions which are invoked from 4 * assembly code and dispatch to the associated handlers. 5 */ 6 #include <linux/kernel.h> 7 #include <linux/kdebug.h> 8 #include <linux/nospec.h> 9 10 #include <asm/desc.h> 11 #include <asm/fred.h> 12 #include <asm/idtentry.h> 13 #include <asm/syscall.h> 14 #include <asm/trapnr.h> 15 #include <asm/traps.h> 16 17 /* FRED EVENT_TYPE_OTHER vector numbers */ 18 #define FRED_SYSCALL 1 19 #define FRED_SYSENTER 2 20 21 static noinstr void fred_bad_type(struct pt_regs *regs, unsigned long error_code) 22 { 23 irqentry_state_t irq_state = irqentry_nmi_enter(regs); 24 25 instrumentation_begin(); 26 27 /* Panic on events from a high stack level */ 28 if (regs->fred_cs.sl > 0) { 29 pr_emerg("PANIC: invalid or fatal FRED event; event type %u " 30 "vector %u error 0x%lx aux 0x%lx at %04x:%016lx\n", 31 regs->fred_ss.type, regs->fred_ss.vector, error_code, 32 fred_event_data(regs), regs->cs, regs->ip); 33 die("invalid or fatal FRED event", regs, error_code); 34 panic("invalid or fatal FRED event"); 35 } else { 36 unsigned long flags = oops_begin(); 37 int sig = SIGKILL; 38 39 pr_alert("BUG: invalid or fatal FRED event; event type %u " 40 "vector %u error 0x%lx aux 0x%lx at %04x:%016lx\n", 41 regs->fred_ss.type, regs->fred_ss.vector, error_code, 42 fred_event_data(regs), regs->cs, regs->ip); 43 44 if (__die("Invalid or fatal FRED event", regs, error_code)) 45 sig = 0; 46 47 oops_end(flags, regs, sig); 48 } 49 50 instrumentation_end(); 51 irqentry_nmi_exit(regs, irq_state); 52 } 53 54 static noinstr void fred_intx(struct pt_regs *regs) 55 { 56 switch (regs->fred_ss.vector) { 57 /* Opcode 0xcd, 0x3, NOT INT3 (opcode 0xcc) */ 58 case X86_TRAP_BP: 59 return exc_int3(regs); 60 61 /* Opcode 0xcd, 0x4, NOT INTO (opcode 0xce) */ 62 case X86_TRAP_OF: 63 return exc_overflow(regs); 64 65 #ifdef CONFIG_IA32_EMULATION 66 /* INT80 */ 67 case IA32_SYSCALL_VECTOR: 68 if (ia32_enabled()) 69 return fred_int80_emulation(regs); 70 fallthrough; 71 #endif 72 73 default: 74 return exc_general_protection(regs, 0); 75 } 76 } 77 78 static __always_inline void fred_other(struct pt_regs *regs) 79 { 80 /* The compiler can fold these conditions into a single test */ 81 if (likely(regs->fred_ss.vector == FRED_SYSCALL && regs->fred_ss.l)) { 82 regs->orig_ax = regs->ax; 83 regs->ax = -ENOSYS; 84 do_syscall_64(regs, regs->orig_ax); 85 return; 86 } else if (ia32_enabled() && 87 likely(regs->fred_ss.vector == FRED_SYSENTER && !regs->fred_ss.l)) { 88 regs->orig_ax = regs->ax; 89 regs->ax = -ENOSYS; 90 do_fast_syscall_32(regs); 91 return; 92 } else { 93 exc_invalid_op(regs); 94 return; 95 } 96 } 97 98 #define SYSVEC(_vector, _function) [_vector - FIRST_SYSTEM_VECTOR] = fred_sysvec_##_function 99 100 static idtentry_t sysvec_table[NR_SYSTEM_VECTORS] __ro_after_init = { 101 SYSVEC(ERROR_APIC_VECTOR, error_interrupt), 102 SYSVEC(SPURIOUS_APIC_VECTOR, spurious_apic_interrupt), 103 SYSVEC(LOCAL_TIMER_VECTOR, apic_timer_interrupt), 104 SYSVEC(X86_PLATFORM_IPI_VECTOR, x86_platform_ipi), 105 106 SYSVEC(RESCHEDULE_VECTOR, reschedule_ipi), 107 SYSVEC(CALL_FUNCTION_SINGLE_VECTOR, call_function_single), 108 SYSVEC(CALL_FUNCTION_VECTOR, call_function), 109 SYSVEC(REBOOT_VECTOR, reboot), 110 111 SYSVEC(THRESHOLD_APIC_VECTOR, threshold), 112 SYSVEC(DEFERRED_ERROR_VECTOR, deferred_error), 113 SYSVEC(THERMAL_APIC_VECTOR, thermal), 114 115 SYSVEC(IRQ_WORK_VECTOR, irq_work), 116 117 SYSVEC(PERF_GUEST_MEDIATED_PMI_VECTOR, perf_guest_mediated_pmi_handler), 118 SYSVEC(POSTED_INTR_VECTOR, kvm_posted_intr_ipi), 119 SYSVEC(POSTED_INTR_WAKEUP_VECTOR, kvm_posted_intr_wakeup_ipi), 120 SYSVEC(POSTED_INTR_NESTED_VECTOR, kvm_posted_intr_nested_ipi), 121 122 SYSVEC(POSTED_MSI_NOTIFICATION_VECTOR, posted_msi_notification), 123 }; 124 125 static bool fred_setup_done __initdata; 126 127 void __init fred_install_sysvec(unsigned int sysvec, idtentry_t handler) 128 { 129 if (WARN_ON_ONCE(sysvec < FIRST_SYSTEM_VECTOR)) 130 return; 131 132 if (WARN_ON_ONCE(fred_setup_done)) 133 return; 134 135 if (!WARN_ON_ONCE(sysvec_table[sysvec - FIRST_SYSTEM_VECTOR])) 136 sysvec_table[sysvec - FIRST_SYSTEM_VECTOR] = handler; 137 } 138 139 static noinstr void fred_handle_spurious_interrupt(struct pt_regs *regs) 140 { 141 spurious_interrupt(regs, regs->fred_ss.vector); 142 } 143 144 void __init fred_complete_exception_setup(void) 145 { 146 unsigned int vector; 147 148 for (vector = 0; vector < FIRST_EXTERNAL_VECTOR; vector++) 149 set_bit(vector, system_vectors); 150 151 for (vector = 0; vector < NR_SYSTEM_VECTORS; vector++) { 152 if (sysvec_table[vector]) 153 set_bit(vector + FIRST_SYSTEM_VECTOR, system_vectors); 154 else 155 sysvec_table[vector] = fred_handle_spurious_interrupt; 156 } 157 fred_setup_done = true; 158 } 159 160 static noinstr void fred_extint(struct pt_regs *regs) 161 { 162 unsigned int vector = regs->fred_ss.vector; 163 164 if (WARN_ON_ONCE(vector < FIRST_EXTERNAL_VECTOR)) 165 return; 166 167 if (likely(vector >= FIRST_SYSTEM_VECTOR)) { 168 irqentry_state_t state = irqentry_enter(regs); 169 170 instrumentation_begin(); 171 sysvec_table[array_index_nospec(vector - FIRST_SYSTEM_VECTOR, 172 NR_SYSTEM_VECTORS)](regs); 173 instrumentation_end(); 174 irqentry_exit(regs, state); 175 } else { 176 common_interrupt(regs, vector); 177 } 178 } 179 180 static noinstr void fred_hwexc(struct pt_regs *regs, unsigned long error_code) 181 { 182 /* Optimize for #PF. That's the only exception which matters performance wise */ 183 if (likely(regs->fred_ss.vector == X86_TRAP_PF)) 184 return exc_page_fault(regs, error_code); 185 186 switch (regs->fred_ss.vector) { 187 case X86_TRAP_DE: return exc_divide_error(regs); 188 case X86_TRAP_DB: return fred_exc_debug(regs); 189 case X86_TRAP_BR: return exc_bounds(regs); 190 case X86_TRAP_UD: return exc_invalid_op(regs); 191 case X86_TRAP_NM: return exc_device_not_available(regs); 192 case X86_TRAP_DF: return exc_double_fault(regs, error_code); 193 case X86_TRAP_TS: return exc_invalid_tss(regs, error_code); 194 case X86_TRAP_NP: return exc_segment_not_present(regs, error_code); 195 case X86_TRAP_SS: return exc_stack_segment(regs, error_code); 196 case X86_TRAP_GP: return exc_general_protection(regs, error_code); 197 case X86_TRAP_MF: return exc_coprocessor_error(regs); 198 case X86_TRAP_AC: return exc_alignment_check(regs, error_code); 199 case X86_TRAP_XF: return exc_simd_coprocessor_error(regs); 200 201 #ifdef CONFIG_X86_MCE 202 case X86_TRAP_MC: return fred_exc_machine_check(regs); 203 #endif 204 #ifdef CONFIG_INTEL_TDX_GUEST 205 case X86_TRAP_VE: return exc_virtualization_exception(regs); 206 #endif 207 #ifdef CONFIG_X86_CET 208 case X86_TRAP_CP: return exc_control_protection(regs, error_code); 209 #endif 210 default: return fred_bad_type(regs, error_code); 211 } 212 213 } 214 215 static noinstr void fred_swexc(struct pt_regs *regs, unsigned long error_code) 216 { 217 switch (regs->fred_ss.vector) { 218 case X86_TRAP_BP: return exc_int3(regs); 219 case X86_TRAP_OF: return exc_overflow(regs); 220 default: return fred_bad_type(regs, error_code); 221 } 222 } 223 224 __visible noinstr void fred_entry_from_user(struct pt_regs *regs) 225 { 226 unsigned long error_code = regs->orig_ax; 227 228 /* Invalidate orig_ax so that syscall_get_nr() works correctly */ 229 regs->orig_ax = -1; 230 231 switch (regs->fred_ss.type) { 232 case EVENT_TYPE_EXTINT: 233 return fred_extint(regs); 234 case EVENT_TYPE_NMI: 235 if (likely(regs->fred_ss.vector == X86_TRAP_NMI)) 236 return fred_exc_nmi(regs); 237 break; 238 case EVENT_TYPE_HWEXC: 239 return fred_hwexc(regs, error_code); 240 case EVENT_TYPE_SWINT: 241 return fred_intx(regs); 242 case EVENT_TYPE_PRIV_SWEXC: 243 if (likely(regs->fred_ss.vector == X86_TRAP_DB)) 244 return fred_exc_debug(regs); 245 break; 246 case EVENT_TYPE_SWEXC: 247 return fred_swexc(regs, error_code); 248 case EVENT_TYPE_OTHER: 249 return fred_other(regs); 250 default: break; 251 } 252 253 return fred_bad_type(regs, error_code); 254 } 255 256 __visible noinstr void fred_entry_from_kernel(struct pt_regs *regs) 257 { 258 unsigned long error_code = regs->orig_ax; 259 260 /* Invalidate orig_ax so that syscall_get_nr() works correctly */ 261 regs->orig_ax = -1; 262 263 switch (regs->fred_ss.type) { 264 case EVENT_TYPE_EXTINT: 265 return fred_extint(regs); 266 case EVENT_TYPE_NMI: 267 if (likely(regs->fred_ss.vector == X86_TRAP_NMI)) 268 return fred_exc_nmi(regs); 269 break; 270 case EVENT_TYPE_HWEXC: 271 return fred_hwexc(regs, error_code); 272 case EVENT_TYPE_PRIV_SWEXC: 273 if (likely(regs->fred_ss.vector == X86_TRAP_DB)) 274 return fred_exc_debug(regs); 275 break; 276 case EVENT_TYPE_SWEXC: 277 return fred_swexc(regs, error_code); 278 default: break; 279 } 280 281 return fred_bad_type(regs, error_code); 282 } 283 284 #if IS_ENABLED(CONFIG_KVM_INTEL) 285 __visible noinstr void __fred_entry_from_kvm(struct pt_regs *regs) 286 { 287 switch (regs->fred_ss.type) { 288 case EVENT_TYPE_EXTINT: 289 return fred_extint(regs); 290 case EVENT_TYPE_NMI: 291 return fred_exc_nmi(regs); 292 default: 293 WARN_ON_ONCE(1); 294 } 295 } 296 #endif 297