1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * The FRED specific kernel/user entry functions which are invoked from 4 * assembly code and dispatch to the associated handlers. 5 */ 6 #include <linux/kernel.h> 7 #include <linux/kdebug.h> 8 #include <linux/nospec.h> 9 10 #include <asm/desc.h> 11 #include <asm/fred.h> 12 #include <asm/idtentry.h> 13 #include <asm/syscall.h> 14 #include <asm/trapnr.h> 15 #include <asm/traps.h> 16 17 /* FRED EVENT_TYPE_OTHER vector numbers */ 18 #define FRED_SYSCALL 1 19 #define FRED_SYSENTER 2 20 21 static noinstr void fred_bad_type(struct pt_regs *regs, unsigned long error_code) 22 { 23 irqentry_state_t irq_state = irqentry_nmi_enter(regs); 24 25 instrumentation_begin(); 26 27 /* Panic on events from a high stack level */ 28 if (regs->fred_cs.sl > 0) { 29 pr_emerg("PANIC: invalid or fatal FRED event; event type %u " 30 "vector %u error 0x%lx aux 0x%lx at %04x:%016lx\n", 31 regs->fred_ss.type, regs->fred_ss.vector, error_code, 32 fred_event_data(regs), regs->cs, regs->ip); 33 die("invalid or fatal FRED event", regs, error_code); 34 panic("invalid or fatal FRED event"); 35 } else { 36 unsigned long flags = oops_begin(); 37 int sig = SIGKILL; 38 39 pr_alert("BUG: invalid or fatal FRED event; event type %u " 40 "vector %u error 0x%lx aux 0x%lx at %04x:%016lx\n", 41 regs->fred_ss.type, regs->fred_ss.vector, error_code, 42 fred_event_data(regs), regs->cs, regs->ip); 43 44 if (__die("Invalid or fatal FRED event", regs, error_code)) 45 sig = 0; 46 47 oops_end(flags, regs, sig); 48 } 49 50 instrumentation_end(); 51 irqentry_nmi_exit(regs, irq_state); 52 } 53 54 static noinstr void fred_intx(struct pt_regs *regs) 55 { 56 switch (regs->fred_ss.vector) { 57 /* Opcode 0xcd, 0x3, NOT INT3 (opcode 0xcc) */ 58 case X86_TRAP_BP: 59 return exc_int3(regs); 60 61 /* Opcode 0xcd, 0x4, NOT INTO (opcode 0xce) */ 62 case X86_TRAP_OF: 63 return exc_overflow(regs); 64 65 #ifdef CONFIG_IA32_EMULATION 66 /* INT80 */ 67 case IA32_SYSCALL_VECTOR: 68 if (ia32_enabled()) 69 return fred_int80_emulation(regs); 70 fallthrough; 71 #endif 72 73 default: 74 return exc_general_protection(regs, 0); 75 } 76 } 77 78 static __always_inline void fred_other(struct pt_regs *regs) 79 { 80 /* The compiler can fold these conditions into a single test */ 81 if (likely(regs->fred_ss.vector == FRED_SYSCALL && regs->fred_ss.l)) { 82 regs->orig_ax = regs->ax; 83 regs->ax = -ENOSYS; 84 do_syscall_64(regs, regs->orig_ax); 85 return; 86 } else if (ia32_enabled() && 87 likely(regs->fred_ss.vector == FRED_SYSENTER && !regs->fred_ss.l)) { 88 regs->orig_ax = regs->ax; 89 regs->ax = -ENOSYS; 90 do_fast_syscall_32(regs); 91 return; 92 } else { 93 exc_invalid_op(regs); 94 return; 95 } 96 } 97 98 #define SYSVEC(_vector, _function) [_vector - FIRST_SYSTEM_VECTOR] = fred_sysvec_##_function 99 100 static idtentry_t sysvec_table[NR_SYSTEM_VECTORS] __ro_after_init = { 101 SYSVEC(ERROR_APIC_VECTOR, error_interrupt), 102 SYSVEC(SPURIOUS_APIC_VECTOR, spurious_apic_interrupt), 103 SYSVEC(LOCAL_TIMER_VECTOR, apic_timer_interrupt), 104 SYSVEC(X86_PLATFORM_IPI_VECTOR, x86_platform_ipi), 105 106 SYSVEC(RESCHEDULE_VECTOR, reschedule_ipi), 107 SYSVEC(CALL_FUNCTION_SINGLE_VECTOR, call_function_single), 108 SYSVEC(CALL_FUNCTION_VECTOR, call_function), 109 SYSVEC(REBOOT_VECTOR, reboot), 110 111 SYSVEC(THRESHOLD_APIC_VECTOR, threshold), 112 SYSVEC(DEFERRED_ERROR_VECTOR, deferred_error), 113 SYSVEC(THERMAL_APIC_VECTOR, thermal), 114 115 SYSVEC(IRQ_WORK_VECTOR, irq_work), 116 117 SYSVEC(PERF_GUEST_MEDIATED_PMI_VECTOR, perf_guest_mediated_pmi_handler), 118 SYSVEC(POSTED_INTR_VECTOR, kvm_posted_intr_ipi), 119 SYSVEC(POSTED_INTR_WAKEUP_VECTOR, kvm_posted_intr_wakeup_ipi), 120 SYSVEC(POSTED_INTR_NESTED_VECTOR, kvm_posted_intr_nested_ipi), 121 122 SYSVEC(POSTED_MSI_NOTIFICATION_VECTOR, posted_msi_notification), 123 }; 124 125 static bool fred_setup_done __initdata; 126 127 void __init fred_install_sysvec(unsigned int sysvec, idtentry_t handler) 128 { 129 if (WARN_ON_ONCE(sysvec < FIRST_SYSTEM_VECTOR)) 130 return; 131 132 if (WARN_ON_ONCE(fred_setup_done)) 133 return; 134 135 if (!WARN_ON_ONCE(sysvec_table[sysvec - FIRST_SYSTEM_VECTOR])) 136 sysvec_table[sysvec - FIRST_SYSTEM_VECTOR] = handler; 137 } 138 139 static noinstr void fred_handle_spurious_interrupt(struct pt_regs *regs) 140 { 141 spurious_interrupt(regs, regs->fred_ss.vector); 142 } 143 144 void __init fred_complete_exception_setup(void) 145 { 146 unsigned int vector; 147 148 for (vector = 0; vector < FIRST_EXTERNAL_VECTOR; vector++) 149 set_bit(vector, system_vectors); 150 151 for (vector = 0; vector < NR_SYSTEM_VECTORS; vector++) { 152 if (sysvec_table[vector]) 153 set_bit(vector + FIRST_SYSTEM_VECTOR, system_vectors); 154 else 155 sysvec_table[vector] = fred_handle_spurious_interrupt; 156 } 157 fred_setup_done = true; 158 } 159 160 static noinstr void fred_extint(struct pt_regs *regs) 161 { 162 unsigned int vector = regs->fred_ss.vector; 163 unsigned int index = array_index_nospec(vector - FIRST_SYSTEM_VECTOR, 164 NR_SYSTEM_VECTORS); 165 166 if (WARN_ON_ONCE(vector < FIRST_EXTERNAL_VECTOR)) 167 return; 168 169 if (likely(vector >= FIRST_SYSTEM_VECTOR)) { 170 irqentry_state_t state = irqentry_enter(regs); 171 172 instrumentation_begin(); 173 sysvec_table[index](regs); 174 instrumentation_end(); 175 irqentry_exit(regs, state); 176 } else { 177 common_interrupt(regs, vector); 178 } 179 } 180 181 static noinstr void fred_hwexc(struct pt_regs *regs, unsigned long error_code) 182 { 183 /* Optimize for #PF. That's the only exception which matters performance wise */ 184 if (likely(regs->fred_ss.vector == X86_TRAP_PF)) 185 return exc_page_fault(regs, error_code); 186 187 switch (regs->fred_ss.vector) { 188 case X86_TRAP_DE: return exc_divide_error(regs); 189 case X86_TRAP_DB: return fred_exc_debug(regs); 190 case X86_TRAP_BR: return exc_bounds(regs); 191 case X86_TRAP_UD: return exc_invalid_op(regs); 192 case X86_TRAP_NM: return exc_device_not_available(regs); 193 case X86_TRAP_DF: return exc_double_fault(regs, error_code); 194 case X86_TRAP_TS: return exc_invalid_tss(regs, error_code); 195 case X86_TRAP_NP: return exc_segment_not_present(regs, error_code); 196 case X86_TRAP_SS: return exc_stack_segment(regs, error_code); 197 case X86_TRAP_GP: return exc_general_protection(regs, error_code); 198 case X86_TRAP_MF: return exc_coprocessor_error(regs); 199 case X86_TRAP_AC: return exc_alignment_check(regs, error_code); 200 case X86_TRAP_XF: return exc_simd_coprocessor_error(regs); 201 202 #ifdef CONFIG_X86_MCE 203 case X86_TRAP_MC: return fred_exc_machine_check(regs); 204 #endif 205 #ifdef CONFIG_INTEL_TDX_GUEST 206 case X86_TRAP_VE: return exc_virtualization_exception(regs); 207 #endif 208 #ifdef CONFIG_X86_CET 209 case X86_TRAP_CP: return exc_control_protection(regs, error_code); 210 #endif 211 default: return fred_bad_type(regs, error_code); 212 } 213 214 } 215 216 static noinstr void fred_swexc(struct pt_regs *regs, unsigned long error_code) 217 { 218 switch (regs->fred_ss.vector) { 219 case X86_TRAP_BP: return exc_int3(regs); 220 case X86_TRAP_OF: return exc_overflow(regs); 221 default: return fred_bad_type(regs, error_code); 222 } 223 } 224 225 __visible noinstr void fred_entry_from_user(struct pt_regs *regs) 226 { 227 unsigned long error_code = regs->orig_ax; 228 229 /* Invalidate orig_ax so that syscall_get_nr() works correctly */ 230 regs->orig_ax = -1; 231 232 switch (regs->fred_ss.type) { 233 case EVENT_TYPE_EXTINT: 234 return fred_extint(regs); 235 case EVENT_TYPE_NMI: 236 if (likely(regs->fred_ss.vector == X86_TRAP_NMI)) 237 return fred_exc_nmi(regs); 238 break; 239 case EVENT_TYPE_HWEXC: 240 return fred_hwexc(regs, error_code); 241 case EVENT_TYPE_SWINT: 242 return fred_intx(regs); 243 case EVENT_TYPE_PRIV_SWEXC: 244 if (likely(regs->fred_ss.vector == X86_TRAP_DB)) 245 return fred_exc_debug(regs); 246 break; 247 case EVENT_TYPE_SWEXC: 248 return fred_swexc(regs, error_code); 249 case EVENT_TYPE_OTHER: 250 return fred_other(regs); 251 default: break; 252 } 253 254 return fred_bad_type(regs, error_code); 255 } 256 257 __visible noinstr void fred_entry_from_kernel(struct pt_regs *regs) 258 { 259 unsigned long error_code = regs->orig_ax; 260 261 /* Invalidate orig_ax so that syscall_get_nr() works correctly */ 262 regs->orig_ax = -1; 263 264 switch (regs->fred_ss.type) { 265 case EVENT_TYPE_EXTINT: 266 return fred_extint(regs); 267 case EVENT_TYPE_NMI: 268 if (likely(regs->fred_ss.vector == X86_TRAP_NMI)) 269 return fred_exc_nmi(regs); 270 break; 271 case EVENT_TYPE_HWEXC: 272 return fred_hwexc(regs, error_code); 273 case EVENT_TYPE_PRIV_SWEXC: 274 if (likely(regs->fred_ss.vector == X86_TRAP_DB)) 275 return fred_exc_debug(regs); 276 break; 277 case EVENT_TYPE_SWEXC: 278 return fred_swexc(regs, error_code); 279 default: break; 280 } 281 282 return fred_bad_type(regs, error_code); 283 } 284 285 #if IS_ENABLED(CONFIG_KVM_INTEL) 286 __visible noinstr void __fred_entry_from_kvm(struct pt_regs *regs) 287 { 288 switch (regs->fred_ss.type) { 289 case EVENT_TYPE_EXTINT: 290 return fred_extint(regs); 291 case EVENT_TYPE_NMI: 292 return fred_exc_nmi(regs); 293 default: 294 WARN_ON_ONCE(1); 295 } 296 } 297 #endif 298