1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * The FRED specific kernel/user entry functions which are invoked from 4 * assembly code and dispatch to the associated handlers. 5 */ 6 #include <linux/kernel.h> 7 #include <linux/kdebug.h> 8 #include <linux/nospec.h> 9 10 #include <asm/desc.h> 11 #include <asm/fred.h> 12 #include <asm/idtentry.h> 13 #include <asm/syscall.h> 14 #include <asm/trapnr.h> 15 #include <asm/traps.h> 16 17 /* FRED EVENT_TYPE_OTHER vector numbers */ 18 #define FRED_SYSCALL 1 19 #define FRED_SYSENTER 2 20 21 static noinstr void fred_bad_type(struct pt_regs *regs, unsigned long error_code) 22 { 23 irqentry_state_t irq_state = irqentry_nmi_enter(regs); 24 25 instrumentation_begin(); 26 27 /* Panic on events from a high stack level */ 28 if (regs->fred_cs.sl > 0) { 29 pr_emerg("PANIC: invalid or fatal FRED event; event type %u " 30 "vector %u error 0x%lx aux 0x%lx at %04x:%016lx\n", 31 regs->fred_ss.type, regs->fred_ss.vector, error_code, 32 fred_event_data(regs), regs->cs, regs->ip); 33 die("invalid or fatal FRED event", regs, error_code); 34 panic("invalid or fatal FRED event"); 35 } else { 36 unsigned long flags = oops_begin(); 37 int sig = SIGKILL; 38 39 pr_alert("BUG: invalid or fatal FRED event; event type %u " 40 "vector %u error 0x%lx aux 0x%lx at %04x:%016lx\n", 41 regs->fred_ss.type, regs->fred_ss.vector, error_code, 42 fred_event_data(regs), regs->cs, regs->ip); 43 44 if (__die("Invalid or fatal FRED event", regs, error_code)) 45 sig = 0; 46 47 oops_end(flags, regs, sig); 48 } 49 50 instrumentation_end(); 51 irqentry_nmi_exit(regs, irq_state); 52 } 53 54 static noinstr void fred_intx(struct pt_regs *regs) 55 { 56 switch (regs->fred_ss.vector) { 57 /* Opcode 0xcd, 0x3, NOT INT3 (opcode 0xcc) */ 58 case X86_TRAP_BP: 59 return exc_int3(regs); 60 61 /* Opcode 0xcd, 0x4, NOT INTO (opcode 0xce) */ 62 case X86_TRAP_OF: 63 return exc_overflow(regs); 64 65 #ifdef CONFIG_IA32_EMULATION 66 /* INT80 */ 67 case IA32_SYSCALL_VECTOR: 68 if (ia32_enabled()) 69 return fred_int80_emulation(regs); 70 fallthrough; 71 #endif 72 73 default: 74 return exc_general_protection(regs, 0); 75 } 76 } 77 78 static __always_inline void fred_other(struct pt_regs *regs) 79 { 80 /* The compiler can fold these conditions into a single test */ 81 if (likely(regs->fred_ss.vector == FRED_SYSCALL && regs->fred_ss.l)) { 82 regs->orig_ax = regs->ax; 83 regs->ax = -ENOSYS; 84 do_syscall_64(regs, regs->orig_ax); 85 return; 86 } else if (ia32_enabled() && 87 likely(regs->fred_ss.vector == FRED_SYSENTER && !regs->fred_ss.l)) { 88 regs->orig_ax = regs->ax; 89 regs->ax = -ENOSYS; 90 do_fast_syscall_32(regs); 91 return; 92 } else { 93 exc_invalid_op(regs); 94 return; 95 } 96 } 97 98 #define SYSVEC(_vector, _function) [_vector - FIRST_SYSTEM_VECTOR] = fred_sysvec_##_function 99 100 static idtentry_t sysvec_table[NR_SYSTEM_VECTORS] __ro_after_init = { 101 SYSVEC(ERROR_APIC_VECTOR, error_interrupt), 102 SYSVEC(SPURIOUS_APIC_VECTOR, spurious_apic_interrupt), 103 SYSVEC(LOCAL_TIMER_VECTOR, apic_timer_interrupt), 104 SYSVEC(X86_PLATFORM_IPI_VECTOR, x86_platform_ipi), 105 106 SYSVEC(RESCHEDULE_VECTOR, reschedule_ipi), 107 SYSVEC(CALL_FUNCTION_SINGLE_VECTOR, call_function_single), 108 SYSVEC(CALL_FUNCTION_VECTOR, call_function), 109 SYSVEC(REBOOT_VECTOR, reboot), 110 111 SYSVEC(THRESHOLD_APIC_VECTOR, threshold), 112 SYSVEC(DEFERRED_ERROR_VECTOR, deferred_error), 113 SYSVEC(THERMAL_APIC_VECTOR, thermal), 114 115 SYSVEC(IRQ_WORK_VECTOR, irq_work), 116 117 SYSVEC(PERF_GUEST_MEDIATED_PMI_VECTOR, perf_guest_mediated_pmi_handler), 118 SYSVEC(POSTED_INTR_VECTOR, kvm_posted_intr_ipi), 119 SYSVEC(POSTED_INTR_WAKEUP_VECTOR, kvm_posted_intr_wakeup_ipi), 120 SYSVEC(POSTED_INTR_NESTED_VECTOR, kvm_posted_intr_nested_ipi), 121 122 SYSVEC(POSTED_MSI_NOTIFICATION_VECTOR, posted_msi_notification), 123 }; 124 125 static bool fred_setup_done __initdata; 126 127 void __init fred_install_sysvec(unsigned int sysvec, idtentry_t handler) 128 { 129 if (WARN_ON_ONCE(sysvec < FIRST_SYSTEM_VECTOR)) 130 return; 131 132 if (WARN_ON_ONCE(fred_setup_done)) 133 return; 134 135 if (!WARN_ON_ONCE(sysvec_table[sysvec - FIRST_SYSTEM_VECTOR])) 136 sysvec_table[sysvec - FIRST_SYSTEM_VECTOR] = handler; 137 } 138 139 static noinstr void fred_handle_spurious_interrupt(struct pt_regs *regs) 140 { 141 spurious_interrupt(regs, regs->fred_ss.vector); 142 } 143 144 void __init fred_complete_exception_setup(void) 145 { 146 unsigned int vector; 147 148 for (vector = 0; vector < FIRST_EXTERNAL_VECTOR; vector++) 149 set_bit(vector, system_vectors); 150 151 for (vector = 0; vector < NR_SYSTEM_VECTORS; vector++) { 152 if (sysvec_table[vector]) 153 set_bit(vector + FIRST_SYSTEM_VECTOR, system_vectors); 154 else 155 sysvec_table[vector] = fred_handle_spurious_interrupt; 156 } 157 fred_setup_done = true; 158 } 159 160 static noinstr void fred_extint(struct pt_regs *regs) 161 { 162 unsigned int vector = regs->fred_ss.vector; 163 164 if (WARN_ON_ONCE(vector < FIRST_EXTERNAL_VECTOR)) 165 return; 166 167 if (likely(vector >= FIRST_SYSTEM_VECTOR)) { 168 irqentry_state_t state = irqentry_enter(regs); 169 170 instrumentation_begin(); 171 sysvec_table[array_index_nospec(vector - FIRST_SYSTEM_VECTOR, 172 NR_SYSTEM_VECTORS)](regs); 173 instrumentation_end(); 174 irqentry_exit(regs, state); 175 } else { 176 common_interrupt(regs, vector); 177 } 178 } 179 180 #ifdef CONFIG_AMD_MEM_ENCRYPT 181 noinstr void exc_vmm_communication(struct pt_regs *regs, unsigned long error_code) 182 { 183 if (user_mode(regs)) 184 return user_exc_vmm_communication(regs, error_code); 185 else 186 return kernel_exc_vmm_communication(regs, error_code); 187 } 188 #endif 189 190 static noinstr void fred_hwexc(struct pt_regs *regs, unsigned long error_code) 191 { 192 /* Optimize for #PF. That's the only exception which matters performance wise */ 193 if (likely(regs->fred_ss.vector == X86_TRAP_PF)) 194 return exc_page_fault(regs, error_code); 195 196 switch (regs->fred_ss.vector) { 197 case X86_TRAP_DE: return exc_divide_error(regs); 198 case X86_TRAP_DB: return fred_exc_debug(regs); 199 case X86_TRAP_BR: return exc_bounds(regs); 200 case X86_TRAP_UD: return exc_invalid_op(regs); 201 case X86_TRAP_NM: return exc_device_not_available(regs); 202 case X86_TRAP_DF: return exc_double_fault(regs, error_code); 203 case X86_TRAP_TS: return exc_invalid_tss(regs, error_code); 204 case X86_TRAP_NP: return exc_segment_not_present(regs, error_code); 205 case X86_TRAP_SS: return exc_stack_segment(regs, error_code); 206 case X86_TRAP_GP: return exc_general_protection(regs, error_code); 207 case X86_TRAP_MF: return exc_coprocessor_error(regs); 208 case X86_TRAP_AC: return exc_alignment_check(regs, error_code); 209 case X86_TRAP_XF: return exc_simd_coprocessor_error(regs); 210 211 #ifdef CONFIG_X86_MCE 212 case X86_TRAP_MC: return fred_exc_machine_check(regs); 213 #endif 214 #ifdef CONFIG_INTEL_TDX_GUEST 215 case X86_TRAP_VE: return exc_virtualization_exception(regs); 216 #endif 217 #ifdef CONFIG_X86_CET 218 case X86_TRAP_CP: return exc_control_protection(regs, error_code); 219 #endif 220 #ifdef CONFIG_AMD_MEM_ENCRYPT 221 case X86_TRAP_VC: return exc_vmm_communication(regs, error_code); 222 #endif 223 224 default: return fred_bad_type(regs, error_code); 225 } 226 227 } 228 229 static noinstr void fred_swexc(struct pt_regs *regs, unsigned long error_code) 230 { 231 switch (regs->fred_ss.vector) { 232 case X86_TRAP_BP: return exc_int3(regs); 233 case X86_TRAP_OF: return exc_overflow(regs); 234 default: return fred_bad_type(regs, error_code); 235 } 236 } 237 238 __visible noinstr void fred_entry_from_user(struct pt_regs *regs) 239 { 240 unsigned long error_code = regs->orig_ax; 241 242 /* Invalidate orig_ax so that syscall_get_nr() works correctly */ 243 regs->orig_ax = -1; 244 245 switch (regs->fred_ss.type) { 246 case EVENT_TYPE_EXTINT: 247 return fred_extint(regs); 248 case EVENT_TYPE_NMI: 249 if (likely(regs->fred_ss.vector == X86_TRAP_NMI)) 250 return fred_exc_nmi(regs); 251 break; 252 case EVENT_TYPE_HWEXC: 253 return fred_hwexc(regs, error_code); 254 case EVENT_TYPE_SWINT: 255 return fred_intx(regs); 256 case EVENT_TYPE_PRIV_SWEXC: 257 if (likely(regs->fred_ss.vector == X86_TRAP_DB)) 258 return fred_exc_debug(regs); 259 break; 260 case EVENT_TYPE_SWEXC: 261 return fred_swexc(regs, error_code); 262 case EVENT_TYPE_OTHER: 263 return fred_other(regs); 264 default: break; 265 } 266 267 return fred_bad_type(regs, error_code); 268 } 269 270 __visible noinstr void fred_entry_from_kernel(struct pt_regs *regs) 271 { 272 unsigned long error_code = regs->orig_ax; 273 274 /* Invalidate orig_ax so that syscall_get_nr() works correctly */ 275 regs->orig_ax = -1; 276 277 switch (regs->fred_ss.type) { 278 case EVENT_TYPE_EXTINT: 279 return fred_extint(regs); 280 case EVENT_TYPE_NMI: 281 if (likely(regs->fred_ss.vector == X86_TRAP_NMI)) 282 return fred_exc_nmi(regs); 283 break; 284 case EVENT_TYPE_HWEXC: 285 return fred_hwexc(regs, error_code); 286 case EVENT_TYPE_PRIV_SWEXC: 287 if (likely(regs->fred_ss.vector == X86_TRAP_DB)) 288 return fred_exc_debug(regs); 289 break; 290 case EVENT_TYPE_SWEXC: 291 return fred_swexc(regs, error_code); 292 default: break; 293 } 294 295 return fred_bad_type(regs, error_code); 296 } 297 298 #if IS_ENABLED(CONFIG_KVM_INTEL) 299 __visible noinstr void __fred_entry_from_kvm(struct pt_regs *regs) 300 { 301 switch (regs->fred_ss.type) { 302 case EVENT_TYPE_EXTINT: 303 return fred_extint(regs); 304 case EVENT_TYPE_NMI: 305 return fred_exc_nmi(regs); 306 default: 307 WARN_ON_ONCE(1); 308 } 309 } 310 #endif 311