1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _ASM_X86_IDTENTRY_H 3 #define _ASM_X86_IDTENTRY_H 4 5 /* Interrupts/Exceptions */ 6 #include <asm/trapnr.h> 7 8 #define IDT_ALIGN (8 * (1 + HAS_KERNEL_IBT)) 9 10 #ifndef __ASSEMBLER__ 11 #include <linux/entry-common.h> 12 #include <linux/hardirq.h> 13 14 #include <asm/irq_stack.h> 15 16 typedef void (*idtentry_t)(struct pt_regs *regs); 17 18 /** 19 * DECLARE_IDTENTRY - Declare functions for simple IDT entry points 20 * No error code pushed by hardware 21 * @vector: Vector number (ignored for C) 22 * @func: Function name of the entry point 23 * 24 * Declares four functions: 25 * - The ASM entry point: asm_##func 26 * - The XEN PV trap entry point: xen_##func (maybe unused) 27 * - The C handler called from the FRED event dispatcher (maybe unused) 28 * - The C handler called from the ASM entry point 29 * 30 * Note: This is the C variant of DECLARE_IDTENTRY(). As the name says it 31 * declares the entry points for usage in C code. There is an ASM variant 32 * as well which is used to emit the entry stubs in entry_32/64.S. 33 */ 34 #define DECLARE_IDTENTRY(vector, func) \ 35 asmlinkage void asm_##func(void); \ 36 asmlinkage void xen_asm_##func(void); \ 37 void fred_##func(struct pt_regs *regs); \ 38 __visible void func(struct pt_regs *regs) 39 40 /** 41 * DEFINE_IDTENTRY - Emit code for simple IDT entry points 42 * @func: Function name of the entry point 43 * 44 * @func is called from ASM entry code with interrupts disabled. 45 * 46 * The macro is written so it acts as function definition. Append the 47 * body with a pair of curly brackets. 48 * 49 * irqentry_enter() contains common code which has to be invoked before 50 * arbitrary code in the body. irqentry_exit() contains common code 51 * which has to run before returning to the low level assembly code. 52 */ 53 #define DEFINE_IDTENTRY(func) \ 54 static __always_inline void __##func(struct pt_regs *regs); \ 55 \ 56 __visible noinstr void func(struct pt_regs *regs) \ 57 { \ 58 irqentry_state_t state = irqentry_enter(regs); \ 59 \ 60 instrumentation_begin(); \ 61 __##func (regs); \ 62 instrumentation_end(); \ 63 irqentry_exit(regs, state); \ 64 } \ 65 \ 66 static __always_inline void __##func(struct pt_regs *regs) 67 68 /* Special case for 32bit IRET 'trap' */ 69 #define DECLARE_IDTENTRY_SW DECLARE_IDTENTRY 70 #define DEFINE_IDTENTRY_SW DEFINE_IDTENTRY 71 72 /** 73 * DECLARE_IDTENTRY_ERRORCODE - Declare functions for simple IDT entry points 74 * Error code pushed by hardware 75 * @vector: Vector number (ignored for C) 76 * @func: Function name of the entry point 77 * 78 * Declares three functions: 79 * - The ASM entry point: asm_##func 80 * - The XEN PV trap entry point: xen_##func (maybe unused) 81 * - The C handler called from the ASM entry point 82 * 83 * Same as DECLARE_IDTENTRY, but has an extra error_code argument for the 84 * C-handler. 85 */ 86 #define DECLARE_IDTENTRY_ERRORCODE(vector, func) \ 87 asmlinkage void asm_##func(void); \ 88 asmlinkage void xen_asm_##func(void); \ 89 __visible void func(struct pt_regs *regs, unsigned long error_code) 90 91 /** 92 * DEFINE_IDTENTRY_ERRORCODE - Emit code for simple IDT entry points 93 * Error code pushed by hardware 94 * @func: Function name of the entry point 95 * 96 * Same as DEFINE_IDTENTRY, but has an extra error_code argument 97 */ 98 #define DEFINE_IDTENTRY_ERRORCODE(func) \ 99 static __always_inline void __##func(struct pt_regs *regs, \ 100 unsigned long error_code); \ 101 \ 102 __visible noinstr void func(struct pt_regs *regs, \ 103 unsigned long error_code) \ 104 { \ 105 irqentry_state_t state = irqentry_enter(regs); \ 106 \ 107 instrumentation_begin(); \ 108 __##func (regs, error_code); \ 109 instrumentation_end(); \ 110 irqentry_exit(regs, state); \ 111 } \ 112 \ 113 static __always_inline void __##func(struct pt_regs *regs, \ 114 unsigned long error_code) 115 116 /** 117 * DECLARE_IDTENTRY_RAW - Declare functions for raw IDT entry points 118 * No error code pushed by hardware 119 * @vector: Vector number (ignored for C) 120 * @func: Function name of the entry point 121 * 122 * Maps to DECLARE_IDTENTRY(). 123 */ 124 #define DECLARE_IDTENTRY_RAW(vector, func) \ 125 DECLARE_IDTENTRY(vector, func) 126 127 /** 128 * DEFINE_IDTENTRY_RAW - Emit code for raw IDT entry points 129 * @func: Function name of the entry point 130 * 131 * @func is called from ASM entry code with interrupts disabled. 132 * 133 * The macro is written so it acts as function definition. Append the 134 * body with a pair of curly brackets. 135 * 136 * Contrary to DEFINE_IDTENTRY() this does not invoke the 137 * idtentry_enter/exit() helpers before and after the body invocation. This 138 * needs to be done in the body itself if applicable. Use if extra work 139 * is required before the enter/exit() helpers are invoked. 140 */ 141 #define DEFINE_IDTENTRY_RAW(func) \ 142 __visible noinstr void func(struct pt_regs *regs) 143 144 /** 145 * DEFINE_FREDENTRY_RAW - Emit code for raw FRED entry points 146 * @func: Function name of the entry point 147 * 148 * @func is called from the FRED event dispatcher with interrupts disabled. 149 * 150 * See @DEFINE_IDTENTRY_RAW for further details. 151 */ 152 #define DEFINE_FREDENTRY_RAW(func) \ 153 noinstr void fred_##func(struct pt_regs *regs) 154 155 /** 156 * DECLARE_IDTENTRY_RAW_ERRORCODE - Declare functions for raw IDT entry points 157 * Error code pushed by hardware 158 * @vector: Vector number (ignored for C) 159 * @func: Function name of the entry point 160 * 161 * Maps to DECLARE_IDTENTRY_ERRORCODE() 162 */ 163 #define DECLARE_IDTENTRY_RAW_ERRORCODE(vector, func) \ 164 DECLARE_IDTENTRY_ERRORCODE(vector, func) 165 166 /** 167 * DEFINE_IDTENTRY_RAW_ERRORCODE - Emit code for raw IDT entry points 168 * @func: Function name of the entry point 169 * 170 * @func is called from ASM entry code with interrupts disabled. 171 * 172 * The macro is written so it acts as function definition. Append the 173 * body with a pair of curly brackets. 174 * 175 * Contrary to DEFINE_IDTENTRY_ERRORCODE() this does not invoke the 176 * irqentry_enter/exit() helpers before and after the body invocation. This 177 * needs to be done in the body itself if applicable. Use if extra work 178 * is required before the enter/exit() helpers are invoked. 179 */ 180 #define DEFINE_IDTENTRY_RAW_ERRORCODE(func) \ 181 __visible noinstr void func(struct pt_regs *regs, unsigned long error_code) 182 183 /** 184 * DECLARE_IDTENTRY_IRQ - Declare functions for device interrupt IDT entry 185 * points (common/spurious) 186 * @vector: Vector number (ignored for C) 187 * @func: Function name of the entry point 188 * 189 * Maps to DECLARE_IDTENTRY_ERRORCODE() 190 */ 191 #define DECLARE_IDTENTRY_IRQ(vector, func) \ 192 DECLARE_IDTENTRY_ERRORCODE(vector, func) 193 194 /** 195 * DEFINE_IDTENTRY_IRQ - Emit code for device interrupt IDT entry points 196 * @func: Function name of the entry point 197 * 198 * The vector number is pushed by the low level entry stub and handed 199 * to the function as error_code argument which needs to be truncated 200 * to an u8 because the push is sign extending. 201 * 202 * irq_enter/exit_rcu() are invoked before the function body and the 203 * KVM L1D flush request is set. Stack switching to the interrupt stack 204 * has to be done in the function body if necessary. 205 */ 206 #define DEFINE_IDTENTRY_IRQ(func) \ 207 static void __##func(struct pt_regs *regs, u32 vector); \ 208 \ 209 __visible noinstr void func(struct pt_regs *regs, \ 210 unsigned long error_code) \ 211 { \ 212 irqentry_state_t state = irqentry_enter(regs); \ 213 u32 vector = (u32)(u8)error_code; \ 214 \ 215 kvm_set_cpu_l1tf_flush_l1d(); \ 216 instrumentation_begin(); \ 217 run_irq_on_irqstack_cond(__##func, regs, vector); \ 218 instrumentation_end(); \ 219 irqentry_exit(regs, state); \ 220 } \ 221 \ 222 static noinline void __##func(struct pt_regs *regs, u32 vector) 223 224 /** 225 * DECLARE_IDTENTRY_SYSVEC - Declare functions for system vector entry points 226 * @vector: Vector number (ignored for C) 227 * @func: Function name of the entry point 228 * 229 * Declares three functions: 230 * - The ASM entry point: asm_##func 231 * - The XEN PV trap entry point: xen_##func (maybe unused) 232 * - The C handler called from the ASM entry point 233 * 234 * Maps to DECLARE_IDTENTRY(). 235 */ 236 #define DECLARE_IDTENTRY_SYSVEC(vector, func) \ 237 DECLARE_IDTENTRY(vector, func) 238 239 /** 240 * DEFINE_IDTENTRY_SYSVEC - Emit code for system vector IDT entry points 241 * @func: Function name of the entry point 242 * 243 * irqentry_enter/exit() and irq_enter/exit_rcu() are invoked before the 244 * function body. KVM L1D flush request is set. 245 * 246 * Runs the function on the interrupt stack if the entry hit kernel mode 247 */ 248 #define DEFINE_IDTENTRY_SYSVEC(func) \ 249 static void __##func(struct pt_regs *regs); \ 250 \ 251 static __always_inline void instr_##func(struct pt_regs *regs) \ 252 { \ 253 run_sysvec_on_irqstack_cond(__##func, regs); \ 254 } \ 255 \ 256 __visible noinstr void func(struct pt_regs *regs) \ 257 { \ 258 irqentry_state_t state = irqentry_enter(regs); \ 259 \ 260 kvm_set_cpu_l1tf_flush_l1d(); \ 261 instrumentation_begin(); \ 262 instr_##func (regs); \ 263 instrumentation_end(); \ 264 irqentry_exit(regs, state); \ 265 } \ 266 \ 267 void fred_##func(struct pt_regs *regs) \ 268 { \ 269 instr_##func (regs); \ 270 } \ 271 \ 272 static noinline void __##func(struct pt_regs *regs) 273 274 /** 275 * DEFINE_IDTENTRY_SYSVEC_SIMPLE - Emit code for simple system vector IDT 276 * entry points 277 * @func: Function name of the entry point 278 * 279 * Runs the function on the interrupted stack. No switch to IRQ stack and 280 * only the minimal __irq_enter/exit() handling. 281 * 282 * Only use for 'empty' vectors like reschedule IPI and KVM posted 283 * interrupt vectors. 284 */ 285 #define DEFINE_IDTENTRY_SYSVEC_SIMPLE(func) \ 286 static __always_inline void __##func(struct pt_regs *regs); \ 287 \ 288 static __always_inline void instr_##func(struct pt_regs *regs) \ 289 { \ 290 __irq_enter_raw(); \ 291 __##func (regs); \ 292 __irq_exit_raw(); \ 293 } \ 294 \ 295 __visible noinstr void func(struct pt_regs *regs) \ 296 { \ 297 irqentry_state_t state = irqentry_enter(regs); \ 298 \ 299 kvm_set_cpu_l1tf_flush_l1d(); \ 300 instrumentation_begin(); \ 301 instr_##func (regs); \ 302 instrumentation_end(); \ 303 irqentry_exit(regs, state); \ 304 } \ 305 \ 306 void fred_##func(struct pt_regs *regs) \ 307 { \ 308 instr_##func (regs); \ 309 } \ 310 \ 311 static __always_inline void __##func(struct pt_regs *regs) 312 313 /** 314 * DECLARE_IDTENTRY_XENCB - Declare functions for XEN HV callback entry point 315 * @vector: Vector number (ignored for C) 316 * @func: Function name of the entry point 317 * 318 * Declares three functions: 319 * - The ASM entry point: asm_##func 320 * - The XEN PV trap entry point: xen_##func (maybe unused) 321 * - The C handler called from the ASM entry point 322 * 323 * Maps to DECLARE_IDTENTRY(). Distinct entry point to handle the 32/64-bit 324 * difference 325 */ 326 #define DECLARE_IDTENTRY_XENCB(vector, func) \ 327 DECLARE_IDTENTRY(vector, func) 328 329 #ifdef CONFIG_X86_64 330 /** 331 * DECLARE_IDTENTRY_IST - Declare functions for IST handling IDT entry points 332 * @vector: Vector number (ignored for C) 333 * @func: Function name of the entry point 334 * 335 * Maps to DECLARE_IDTENTRY_RAW, but declares also the NOIST C handler 336 * which is called from the ASM entry point on user mode entry 337 */ 338 #define DECLARE_IDTENTRY_IST(vector, func) \ 339 DECLARE_IDTENTRY_RAW(vector, func); \ 340 __visible void noist_##func(struct pt_regs *regs) 341 342 /** 343 * DECLARE_IDTENTRY_VC - Declare functions for the VC entry point 344 * @vector: Vector number (ignored for C) 345 * @func: Function name of the entry point 346 * 347 * Maps to DECLARE_IDTENTRY_RAW_ERRORCODE, but declares also the 348 * safe_stack C handler. 349 */ 350 #define DECLARE_IDTENTRY_VC(vector, func) \ 351 DECLARE_IDTENTRY_RAW_ERRORCODE(vector, func); \ 352 __visible noinstr void kernel_##func(struct pt_regs *regs, unsigned long error_code); \ 353 __visible noinstr void user_##func(struct pt_regs *regs, unsigned long error_code) 354 355 /** 356 * DEFINE_IDTENTRY_IST - Emit code for IST entry points 357 * @func: Function name of the entry point 358 * 359 * Maps to DEFINE_IDTENTRY_RAW 360 */ 361 #define DEFINE_IDTENTRY_IST(func) \ 362 DEFINE_IDTENTRY_RAW(func) 363 364 /** 365 * DEFINE_IDTENTRY_NOIST - Emit code for NOIST entry points which 366 * belong to a IST entry point (MCE, DB) 367 * @func: Function name of the entry point. Must be the same as 368 * the function name of the corresponding IST variant 369 * 370 * Maps to DEFINE_IDTENTRY_RAW(). 371 */ 372 #define DEFINE_IDTENTRY_NOIST(func) \ 373 DEFINE_IDTENTRY_RAW(noist_##func) 374 375 /** 376 * DECLARE_IDTENTRY_DF - Declare functions for double fault 377 * @vector: Vector number (ignored for C) 378 * @func: Function name of the entry point 379 * 380 * Maps to DECLARE_IDTENTRY_RAW_ERRORCODE 381 */ 382 #define DECLARE_IDTENTRY_DF(vector, func) \ 383 DECLARE_IDTENTRY_RAW_ERRORCODE(vector, func) 384 385 /** 386 * DEFINE_IDTENTRY_DF - Emit code for double fault 387 * @func: Function name of the entry point 388 * 389 * Maps to DEFINE_IDTENTRY_RAW_ERRORCODE 390 */ 391 #define DEFINE_IDTENTRY_DF(func) \ 392 DEFINE_IDTENTRY_RAW_ERRORCODE(func) 393 394 /** 395 * DEFINE_IDTENTRY_VC_KERNEL - Emit code for VMM communication handler 396 when raised from kernel mode 397 * @func: Function name of the entry point 398 * 399 * Maps to DEFINE_IDTENTRY_RAW_ERRORCODE 400 */ 401 #define DEFINE_IDTENTRY_VC_KERNEL(func) \ 402 DEFINE_IDTENTRY_RAW_ERRORCODE(kernel_##func) 403 404 /** 405 * DEFINE_IDTENTRY_VC_USER - Emit code for VMM communication handler 406 when raised from user mode 407 * @func: Function name of the entry point 408 * 409 * Maps to DEFINE_IDTENTRY_RAW_ERRORCODE 410 */ 411 #define DEFINE_IDTENTRY_VC_USER(func) \ 412 DEFINE_IDTENTRY_RAW_ERRORCODE(user_##func) 413 414 #else /* CONFIG_X86_64 */ 415 416 /** 417 * DECLARE_IDTENTRY_DF - Declare functions for double fault 32bit variant 418 * @vector: Vector number (ignored for C) 419 * @func: Function name of the entry point 420 * 421 * Declares two functions: 422 * - The ASM entry point: asm_##func 423 * - The C handler called from the C shim 424 */ 425 #define DECLARE_IDTENTRY_DF(vector, func) \ 426 asmlinkage void asm_##func(void); \ 427 __visible void func(struct pt_regs *regs, \ 428 unsigned long error_code, \ 429 unsigned long address) 430 431 /** 432 * DEFINE_IDTENTRY_DF - Emit code for double fault on 32bit 433 * @func: Function name of the entry point 434 * 435 * This is called through the doublefault shim which already provides 436 * cr2 in the address argument. 437 */ 438 #define DEFINE_IDTENTRY_DF(func) \ 439 __visible noinstr void func(struct pt_regs *regs, \ 440 unsigned long error_code, \ 441 unsigned long address) 442 443 #endif /* !CONFIG_X86_64 */ 444 445 /* C-Code mapping */ 446 #define DECLARE_IDTENTRY_NMI DECLARE_IDTENTRY_RAW 447 #define DEFINE_IDTENTRY_NMI DEFINE_IDTENTRY_RAW 448 #define DEFINE_FREDENTRY_NMI DEFINE_FREDENTRY_RAW 449 450 #ifdef CONFIG_X86_64 451 #define DECLARE_IDTENTRY_MCE DECLARE_IDTENTRY_IST 452 #define DEFINE_IDTENTRY_MCE DEFINE_IDTENTRY_IST 453 #define DEFINE_IDTENTRY_MCE_USER DEFINE_IDTENTRY_NOIST 454 #define DEFINE_FREDENTRY_MCE DEFINE_FREDENTRY_RAW 455 456 #define DECLARE_IDTENTRY_DEBUG DECLARE_IDTENTRY_IST 457 #define DEFINE_IDTENTRY_DEBUG DEFINE_IDTENTRY_IST 458 #define DEFINE_IDTENTRY_DEBUG_USER DEFINE_IDTENTRY_NOIST 459 #define DEFINE_FREDENTRY_DEBUG DEFINE_FREDENTRY_RAW 460 #endif 461 462 void idt_install_sysvec(unsigned int n, const void *function); 463 void fred_install_sysvec(unsigned int vector, const idtentry_t function); 464 465 #define sysvec_install(vector, function) { \ 466 if (IS_ENABLED(CONFIG_X86_FRED)) \ 467 fred_install_sysvec(vector, function); \ 468 if (!cpu_feature_enabled(X86_FEATURE_FRED)) \ 469 idt_install_sysvec(vector, asm_##function); \ 470 } 471 472 #else /* !__ASSEMBLER__ */ 473 474 /* 475 * The ASM variants for DECLARE_IDTENTRY*() which emit the ASM entry stubs. 476 */ 477 #define DECLARE_IDTENTRY(vector, func) \ 478 idtentry vector asm_##func func has_error_code=0 479 480 #define DECLARE_IDTENTRY_ERRORCODE(vector, func) \ 481 idtentry vector asm_##func func has_error_code=1 482 483 /* Special case for 32bit IRET 'trap'. Do not emit ASM code */ 484 #define DECLARE_IDTENTRY_SW(vector, func) 485 486 #define DECLARE_IDTENTRY_RAW(vector, func) \ 487 DECLARE_IDTENTRY(vector, func) 488 489 #define DECLARE_IDTENTRY_RAW_ERRORCODE(vector, func) \ 490 DECLARE_IDTENTRY_ERRORCODE(vector, func) 491 492 /* Entries for common/spurious (device) interrupts */ 493 #define DECLARE_IDTENTRY_IRQ(vector, func) \ 494 idtentry_irq vector func 495 496 /* System vector entries */ 497 #define DECLARE_IDTENTRY_SYSVEC(vector, func) \ 498 DECLARE_IDTENTRY(vector, func) 499 500 #ifdef CONFIG_X86_64 501 # define DECLARE_IDTENTRY_MCE(vector, func) \ 502 idtentry_mce_db vector asm_##func func 503 504 # define DECLARE_IDTENTRY_DEBUG(vector, func) \ 505 idtentry_mce_db vector asm_##func func 506 507 # define DECLARE_IDTENTRY_DF(vector, func) \ 508 idtentry_df vector asm_##func func 509 510 # define DECLARE_IDTENTRY_XENCB(vector, func) \ 511 DECLARE_IDTENTRY(vector, func) 512 513 # define DECLARE_IDTENTRY_VC(vector, func) \ 514 idtentry_vc vector asm_##func func 515 516 #else 517 # define DECLARE_IDTENTRY_MCE(vector, func) \ 518 DECLARE_IDTENTRY(vector, func) 519 520 /* No ASM emitted for DF as this goes through a C shim */ 521 # define DECLARE_IDTENTRY_DF(vector, func) 522 523 /* No ASM emitted for XEN hypervisor callback */ 524 # define DECLARE_IDTENTRY_XENCB(vector, func) 525 526 #endif 527 528 /* No ASM code emitted for NMI */ 529 #define DECLARE_IDTENTRY_NMI(vector, func) 530 531 /* 532 * ASM code to emit the common vector entry stubs where each stub is 533 * packed into IDT_ALIGN bytes. 534 * 535 * Note, that the 'pushq imm8' is emitted via '.byte 0x6a, vector' because 536 * GCC treats the local vector variable as unsigned int and would expand 537 * all vectors above 0x7F to a 5 byte push. The original code did an 538 * adjustment of the vector number to be in the signed byte range to avoid 539 * this. While clever it's mindboggling counterintuitive and requires the 540 * odd conversion back to a real vector number in the C entry points. Using 541 * .byte achieves the same thing and the only fixup needed in the C entry 542 * point is to mask off the bits above bit 7 because the push is sign 543 * extending. 544 */ 545 .align IDT_ALIGN 546 SYM_CODE_START(irq_entries_start) 547 vector=FIRST_EXTERNAL_VECTOR 548 .rept NR_EXTERNAL_VECTORS 549 UNWIND_HINT_IRET_REGS 550 0 : 551 ENDBR 552 .byte 0x6a, vector 553 jmp asm_common_interrupt 554 /* Ensure that the above is IDT_ALIGN bytes max */ 555 .fill 0b + IDT_ALIGN - ., 1, 0xcc 556 vector = vector+1 557 .endr 558 SYM_CODE_END(irq_entries_start) 559 560 #ifdef CONFIG_X86_LOCAL_APIC 561 .align IDT_ALIGN 562 SYM_CODE_START(spurious_entries_start) 563 vector=FIRST_SYSTEM_VECTOR 564 .rept NR_SYSTEM_VECTORS 565 UNWIND_HINT_IRET_REGS 566 0 : 567 ENDBR 568 .byte 0x6a, vector 569 jmp asm_spurious_interrupt 570 /* Ensure that the above is IDT_ALIGN bytes max */ 571 .fill 0b + IDT_ALIGN - ., 1, 0xcc 572 vector = vector+1 573 .endr 574 SYM_CODE_END(spurious_entries_start) 575 #endif 576 577 #endif /* __ASSEMBLER__ */ 578 579 /* 580 * The actual entry points. Note that DECLARE_IDTENTRY*() serves two 581 * purposes: 582 * - provide the function declarations when included from C-Code 583 * - emit the ASM stubs when included from entry_32/64.S 584 * 585 * This avoids duplicate defines and ensures that everything is consistent. 586 */ 587 588 /* 589 * Dummy trap number so the low level ASM macro vector number checks do not 590 * match which results in emitting plain IDTENTRY stubs without bells and 591 * whistles. 592 */ 593 #define X86_TRAP_OTHER 0xFFFF 594 595 /* Simple exception entry points. No hardware error code */ 596 DECLARE_IDTENTRY(X86_TRAP_DE, exc_divide_error); 597 DECLARE_IDTENTRY(X86_TRAP_OF, exc_overflow); 598 DECLARE_IDTENTRY(X86_TRAP_BR, exc_bounds); 599 DECLARE_IDTENTRY(X86_TRAP_NM, exc_device_not_available); 600 DECLARE_IDTENTRY(X86_TRAP_OLD_MF, exc_coproc_segment_overrun); 601 DECLARE_IDTENTRY(X86_TRAP_SPURIOUS, exc_spurious_interrupt_bug); 602 DECLARE_IDTENTRY(X86_TRAP_MF, exc_coprocessor_error); 603 DECLARE_IDTENTRY(X86_TRAP_XF, exc_simd_coprocessor_error); 604 605 /* 32bit software IRET trap. Do not emit ASM code */ 606 DECLARE_IDTENTRY_SW(X86_TRAP_IRET, iret_error); 607 608 /* Simple exception entries with error code pushed by hardware */ 609 DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_TS, exc_invalid_tss); 610 DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_NP, exc_segment_not_present); 611 DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_SS, exc_stack_segment); 612 DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_GP, exc_general_protection); 613 DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_AC, exc_alignment_check); 614 615 /* Raw exception entries which need extra work */ 616 DECLARE_IDTENTRY_RAW(X86_TRAP_UD, exc_invalid_op); 617 DECLARE_IDTENTRY_RAW(X86_TRAP_BP, exc_int3); 618 DECLARE_IDTENTRY_RAW_ERRORCODE(X86_TRAP_PF, exc_page_fault); 619 620 #if defined(CONFIG_IA32_EMULATION) 621 DECLARE_IDTENTRY_RAW(IA32_SYSCALL_VECTOR, int80_emulation); 622 #endif 623 624 #ifdef CONFIG_X86_MCE 625 #ifdef CONFIG_X86_64 626 DECLARE_IDTENTRY_MCE(X86_TRAP_MC, exc_machine_check); 627 #else 628 DECLARE_IDTENTRY_RAW(X86_TRAP_MC, exc_machine_check); 629 #endif 630 #ifdef CONFIG_XEN_PV 631 DECLARE_IDTENTRY_RAW(X86_TRAP_MC, xenpv_exc_machine_check); 632 #endif 633 #endif 634 635 /* NMI */ 636 637 #if IS_ENABLED(CONFIG_KVM_INTEL) 638 /* 639 * Special entry point for VMX which invokes this on the kernel stack, even for 640 * 64-bit, i.e. without using an IST. asm_exc_nmi() requires an IST to work 641 * correctly vs. the NMI 'executing' marker. Used for 32-bit kernels as well 642 * to avoid more ifdeffery. 643 */ 644 DECLARE_IDTENTRY(X86_TRAP_NMI, exc_nmi_kvm_vmx); 645 #endif 646 647 DECLARE_IDTENTRY_NMI(X86_TRAP_NMI, exc_nmi); 648 #ifdef CONFIG_XEN_PV 649 DECLARE_IDTENTRY_RAW(X86_TRAP_NMI, xenpv_exc_nmi); 650 #endif 651 652 /* #DB */ 653 #ifdef CONFIG_X86_64 654 DECLARE_IDTENTRY_DEBUG(X86_TRAP_DB, exc_debug); 655 #else 656 DECLARE_IDTENTRY_RAW(X86_TRAP_DB, exc_debug); 657 #endif 658 #ifdef CONFIG_XEN_PV 659 DECLARE_IDTENTRY_RAW(X86_TRAP_DB, xenpv_exc_debug); 660 #endif 661 662 /* #DF */ 663 DECLARE_IDTENTRY_DF(X86_TRAP_DF, exc_double_fault); 664 #ifdef CONFIG_XEN_PV 665 DECLARE_IDTENTRY_RAW_ERRORCODE(X86_TRAP_DF, xenpv_exc_double_fault); 666 #endif 667 668 /* #CP */ 669 #ifdef CONFIG_X86_CET 670 DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_CP, exc_control_protection); 671 #endif 672 673 /* #VC */ 674 #ifdef CONFIG_AMD_MEM_ENCRYPT 675 DECLARE_IDTENTRY_VC(X86_TRAP_VC, exc_vmm_communication); 676 #endif 677 678 #ifdef CONFIG_XEN_PV 679 DECLARE_IDTENTRY_XENCB(X86_TRAP_OTHER, exc_xen_hypervisor_callback); 680 DECLARE_IDTENTRY_RAW(X86_TRAP_OTHER, exc_xen_unknown_trap); 681 #endif 682 683 #ifdef CONFIG_INTEL_TDX_GUEST 684 DECLARE_IDTENTRY(X86_TRAP_VE, exc_virtualization_exception); 685 #endif 686 687 /* Device interrupts common/spurious */ 688 DECLARE_IDTENTRY_IRQ(X86_TRAP_OTHER, common_interrupt); 689 #ifdef CONFIG_X86_LOCAL_APIC 690 DECLARE_IDTENTRY_IRQ(X86_TRAP_OTHER, spurious_interrupt); 691 #endif 692 693 /* System vector entry points */ 694 #ifdef CONFIG_X86_LOCAL_APIC 695 DECLARE_IDTENTRY_SYSVEC(ERROR_APIC_VECTOR, sysvec_error_interrupt); 696 DECLARE_IDTENTRY_SYSVEC(SPURIOUS_APIC_VECTOR, sysvec_spurious_apic_interrupt); 697 DECLARE_IDTENTRY_SYSVEC(LOCAL_TIMER_VECTOR, sysvec_apic_timer_interrupt); 698 DECLARE_IDTENTRY_SYSVEC(X86_PLATFORM_IPI_VECTOR, sysvec_x86_platform_ipi); 699 #endif 700 701 #ifdef CONFIG_SMP 702 DECLARE_IDTENTRY(RESCHEDULE_VECTOR, sysvec_reschedule_ipi); 703 DECLARE_IDTENTRY_SYSVEC(REBOOT_VECTOR, sysvec_reboot); 704 DECLARE_IDTENTRY_SYSVEC(CALL_FUNCTION_SINGLE_VECTOR, sysvec_call_function_single); 705 DECLARE_IDTENTRY_SYSVEC(CALL_FUNCTION_VECTOR, sysvec_call_function); 706 #else 707 # define fred_sysvec_reschedule_ipi NULL 708 # define fred_sysvec_reboot NULL 709 # define fred_sysvec_call_function_single NULL 710 # define fred_sysvec_call_function NULL 711 #endif 712 713 #ifdef CONFIG_X86_LOCAL_APIC 714 # ifdef CONFIG_X86_MCE_THRESHOLD 715 DECLARE_IDTENTRY_SYSVEC(THRESHOLD_APIC_VECTOR, sysvec_threshold); 716 # else 717 # define fred_sysvec_threshold NULL 718 # endif 719 720 # ifdef CONFIG_X86_MCE_AMD 721 DECLARE_IDTENTRY_SYSVEC(DEFERRED_ERROR_VECTOR, sysvec_deferred_error); 722 # else 723 # define fred_sysvec_deferred_error NULL 724 # endif 725 726 # ifdef CONFIG_X86_THERMAL_VECTOR 727 DECLARE_IDTENTRY_SYSVEC(THERMAL_APIC_VECTOR, sysvec_thermal); 728 # else 729 # define fred_sysvec_thermal NULL 730 # endif 731 732 # ifdef CONFIG_IRQ_WORK 733 DECLARE_IDTENTRY_SYSVEC(IRQ_WORK_VECTOR, sysvec_irq_work); 734 # else 735 # define fred_sysvec_irq_work NULL 736 # endif 737 #endif 738 739 #if IS_ENABLED(CONFIG_KVM) 740 DECLARE_IDTENTRY_SYSVEC(POSTED_INTR_VECTOR, sysvec_kvm_posted_intr_ipi); 741 DECLARE_IDTENTRY_SYSVEC(POSTED_INTR_WAKEUP_VECTOR, sysvec_kvm_posted_intr_wakeup_ipi); 742 DECLARE_IDTENTRY_SYSVEC(POSTED_INTR_NESTED_VECTOR, sysvec_kvm_posted_intr_nested_ipi); 743 #else 744 # define fred_sysvec_kvm_posted_intr_ipi NULL 745 # define fred_sysvec_kvm_posted_intr_wakeup_ipi NULL 746 # define fred_sysvec_kvm_posted_intr_nested_ipi NULL 747 #endif 748 749 # ifdef CONFIG_X86_POSTED_MSI 750 DECLARE_IDTENTRY_SYSVEC(POSTED_MSI_NOTIFICATION_VECTOR, sysvec_posted_msi_notification); 751 #else 752 # define fred_sysvec_posted_msi_notification NULL 753 # endif 754 755 #if IS_ENABLED(CONFIG_HYPERV) 756 DECLARE_IDTENTRY_SYSVEC(HYPERVISOR_CALLBACK_VECTOR, sysvec_hyperv_callback); 757 DECLARE_IDTENTRY_SYSVEC(HYPERV_REENLIGHTENMENT_VECTOR, sysvec_hyperv_reenlightenment); 758 DECLARE_IDTENTRY_SYSVEC(HYPERV_STIMER0_VECTOR, sysvec_hyperv_stimer0); 759 #endif 760 761 #if IS_ENABLED(CONFIG_ACRN_GUEST) 762 DECLARE_IDTENTRY_SYSVEC(HYPERVISOR_CALLBACK_VECTOR, sysvec_acrn_hv_callback); 763 #endif 764 765 #ifdef CONFIG_XEN_PVHVM 766 DECLARE_IDTENTRY_SYSVEC(HYPERVISOR_CALLBACK_VECTOR, sysvec_xen_hvm_callback); 767 #endif 768 769 #ifdef CONFIG_KVM_GUEST 770 DECLARE_IDTENTRY_SYSVEC(HYPERVISOR_CALLBACK_VECTOR, sysvec_kvm_asyncpf_interrupt); 771 #endif 772 773 #undef X86_TRAP_OTHER 774 775 #endif 776