1/* SPDX-License-Identifier: GPL-2.0 */ 2/* 3 * The actual FRED entry points. 4 */ 5 6#include <linux/export.h> 7 8#include <asm/asm.h> 9#include <asm/fred.h> 10#include <asm/segment.h> 11 12#include "calling.h" 13 14 .code64 15 .section .noinstr.text, "ax" 16 17.macro FRED_ENTER 18 UNWIND_HINT_END_OF_STACK 19 ANNOTATE_NOENDBR 20 PUSH_AND_CLEAR_REGS 21 movq %rsp, %rdi /* %rdi -> pt_regs */ 22.endm 23 24.macro FRED_EXIT 25 UNWIND_HINT_REGS 26 POP_REGS 27.endm 28 29/* 30 * The new RIP value that FRED event delivery establishes is 31 * IA32_FRED_CONFIG & ~FFFH for events that occur in ring 3. 32 * Thus the FRED ring 3 entry point must be 4K page aligned. 33 */ 34 .align 4096 35 36SYM_CODE_START_NOALIGN(asm_fred_entrypoint_user) 37 FRED_ENTER 38 call fred_entry_from_user 39SYM_INNER_LABEL(asm_fred_exit_user, SYM_L_GLOBAL) 40 FRED_EXIT 411: ERETU 42 43 _ASM_EXTABLE_TYPE(1b, asm_fred_entrypoint_user, EX_TYPE_ERETU) 44SYM_CODE_END(asm_fred_entrypoint_user) 45 46/* 47 * The new RIP value that FRED event delivery establishes is 48 * (IA32_FRED_CONFIG & ~FFFH) + 256 for events that occur in 49 * ring 0, i.e., asm_fred_entrypoint_user + 256. 50 */ 51 .org asm_fred_entrypoint_user + 256, 0xcc 52SYM_CODE_START_NOALIGN(asm_fred_entrypoint_kernel) 53 FRED_ENTER 54 call fred_entry_from_kernel 55 FRED_EXIT 56 ERETS 57SYM_CODE_END(asm_fred_entrypoint_kernel) 58 59#if IS_ENABLED(CONFIG_KVM_INTEL) 60SYM_FUNC_START(asm_fred_entry_from_kvm) 61 ANNOTATE_NOENDBR 62 push %rbp 63 mov %rsp, %rbp 64 65 UNWIND_HINT_SAVE 66 67 /* 68 * Both IRQ and NMI from VMX can be handled on current task stack 69 * because there is no need to protect from reentrancy and the call 70 * stack leading to this helper is effectively constant and shallow 71 * (relatively speaking). Do the same when FRED is active, i.e., no 72 * need to check current stack level for a stack switch. 73 * 74 * Emulate the FRED-defined redzone and stack alignment. 75 */ 76 sub $(FRED_CONFIG_REDZONE_AMOUNT << 6), %rsp 77 and $FRED_STACK_FRAME_RSP_MASK, %rsp 78 79 /* 80 * Start to push a FRED stack frame, which is always 64 bytes: 81 * 82 * +--------+-----------------+ 83 * | Bytes | Usage | 84 * +--------+-----------------+ 85 * | 63:56 | Reserved | 86 * | 55:48 | Event Data | 87 * | 47:40 | SS + Event Info | 88 * | 39:32 | RSP | 89 * | 31:24 | RFLAGS | 90 * | 23:16 | CS + Aux Info | 91 * | 15:8 | RIP | 92 * | 7:0 | Error Code | 93 * +--------+-----------------+ 94 */ 95 push $0 /* Reserved, must be 0 */ 96 push $0 /* Event data, 0 for IRQ/NMI */ 97 push %rdi /* fred_ss handed in by the caller */ 98 push %rbp 99 pushf 100 push $__KERNEL_CS 101 102 /* 103 * Unlike the IDT event delivery, FRED _always_ pushes an error code 104 * after pushing the return RIP, thus the CALL instruction CANNOT be 105 * used here to push the return RIP, otherwise there is no chance to 106 * push an error code before invoking the IRQ/NMI handler. 107 * 108 * Use LEA to get the return RIP and push it, then push an error code. 109 */ 110 lea 1f(%rip), %rax 111 push %rax /* Return RIP */ 112 push $0 /* Error code, 0 for IRQ/NMI */ 113 114 PUSH_AND_CLEAR_REGS clear_callee=0 unwind_hint=0 115 116 movq %rsp, %rdi /* %rdi -> pt_regs */ 117 /* 118 * At this point: {rdi, rsi, rdx, rcx, r8, r9}, {r10, r11}, {rax, rdx} 119 * are clobbered, which corresponds to: arguments, extra caller-saved 120 * and return. All registers a C function is allowed to clobber. 121 * 122 * Notably, the callee-saved registers: {rbx, r12, r13, r14, r15} 123 * are untouched, with the exception of rbp, which carries the stack 124 * frame and will be restored before exit. 125 * 126 * Further calling another C function will not alter this state. 127 */ 128 call __fred_entry_from_kvm /* Call the C entry point */ 129 130 /* 131 * When FRED, use ERETS to potentially clear NMIs, otherwise simply 132 * restore the stack pointer. 133 */ 134 ALTERNATIVE "nop; nop; mov %rbp, %rsp", \ 135 __stringify(add $C_PTREGS_SIZE, %rsp; ERETS), \ 136 X86_FEATURE_FRED 137 1381: /* 139 * Objtool doesn't understand ERETS, and the cfi register state is 140 * different from initial_func_cfi due to PUSH_REGS. Tell it the state 141 * is similar to where UNWIND_HINT_SAVE is. 142 */ 143 UNWIND_HINT_RESTORE 144 145 pop %rbp 146 RET 147 148SYM_FUNC_END(asm_fred_entry_from_kvm) 149EXPORT_SYMBOL_GPL(asm_fred_entry_from_kvm); 150#endif 151