1/* SPDX-License-Identifier: GPL-2.0 */ 2/* 3 * The actual FRED entry points. 4 */ 5 6#include <linux/export.h> 7#include <linux/kvm_types.h> 8 9#include <asm/asm.h> 10#include <asm/fred.h> 11#include <asm/segment.h> 12 13#include "calling.h" 14 15 .code64 16 .section .noinstr.text, "ax" 17 18.macro FRED_ENTER 19 UNWIND_HINT_END_OF_STACK 20 ANNOTATE_NOENDBR 21 PUSH_AND_CLEAR_REGS 22 movq %rsp, %rdi /* %rdi -> pt_regs */ 23.endm 24 25.macro FRED_EXIT 26 UNWIND_HINT_REGS 27 POP_REGS 28.endm 29 30/* 31 * The new RIP value that FRED event delivery establishes is 32 * IA32_FRED_CONFIG & ~FFFH for events that occur in ring 3. 33 * Thus the FRED ring 3 entry point must be 4K page aligned. 34 */ 35 .align 4096 36 37SYM_CODE_START_NOALIGN(asm_fred_entrypoint_user) 38 FRED_ENTER 39 call fred_entry_from_user 40SYM_INNER_LABEL(asm_fred_exit_user, SYM_L_GLOBAL) 41 FRED_EXIT 421: ERETU 43 44 _ASM_EXTABLE_TYPE(1b, asm_fred_entrypoint_user, EX_TYPE_ERETU) 45SYM_CODE_END(asm_fred_entrypoint_user) 46 47/* 48 * The new RIP value that FRED event delivery establishes is 49 * (IA32_FRED_CONFIG & ~FFFH) + 256 for events that occur in 50 * ring 0, i.e., asm_fred_entrypoint_user + 256. 51 */ 52 .org asm_fred_entrypoint_user + 256, 0xcc 53SYM_CODE_START_NOALIGN(asm_fred_entrypoint_kernel) 54 FRED_ENTER 55 call fred_entry_from_kernel 56 FRED_EXIT 57 ERETS 58SYM_CODE_END(asm_fred_entrypoint_kernel) 59 60#if IS_ENABLED(CONFIG_KVM_INTEL) 61SYM_FUNC_START(asm_fred_entry_from_kvm) 62 ANNOTATE_NOENDBR 63 push %rbp 64 mov %rsp, %rbp 65 66 UNWIND_HINT_SAVE 67 68 /* 69 * Both IRQ and NMI from VMX can be handled on current task stack 70 * because there is no need to protect from reentrancy and the call 71 * stack leading to this helper is effectively constant and shallow 72 * (relatively speaking). Do the same when FRED is active, i.e., no 73 * need to check current stack level for a stack switch. 74 * 75 * Emulate the FRED-defined redzone and stack alignment. 76 */ 77 sub $(FRED_CONFIG_REDZONE_AMOUNT << 6), %rsp 78 and $FRED_STACK_FRAME_RSP_MASK, %rsp 79 80 /* 81 * Start to push a FRED stack frame, which is always 64 bytes: 82 * 83 * +--------+-----------------+ 84 * | Bytes | Usage | 85 * +--------+-----------------+ 86 * | 63:56 | Reserved | 87 * | 55:48 | Event Data | 88 * | 47:40 | SS + Event Info | 89 * | 39:32 | RSP | 90 * | 31:24 | RFLAGS | 91 * | 23:16 | CS + Aux Info | 92 * | 15:8 | RIP | 93 * | 7:0 | Error Code | 94 * +--------+-----------------+ 95 */ 96 push $0 /* Reserved, must be 0 */ 97 push $0 /* Event data, 0 for IRQ/NMI */ 98 push %rdi /* fred_ss handed in by the caller */ 99 push %rbp 100 pushf 101 push $__KERNEL_CS 102 103 /* 104 * Unlike the IDT event delivery, FRED _always_ pushes an error code 105 * after pushing the return RIP, thus the CALL instruction CANNOT be 106 * used here to push the return RIP, otherwise there is no chance to 107 * push an error code before invoking the IRQ/NMI handler. 108 * 109 * Use LEA to get the return RIP and push it, then push an error code. 110 */ 111 lea 1f(%rip), %rax 112 push %rax /* Return RIP */ 113 push $0 /* Error code, 0 for IRQ/NMI */ 114 115 PUSH_AND_CLEAR_REGS clear_callee=0 unwind_hint=0 116 117 movq %rsp, %rdi /* %rdi -> pt_regs */ 118 /* 119 * At this point: {rdi, rsi, rdx, rcx, r8, r9}, {r10, r11}, {rax, rdx} 120 * are clobbered, which corresponds to: arguments, extra caller-saved 121 * and return. All registers a C function is allowed to clobber. 122 * 123 * Notably, the callee-saved registers: {rbx, r12, r13, r14, r15} 124 * are untouched, with the exception of rbp, which carries the stack 125 * frame and will be restored before exit. 126 * 127 * Further calling another C function will not alter this state. 128 */ 129 call __fred_entry_from_kvm /* Call the C entry point */ 130 131 /* 132 * When FRED, use ERETS to potentially clear NMIs, otherwise simply 133 * restore the stack pointer. 134 */ 135 ALTERNATIVE "nop; nop; mov %rbp, %rsp", \ 136 __stringify(add $C_PTREGS_SIZE, %rsp; ERETS), \ 137 X86_FEATURE_FRED 138 1391: /* 140 * Objtool doesn't understand ERETS, and the cfi register state is 141 * different from initial_func_cfi due to PUSH_REGS. Tell it the state 142 * is similar to where UNWIND_HINT_SAVE is. 143 */ 144 UNWIND_HINT_RESTORE 145 146 pop %rbp 147 RET 148 149SYM_FUNC_END(asm_fred_entry_from_kvm) 150EXPORT_SYMBOL_FOR_KVM(asm_fred_entry_from_kvm); 151#endif 152