1/* SPDX-License-Identifier: GPL-2.0 */ 2/* 3 * The actual FRED entry points. 4 */ 5 6#include <linux/export.h> 7#include <linux/kvm_types.h> 8 9#include <asm/asm.h> 10#include <asm/frame.h> 11#include <asm/fred.h> 12#include <asm/segment.h> 13 14#include "calling.h" 15 16 .code64 17 .section .noinstr.text, "ax" 18 19.macro FRED_ENTER 20 UNWIND_HINT_END_OF_STACK 21 ANNOTATE_NOENDBR 22 PUSH_AND_CLEAR_REGS 23 ENCODE_FRAME_POINTER 24 movq %rsp, %rdi /* %rdi -> pt_regs */ 25.endm 26 27.macro FRED_EXIT 28 UNWIND_HINT_REGS 29 POP_REGS 30.endm 31 32/* 33 * The new RIP value that FRED event delivery establishes is 34 * IA32_FRED_CONFIG & ~FFFH for events that occur in ring 3. 35 * Thus the FRED ring 3 entry point must be 4K page aligned. 36 */ 37 .align 4096 38 39SYM_CODE_START_NOALIGN(asm_fred_entrypoint_user) 40 FRED_ENTER 41 call fred_entry_from_user 42SYM_INNER_LABEL(asm_fred_exit_user, SYM_L_GLOBAL) 43 FRED_EXIT 441: ERETU 45 46 _ASM_EXTABLE_TYPE(1b, asm_fred_entrypoint_user, EX_TYPE_ERETU) 47SYM_CODE_END(asm_fred_entrypoint_user) 48 49/* 50 * The new RIP value that FRED event delivery establishes is 51 * (IA32_FRED_CONFIG & ~FFFH) + 256 for events that occur in 52 * ring 0, i.e., asm_fred_entrypoint_user + 256. 53 */ 54 .org asm_fred_entrypoint_user + 256, 0xcc 55SYM_CODE_START_NOALIGN(asm_fred_entrypoint_kernel) 56 FRED_ENTER 57 call fred_entry_from_kernel 58 FRED_EXIT 59 ERETS 60SYM_CODE_END(asm_fred_entrypoint_kernel) 61 62#if IS_ENABLED(CONFIG_KVM_INTEL) 63SYM_FUNC_START(asm_fred_entry_from_kvm) 64 ANNOTATE_NOENDBR 65 push %rbp 66 mov %rsp, %rbp 67 68 UNWIND_HINT_SAVE 69 70 /* 71 * Both IRQ and NMI from VMX can be handled on current task stack 72 * because there is no need to protect from reentrancy and the call 73 * stack leading to this helper is effectively constant and shallow 74 * (relatively speaking). Do the same when FRED is active, i.e., no 75 * need to check current stack level for a stack switch. 76 * 77 * Emulate the FRED-defined redzone and stack alignment. 78 */ 79 sub $(FRED_CONFIG_REDZONE_AMOUNT << 6), %rsp 80 and $FRED_STACK_FRAME_RSP_MASK, %rsp 81 82 /* 83 * Start to push a FRED stack frame, which is always 64 bytes: 84 * 85 * +--------+-----------------+ 86 * | Bytes | Usage | 87 * +--------+-----------------+ 88 * | 63:56 | Reserved | 89 * | 55:48 | Event Data | 90 * | 47:40 | SS + Event Info | 91 * | 39:32 | RSP | 92 * | 31:24 | RFLAGS | 93 * | 23:16 | CS + Aux Info | 94 * | 15:8 | RIP | 95 * | 7:0 | Error Code | 96 * +--------+-----------------+ 97 */ 98 push $0 /* Reserved, must be 0 */ 99 push $0 /* Event data, 0 for IRQ/NMI */ 100 push %rdi /* fred_ss handed in by the caller */ 101 push %rbp 102 pushf 103 push $__KERNEL_CS 104 105 /* 106 * Unlike the IDT event delivery, FRED _always_ pushes an error code 107 * after pushing the return RIP, thus the CALL instruction CANNOT be 108 * used here to push the return RIP, otherwise there is no chance to 109 * push an error code before invoking the IRQ/NMI handler. 110 * 111 * Use LEA to get the return RIP and push it, then push an error code. 112 */ 113 lea 1f(%rip), %rax 114 push %rax /* Return RIP */ 115 push $0 /* Error code, 0 for IRQ/NMI */ 116 117 PUSH_AND_CLEAR_REGS clear_callee=0 unwind_hint=0 118 119 movq %rsp, %rdi /* %rdi -> pt_regs */ 120 /* 121 * At this point: {rdi, rsi, rdx, rcx, r8, r9}, {r10, r11}, {rax, rdx} 122 * are clobbered, which corresponds to: arguments, extra caller-saved 123 * and return. All registers a C function is allowed to clobber. 124 * 125 * Notably, the callee-saved registers: {rbx, r12, r13, r14, r15} 126 * are untouched, with the exception of rbp, which carries the stack 127 * frame and will be restored before exit. 128 * 129 * Further calling another C function will not alter this state. 130 */ 131 call __fred_entry_from_kvm /* Call the C entry point */ 132 133 /* 134 * When FRED, use ERETS to potentially clear NMIs, otherwise simply 135 * restore the stack pointer. 136 */ 137 ALTERNATIVE "nop; nop; mov %rbp, %rsp", \ 138 __stringify(add $C_PTREGS_SIZE, %rsp; ERETS), \ 139 X86_FEATURE_FRED 140 1411: /* 142 * Objtool doesn't understand ERETS, and the cfi register state is 143 * different from initial_func_cfi due to PUSH_REGS. Tell it the state 144 * is similar to where UNWIND_HINT_SAVE is. 145 */ 146 UNWIND_HINT_RESTORE 147 148 pop %rbp 149 RET 150 151SYM_FUNC_END(asm_fred_entry_from_kvm) 152#endif 153