1/* 2 * This file and its contents are supplied under the terms of the 3 * Common Development and Distribution License ("CDDL"), version 1.0. 4 * You may only use this file in accordance with the terms of version 5 * 1.0 of the CDDL. 6 * 7 * A full copy of the text of the CDDL should have accompanied this 8 * source. A copy of the CDDL is also available via the Internet at 9 * http://www.illumos.org/license/CDDL. 10 */ 11 12/* 13 * Copyright 2019 Joyent, Inc. 14 */ 15 16 .file "retpoline.s" 17 18/* 19 * This file implements the various hooks that are needed for retpolines and 20 * return stack buffer (RSB) stuffing. For more information, please see the 21 * 'Speculative Execution CPU Side Channel Security' section of the 22 * uts/i86pc/os/cpuid.c big theory statement. 23 */ 24 25#include <sys/asm_linkage.h> 26#include <sys/x86_archext.h> 27 28#if defined(__amd64) 29 30/* 31 * This macro generates the default retpoline entry point that the compiler 32 * expects. It implements the expected retpoline form. 33 */ 34#define RETPOLINE_MKTHUNK(reg) \ 35 ENTRY(__x86_indirect_thunk_##reg) \ 36 call 2f; \ 371: \ 38 pause; \ 39 lfence; \ 40 jmp 1b; \ 412: \ 42 movq %##reg, (%rsp); \ 43 ret; \ 44 SET_SIZE(__x86_indirect_thunk_##reg) 45 46/* 47 * This macro generates the default retpoline form. It exists in addition to the 48 * thunk so if we need to restore the default retpoline behavior to the thunk 49 * we can. 50 */ 51#define RETPOLINE_MKGENERIC(reg) \ 52 ENTRY(__x86_indirect_thunk_gen_##reg) \ 53 call 2f; \ 541: \ 55 pause; \ 56 lfence; \ 57 jmp 1b; \ 582: \ 59 movq %##reg, (%rsp); \ 60 ret; \ 61 SET_SIZE(__x86_indirect_thunk_gen_##reg) 62 63/* 64 * This macro generates the no-op form of the retpoline which will be used if we 65 * either need to disable retpolines because we have enhanced IBRS or because we 66 * have been asked to disable mitigations. 67 */ 68#define RETPOLINE_MKJUMP(reg) \ 69 ENTRY(__x86_indirect_thunk_jmp_##reg) \ 70 jmp *%##reg; \ 71 SET_SIZE(__x86_indirect_thunk_jmp_##reg) 72 73 RETPOLINE_MKTHUNK(rax) 74 RETPOLINE_MKTHUNK(rbx) 75 RETPOLINE_MKTHUNK(rcx) 76 RETPOLINE_MKTHUNK(rdx) 77 RETPOLINE_MKTHUNK(rdi) 78 RETPOLINE_MKTHUNK(rsi) 79 RETPOLINE_MKTHUNK(rbp) 80 RETPOLINE_MKTHUNK(r8) 81 RETPOLINE_MKTHUNK(r9) 82 RETPOLINE_MKTHUNK(r10) 83 RETPOLINE_MKTHUNK(r11) 84 RETPOLINE_MKTHUNK(r12) 85 RETPOLINE_MKTHUNK(r13) 86 RETPOLINE_MKTHUNK(r14) 87 RETPOLINE_MKTHUNK(r15) 88 89 RETPOLINE_MKGENERIC(rax) 90 RETPOLINE_MKGENERIC(rbx) 91 RETPOLINE_MKGENERIC(rcx) 92 RETPOLINE_MKGENERIC(rdx) 93 RETPOLINE_MKGENERIC(rdi) 94 RETPOLINE_MKGENERIC(rsi) 95 RETPOLINE_MKGENERIC(rbp) 96 RETPOLINE_MKGENERIC(r8) 97 RETPOLINE_MKGENERIC(r9) 98 RETPOLINE_MKGENERIC(r10) 99 RETPOLINE_MKGENERIC(r11) 100 RETPOLINE_MKGENERIC(r12) 101 RETPOLINE_MKGENERIC(r13) 102 RETPOLINE_MKGENERIC(r14) 103 RETPOLINE_MKGENERIC(r15) 104 105 RETPOLINE_MKJUMP(rax) 106 RETPOLINE_MKJUMP(rbx) 107 RETPOLINE_MKJUMP(rcx) 108 RETPOLINE_MKJUMP(rdx) 109 RETPOLINE_MKJUMP(rdi) 110 RETPOLINE_MKJUMP(rsi) 111 RETPOLINE_MKJUMP(rbp) 112 RETPOLINE_MKJUMP(r8) 113 RETPOLINE_MKJUMP(r9) 114 RETPOLINE_MKJUMP(r10) 115 RETPOLINE_MKJUMP(r11) 116 RETPOLINE_MKJUMP(r12) 117 RETPOLINE_MKJUMP(r13) 118 RETPOLINE_MKJUMP(r14) 119 RETPOLINE_MKJUMP(r15) 120 121 /* 122 * The x86_rsb_stuff function is called from pretty arbitrary 123 * contexts. It's much easier for us to save and restore all the 124 * registers we touch rather than clobber them for callers. You must 125 * preserve this property or the system will panic at best. 126 */ 127 ENTRY(x86_rsb_stuff) 128 /* 129 * These nops are present so we can patch a ret instruction if we need 130 * to disable RSB stuffing because enhanced IBRS is present or we're 131 * disabling mitigations. 132 */ 133 nop 134 nop 135 pushq %rdi 136 pushq %rax 137 movl $16, %edi 138 movq %rsp, %rax 139rsb_loop: 140 call 2f 1411: 142 pause 143 call 1b 1442: 145 call 2f 1461: 147 pause 148 call 1b 1492: 150 subl $1, %edi 151 jnz rsb_loop 152 movq %rax, %rsp 153 popq %rax 154 popq %rdi 155 ret 156 SET_SIZE(x86_rsb_stuff) 157 158#elif defined(__i386) 159 160/* 161 * While the kernel is 64-bit only, dboot is still 32-bit, so there are a 162 * limited number of variants that are used for 32-bit. However as dboot is 163 * short lived and uses them sparingly, we only do the full variant and do not 164 * have an AMD specific version. 165 */ 166 167#define RETPOLINE_MKTHUNK(reg) \ 168 ENTRY(__x86_indirect_thunk_##reg) \ 169 call 2f; \ 1701: \ 171 pause; \ 172 lfence; \ 173 jmp 1b; \ 1742: \ 175 movl %##reg, (%esp); \ 176 ret; \ 177 SET_SIZE(__x86_indirect_thunk_##reg) 178 179 RETPOLINE_MKTHUNK(edi) 180 RETPOLINE_MKTHUNK(eax) 181 182#else 183#error "Your architecture is in another castle." 184#endif 185