1 /* -*- mode: asm -*- */ 2 /*- 3 * SPDX-License-Identifier: BSD-3-Clause 4 * 5 * Copyright (c) 1993 The Regents of the University of California. 6 * All rights reserved. 7 * 8 * Copyright (c) 2018 The FreeBSD Foundation 9 * All rights reserved. 10 * 11 * Portions of this software were developed by 12 * Konstantin Belousov <kib@FreeBSD.org> under sponsorship from 13 * the FreeBSD Foundation. 14 * 15 * Redistribution and use in source and binary forms, with or without 16 * modification, are permitted provided that the following conditions 17 * are met: 18 * 1. Redistributions of source code must retain the above copyright 19 * notice, this list of conditions and the following disclaimer. 20 * 2. Redistributions in binary form must reproduce the above copyright 21 * notice, this list of conditions and the following disclaimer in the 22 * documentation and/or other materials provided with the distribution. 23 * 3. Neither the name of the University nor the names of its contributors 24 * may be used to endorse or promote products derived from this software 25 * without specific prior written permission. 26 * 27 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 28 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 29 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 30 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 31 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 32 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 33 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 34 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 35 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 36 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 37 * SUCH DAMAGE. 38 * 39 * $FreeBSD$ 40 */ 41 42 #if defined(__i386__) 43 #include <i386/asmacros.h> 44 #else /* !__i386__ */ 45 46 #ifndef _MACHINE_ASMACROS_H_ 47 #define _MACHINE_ASMACROS_H_ 48 49 #include <sys/cdefs.h> 50 51 /* XXX too much duplication in various asm*.h's. */ 52 53 /* 54 * CNAME is used to manage the relationship between symbol names in C 55 * and the equivalent assembly language names. CNAME is given a name as 56 * it would be used in a C program. It expands to the equivalent assembly 57 * language name. 58 */ 59 #define CNAME(csym) csym 60 61 #define ALIGN_DATA .p2align 3 /* 8 byte alignment, zero filled */ 62 #define ALIGN_TEXT .p2align 4,0x90 /* 16-byte alignment, nop filled */ 63 #define SUPERALIGN_TEXT .p2align 4,0x90 /* 16-byte alignment, nop filled */ 64 65 #define GEN_ENTRY(name) ALIGN_TEXT; .globl CNAME(name); \ 66 .type CNAME(name),@function; CNAME(name): 67 #define ENTRY(name) GEN_ENTRY(name) 68 #define ALTENTRY(name) GEN_ENTRY(name) 69 #define END(name) .size name, . - name 70 71 /* 72 * Convenience for adding frame pointers to hand-coded ASM. Useful for 73 * DTrace, HWPMC, and KDB. 74 */ 75 #define PUSH_FRAME_POINTER \ 76 pushq %rbp ; \ 77 movq %rsp, %rbp ; 78 #define POP_FRAME_POINTER \ 79 popq %rbp 80 81 #ifdef LOCORE 82 /* 83 * Access per-CPU data. 84 */ 85 #define PCPU(member) %gs:PC_ ## member 86 #define PCPU_ADDR(member, reg) \ 87 movq %gs:PC_PRVSPACE, reg ; \ 88 addq $PC_ ## member, reg 89 90 /* 91 * Convenience macro for declaring interrupt entry points. 92 */ 93 #define IDTVEC(name) ALIGN_TEXT; .globl __CONCAT(X,name); \ 94 .type __CONCAT(X,name),@function; __CONCAT(X,name): 95 96 .macro SAVE_SEGS 97 movw %fs,TF_FS(%rsp) 98 movw %gs,TF_GS(%rsp) 99 movw %es,TF_ES(%rsp) 100 movw %ds,TF_DS(%rsp) 101 .endm 102 103 .macro MOVE_STACKS qw 104 .L.offset=0 105 .rept \qw 106 movq .L.offset(%rsp),%rdx 107 movq %rdx,.L.offset(%rax) 108 .L.offset=.L.offset+8 109 .endr 110 .endm 111 112 .macro PTI_UUENTRY has_err 113 movq PCPU(KCR3),%rax 114 movq %rax,%cr3 115 movq PCPU(RSP0),%rax 116 subq $PTI_SIZE - 8 * (1 - \has_err),%rax 117 MOVE_STACKS ((PTI_SIZE / 8) - 1 + \has_err) 118 movq %rax,%rsp 119 popq %rdx 120 popq %rax 121 .endm 122 123 .macro PTI_UENTRY has_err 124 swapgs 125 lfence 126 cmpq $~0,PCPU(UCR3) 127 je 1f 128 pushq %rax 129 pushq %rdx 130 PTI_UUENTRY \has_err 131 1: 132 .endm 133 134 .macro PTI_ENTRY name, contk, contu, has_err=0 135 ALIGN_TEXT 136 .globl X\name\()_pti 137 .type X\name\()_pti,@function 138 X\name\()_pti: 139 /* %rax, %rdx, and possibly err are not yet pushed */ 140 testb $SEL_RPL_MASK,PTI_CS-PTI_ERR-((1-\has_err)*8)(%rsp) 141 jz \contk 142 PTI_UENTRY \has_err 143 jmp \contu 144 .endm 145 146 .macro PTI_INTRENTRY vec_name 147 SUPERALIGN_TEXT 148 .globl X\vec_name\()_pti 149 .type X\vec_name\()_pti,@function 150 X\vec_name\()_pti: 151 testb $SEL_RPL_MASK,PTI_CS-3*8(%rsp) /* err, %rax, %rdx not pushed */ 152 jz .L\vec_name\()_u 153 PTI_UENTRY has_err=0 154 jmp .L\vec_name\()_u 155 .endm 156 157 .macro INTR_PUSH_FRAME vec_name 158 SUPERALIGN_TEXT 159 .globl X\vec_name 160 .type X\vec_name,@function 161 X\vec_name: 162 testb $SEL_RPL_MASK,PTI_CS-3*8(%rsp) /* come from kernel? */ 163 jz .L\vec_name\()_u /* Yes, dont swapgs again */ 164 swapgs 165 .L\vec_name\()_u: 166 lfence 167 subq $TF_RIP,%rsp /* skip dummy tf_err and tf_trapno */ 168 movq %rdi,TF_RDI(%rsp) 169 movq %rsi,TF_RSI(%rsp) 170 movq %rdx,TF_RDX(%rsp) 171 movq %rcx,TF_RCX(%rsp) 172 movq %r8,TF_R8(%rsp) 173 movq %r9,TF_R9(%rsp) 174 movq %rax,TF_RAX(%rsp) 175 movq %rbx,TF_RBX(%rsp) 176 movq %rbp,TF_RBP(%rsp) 177 movq %r10,TF_R10(%rsp) 178 movq %r11,TF_R11(%rsp) 179 movq %r12,TF_R12(%rsp) 180 movq %r13,TF_R13(%rsp) 181 movq %r14,TF_R14(%rsp) 182 movq %r15,TF_R15(%rsp) 183 SAVE_SEGS 184 movl $TF_HASSEGS,TF_FLAGS(%rsp) 185 pushfq 186 andq $~(PSL_D|PSL_AC),(%rsp) 187 popfq 188 testb $SEL_RPL_MASK,TF_CS(%rsp) /* come from kernel ? */ 189 jz 1f /* yes, leave PCB_FULL_IRET alone */ 190 movq PCPU(CURPCB),%r8 191 andl $~PCB_FULL_IRET,PCB_FLAGS(%r8) 192 call handle_ibrs_entry 193 1: 194 .endm 195 196 .macro INTR_HANDLER vec_name 197 .text 198 PTI_INTRENTRY \vec_name 199 INTR_PUSH_FRAME \vec_name 200 .endm 201 202 .macro RESTORE_REGS 203 movq TF_RDI(%rsp),%rdi 204 movq TF_RSI(%rsp),%rsi 205 movq TF_RDX(%rsp),%rdx 206 movq TF_RCX(%rsp),%rcx 207 movq TF_R8(%rsp),%r8 208 movq TF_R9(%rsp),%r9 209 movq TF_RAX(%rsp),%rax 210 movq TF_RBX(%rsp),%rbx 211 movq TF_RBP(%rsp),%rbp 212 movq TF_R10(%rsp),%r10 213 movq TF_R11(%rsp),%r11 214 movq TF_R12(%rsp),%r12 215 movq TF_R13(%rsp),%r13 216 movq TF_R14(%rsp),%r14 217 movq TF_R15(%rsp),%r15 218 .endm 219 220 #ifdef KMSAN 221 /* 222 * The KMSAN runtime relies on a TLS block to track initialization and origin 223 * state for function parameters and return values. To keep this state 224 * consistent in the face of asynchronous kernel-mode traps, the runtime 225 * maintains a stack of blocks: when handling an exception or interrupt, 226 * kmsan_intr_enter() pushes the new block to be used until the handler is 227 * complete, at which point kmsan_intr_leave() restores the previous block. 228 * 229 * Thus, KMSAN_ENTER/LEAVE hooks are required only in handlers for events that 230 * may have happened while in kernel-mode. In particular, they are not required 231 * around amd64_syscall() or ast() calls. Otherwise, kmsan_intr_enter() can be 232 * called unconditionally, without distinguishing between entry from user-mode 233 * or kernel-mode. 234 */ 235 #define KMSAN_ENTER callq kmsan_intr_enter 236 #define KMSAN_LEAVE callq kmsan_intr_leave 237 #else 238 #define KMSAN_ENTER 239 #define KMSAN_LEAVE 240 #endif 241 242 #endif /* LOCORE */ 243 244 #ifdef __STDC__ 245 #define ELFNOTE(name, type, desctype, descdata...) \ 246 .pushsection .note.name, "a", @note ; \ 247 .align 4 ; \ 248 .long 2f - 1f /* namesz */ ; \ 249 .long 4f - 3f /* descsz */ ; \ 250 .long type ; \ 251 1:.asciz #name ; \ 252 2:.align 4 ; \ 253 3:desctype descdata ; \ 254 4:.align 4 ; \ 255 .popsection 256 #else /* !__STDC__, i.e. -traditional */ 257 #define ELFNOTE(name, type, desctype, descdata) \ 258 .pushsection .note.name, "a", @note ; \ 259 .align 4 ; \ 260 .long 2f - 1f /* namesz */ ; \ 261 .long 4f - 3f /* descsz */ ; \ 262 .long type ; \ 263 1:.asciz "name" ; \ 264 2:.align 4 ; \ 265 3:desctype descdata ; \ 266 4:.align 4 ; \ 267 .popsection 268 #endif /* __STDC__ */ 269 270 #endif /* !_MACHINE_ASMACROS_H_ */ 271 272 #endif /* __i386__ */ 273