1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */ 2 /* 3 * rseq-x86.h 4 * 5 * (C) Copyright 2016-2022 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> 6 */ 7 8 #ifndef RSEQ_H 9 #error "Never use <rseq-x86.h> directly; include <rseq.h> instead." 10 #endif 11 12 #include <stdint.h> 13 14 /* 15 * RSEQ_SIG is used with the following reserved undefined instructions, which 16 * trap in user-space: 17 * 18 * x86-32: 0f b9 3d 53 30 05 53 ud1 0x53053053,%edi 19 * x86-64: 0f b9 3d 53 30 05 53 ud1 0x53053053(%rip),%edi 20 */ 21 #define RSEQ_SIG 0x53053053 22 23 /* 24 * Due to a compiler optimization bug in gcc-8 with asm goto and TLS asm input 25 * operands, we cannot use "m" input operands, and rather pass the __rseq_abi 26 * address through a "r" input operand. 27 */ 28 29 /* Offset of cpu_id, rseq_cs, and mm_cid fields in struct rseq. */ 30 #define RSEQ_CPU_ID_OFFSET 4 31 #define RSEQ_CS_OFFSET 8 32 #define RSEQ_MM_CID_OFFSET 24 33 34 #ifdef __x86_64__ 35 36 #define RSEQ_ASM_TP_SEGMENT %%fs 37 38 #define rseq_smp_mb() \ 39 __asm__ __volatile__ ("lock; addl $0,-128(%%rsp)" ::: "memory", "cc") 40 #define rseq_smp_rmb() rseq_barrier() 41 #define rseq_smp_wmb() rseq_barrier() 42 43 #define rseq_smp_load_acquire(p) \ 44 __extension__ ({ \ 45 rseq_unqual_scalar_typeof(*(p)) ____p1 = RSEQ_READ_ONCE(*(p)); \ 46 rseq_barrier(); \ 47 ____p1; \ 48 }) 49 50 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb() 51 52 #define rseq_smp_store_release(p, v) \ 53 do { \ 54 rseq_barrier(); \ 55 RSEQ_WRITE_ONCE(*(p), v); \ 56 } while (0) 57 58 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ 59 start_ip, post_commit_offset, abort_ip) \ 60 ".pushsection __rseq_cs, \"aw\"\n\t" \ 61 ".balign 32\n\t" \ 62 __rseq_str(label) ":\n\t" \ 63 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ 64 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \ 65 ".popsection\n\t" \ 66 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \ 67 ".quad " __rseq_str(label) "b\n\t" \ 68 ".popsection\n\t" 69 70 71 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ 72 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ 73 (post_commit_ip - start_ip), abort_ip) 74 75 /* 76 * Exit points of a rseq critical section consist of all instructions outside 77 * of the critical section where a critical section can either branch to or 78 * reach through the normal course of its execution. The abort IP and the 79 * post-commit IP are already part of the __rseq_cs section and should not be 80 * explicitly defined as additional exit points. Knowing all exit points is 81 * useful to assist debuggers stepping over the critical section. 82 */ 83 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ 84 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \ 85 ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \ 86 ".popsection\n\t" 87 88 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ 89 RSEQ_INJECT_ASM(1) \ 90 "leaq " __rseq_str(cs_label) "(%%rip), %%rax\n\t" \ 91 "movq %%rax, " __rseq_str(rseq_cs) "\n\t" \ 92 __rseq_str(label) ":\n\t" 93 94 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ 95 RSEQ_INJECT_ASM(2) \ 96 "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \ 97 "jnz " __rseq_str(label) "\n\t" 98 99 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \ 100 ".pushsection __rseq_failure, \"ax\"\n\t" \ 101 /* Disassembler-friendly signature: ud1 <sig>(%rip),%edi. */ \ 102 ".byte 0x0f, 0xb9, 0x3d\n\t" \ 103 ".long " __rseq_str(RSEQ_SIG) "\n\t" \ 104 __rseq_str(label) ":\n\t" \ 105 teardown \ 106 "jmp %l[" __rseq_str(abort_label) "]\n\t" \ 107 ".popsection\n\t" 108 109 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \ 110 ".pushsection __rseq_failure, \"ax\"\n\t" \ 111 __rseq_str(label) ":\n\t" \ 112 teardown \ 113 "jmp %l[" __rseq_str(cmpfail_label) "]\n\t" \ 114 ".popsection\n\t" 115 116 #elif defined(__i386__) 117 118 #define RSEQ_ASM_TP_SEGMENT %%gs 119 120 #define rseq_smp_mb() \ 121 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc") 122 #define rseq_smp_rmb() \ 123 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc") 124 #define rseq_smp_wmb() \ 125 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc") 126 127 #define rseq_smp_load_acquire(p) \ 128 __extension__ ({ \ 129 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \ 130 rseq_smp_mb(); \ 131 ____p1; \ 132 }) 133 134 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb() 135 136 #define rseq_smp_store_release(p, v) \ 137 do { \ 138 rseq_smp_mb(); \ 139 RSEQ_WRITE_ONCE(*p, v); \ 140 } while (0) 141 142 /* 143 * Use eax as scratch register and take memory operands as input to 144 * lessen register pressure. Especially needed when compiling in O0. 145 */ 146 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ 147 start_ip, post_commit_offset, abort_ip) \ 148 ".pushsection __rseq_cs, \"aw\"\n\t" \ 149 ".balign 32\n\t" \ 150 __rseq_str(label) ":\n\t" \ 151 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ 152 ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \ 153 ".popsection\n\t" \ 154 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \ 155 ".long " __rseq_str(label) "b, 0x0\n\t" \ 156 ".popsection\n\t" 157 158 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ 159 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ 160 (post_commit_ip - start_ip), abort_ip) 161 162 /* 163 * Exit points of a rseq critical section consist of all instructions outside 164 * of the critical section where a critical section can either branch to or 165 * reach through the normal course of its execution. The abort IP and the 166 * post-commit IP are already part of the __rseq_cs section and should not be 167 * explicitly defined as additional exit points. Knowing all exit points is 168 * useful to assist debuggers stepping over the critical section. 169 */ 170 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ 171 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \ 172 ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \ 173 ".popsection\n\t" 174 175 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ 176 RSEQ_INJECT_ASM(1) \ 177 "movl $" __rseq_str(cs_label) ", " __rseq_str(rseq_cs) "\n\t" \ 178 __rseq_str(label) ":\n\t" 179 180 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ 181 RSEQ_INJECT_ASM(2) \ 182 "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \ 183 "jnz " __rseq_str(label) "\n\t" 184 185 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \ 186 ".pushsection __rseq_failure, \"ax\"\n\t" \ 187 /* Disassembler-friendly signature: ud1 <sig>,%edi. */ \ 188 ".byte 0x0f, 0xb9, 0x3d\n\t" \ 189 ".long " __rseq_str(RSEQ_SIG) "\n\t" \ 190 __rseq_str(label) ":\n\t" \ 191 teardown \ 192 "jmp %l[" __rseq_str(abort_label) "]\n\t" \ 193 ".popsection\n\t" 194 195 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \ 196 ".pushsection __rseq_failure, \"ax\"\n\t" \ 197 __rseq_str(label) ":\n\t" \ 198 teardown \ 199 "jmp %l[" __rseq_str(cmpfail_label) "]\n\t" \ 200 ".popsection\n\t" 201 202 #endif 203 204 /* Per-cpu-id indexing. */ 205 206 #define RSEQ_TEMPLATE_CPU_ID 207 #define RSEQ_TEMPLATE_MO_RELAXED 208 #include "rseq-x86-bits.h" 209 #undef RSEQ_TEMPLATE_MO_RELAXED 210 211 #define RSEQ_TEMPLATE_MO_RELEASE 212 #include "rseq-x86-bits.h" 213 #undef RSEQ_TEMPLATE_MO_RELEASE 214 #undef RSEQ_TEMPLATE_CPU_ID 215 216 /* Per-mm-cid indexing. */ 217 218 #define RSEQ_TEMPLATE_MM_CID 219 #define RSEQ_TEMPLATE_MO_RELAXED 220 #include "rseq-x86-bits.h" 221 #undef RSEQ_TEMPLATE_MO_RELAXED 222 223 #define RSEQ_TEMPLATE_MO_RELEASE 224 #include "rseq-x86-bits.h" 225 #undef RSEQ_TEMPLATE_MO_RELEASE 226 #undef RSEQ_TEMPLATE_MM_CID 227 228 /* APIs which are not based on cpu ids. */ 229 230 #define RSEQ_TEMPLATE_CPU_ID_NONE 231 #define RSEQ_TEMPLATE_MO_RELAXED 232 #include "rseq-x86-bits.h" 233 #undef RSEQ_TEMPLATE_MO_RELAXED 234 #undef RSEQ_TEMPLATE_CPU_ID_NONE 235