1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */ 2 /* 3 * rseq-arm64.h 4 * 5 * (C) Copyright 2016-2022 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> 6 * (C) Copyright 2018 - Will Deacon <will.deacon@arm.com> 7 */ 8 9 /* 10 * aarch64 -mbig-endian generates mixed endianness code vs data: 11 * little-endian code and big-endian data. Ensure the RSEQ_SIG signature 12 * matches code endianness. 13 */ 14 #define RSEQ_SIG_CODE 0xd428bc00 /* BRK #0x45E0. */ 15 #define RSEQ_SIG_DATA RSEQ_SIG_CODE 16 17 #define RSEQ_SIG RSEQ_SIG_DATA 18 19 #define rseq_smp_mb() __asm__ __volatile__ ("dmb ish" ::: "memory") 20 #define rseq_smp_rmb() __asm__ __volatile__ ("dmb ishld" ::: "memory") 21 #define rseq_smp_wmb() __asm__ __volatile__ ("dmb ishst" ::: "memory") 22 23 #define rseq_smp_load_acquire(p) \ 24 __extension__ ({ \ 25 union { rseq_unqual_scalar_typeof(*(p)) __val; char __c[sizeof(*(p))]; } __u; \ 26 switch (sizeof(*(p))) { \ 27 case 1: \ 28 __asm__ __volatile__ ("ldarb %w0, %1" \ 29 : "=r" (*(__u8 *)__u.__c) \ 30 : "Q" (*(p)) : "memory"); \ 31 break; \ 32 case 2: \ 33 __asm__ __volatile__ ("ldarh %w0, %1" \ 34 : "=r" (*(__u16 *)__u.__c) \ 35 : "Q" (*(p)) : "memory"); \ 36 break; \ 37 case 4: \ 38 __asm__ __volatile__ ("ldar %w0, %1" \ 39 : "=r" (*(__u32 *)__u.__c) \ 40 : "Q" (*(p)) : "memory"); \ 41 break; \ 42 case 8: \ 43 __asm__ __volatile__ ("ldar %0, %1" \ 44 : "=r" (*(__u64 *)__u.__c) \ 45 : "Q" (*(p)) : "memory"); \ 46 break; \ 47 } \ 48 (rseq_unqual_scalar_typeof(*(p)))__u.__val; \ 49 }) 50 51 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb() 52 53 #define rseq_smp_store_release(p, v) \ 54 do { \ 55 union { rseq_unqual_scalar_typeof(*(p)) __val; char __c[sizeof(*(p))]; } __u = \ 56 { .__val = (rseq_unqual_scalar_typeof(*(p))) (v) }; \ 57 switch (sizeof(*(p))) { \ 58 case 1: \ 59 __asm__ __volatile__ ("stlrb %w1, %0" \ 60 : "=Q" (*(p)) \ 61 : "r" (*(__u8 *)__u.__c) \ 62 : "memory"); \ 63 break; \ 64 case 2: \ 65 __asm__ __volatile__ ("stlrh %w1, %0" \ 66 : "=Q" (*(p)) \ 67 : "r" (*(__u16 *)__u.__c) \ 68 : "memory"); \ 69 break; \ 70 case 4: \ 71 __asm__ __volatile__ ("stlr %w1, %0" \ 72 : "=Q" (*(p)) \ 73 : "r" (*(__u32 *)__u.__c) \ 74 : "memory"); \ 75 break; \ 76 case 8: \ 77 __asm__ __volatile__ ("stlr %1, %0" \ 78 : "=Q" (*(p)) \ 79 : "r" (*(__u64 *)__u.__c) \ 80 : "memory"); \ 81 break; \ 82 } \ 83 } while (0) 84 85 #define RSEQ_ASM_TMP_REG32 "w15" 86 #define RSEQ_ASM_TMP_REG "x15" 87 #define RSEQ_ASM_TMP_REG_2 "x14" 88 89 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip, \ 90 post_commit_offset, abort_ip) \ 91 " .pushsection __rseq_cs, \"aw\"\n" \ 92 " .balign 32\n" \ 93 __rseq_str(label) ":\n" \ 94 " .long " __rseq_str(version) ", " __rseq_str(flags) "\n" \ 95 " .quad " __rseq_str(start_ip) ", " \ 96 __rseq_str(post_commit_offset) ", " \ 97 __rseq_str(abort_ip) "\n" \ 98 " .popsection\n\t" \ 99 " .pushsection __rseq_cs_ptr_array, \"aw\"\n" \ 100 " .quad " __rseq_str(label) "b\n" \ 101 " .popsection\n" 102 103 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ 104 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ 105 (post_commit_ip - start_ip), abort_ip) 106 107 /* 108 * Exit points of a rseq critical section consist of all instructions outside 109 * of the critical section where a critical section can either branch to or 110 * reach through the normal course of its execution. The abort IP and the 111 * post-commit IP are already part of the __rseq_cs section and should not be 112 * explicitly defined as additional exit points. Knowing all exit points is 113 * useful to assist debuggers stepping over the critical section. 114 */ 115 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ 116 " .pushsection __rseq_exit_point_array, \"aw\"\n" \ 117 " .quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n" \ 118 " .popsection\n" 119 120 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ 121 RSEQ_INJECT_ASM(1) \ 122 " adrp " RSEQ_ASM_TMP_REG ", " __rseq_str(cs_label) "\n" \ 123 " add " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \ 124 ", :lo12:" __rseq_str(cs_label) "\n" \ 125 " str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(rseq_cs) "]\n" \ 126 __rseq_str(label) ":\n" 127 128 #define RSEQ_ASM_DEFINE_ABORT(label, abort_label) \ 129 " b 222f\n" \ 130 " .inst " __rseq_str(RSEQ_SIG_CODE) "\n" \ 131 __rseq_str(label) ":\n" \ 132 " b %l[" __rseq_str(abort_label) "]\n" \ 133 "222:\n" 134 135 #define RSEQ_ASM_OP_STORE(value, var) \ 136 " str %[" __rseq_str(value) "], %[" __rseq_str(var) "]\n" 137 138 #define RSEQ_ASM_OP_STORE_RELEASE(value, var) \ 139 " stlr %[" __rseq_str(value) "], %[" __rseq_str(var) "]\n" 140 141 #define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label) \ 142 RSEQ_ASM_OP_STORE(value, var) \ 143 __rseq_str(post_commit_label) ":\n" 144 145 #define RSEQ_ASM_OP_FINAL_STORE_RELEASE(value, var, post_commit_label) \ 146 RSEQ_ASM_OP_STORE_RELEASE(value, var) \ 147 __rseq_str(post_commit_label) ":\n" 148 149 #define RSEQ_ASM_OP_CMPEQ(var, expect, label) \ 150 " ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \ 151 " sub " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \ 152 ", %[" __rseq_str(expect) "]\n" \ 153 " cbnz " RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n" 154 155 #define RSEQ_ASM_OP_CMPEQ32(var, expect, label) \ 156 " ldr " RSEQ_ASM_TMP_REG32 ", %[" __rseq_str(var) "]\n" \ 157 " sub " RSEQ_ASM_TMP_REG32 ", " RSEQ_ASM_TMP_REG32 \ 158 ", %w[" __rseq_str(expect) "]\n" \ 159 " cbnz " RSEQ_ASM_TMP_REG32 ", " __rseq_str(label) "\n" 160 161 #define RSEQ_ASM_OP_CMPNE(var, expect, label) \ 162 " ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \ 163 " sub " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \ 164 ", %[" __rseq_str(expect) "]\n" \ 165 " cbz " RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n" 166 167 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ 168 RSEQ_INJECT_ASM(2) \ 169 RSEQ_ASM_OP_CMPEQ32(current_cpu_id, cpu_id, label) 170 171 #define RSEQ_ASM_OP_R_LOAD(var) \ 172 " ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" 173 174 #define RSEQ_ASM_OP_R_STORE(var) \ 175 " str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" 176 177 #define RSEQ_ASM_OP_R_LOAD_OFF(offset) \ 178 " ldr " RSEQ_ASM_TMP_REG ", [" RSEQ_ASM_TMP_REG \ 179 ", %[" __rseq_str(offset) "]]\n" 180 181 #define RSEQ_ASM_OP_R_ADD(count) \ 182 " add " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \ 183 ", %[" __rseq_str(count) "]\n" 184 185 #define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label) \ 186 " str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \ 187 __rseq_str(post_commit_label) ":\n" 188 189 #define RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len) \ 190 " cbz %[" __rseq_str(len) "], 333f\n" \ 191 " mov " RSEQ_ASM_TMP_REG_2 ", %[" __rseq_str(len) "]\n" \ 192 "222: sub " RSEQ_ASM_TMP_REG_2 ", " RSEQ_ASM_TMP_REG_2 ", #1\n" \ 193 " ldrb " RSEQ_ASM_TMP_REG32 ", [%[" __rseq_str(src) "]" \ 194 ", " RSEQ_ASM_TMP_REG_2 "]\n" \ 195 " strb " RSEQ_ASM_TMP_REG32 ", [%[" __rseq_str(dst) "]" \ 196 ", " RSEQ_ASM_TMP_REG_2 "]\n" \ 197 " cbnz " RSEQ_ASM_TMP_REG_2 ", 222b\n" \ 198 "333:\n" 199 200 /* Per-cpu-id indexing. */ 201 202 #define RSEQ_TEMPLATE_CPU_ID 203 #define RSEQ_TEMPLATE_MO_RELAXED 204 #include "rseq-arm64-bits.h" 205 #undef RSEQ_TEMPLATE_MO_RELAXED 206 207 #define RSEQ_TEMPLATE_MO_RELEASE 208 #include "rseq-arm64-bits.h" 209 #undef RSEQ_TEMPLATE_MO_RELEASE 210 #undef RSEQ_TEMPLATE_CPU_ID 211 212 /* Per-mm-cid indexing. */ 213 214 #define RSEQ_TEMPLATE_MM_CID 215 #define RSEQ_TEMPLATE_MO_RELAXED 216 #include "rseq-arm64-bits.h" 217 #undef RSEQ_TEMPLATE_MO_RELAXED 218 219 #define RSEQ_TEMPLATE_MO_RELEASE 220 #include "rseq-arm64-bits.h" 221 #undef RSEQ_TEMPLATE_MO_RELEASE 222 #undef RSEQ_TEMPLATE_MM_CID 223 224 /* APIs which are not based on cpu ids. */ 225 226 #define RSEQ_TEMPLATE_CPU_ID_NONE 227 #define RSEQ_TEMPLATE_MO_RELAXED 228 #include "rseq-arm64-bits.h" 229 #undef RSEQ_TEMPLATE_MO_RELAXED 230 #undef RSEQ_TEMPLATE_CPU_ID_NONE 231