1// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 2// See https://llvm.org/LICENSE.txt for license information. 3// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 4 5#include "assembly.h" 6 7// Out-of-line LSE atomics helpers. Ported from libgcc library. 8// N = {1, 2, 4, 8} 9// M = {1, 2, 4, 8, 16} 10// ORDER = {'relax', 'acq', 'rel', 'acq_rel'} 11// Routines implemented: 12// 13// iM __aarch64_casM_ORDER(iM expected, iM desired, iM *ptr) 14// iN __aarch64_swpN_ORDER(iN val, iN *ptr) 15// iN __aarch64_ldaddN_ORDER(iN val, iN *ptr) 16// iN __aarch64_ldclrN_ORDER(iN val, iN *ptr) 17// iN __aarch64_ldeorN_ORDER(iN val, iN *ptr) 18// iN __aarch64_ldsetN_ORDER(iN val, iN *ptr) 19// 20// Routines may modify temporary registers tmp0, tmp1, tmp2, 21// return value x0 and the flags only. 22 23#ifdef __aarch64__ 24 25#ifdef HAS_ASM_LSE 26.arch armv8-a+lse 27#else 28.arch armv8-a 29#endif 30 31#if !defined(__APPLE__) 32HIDDEN(__aarch64_have_lse_atomics) 33#else 34HIDDEN(___aarch64_have_lse_atomics) 35#endif 36 37// Generate mnemonics for 38// L_cas: SIZE: 1,2,4,8,16 MODEL: 1,2,3,4 39// L_swp L_ldadd L_ldclr L_ldeor L_ldset: SIZE: 1,2,4,8 MODEL: 1,2,3,4 40 41#if SIZE == 1 42#define S b 43#define UXT uxtb 44#define B 0x00000000 45#elif SIZE == 2 46#define S h 47#define UXT uxth 48#define B 0x40000000 49#elif SIZE == 4 || SIZE == 8 || SIZE == 16 50#define S 51#define UXT mov 52#if SIZE == 4 53#define B 0x80000000 54#elif SIZE == 8 55#define B 0xc0000000 56#endif 57#else 58#error 59#endif // SIZE 60 61#if MODEL == 1 62#define SUFF _relax 63#define A 64#define L 65#define M 0x000000 66#define N 0x000000 67#elif MODEL == 2 68#define SUFF _acq 69#define A a 70#define L 71#define M 0x400000 72#define N 0x800000 73#elif MODEL == 3 74#define SUFF _rel 75#define A 76#define L l 77#define M 0x008000 78#define N 0x400000 79#elif MODEL == 4 80#define SUFF _acq_rel 81#define A a 82#define L l 83#define M 0x408000 84#define N 0xc00000 85#else 86#error 87#endif // MODEL 88 89// Define register size. 90#define x(N) GLUE2(x, N) 91#define w(N) GLUE2(w, N) 92#if SIZE < 8 93#define s(N) w(N) 94#else 95#define s(N) x(N) 96#endif 97 98#define NAME(BASE) GLUE4(__aarch64_, BASE, SIZE, SUFF) 99#define LDXR GLUE4(ld, A, xr, S) 100#define STXR GLUE4(st, L, xr, S) 101 102// Define temporary registers. 103#define tmp0 16 104#define tmp1 17 105#define tmp2 15 106 107// Macro for branch to label if no LSE available 108.macro JUMP_IF_NOT_LSE label 109#if !defined(__APPLE__) 110 adrp x(tmp0), __aarch64_have_lse_atomics 111 ldrb w(tmp0), [x(tmp0), :lo12:__aarch64_have_lse_atomics] 112#else 113 adrp x(tmp0), ___aarch64_have_lse_atomics@page 114 ldrb w(tmp0), [x(tmp0), ___aarch64_have_lse_atomics@pageoff] 115#endif 116 cbz w(tmp0), \label 117.endm 118 119#ifdef L_cas 120DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(NAME(cas)) 121 JUMP_IF_NOT_LSE 8f 122#if SIZE < 16 123#ifdef HAS_ASM_LSE 124#define CAS GLUE4(cas, A, L, S) s(0), s(1), [x2] 125#else 126#define CAS .inst 0x08a07c41 + B + M 127#endif 128 CAS // s(0), s(1), [x2] 129 ret 1308: 131 UXT s(tmp0), s(0) 1320: 133 LDXR s(0), [x2] 134 cmp s(0), s(tmp0) 135 bne 1f 136 STXR w(tmp1), s(1), [x2] 137 cbnz w(tmp1), 0b 1381: 139 ret 140#else 141#define LDXP GLUE3(ld, A, xp) 142#define STXP GLUE3(st, L, xp) 143#ifdef HAS_ASM_LSE 144#define CASP GLUE3(casp, A, L) x0, x1, x2, x3, [x4] 145#else 146#define CASP .inst 0x48207c82 + M 147#endif 148 149 CASP // x0, x1, x2, x3, [x4] 150 ret 1518: 152 mov x(tmp0), x0 153 mov x(tmp1), x1 1540: 155 LDXP x0, x1, [x4] 156 cmp x0, x(tmp0) 157 ccmp x1, x(tmp1), #0, eq 158 bne 1f 159 STXP w(tmp2), x2, x3, [x4] 160 cbnz w(tmp2), 0b 1611: 162 ret 163#endif 164END_COMPILERRT_OUTLINE_FUNCTION(NAME(cas)) 165#endif // L_cas 166 167#ifdef L_swp 168#ifdef HAS_ASM_LSE 169#define SWP GLUE4(swp, A, L, S) s(0), s(0), [x1] 170#else 171#define SWP .inst 0x38208020 + B + N 172#endif 173DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(NAME(swp)) 174 JUMP_IF_NOT_LSE 8f 175 SWP // s(0), s(0), [x1] 176 ret 1778: 178 mov s(tmp0), s(0) 1790: 180 LDXR s(0), [x1] 181 STXR w(tmp1), s(tmp0), [x1] 182 cbnz w(tmp1), 0b 183 ret 184END_COMPILERRT_OUTLINE_FUNCTION(NAME(swp)) 185#endif // L_swp 186 187#if defined(L_ldadd) || defined(L_ldclr) || \ 188 defined(L_ldeor) || defined(L_ldset) 189 190#ifdef L_ldadd 191#define LDNM ldadd 192#define OP add 193#define OPN 0x0000 194#elif defined(L_ldclr) 195#define LDNM ldclr 196#define OP bic 197#define OPN 0x1000 198#elif defined(L_ldeor) 199#define LDNM ldeor 200#define OP eor 201#define OPN 0x2000 202#elif defined(L_ldset) 203#define LDNM ldset 204#define OP orr 205#define OPN 0x3000 206#else 207#error 208#endif 209 210#ifdef HAS_ASM_LSE 211#define LDOP GLUE4(LDNM, A, L, S) s(0), s(0), [x1] 212#else 213#define LDOP .inst 0x38200020 + OPN + B + N 214#endif 215 216DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(NAME(LDNM)) 217 JUMP_IF_NOT_LSE 8f 218 LDOP // s(0), s(0), [x1] 219 ret 2208: 221 mov s(tmp0), s(0) 2220: 223 LDXR s(0), [x1] 224 OP s(tmp1), s(0), s(tmp0) 225 STXR w(tmp2), s(tmp1), [x1] 226 cbnz w(tmp2), 0b 227 ret 228END_COMPILERRT_OUTLINE_FUNCTION(NAME(LDNM)) 229#endif // L_ldadd L_ldclr L_ldeor L_ldset 230 231NO_EXEC_STACK_DIRECTIVE 232 233// GNU property note for BTI and PAC 234GNU_PROPERTY_BTI_PAC 235 236#endif // __aarch64__ 237