1 /* 2 * Copyright 2009-2016 Samy Al Bahra. 3 * Copyright 2013-2016 Olivier Houchard. 4 * Copyright 2016 Alexey Kopytov. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #ifndef CK_PR_AARCH64_LSE_H 30 #define CK_PR_AARCH64_LSE_H 31 32 #ifndef CK_PR_H 33 #error Do not include this file directly, use ck_pr.h 34 #endif 35 36 CK_CC_INLINE static bool 37 ck_pr_cas_64_2_value(uint64_t target[2], uint64_t compare[2], uint64_t set[2], uint64_t value[2]) 38 { 39 uint64_t tmp1; 40 uint64_t tmp2; 41 register uint64_t x0 __asm__ ("x0") = compare[0]; 42 register uint64_t x1 __asm__ ("x1") = compare[1]; 43 register uint64_t x2 __asm__ ("x2") = set[0]; 44 register uint64_t x3 __asm__ ("x3") = set[1]; 45 46 __asm__ __volatile__("casp %0, %1, %4, %5, [%6];" 47 "eor %2, %0, %7;" 48 "eor %3, %1, %8;" 49 "orr %2, %2, %3;" 50 : "+&r" (x0), "+&r" (x1), "=&r" (tmp1), "=&r" (tmp2) 51 : "r" (x2), "r" (x3), "r" (target), "r" (compare[0]), "r" (compare[1]) 52 : "memory"); 53 54 value[0] = x0; 55 value[1] = x1; 56 57 return (!!tmp1); 58 } 59 60 CK_CC_INLINE static bool 61 ck_pr_cas_ptr_2_value(void *target, void *compare, void *set, void *value) 62 { 63 return (ck_pr_cas_64_2_value(CK_CPP_CAST(uint64_t *, target), 64 CK_CPP_CAST(uint64_t *, compare), 65 CK_CPP_CAST(uint64_t *, set), 66 CK_CPP_CAST(uint64_t *, value))); 67 } 68 69 CK_CC_INLINE static bool 70 ck_pr_cas_64_2(uint64_t target[2], uint64_t compare[2], uint64_t set[2]) 71 { 72 register uint64_t x0 __asm__ ("x0") = compare[0]; 73 register uint64_t x1 __asm__ ("x1") = compare[1]; 74 register uint64_t x2 __asm__ ("x2") = set[0]; 75 register uint64_t x3 __asm__ ("x3") = set[1]; 76 77 __asm__ __volatile__("casp %0, %1, %2, %3, [%4];" 78 "eor %0, %0, %5;" 79 "eor %1, %1, %6;" 80 "orr %0, %0, %1;" 81 : "+&r" (x0), "+&r" (x1) 82 : "r" (x2), "r" (x3), "r" (target), "r" (compare[0]), "r" (compare[1]) 83 : "memory"); 84 85 return (!!x0); 86 } 87 CK_CC_INLINE static bool 88 ck_pr_cas_ptr_2(void *target, void *compare, void *set) 89 { 90 return (ck_pr_cas_64_2(CK_CPP_CAST(uint64_t *, target), 91 CK_CPP_CAST(uint64_t *, compare), 92 CK_CPP_CAST(uint64_t *, set))); 93 } 94 95 96 #define CK_PR_CAS(N, M, T, W, R) \ 97 CK_CC_INLINE static bool \ 98 ck_pr_cas_##N##_value(M *target, T compare, T set, M *value) \ 99 { \ 100 *(T *)value = compare; \ 101 __asm__ __volatile__( \ 102 "cas" W " %" R "0, %" R "2, [%1];" \ 103 : "+&r" (*(T *)value) \ 104 : "r" (target), \ 105 "r" (set) \ 106 : "memory"); \ 107 return (*(T *)value == compare); \ 108 } \ 109 CK_CC_INLINE static bool \ 110 ck_pr_cas_##N(M *target, T compare, T set) \ 111 { \ 112 T previous = compare; \ 113 __asm__ __volatile__( \ 114 "cas" W " %" R "0, %" R "2, [%1];" \ 115 : "+&r" (previous) \ 116 : "r" (target), \ 117 "r" (set) \ 118 : "memory"); \ 119 return (previous == compare); \ 120 } 121 122 CK_PR_CAS(ptr, void, void *, "", "") 123 124 #define CK_PR_CAS_S(N, M, W, R) CK_PR_CAS(N, M, M, W, R) 125 CK_PR_CAS_S(64, uint64_t, "", "") 126 #ifndef CK_PR_DISABLE_DOUBLE 127 CK_PR_CAS_S(double, double, "", "") 128 #endif 129 CK_PR_CAS_S(32, uint32_t, "", "w") 130 CK_PR_CAS_S(uint, unsigned int, "", "w") 131 CK_PR_CAS_S(int, int, "", "w") 132 CK_PR_CAS_S(16, uint16_t, "h", "w") 133 CK_PR_CAS_S(8, uint8_t, "b", "w") 134 CK_PR_CAS_S(short, short, "h", "w") 135 CK_PR_CAS_S(char, char, "b", "w") 136 137 138 #undef CK_PR_CAS_S 139 #undef CK_PR_CAS 140 141 #define CK_PR_FAS(N, M, T, W, R) \ 142 CK_CC_INLINE static T \ 143 ck_pr_fas_##N(M *target, T v) \ 144 { \ 145 T previous; \ 146 __asm__ __volatile__( \ 147 "swp" W " %" R "2, %" R "0, [%1];" \ 148 : "=&r" (previous) \ 149 : "r" (target), \ 150 "r" (v) \ 151 : "memory"); \ 152 return (previous); \ 153 } 154 155 CK_PR_FAS(64, uint64_t, uint64_t, "", "") 156 CK_PR_FAS(32, uint32_t, uint32_t, "", "w") 157 CK_PR_FAS(ptr, void, void *, "", "") 158 CK_PR_FAS(int, int, int, "", "w") 159 CK_PR_FAS(uint, unsigned int, unsigned int, "", "w") 160 CK_PR_FAS(16, uint16_t, uint16_t, "h", "w") 161 CK_PR_FAS(8, uint8_t, uint8_t, "b", "w") 162 CK_PR_FAS(short, short, short, "h", "w") 163 CK_PR_FAS(char, char, char, "b", "w") 164 165 166 #undef CK_PR_FAS 167 168 #define CK_PR_UNARY(O, N, M, T, I, W, R, S) \ 169 CK_CC_INLINE static void \ 170 ck_pr_##O##_##N(M *target) \ 171 { \ 172 __asm__ __volatile__(I ";" \ 173 "st" S W " " R "0, [%0];" \ 174 : \ 175 : "r" (target) \ 176 : "x0", "memory"); \ 177 return; \ 178 } 179 180 CK_PR_UNARY(inc, ptr, void, void *, "mov x0, 1", "", "x", "add") 181 CK_PR_UNARY(dec, ptr, void, void *, "mov x0, -1", "", "x", "add") 182 CK_PR_UNARY(not, ptr, void, void *, "mov x0, -1", "", "x", "eor") 183 CK_PR_UNARY(inc, 64, uint64_t, uint64_t, "mov x0, 1", "", "x", "add") 184 CK_PR_UNARY(dec, 64, uint64_t, uint64_t, "mov x0, -1", "", "x", "add") 185 CK_PR_UNARY(not, 64, uint64_t, uint64_t, "mov x0, -1", "", "x", "eor") 186 187 #define CK_PR_UNARY_S(S, T, W) \ 188 CK_PR_UNARY(inc, S, T, T, "mov w0, 1", W, "w", "add") \ 189 CK_PR_UNARY(dec, S, T, T, "mov w0, -1", W, "w", "add") \ 190 CK_PR_UNARY(not, S, T, T, "mov w0, -1", W, "w", "eor") \ 191 192 CK_PR_UNARY_S(32, uint32_t, "") 193 CK_PR_UNARY_S(uint, unsigned int, "") 194 CK_PR_UNARY_S(int, int, "") 195 CK_PR_UNARY_S(16, uint16_t, "h") 196 CK_PR_UNARY_S(8, uint8_t, "b") 197 CK_PR_UNARY_S(short, short, "h") 198 CK_PR_UNARY_S(char, char, "b") 199 200 #undef CK_PR_UNARY_S 201 #undef CK_PR_UNARY 202 203 #define CK_PR_BINARY(O, N, M, T, S, W, R, I) \ 204 CK_CC_INLINE static void \ 205 ck_pr_##O##_##N(M *target, T delta) \ 206 { \ 207 __asm__ __volatile__(I ";" \ 208 "st" S W " %" R "0, [%1];" \ 209 : "+&r" (delta) \ 210 : "r" (target) \ 211 : "memory"); \ 212 return; \ 213 } 214 215 CK_PR_BINARY(and, ptr, void, uintptr_t, "clr", "", "", "mvn %0, %0") 216 CK_PR_BINARY(add, ptr, void, uintptr_t, "add", "", "", "") 217 CK_PR_BINARY(or, ptr, void, uintptr_t, "set", "", "", "") 218 CK_PR_BINARY(sub, ptr, void, uintptr_t, "add", "", "", "neg %0, %0") 219 CK_PR_BINARY(xor, ptr, void, uintptr_t, "eor", "", "", "") 220 CK_PR_BINARY(and, 64, uint64_t, uint64_t, "clr", "", "", "mvn %0, %0") 221 CK_PR_BINARY(add, 64, uint64_t, uint64_t, "add", "", "", "") 222 CK_PR_BINARY(or, 64, uint64_t, uint64_t, "set", "", "", "") 223 CK_PR_BINARY(sub, 64, uint64_t, uint64_t, "add", "", "", "neg %0, %0") 224 CK_PR_BINARY(xor, 64, uint64_t, uint64_t, "eor", "", "", "") 225 226 #define CK_PR_BINARY_S(S, T, W) \ 227 CK_PR_BINARY(and, S, T, T, "clr", W, "w", "mvn %w0, %w0") \ 228 CK_PR_BINARY(add, S, T, T, "add", W, "w", "") \ 229 CK_PR_BINARY(or, S, T, T, "set", W, "w", "") \ 230 CK_PR_BINARY(sub, S, T, T, "add", W, "w", "neg %w0, %w0") \ 231 CK_PR_BINARY(xor, S, T, T, "eor", W, "w", "") 232 233 CK_PR_BINARY_S(32, uint32_t, "") 234 CK_PR_BINARY_S(uint, unsigned int, "") 235 CK_PR_BINARY_S(int, int, "") 236 CK_PR_BINARY_S(16, uint16_t, "h") 237 CK_PR_BINARY_S(8, uint8_t, "b") 238 CK_PR_BINARY_S(short, short, "h") 239 CK_PR_BINARY_S(char, char, "b") 240 241 #undef CK_PR_BINARY_S 242 #undef CK_PR_BINARY 243 244 CK_CC_INLINE static void * 245 ck_pr_faa_ptr(void *target, uintptr_t delta) 246 { 247 uintptr_t previous; 248 249 __asm__ __volatile__( 250 "ldadd %2, %0, [%1];" 251 : "=r" (previous) 252 : "r" (target), 253 "r" (delta) 254 : "memory"); 255 256 return (void *)(previous); 257 } 258 259 CK_CC_INLINE static uint64_t 260 ck_pr_faa_64(uint64_t *target, uint64_t delta) 261 { 262 uint64_t previous; 263 264 __asm__ __volatile__( 265 "ldadd %2, %0, [%1];" 266 : "=r" (previous) 267 : "r" (target), 268 "r" (delta) 269 : "memory"); 270 271 return (previous); 272 } 273 274 #define CK_PR_FAA(S, T, W) \ 275 CK_CC_INLINE static T \ 276 ck_pr_faa_##S(T *target, T delta) \ 277 { \ 278 T previous; \ 279 __asm__ __volatile__( \ 280 "ldadd" W " %w2, %w0, [%1];" \ 281 : "=r" (previous) \ 282 : "r" (target), \ 283 "r" (delta) \ 284 : "memory"); \ 285 return (previous); \ 286 } 287 288 CK_PR_FAA(32, uint32_t, "") 289 CK_PR_FAA(uint, unsigned int, "") 290 CK_PR_FAA(int, int, "") 291 CK_PR_FAA(16, uint16_t, "h") 292 CK_PR_FAA(8, uint8_t, "b") 293 CK_PR_FAA(short, short, "h") 294 CK_PR_FAA(char, char, "b") 295 296 #undef CK_PR_FAA 297 298 #endif /* CK_PR_AARCH64_LSE_H */ 299