1 /* 2 * Copyright 2009-2015 Samy Al Bahra. 3 * Copyright 2013-2015 Olivier Houchard. 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #ifndef CK_PR_ARM_H 29 #define CK_PR_ARM_H 30 31 #ifndef CK_PR_H 32 #error Do not include this file directly, use ck_pr.h 33 #endif 34 35 #include <ck_cc.h> 36 #include <ck_md.h> 37 38 /* 39 * The following represent supported atomic operations. 40 * These operations may be emulated. 41 */ 42 #include "ck_f_pr.h" 43 44 /* 45 * Minimum interface requirement met. 46 */ 47 #define CK_F_PR 48 49 CK_CC_INLINE static void 50 ck_pr_stall(void) 51 { 52 53 __asm__ __volatile__("" ::: "memory"); 54 return; 55 } 56 57 #if defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) 58 #define CK_ISB __asm __volatile("isb" : : "r" (0) : "memory") 59 #define CK_DMB __asm __volatile("dmb" : : "r" (0) : "memory") 60 #define CK_DSB __asm __volatile("dsb" : : "r" (0) : "memory") 61 /* FreeBSD's toolchain doesn't accept dmb st, so use the opcode instead */ 62 #ifdef __FreeBSD__ 63 #define CK_DMB_ST __asm __volatile(".word 0xf57ff05e" : : "r" (0) : "memory") 64 #else 65 #define CK_DMB_ST __asm __volatile("dmb st" : : "r" (0) : "memory") 66 #endif /* __FreeBSD__ */ 67 #else 68 /* armv6 doesn't have dsb/dmb/isb, and no way to wait only for stores */ 69 #define CK_ISB \ 70 __asm __volatile("mcr p15, 0, %0, c7, c5, 4" : : "r" (0) : "memory") 71 #define CK_DSB \ 72 __asm __volatile("mcr p15, 0, %0, c7, c10, 4" : : "r" (0) : "memory") 73 #define CK_DMB \ 74 __asm __volatile("mcr p15, 0, %0, c7, c10, 5" : : "r" (0) : "memory") 75 #define CK_DMB_ST CK_DMB 76 #endif 77 78 #define CK_PR_FENCE(T, I) \ 79 CK_CC_INLINE static void \ 80 ck_pr_fence_strict_##T(void) \ 81 { \ 82 I; \ 83 } 84 85 CK_PR_FENCE(atomic, CK_DMB_ST) 86 CK_PR_FENCE(atomic_store, CK_DMB_ST) 87 CK_PR_FENCE(atomic_load, CK_DMB_ST) 88 CK_PR_FENCE(store_atomic, CK_DMB_ST) 89 CK_PR_FENCE(load_atomic, CK_DMB) 90 CK_PR_FENCE(store, CK_DMB_ST) 91 CK_PR_FENCE(store_load, CK_DMB) 92 CK_PR_FENCE(load, CK_DMB) 93 CK_PR_FENCE(load_store, CK_DMB) 94 CK_PR_FENCE(memory, CK_DMB) 95 CK_PR_FENCE(acquire, CK_DMB) 96 CK_PR_FENCE(release, CK_DMB) 97 CK_PR_FENCE(acqrel, CK_DMB) 98 CK_PR_FENCE(lock, CK_DMB) 99 CK_PR_FENCE(unlock, CK_DMB) 100 101 #undef CK_PR_FENCE 102 103 #undef CK_ISB 104 #undef CK_DSB 105 #undef CK_DMB 106 #undef CK_DMB_ST 107 108 #define CK_PR_LOAD(S, M, T, C, I) \ 109 CK_CC_INLINE static T \ 110 ck_pr_md_load_##S(const M *target) \ 111 { \ 112 long r = 0; \ 113 __asm__ __volatile__(I " %0, [%1];" \ 114 : "=r" (r) \ 115 : "r" (target) \ 116 : "memory"); \ 117 return ((T)r); \ 118 } 119 120 CK_PR_LOAD(ptr, void, void *, uint32_t, "ldr") 121 122 #define CK_PR_LOAD_S(S, T, I) CK_PR_LOAD(S, T, T, T, I) 123 124 CK_PR_LOAD_S(32, uint32_t, "ldr") 125 CK_PR_LOAD_S(16, uint16_t, "ldrh") 126 CK_PR_LOAD_S(8, uint8_t, "ldrb") 127 CK_PR_LOAD_S(uint, unsigned int, "ldr") 128 CK_PR_LOAD_S(int, int, "ldr") 129 CK_PR_LOAD_S(short, short, "ldrh") 130 CK_PR_LOAD_S(char, char, "ldrb") 131 132 #undef CK_PR_LOAD_S 133 #undef CK_PR_LOAD 134 135 #if defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) 136 137 #define CK_PR_DOUBLE_LOAD(T, N) \ 138 CK_CC_INLINE static T \ 139 ck_pr_md_load_##N(const T *target) \ 140 { \ 141 register T ret; \ 142 \ 143 __asm __volatile("ldrexd %0, [%1]" \ 144 : "=&r" (ret) \ 145 : "r" (target) \ 146 : "memory", "cc"); \ 147 return (ret); \ 148 } 149 150 CK_PR_DOUBLE_LOAD(uint64_t, 64) 151 #ifndef CK_PR_DISABLE_DOUBLE 152 CK_PR_DOUBLE_LOAD(double, double) 153 #endif 154 #undef CK_PR_DOUBLE_LOAD 155 #endif 156 157 #define CK_PR_STORE(S, M, T, C, I) \ 158 CK_CC_INLINE static void \ 159 ck_pr_md_store_##S(M *target, T v) \ 160 { \ 161 __asm__ __volatile__(I " %1, [%0]" \ 162 : \ 163 : "r" (target), \ 164 "r" (v) \ 165 : "memory"); \ 166 return; \ 167 } 168 169 CK_PR_STORE(ptr, void, const void *, uint32_t, "str") 170 171 #define CK_PR_STORE_S(S, T, I) CK_PR_STORE(S, T, T, T, I) 172 173 CK_PR_STORE_S(32, uint32_t, "str") 174 CK_PR_STORE_S(16, uint16_t, "strh") 175 CK_PR_STORE_S(8, uint8_t, "strb") 176 CK_PR_STORE_S(uint, unsigned int, "str") 177 CK_PR_STORE_S(int, int, "str") 178 CK_PR_STORE_S(short, short, "strh") 179 CK_PR_STORE_S(char, char, "strb") 180 181 #undef CK_PR_STORE_S 182 #undef CK_PR_STORE 183 184 #if defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) 185 186 #define CK_PR_DOUBLE_STORE(T, N) \ 187 CK_CC_INLINE static void \ 188 ck_pr_md_store_##N(const T *target, T value) \ 189 { \ 190 T tmp; \ 191 uint32_t flag; \ 192 __asm __volatile("1: \n" \ 193 "ldrexd %0, [%2]\n" \ 194 "strexd %1, %3, [%2]\n" \ 195 "teq %1, #0\n" \ 196 "it ne \n" \ 197 "bne 1b\n" \ 198 : "=&r" (tmp), "=&r" (flag) \ 199 : "r" (target), "r" (value) \ 200 : "memory", "cc"); \ 201 } 202 203 CK_PR_DOUBLE_STORE(uint64_t, 64) 204 #ifndef CK_PR_DISABLE_DOUBLE 205 CK_PR_DOUBLE_STORE(double, double) 206 #endif 207 208 #undef CK_PR_DOUBLE_STORE 209 210 #define CK_PR_DOUBLE_CAS_VALUE(T, N) \ 211 CK_CC_INLINE static bool \ 212 ck_pr_cas_##N##_value(T *target, T compare, T set, T *value) \ 213 { \ 214 T previous; \ 215 int tmp; \ 216 \ 217 __asm__ __volatile__("1:" \ 218 "ldrexd %0, [%4];" \ 219 "cmp %Q0, %Q2;" \ 220 "ittt eq;" \ 221 "cmpeq %R0, %R2;" \ 222 "strexdeq %1, %3, [%4];" \ 223 "cmpeq %1, #1;" \ 224 "beq 1b;" \ 225 :"=&r" (previous), "=&r" (tmp) \ 226 : "r" (compare), "r" (set) , \ 227 "r"(target) \ 228 : "memory", "cc"); \ 229 *value = previous; \ 230 return (*value == compare); \ 231 } 232 233 CK_PR_DOUBLE_CAS_VALUE(uint64_t, 64) 234 #ifndef CK_PR_DISABLE_DOUBLE 235 CK_PR_DOUBLE_CAS_VALUE(double, double) 236 #endif 237 238 #undef CK_PR_DOUBLE_CAS_VALUE 239 240 CK_CC_INLINE static bool 241 ck_pr_cas_ptr_2_value(void *target, void *compare, void *set, void *value) 242 { 243 uint32_t *_compare = CK_CPP_CAST(uint32_t *, compare); 244 uint32_t *_set = CK_CPP_CAST(uint32_t *, set); 245 uint64_t __compare = ((uint64_t)_compare[0]) | ((uint64_t)_compare[1] << 32); 246 uint64_t __set = ((uint64_t)_set[0]) | ((uint64_t)_set[1] << 32); 247 248 return (ck_pr_cas_64_value(CK_CPP_CAST(uint64_t *, target), 249 __compare, 250 __set, 251 CK_CPP_CAST(uint64_t *, value))); 252 } 253 254 #define CK_PR_DOUBLE_CAS(T, N) \ 255 CK_CC_INLINE static bool \ 256 ck_pr_cas_##N(T *target, T compare, T set) \ 257 { \ 258 int ret; \ 259 T tmp; \ 260 \ 261 __asm__ __volatile__("1:" \ 262 "mov %0, #0;" \ 263 "ldrexd %1, [%4];" \ 264 "cmp %Q1, %Q2;" \ 265 "itttt eq;" \ 266 "cmpeq %R1, %R2;" \ 267 "strexdeq %1, %3, [%4];" \ 268 "moveq %0, #1;" \ 269 "cmpeq %1, #1;" \ 270 "beq 1b;" \ 271 : "=&r" (ret), "=&r" (tmp) \ 272 : "r" (compare), "r" (set) , \ 273 "r"(target) \ 274 : "memory", "cc"); \ 275 \ 276 return (ret); \ 277 } 278 279 CK_PR_DOUBLE_CAS(uint64_t, 64) 280 #ifndef CK_PR_DISABLE_DOUBLE 281 CK_PR_DOUBLE_CAS(double, double) 282 #endif 283 284 CK_CC_INLINE static bool 285 ck_pr_cas_ptr_2(void *target, void *compare, void *set) 286 { 287 uint32_t *_compare = CK_CPP_CAST(uint32_t *, compare); 288 uint32_t *_set = CK_CPP_CAST(uint32_t *, set); 289 uint64_t __compare = ((uint64_t)_compare[0]) | ((uint64_t)_compare[1] << 32); 290 uint64_t __set = ((uint64_t)_set[0]) | ((uint64_t)_set[1] << 32); 291 return (ck_pr_cas_64(CK_CPP_CAST(uint64_t *, target), 292 __compare, 293 __set)); 294 } 295 296 #endif 297 298 CK_CC_INLINE static bool 299 ck_pr_cas_ptr_value(void *target, void *compare, void *set, void *value) 300 { 301 void *previous, *tmp; 302 __asm__ __volatile__("1:" 303 "ldrex %0, [%2];" 304 "cmp %0, %4;" 305 "itt eq;" 306 "strexeq %1, %3, [%2];" 307 "cmpeq %1, #1;" 308 "beq 1b;" 309 : "=&r" (previous), 310 "=&r" (tmp) 311 : "r" (target), 312 "r" (set), 313 "r" (compare) 314 : "memory", "cc"); 315 *(void **)value = previous; 316 return (previous == compare); 317 } 318 319 CK_CC_INLINE static bool 320 ck_pr_cas_ptr(void *target, void *compare, void *set) 321 { 322 void *previous, *tmp; 323 __asm__ __volatile__("1:" 324 "ldrex %0, [%2];" 325 "cmp %0, %4;" 326 "itt eq;" 327 "strexeq %1, %3, [%2];" 328 "cmpeq %1, #1;" 329 "beq 1b;" 330 : "=&r" (previous), 331 "=&r" (tmp) 332 : "r" (target), 333 "r" (set), 334 "r" (compare) 335 : "memory", "cc"); 336 return (previous == compare); 337 } 338 339 #define CK_PR_CAS(N, T, W) \ 340 CK_CC_INLINE static bool \ 341 ck_pr_cas_##N##_value(T *target, T compare, T set, T *value) \ 342 { \ 343 T previous = 0, tmp = 0; \ 344 __asm__ __volatile__("1:" \ 345 "ldrex" W " %0, [%2];" \ 346 "cmp %0, %4;" \ 347 "itt eq;" \ 348 "strex" W "eq %1, %3, [%2];" \ 349 "cmpeq %1, #1;" \ 350 "beq 1b;" \ 351 /* \ 352 * Using "+&" instead of "=&" to avoid bogus \ 353 * clang warnings. \ 354 */ \ 355 : "+&r" (previous), \ 356 "+&r" (tmp) \ 357 : "r" (target), \ 358 "r" (set), \ 359 "r" (compare) \ 360 : "memory", "cc"); \ 361 *value = previous; \ 362 return (previous == compare); \ 363 } \ 364 CK_CC_INLINE static bool \ 365 ck_pr_cas_##N(T *target, T compare, T set) \ 366 { \ 367 T previous = 0, tmp = 0; \ 368 __asm__ __volatile__("1:" \ 369 "ldrex" W " %0, [%2];" \ 370 "cmp %0, %4;" \ 371 "itt eq;" \ 372 "strex" W "eq %1, %3, [%2];" \ 373 "cmpeq %1, #1;" \ 374 "beq 1b;" \ 375 : "+&r" (previous), \ 376 "+&r" (tmp) \ 377 : "r" (target), \ 378 "r" (set), \ 379 "r" (compare) \ 380 : "memory", "cc"); \ 381 return (previous == compare); \ 382 } 383 384 CK_PR_CAS(32, uint32_t, "") 385 CK_PR_CAS(uint, unsigned int, "") 386 CK_PR_CAS(int, int, "") 387 CK_PR_CAS(16, uint16_t, "h") 388 CK_PR_CAS(8, uint8_t, "b") 389 CK_PR_CAS(short, short, "h") 390 CK_PR_CAS(char, char, "b") 391 392 393 #undef CK_PR_CAS 394 395 #define CK_PR_FAS(N, M, T, W) \ 396 CK_CC_INLINE static T \ 397 ck_pr_fas_##N(M *target, T v) \ 398 { \ 399 T previous = 0; \ 400 T tmp = 0; \ 401 __asm__ __volatile__("1:" \ 402 "ldrex" W " %0, [%2];" \ 403 "strex" W " %1, %3, [%2];" \ 404 "cmp %1, #0;" \ 405 "bne 1b;" \ 406 : "+&r" (previous), \ 407 "+&r" (tmp) \ 408 : "r" (target), \ 409 "r" (v) \ 410 : "memory", "cc"); \ 411 return (previous); \ 412 } 413 414 CK_PR_FAS(32, uint32_t, uint32_t, "") 415 CK_PR_FAS(ptr, void, void *, "") 416 CK_PR_FAS(int, int, int, "") 417 CK_PR_FAS(uint, unsigned int, unsigned int, "") 418 CK_PR_FAS(16, uint16_t, uint16_t, "h") 419 CK_PR_FAS(8, uint8_t, uint8_t, "b") 420 CK_PR_FAS(short, short, short, "h") 421 CK_PR_FAS(char, char, char, "b") 422 423 424 #undef CK_PR_FAS 425 426 #define CK_PR_UNARY(O, N, M, T, I, W) \ 427 CK_CC_INLINE static void \ 428 ck_pr_##O##_##N(M *target) \ 429 { \ 430 T previous = 0; \ 431 T tmp = 0; \ 432 __asm__ __volatile__("1:" \ 433 "ldrex" W " %0, [%2];" \ 434 I ";" \ 435 "strex" W " %1, %0, [%2];" \ 436 "cmp %1, #0;" \ 437 "bne 1b;" \ 438 : "+&r" (previous), \ 439 "+&r" (tmp) \ 440 : "r" (target) \ 441 : "memory", "cc"); \ 442 return; \ 443 } 444 445 CK_PR_UNARY(inc, ptr, void, void *, "add %0, %0, #1", "") 446 CK_PR_UNARY(dec, ptr, void, void *, "sub %0, %0, #1", "") 447 CK_PR_UNARY(not, ptr, void, void *, "mvn %0, %0", "") 448 CK_PR_UNARY(neg, ptr, void, void *, "neg %0, %0", "") 449 450 #define CK_PR_UNARY_S(S, T, W) \ 451 CK_PR_UNARY(inc, S, T, T, "add %0, %0, #1", W) \ 452 CK_PR_UNARY(dec, S, T, T, "sub %0, %0, #1", W) \ 453 CK_PR_UNARY(not, S, T, T, "mvn %0, %0", W) \ 454 CK_PR_UNARY(neg, S, T, T, "neg %0, %0", W) \ 455 456 CK_PR_UNARY_S(32, uint32_t, "") 457 CK_PR_UNARY_S(uint, unsigned int, "") 458 CK_PR_UNARY_S(int, int, "") 459 CK_PR_UNARY_S(16, uint16_t, "h") 460 CK_PR_UNARY_S(8, uint8_t, "b") 461 CK_PR_UNARY_S(short, short, "h") 462 CK_PR_UNARY_S(char, char, "b") 463 464 #undef CK_PR_UNARY_S 465 #undef CK_PR_UNARY 466 467 #define CK_PR_BINARY(O, N, M, T, I, W) \ 468 CK_CC_INLINE static void \ 469 ck_pr_##O##_##N(M *target, T delta) \ 470 { \ 471 T previous = 0; \ 472 T tmp = 0; \ 473 __asm__ __volatile__("1:" \ 474 "ldrex" W " %0, [%2];" \ 475 I " %0, %0, %3;" \ 476 "strex" W " %1, %0, [%2];" \ 477 "cmp %1, #0;" \ 478 "bne 1b;" \ 479 : "+&r" (previous), \ 480 "+&r" (tmp) \ 481 : "r" (target), \ 482 "r" (delta) \ 483 : "memory", "cc"); \ 484 return; \ 485 } 486 487 CK_PR_BINARY(and, ptr, void, uintptr_t, "and", "") 488 CK_PR_BINARY(add, ptr, void, uintptr_t, "add", "") 489 CK_PR_BINARY(or, ptr, void, uintptr_t, "orr", "") 490 CK_PR_BINARY(sub, ptr, void, uintptr_t, "sub", "") 491 CK_PR_BINARY(xor, ptr, void, uintptr_t, "eor", "") 492 493 #define CK_PR_BINARY_S(S, T, W) \ 494 CK_PR_BINARY(and, S, T, T, "and", W) \ 495 CK_PR_BINARY(add, S, T, T, "add", W) \ 496 CK_PR_BINARY(or, S, T, T, "orr", W) \ 497 CK_PR_BINARY(sub, S, T, T, "sub", W) \ 498 CK_PR_BINARY(xor, S, T, T, "eor", W) 499 500 CK_PR_BINARY_S(32, uint32_t, "") 501 CK_PR_BINARY_S(uint, unsigned int, "") 502 CK_PR_BINARY_S(int, int, "") 503 CK_PR_BINARY_S(16, uint16_t, "h") 504 CK_PR_BINARY_S(8, uint8_t, "b") 505 CK_PR_BINARY_S(short, short, "h") 506 CK_PR_BINARY_S(char, char, "b") 507 508 #undef CK_PR_BINARY_S 509 #undef CK_PR_BINARY 510 511 CK_CC_INLINE static void * 512 ck_pr_faa_ptr(void *target, uintptr_t delta) 513 { 514 uintptr_t previous, r, tmp; 515 516 __asm__ __volatile__("1:" 517 "ldrex %0, [%3];" 518 "add %1, %4, %0;" 519 "strex %2, %1, [%3];" 520 "cmp %2, #0;" 521 "bne 1b;" 522 : "=&r" (previous), 523 "=&r" (r), 524 "=&r" (tmp) 525 : "r" (target), 526 "r" (delta) 527 : "memory", "cc"); 528 529 return (void *)(previous); 530 } 531 532 #define CK_PR_FAA(S, T, W) \ 533 CK_CC_INLINE static T \ 534 ck_pr_faa_##S(T *target, T delta) \ 535 { \ 536 T previous = 0, r = 0, tmp = 0; \ 537 __asm__ __volatile__("1:" \ 538 "ldrex" W " %0, [%3];" \ 539 "add %1, %4, %0;" \ 540 "strex" W " %2, %1, [%3];" \ 541 "cmp %2, #0;" \ 542 "bne 1b;" \ 543 : "+&r" (previous), \ 544 "+&r" (r), \ 545 "+&r" (tmp) \ 546 : "r" (target), \ 547 "r" (delta) \ 548 : "memory", "cc"); \ 549 return (previous); \ 550 } 551 552 CK_PR_FAA(32, uint32_t, "") 553 CK_PR_FAA(uint, unsigned int, "") 554 CK_PR_FAA(int, int, "") 555 CK_PR_FAA(16, uint16_t, "h") 556 CK_PR_FAA(8, uint8_t, "b") 557 CK_PR_FAA(short, short, "h") 558 CK_PR_FAA(char, char, "b") 559 560 #undef CK_PR_FAA 561 562 #endif /* CK_PR_ARM_H */ 563 564