1 /* 2 * Copyright 2009-2015 Samy Al Bahra. 3 * Copyright 2011 David Joseph. 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #ifndef CK_PR_H 29 #define CK_PR_H 30 31 #include <ck_cc.h> 32 #include <ck_limits.h> 33 #include <ck_md.h> 34 #include <ck_stdint.h> 35 #include <ck_stdbool.h> 36 37 #ifndef CK_USE_CC_BUILTINS 38 #if defined(__x86_64__) 39 #include "gcc/x86_64/ck_pr.h" 40 #elif defined(__x86__) 41 #include "gcc/x86/ck_pr.h" 42 #elif defined(__sparcv9__) 43 #include "gcc/sparcv9/ck_pr.h" 44 #elif defined(__ppc64__) 45 #include "gcc/ppc64/ck_pr.h" 46 #elif defined(__s390x__) 47 #include "gcc/s390x/ck_pr.h" 48 #elif defined(__ppc__) 49 #include "gcc/ppc/ck_pr.h" 50 #elif defined(__arm__) 51 #if __ARM_ARCH >= 6 52 #include "gcc/arm/ck_pr.h" 53 #else 54 #include "gcc/arm/ck_pr_armv4.h" 55 #endif 56 #elif defined(__aarch64__) 57 #include "gcc/aarch64/ck_pr.h" 58 #elif !defined(__GNUC__) 59 #error Your platform is unsupported 60 #endif 61 #endif /* !CK_USE_CC_BUILTINS */ 62 63 #if defined(__GNUC__) 64 #include "gcc/ck_pr.h" 65 #endif 66 67 #define CK_PR_FENCE_EMIT(T) \ 68 CK_CC_INLINE static void \ 69 ck_pr_fence_##T(void) \ 70 { \ 71 ck_pr_fence_strict_##T(); \ 72 return; \ 73 } 74 #define CK_PR_FENCE_NOOP(T) \ 75 CK_CC_INLINE static void \ 76 ck_pr_fence_##T(void) \ 77 { \ 78 ck_pr_barrier(); \ 79 return; \ 80 } 81 82 /* 83 * None of the currently supported platforms allow for data-dependent 84 * load ordering. 85 */ 86 CK_PR_FENCE_NOOP(load_depends) 87 #define ck_pr_fence_strict_load_depends ck_pr_fence_load_depends 88 89 /* 90 * In memory models where atomic operations do not have serializing 91 * effects, atomic read-modify-write operations are modeled as stores. 92 */ 93 #if defined(CK_MD_RMO) 94 /* 95 * Only stores to the same location have a global 96 * ordering. 97 */ 98 CK_PR_FENCE_EMIT(atomic) 99 CK_PR_FENCE_EMIT(atomic_load) 100 CK_PR_FENCE_EMIT(atomic_store) 101 CK_PR_FENCE_EMIT(store_atomic) 102 CK_PR_FENCE_EMIT(load_atomic) 103 CK_PR_FENCE_EMIT(load_store) 104 CK_PR_FENCE_EMIT(store_load) 105 CK_PR_FENCE_EMIT(load) 106 CK_PR_FENCE_EMIT(store) 107 CK_PR_FENCE_EMIT(memory) 108 CK_PR_FENCE_EMIT(acquire) 109 CK_PR_FENCE_EMIT(release) 110 CK_PR_FENCE_EMIT(acqrel) 111 CK_PR_FENCE_EMIT(lock) 112 CK_PR_FENCE_EMIT(unlock) 113 #elif defined(CK_MD_PSO) 114 /* 115 * Anything can be re-ordered with respect to stores. 116 * Otherwise, loads are executed in-order. 117 */ 118 CK_PR_FENCE_EMIT(atomic) 119 CK_PR_FENCE_NOOP(atomic_load) 120 CK_PR_FENCE_EMIT(atomic_store) 121 CK_PR_FENCE_EMIT(store_atomic) 122 CK_PR_FENCE_NOOP(load_atomic) 123 CK_PR_FENCE_EMIT(load_store) 124 CK_PR_FENCE_EMIT(store_load) 125 CK_PR_FENCE_NOOP(load) 126 CK_PR_FENCE_EMIT(store) 127 CK_PR_FENCE_EMIT(memory) 128 CK_PR_FENCE_EMIT(acquire) 129 CK_PR_FENCE_EMIT(release) 130 CK_PR_FENCE_EMIT(acqrel) 131 CK_PR_FENCE_EMIT(lock) 132 CK_PR_FENCE_EMIT(unlock) 133 #elif defined(CK_MD_TSO) 134 /* 135 * Only loads are re-ordered and only with respect to 136 * prior stores. Atomic operations are serializing. 137 */ 138 CK_PR_FENCE_NOOP(atomic) 139 CK_PR_FENCE_NOOP(atomic_load) 140 CK_PR_FENCE_NOOP(atomic_store) 141 CK_PR_FENCE_NOOP(store_atomic) 142 CK_PR_FENCE_NOOP(load_atomic) 143 CK_PR_FENCE_NOOP(load_store) 144 CK_PR_FENCE_EMIT(store_load) 145 CK_PR_FENCE_NOOP(load) 146 CK_PR_FENCE_NOOP(store) 147 CK_PR_FENCE_EMIT(memory) 148 CK_PR_FENCE_NOOP(acquire) 149 CK_PR_FENCE_NOOP(release) 150 CK_PR_FENCE_NOOP(acqrel) 151 CK_PR_FENCE_NOOP(lock) 152 CK_PR_FENCE_NOOP(unlock) 153 #else 154 #error "No memory model has been defined." 155 #endif /* CK_MD_TSO */ 156 157 #undef CK_PR_FENCE_EMIT 158 #undef CK_PR_FENCE_NOOP 159 160 #ifndef CK_F_PR_RFO 161 #define CK_F_PR_RFO 162 CK_CC_INLINE static void 163 ck_pr_rfo(const void *m) 164 { 165 166 (void)m; 167 return; 168 } 169 #endif /* CK_F_PR_RFO */ 170 171 #define CK_PR_STORE_SAFE(DST, VAL, TYPE) \ 172 ck_pr_md_store_##TYPE( \ 173 ((void)sizeof(*(DST) = (VAL)), (DST)), \ 174 (VAL)) 175 176 #define ck_pr_store_ptr(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), ptr) 177 #define ck_pr_store_char(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), char) 178 #ifndef CK_PR_DISABLE_DOUBLE 179 #define ck_pr_store_double(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), double) 180 #endif 181 #define ck_pr_store_uint(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), uint) 182 #define ck_pr_store_int(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), int) 183 #define ck_pr_store_32(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 32) 184 #define ck_pr_store_16(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 16) 185 #define ck_pr_store_8(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 8) 186 187 #define ck_pr_store_ptr_unsafe(DST, VAL) ck_pr_md_store_ptr((DST), (VAL)) 188 189 #ifdef CK_F_PR_LOAD_64 190 #define ck_pr_store_64(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 64) 191 #endif /* CK_F_PR_LOAD_64 */ 192 193 #define CK_PR_LOAD_PTR_SAFE(SRC) (CK_CC_TYPEOF(*(SRC), (void *)))ck_pr_md_load_ptr((SRC)) 194 #define ck_pr_load_ptr(SRC) CK_PR_LOAD_PTR_SAFE((SRC)) 195 196 #define CK_PR_LOAD_SAFE(SRC, TYPE) ck_pr_md_load_##TYPE((SRC)) 197 #define ck_pr_load_char(SRC) CK_PR_LOAD_SAFE((SRC), char) 198 #ifndef CK_PR_DISABLE_DOUBLE 199 #define ck_pr_load_double(SRC) CK_PR_LOAD_SAFE((SRC), double) 200 #endif 201 #define ck_pr_load_uint(SRC) CK_PR_LOAD_SAFE((SRC), uint) 202 #define ck_pr_load_int(SRC) CK_PR_LOAD_SAFE((SRC), int) 203 #define ck_pr_load_32(SRC) CK_PR_LOAD_SAFE((SRC), 32) 204 #define ck_pr_load_16(SRC) CK_PR_LOAD_SAFE((SRC), 16) 205 #define ck_pr_load_8(SRC) CK_PR_LOAD_SAFE((SRC), 8) 206 207 #ifdef CK_F_PR_LOAD_64 208 #define ck_pr_load_64(SRC) CK_PR_LOAD_SAFE((SRC), 64) 209 #endif /* CK_F_PR_LOAD_64 */ 210 211 #define CK_PR_BIN(K, S, M, T, P, C) \ 212 CK_CC_INLINE static void \ 213 ck_pr_##K##_##S(M *target, T value) \ 214 { \ 215 T previous; \ 216 C punt; \ 217 punt = ck_pr_md_load_##S(target); \ 218 previous = (T)punt; \ 219 while (ck_pr_cas_##S##_value(target, \ 220 (C)previous, \ 221 (C)(previous P value), \ 222 &previous) == false) \ 223 ck_pr_stall(); \ 224 \ 225 return; \ 226 } 227 228 #define CK_PR_BIN_S(K, S, T, P) CK_PR_BIN(K, S, T, T, P, T) 229 230 #if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE) 231 232 #ifndef CK_F_PR_ADD_CHAR 233 #define CK_F_PR_ADD_CHAR 234 CK_PR_BIN_S(add, char, char, +) 235 #endif /* CK_F_PR_ADD_CHAR */ 236 237 #ifndef CK_F_PR_SUB_CHAR 238 #define CK_F_PR_SUB_CHAR 239 CK_PR_BIN_S(sub, char, char, -) 240 #endif /* CK_F_PR_SUB_CHAR */ 241 242 #ifndef CK_F_PR_AND_CHAR 243 #define CK_F_PR_AND_CHAR 244 CK_PR_BIN_S(and, char, char, &) 245 #endif /* CK_F_PR_AND_CHAR */ 246 247 #ifndef CK_F_PR_XOR_CHAR 248 #define CK_F_PR_XOR_CHAR 249 CK_PR_BIN_S(xor, char, char, ^) 250 #endif /* CK_F_PR_XOR_CHAR */ 251 252 #ifndef CK_F_PR_OR_CHAR 253 #define CK_F_PR_OR_CHAR 254 CK_PR_BIN_S(or, char, char, |) 255 #endif /* CK_F_PR_OR_CHAR */ 256 257 #endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */ 258 259 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) 260 261 #ifndef CK_F_PR_ADD_INT 262 #define CK_F_PR_ADD_INT 263 CK_PR_BIN_S(add, int, int, +) 264 #endif /* CK_F_PR_ADD_INT */ 265 266 #ifndef CK_F_PR_SUB_INT 267 #define CK_F_PR_SUB_INT 268 CK_PR_BIN_S(sub, int, int, -) 269 #endif /* CK_F_PR_SUB_INT */ 270 271 #ifndef CK_F_PR_AND_INT 272 #define CK_F_PR_AND_INT 273 CK_PR_BIN_S(and, int, int, &) 274 #endif /* CK_F_PR_AND_INT */ 275 276 #ifndef CK_F_PR_XOR_INT 277 #define CK_F_PR_XOR_INT 278 CK_PR_BIN_S(xor, int, int, ^) 279 #endif /* CK_F_PR_XOR_INT */ 280 281 #ifndef CK_F_PR_OR_INT 282 #define CK_F_PR_OR_INT 283 CK_PR_BIN_S(or, int, int, |) 284 #endif /* CK_F_PR_OR_INT */ 285 286 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ 287 288 #if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \ 289 !defined(CK_PR_DISABLE_DOUBLE) 290 291 #ifndef CK_F_PR_ADD_DOUBLE 292 #define CK_F_PR_ADD_DOUBLE 293 CK_PR_BIN_S(add, double, double, +) 294 #endif /* CK_F_PR_ADD_DOUBLE */ 295 296 #ifndef CK_F_PR_SUB_DOUBLE 297 #define CK_F_PR_SUB_DOUBLE 298 CK_PR_BIN_S(sub, double, double, -) 299 #endif /* CK_F_PR_SUB_DOUBLE */ 300 301 #endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */ 302 303 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) 304 305 #ifndef CK_F_PR_ADD_UINT 306 #define CK_F_PR_ADD_UINT 307 CK_PR_BIN_S(add, uint, unsigned int, +) 308 #endif /* CK_F_PR_ADD_UINT */ 309 310 #ifndef CK_F_PR_SUB_UINT 311 #define CK_F_PR_SUB_UINT 312 CK_PR_BIN_S(sub, uint, unsigned int, -) 313 #endif /* CK_F_PR_SUB_UINT */ 314 315 #ifndef CK_F_PR_AND_UINT 316 #define CK_F_PR_AND_UINT 317 CK_PR_BIN_S(and, uint, unsigned int, &) 318 #endif /* CK_F_PR_AND_UINT */ 319 320 #ifndef CK_F_PR_XOR_UINT 321 #define CK_F_PR_XOR_UINT 322 CK_PR_BIN_S(xor, uint, unsigned int, ^) 323 #endif /* CK_F_PR_XOR_UINT */ 324 325 #ifndef CK_F_PR_OR_UINT 326 #define CK_F_PR_OR_UINT 327 CK_PR_BIN_S(or, uint, unsigned int, |) 328 #endif /* CK_F_PR_OR_UINT */ 329 330 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ 331 332 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) 333 334 #ifndef CK_F_PR_ADD_PTR 335 #define CK_F_PR_ADD_PTR 336 CK_PR_BIN(add, ptr, void, uintptr_t, +, void *) 337 #endif /* CK_F_PR_ADD_PTR */ 338 339 #ifndef CK_F_PR_SUB_PTR 340 #define CK_F_PR_SUB_PTR 341 CK_PR_BIN(sub, ptr, void, uintptr_t, -, void *) 342 #endif /* CK_F_PR_SUB_PTR */ 343 344 #ifndef CK_F_PR_AND_PTR 345 #define CK_F_PR_AND_PTR 346 CK_PR_BIN(and, ptr, void, uintptr_t, &, void *) 347 #endif /* CK_F_PR_AND_PTR */ 348 349 #ifndef CK_F_PR_XOR_PTR 350 #define CK_F_PR_XOR_PTR 351 CK_PR_BIN(xor, ptr, void, uintptr_t, ^, void *) 352 #endif /* CK_F_PR_XOR_PTR */ 353 354 #ifndef CK_F_PR_OR_PTR 355 #define CK_F_PR_OR_PTR 356 CK_PR_BIN(or, ptr, void, uintptr_t, |, void *) 357 #endif /* CK_F_PR_OR_PTR */ 358 359 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ 360 361 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) 362 363 #ifndef CK_F_PR_ADD_64 364 #define CK_F_PR_ADD_64 365 CK_PR_BIN_S(add, 64, uint64_t, +) 366 #endif /* CK_F_PR_ADD_64 */ 367 368 #ifndef CK_F_PR_SUB_64 369 #define CK_F_PR_SUB_64 370 CK_PR_BIN_S(sub, 64, uint64_t, -) 371 #endif /* CK_F_PR_SUB_64 */ 372 373 #ifndef CK_F_PR_AND_64 374 #define CK_F_PR_AND_64 375 CK_PR_BIN_S(and, 64, uint64_t, &) 376 #endif /* CK_F_PR_AND_64 */ 377 378 #ifndef CK_F_PR_XOR_64 379 #define CK_F_PR_XOR_64 380 CK_PR_BIN_S(xor, 64, uint64_t, ^) 381 #endif /* CK_F_PR_XOR_64 */ 382 383 #ifndef CK_F_PR_OR_64 384 #define CK_F_PR_OR_64 385 CK_PR_BIN_S(or, 64, uint64_t, |) 386 #endif /* CK_F_PR_OR_64 */ 387 388 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ 389 390 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) 391 392 #ifndef CK_F_PR_ADD_32 393 #define CK_F_PR_ADD_32 394 CK_PR_BIN_S(add, 32, uint32_t, +) 395 #endif /* CK_F_PR_ADD_32 */ 396 397 #ifndef CK_F_PR_SUB_32 398 #define CK_F_PR_SUB_32 399 CK_PR_BIN_S(sub, 32, uint32_t, -) 400 #endif /* CK_F_PR_SUB_32 */ 401 402 #ifndef CK_F_PR_AND_32 403 #define CK_F_PR_AND_32 404 CK_PR_BIN_S(and, 32, uint32_t, &) 405 #endif /* CK_F_PR_AND_32 */ 406 407 #ifndef CK_F_PR_XOR_32 408 #define CK_F_PR_XOR_32 409 CK_PR_BIN_S(xor, 32, uint32_t, ^) 410 #endif /* CK_F_PR_XOR_32 */ 411 412 #ifndef CK_F_PR_OR_32 413 #define CK_F_PR_OR_32 414 CK_PR_BIN_S(or, 32, uint32_t, |) 415 #endif /* CK_F_PR_OR_32 */ 416 417 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ 418 419 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) 420 421 #ifndef CK_F_PR_ADD_16 422 #define CK_F_PR_ADD_16 423 CK_PR_BIN_S(add, 16, uint16_t, +) 424 #endif /* CK_F_PR_ADD_16 */ 425 426 #ifndef CK_F_PR_SUB_16 427 #define CK_F_PR_SUB_16 428 CK_PR_BIN_S(sub, 16, uint16_t, -) 429 #endif /* CK_F_PR_SUB_16 */ 430 431 #ifndef CK_F_PR_AND_16 432 #define CK_F_PR_AND_16 433 CK_PR_BIN_S(and, 16, uint16_t, &) 434 #endif /* CK_F_PR_AND_16 */ 435 436 #ifndef CK_F_PR_XOR_16 437 #define CK_F_PR_XOR_16 438 CK_PR_BIN_S(xor, 16, uint16_t, ^) 439 #endif /* CK_F_PR_XOR_16 */ 440 441 #ifndef CK_F_PR_OR_16 442 #define CK_F_PR_OR_16 443 CK_PR_BIN_S(or, 16, uint16_t, |) 444 #endif /* CK_F_PR_OR_16 */ 445 446 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ 447 448 #if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE) 449 450 #ifndef CK_F_PR_ADD_8 451 #define CK_F_PR_ADD_8 452 CK_PR_BIN_S(add, 8, uint8_t, +) 453 #endif /* CK_F_PR_ADD_8 */ 454 455 #ifndef CK_F_PR_SUB_8 456 #define CK_F_PR_SUB_8 457 CK_PR_BIN_S(sub, 8, uint8_t, -) 458 #endif /* CK_F_PR_SUB_8 */ 459 460 #ifndef CK_F_PR_AND_8 461 #define CK_F_PR_AND_8 462 CK_PR_BIN_S(and, 8, uint8_t, &) 463 #endif /* CK_F_PR_AND_8 */ 464 465 #ifndef CK_F_PR_XOR_8 466 #define CK_F_PR_XOR_8 467 CK_PR_BIN_S(xor, 8, uint8_t, ^) 468 #endif /* CK_F_PR_XOR_8 */ 469 470 #ifndef CK_F_PR_OR_8 471 #define CK_F_PR_OR_8 472 CK_PR_BIN_S(or, 8, uint8_t, |) 473 #endif /* CK_F_PR_OR_8 */ 474 475 #endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */ 476 477 #undef CK_PR_BIN_S 478 #undef CK_PR_BIN 479 480 #define CK_PR_BTX(K, S, M, T, P, C, R) \ 481 CK_CC_INLINE static bool \ 482 ck_pr_##K##_##S(M *target, unsigned int offset) \ 483 { \ 484 T previous; \ 485 C punt; \ 486 punt = ck_pr_md_load_##S(target); \ 487 previous = (T)punt; \ 488 while (ck_pr_cas_##S##_value(target, (C)previous, \ 489 (C)(previous P (R ((T)1 << offset))), &previous) == false) \ 490 ck_pr_stall(); \ 491 return ((previous >> offset) & 1); \ 492 } 493 494 #define CK_PR_BTX_S(K, S, T, P, R) CK_PR_BTX(K, S, T, T, P, T, R) 495 496 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) 497 498 #ifndef CK_F_PR_BTC_INT 499 #define CK_F_PR_BTC_INT 500 CK_PR_BTX_S(btc, int, int, ^,) 501 #endif /* CK_F_PR_BTC_INT */ 502 503 #ifndef CK_F_PR_BTR_INT 504 #define CK_F_PR_BTR_INT 505 CK_PR_BTX_S(btr, int, int, &, ~) 506 #endif /* CK_F_PR_BTR_INT */ 507 508 #ifndef CK_F_PR_BTS_INT 509 #define CK_F_PR_BTS_INT 510 CK_PR_BTX_S(bts, int, int, |,) 511 #endif /* CK_F_PR_BTS_INT */ 512 513 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ 514 515 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) 516 517 #ifndef CK_F_PR_BTC_UINT 518 #define CK_F_PR_BTC_UINT 519 CK_PR_BTX_S(btc, uint, unsigned int, ^,) 520 #endif /* CK_F_PR_BTC_UINT */ 521 522 #ifndef CK_F_PR_BTR_UINT 523 #define CK_F_PR_BTR_UINT 524 CK_PR_BTX_S(btr, uint, unsigned int, &, ~) 525 #endif /* CK_F_PR_BTR_UINT */ 526 527 #ifndef CK_F_PR_BTS_UINT 528 #define CK_F_PR_BTS_UINT 529 CK_PR_BTX_S(bts, uint, unsigned int, |,) 530 #endif /* CK_F_PR_BTS_UINT */ 531 532 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ 533 534 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) 535 536 #ifndef CK_F_PR_BTC_PTR 537 #define CK_F_PR_BTC_PTR 538 CK_PR_BTX(btc, ptr, void, uintptr_t, ^, void *,) 539 #endif /* CK_F_PR_BTC_PTR */ 540 541 #ifndef CK_F_PR_BTR_PTR 542 #define CK_F_PR_BTR_PTR 543 CK_PR_BTX(btr, ptr, void, uintptr_t, &, void *, ~) 544 #endif /* CK_F_PR_BTR_PTR */ 545 546 #ifndef CK_F_PR_BTS_PTR 547 #define CK_F_PR_BTS_PTR 548 CK_PR_BTX(bts, ptr, void, uintptr_t, |, void *,) 549 #endif /* CK_F_PR_BTS_PTR */ 550 551 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ 552 553 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) 554 555 #ifndef CK_F_PR_BTC_64 556 #define CK_F_PR_BTC_64 557 CK_PR_BTX_S(btc, 64, uint64_t, ^,) 558 #endif /* CK_F_PR_BTC_64 */ 559 560 #ifndef CK_F_PR_BTR_64 561 #define CK_F_PR_BTR_64 562 CK_PR_BTX_S(btr, 64, uint64_t, &, ~) 563 #endif /* CK_F_PR_BTR_64 */ 564 565 #ifndef CK_F_PR_BTS_64 566 #define CK_F_PR_BTS_64 567 CK_PR_BTX_S(bts, 64, uint64_t, |,) 568 #endif /* CK_F_PR_BTS_64 */ 569 570 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ 571 572 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) 573 574 #ifndef CK_F_PR_BTC_32 575 #define CK_F_PR_BTC_32 576 CK_PR_BTX_S(btc, 32, uint32_t, ^,) 577 #endif /* CK_F_PR_BTC_32 */ 578 579 #ifndef CK_F_PR_BTR_32 580 #define CK_F_PR_BTR_32 581 CK_PR_BTX_S(btr, 32, uint32_t, &, ~) 582 #endif /* CK_F_PR_BTR_32 */ 583 584 #ifndef CK_F_PR_BTS_32 585 #define CK_F_PR_BTS_32 586 CK_PR_BTX_S(bts, 32, uint32_t, |,) 587 #endif /* CK_F_PR_BTS_32 */ 588 589 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ 590 591 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) 592 593 #ifndef CK_F_PR_BTC_16 594 #define CK_F_PR_BTC_16 595 CK_PR_BTX_S(btc, 16, uint16_t, ^,) 596 #endif /* CK_F_PR_BTC_16 */ 597 598 #ifndef CK_F_PR_BTR_16 599 #define CK_F_PR_BTR_16 600 CK_PR_BTX_S(btr, 16, uint16_t, &, ~) 601 #endif /* CK_F_PR_BTR_16 */ 602 603 #ifndef CK_F_PR_BTS_16 604 #define CK_F_PR_BTS_16 605 CK_PR_BTX_S(bts, 16, uint16_t, |,) 606 #endif /* CK_F_PR_BTS_16 */ 607 608 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ 609 610 #undef CK_PR_BTX_S 611 #undef CK_PR_BTX 612 613 #define CK_PR_UNARY(K, X, S, M, T) \ 614 CK_CC_INLINE static void \ 615 ck_pr_##K##_##S(M *target) \ 616 { \ 617 ck_pr_##X##_##S(target, (T)1); \ 618 return; \ 619 } 620 621 #define CK_PR_UNARY_Z(K, S, M, T, P, C, Z) \ 622 CK_CC_INLINE static void \ 623 ck_pr_##K##_##S##_zero(M *target, bool *zero) \ 624 { \ 625 T previous; \ 626 C punt; \ 627 punt = (C)ck_pr_md_load_##S(target); \ 628 previous = (T)punt; \ 629 while (ck_pr_cas_##S##_value(target, \ 630 (C)previous, \ 631 (C)(previous P 1), \ 632 &previous) == false) \ 633 ck_pr_stall(); \ 634 *zero = previous == (T)Z; \ 635 return; \ 636 } 637 638 #define CK_PR_UNARY_S(K, X, S, M) CK_PR_UNARY(K, X, S, M, M) 639 #define CK_PR_UNARY_Z_S(K, S, M, P, Z) CK_PR_UNARY_Z(K, S, M, M, P, M, Z) 640 641 #if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE) 642 643 #ifndef CK_F_PR_INC_CHAR 644 #define CK_F_PR_INC_CHAR 645 CK_PR_UNARY_S(inc, add, char, char) 646 #endif /* CK_F_PR_INC_CHAR */ 647 648 #ifndef CK_F_PR_INC_CHAR_ZERO 649 #define CK_F_PR_INC_CHAR_ZERO 650 CK_PR_UNARY_Z_S(inc, char, char, +, -1) 651 #endif /* CK_F_PR_INC_CHAR_ZERO */ 652 653 #ifndef CK_F_PR_DEC_CHAR 654 #define CK_F_PR_DEC_CHAR 655 CK_PR_UNARY_S(dec, sub, char, char) 656 #endif /* CK_F_PR_DEC_CHAR */ 657 658 #ifndef CK_F_PR_DEC_CHAR_ZERO 659 #define CK_F_PR_DEC_CHAR_ZERO 660 CK_PR_UNARY_Z_S(dec, char, char, -, 1) 661 #endif /* CK_F_PR_DEC_CHAR_ZERO */ 662 663 #endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */ 664 665 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) 666 667 #ifndef CK_F_PR_INC_INT 668 #define CK_F_PR_INC_INT 669 CK_PR_UNARY_S(inc, add, int, int) 670 #endif /* CK_F_PR_INC_INT */ 671 672 #ifndef CK_F_PR_INC_INT_ZERO 673 #define CK_F_PR_INC_INT_ZERO 674 CK_PR_UNARY_Z_S(inc, int, int, +, -1) 675 #endif /* CK_F_PR_INC_INT_ZERO */ 676 677 #ifndef CK_F_PR_DEC_INT 678 #define CK_F_PR_DEC_INT 679 CK_PR_UNARY_S(dec, sub, int, int) 680 #endif /* CK_F_PR_DEC_INT */ 681 682 #ifndef CK_F_PR_DEC_INT_ZERO 683 #define CK_F_PR_DEC_INT_ZERO 684 CK_PR_UNARY_Z_S(dec, int, int, -, 1) 685 #endif /* CK_F_PR_DEC_INT_ZERO */ 686 687 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ 688 689 #if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \ 690 !defined(CK_PR_DISABLE_DOUBLE) 691 692 #ifndef CK_F_PR_INC_DOUBLE 693 #define CK_F_PR_INC_DOUBLE 694 CK_PR_UNARY_S(inc, add, double, double) 695 #endif /* CK_F_PR_INC_DOUBLE */ 696 697 #ifndef CK_F_PR_DEC_DOUBLE 698 #define CK_F_PR_DEC_DOUBLE 699 CK_PR_UNARY_S(dec, sub, double, double) 700 #endif /* CK_F_PR_DEC_DOUBLE */ 701 702 #endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */ 703 704 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) 705 706 #ifndef CK_F_PR_INC_UINT 707 #define CK_F_PR_INC_UINT 708 CK_PR_UNARY_S(inc, add, uint, unsigned int) 709 #endif /* CK_F_PR_INC_UINT */ 710 711 #ifndef CK_F_PR_INC_UINT_ZERO 712 #define CK_F_PR_INC_UINT_ZERO 713 CK_PR_UNARY_Z_S(inc, uint, unsigned int, +, UINT_MAX) 714 #endif /* CK_F_PR_INC_UINT_ZERO */ 715 716 #ifndef CK_F_PR_DEC_UINT 717 #define CK_F_PR_DEC_UINT 718 CK_PR_UNARY_S(dec, sub, uint, unsigned int) 719 #endif /* CK_F_PR_DEC_UINT */ 720 721 #ifndef CK_F_PR_DEC_UINT_ZERO 722 #define CK_F_PR_DEC_UINT_ZERO 723 CK_PR_UNARY_Z_S(dec, uint, unsigned int, -, 1) 724 #endif /* CK_F_PR_DEC_UINT_ZERO */ 725 726 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ 727 728 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) 729 730 #ifndef CK_F_PR_INC_PTR 731 #define CK_F_PR_INC_PTR 732 CK_PR_UNARY(inc, add, ptr, void, uintptr_t) 733 #endif /* CK_F_PR_INC_PTR */ 734 735 #ifndef CK_F_PR_INC_PTR_ZERO 736 #define CK_F_PR_INC_PTR_ZERO 737 CK_PR_UNARY_Z(inc, ptr, void, uintptr_t, +, void *, UINT_MAX) 738 #endif /* CK_F_PR_INC_PTR_ZERO */ 739 740 #ifndef CK_F_PR_DEC_PTR 741 #define CK_F_PR_DEC_PTR 742 CK_PR_UNARY(dec, sub, ptr, void, uintptr_t) 743 #endif /* CK_F_PR_DEC_PTR */ 744 745 #ifndef CK_F_PR_DEC_PTR_ZERO 746 #define CK_F_PR_DEC_PTR_ZERO 747 CK_PR_UNARY_Z(dec, ptr, void, uintptr_t, -, void *, 1) 748 #endif /* CK_F_PR_DEC_PTR_ZERO */ 749 750 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ 751 752 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) 753 754 #ifndef CK_F_PR_INC_64 755 #define CK_F_PR_INC_64 756 CK_PR_UNARY_S(inc, add, 64, uint64_t) 757 #endif /* CK_F_PR_INC_64 */ 758 759 #ifndef CK_F_PR_INC_64_ZERO 760 #define CK_F_PR_INC_64_ZERO 761 CK_PR_UNARY_Z_S(inc, 64, uint64_t, +, UINT64_MAX) 762 #endif /* CK_F_PR_INC_64_ZERO */ 763 764 #ifndef CK_F_PR_DEC_64 765 #define CK_F_PR_DEC_64 766 CK_PR_UNARY_S(dec, sub, 64, uint64_t) 767 #endif /* CK_F_PR_DEC_64 */ 768 769 #ifndef CK_F_PR_DEC_64_ZERO 770 #define CK_F_PR_DEC_64_ZERO 771 CK_PR_UNARY_Z_S(dec, 64, uint64_t, -, 1) 772 #endif /* CK_F_PR_DEC_64_ZERO */ 773 774 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ 775 776 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) 777 778 #ifndef CK_F_PR_INC_32 779 #define CK_F_PR_INC_32 780 CK_PR_UNARY_S(inc, add, 32, uint32_t) 781 #endif /* CK_F_PR_INC_32 */ 782 783 #ifndef CK_F_PR_INC_32_ZERO 784 #define CK_F_PR_INC_32_ZERO 785 CK_PR_UNARY_Z_S(inc, 32, uint32_t, +, UINT32_MAX) 786 #endif /* CK_F_PR_INC_32_ZERO */ 787 788 #ifndef CK_F_PR_DEC_32 789 #define CK_F_PR_DEC_32 790 CK_PR_UNARY_S(dec, sub, 32, uint32_t) 791 #endif /* CK_F_PR_DEC_32 */ 792 793 #ifndef CK_F_PR_DEC_32_ZERO 794 #define CK_F_PR_DEC_32_ZERO 795 CK_PR_UNARY_Z_S(dec, 32, uint32_t, -, 1) 796 #endif /* CK_F_PR_DEC_32_ZERO */ 797 798 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ 799 800 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) 801 802 #ifndef CK_F_PR_INC_16 803 #define CK_F_PR_INC_16 804 CK_PR_UNARY_S(inc, add, 16, uint16_t) 805 #endif /* CK_F_PR_INC_16 */ 806 807 #ifndef CK_F_PR_INC_16_ZERO 808 #define CK_F_PR_INC_16_ZERO 809 CK_PR_UNARY_Z_S(inc, 16, uint16_t, +, UINT16_MAX) 810 #endif /* CK_F_PR_INC_16_ZERO */ 811 812 #ifndef CK_F_PR_DEC_16 813 #define CK_F_PR_DEC_16 814 CK_PR_UNARY_S(dec, sub, 16, uint16_t) 815 #endif /* CK_F_PR_DEC_16 */ 816 817 #ifndef CK_F_PR_DEC_16_ZERO 818 #define CK_F_PR_DEC_16_ZERO 819 CK_PR_UNARY_Z_S(dec, 16, uint16_t, -, 1) 820 #endif /* CK_F_PR_DEC_16_ZERO */ 821 822 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ 823 824 #if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE) 825 826 #ifndef CK_F_PR_INC_8 827 #define CK_F_PR_INC_8 828 CK_PR_UNARY_S(inc, add, 8, uint8_t) 829 #endif /* CK_F_PR_INC_8 */ 830 831 #ifndef CK_F_PR_INC_8_ZERO 832 #define CK_F_PR_INC_8_ZERO 833 CK_PR_UNARY_Z_S(inc, 8, uint8_t, +, UINT8_MAX) 834 #endif /* CK_F_PR_INC_8_ZERO */ 835 836 #ifndef CK_F_PR_DEC_8 837 #define CK_F_PR_DEC_8 838 CK_PR_UNARY_S(dec, sub, 8, uint8_t) 839 #endif /* CK_F_PR_DEC_8 */ 840 841 #ifndef CK_F_PR_DEC_8_ZERO 842 #define CK_F_PR_DEC_8_ZERO 843 CK_PR_UNARY_Z_S(dec, 8, uint8_t, -, 1) 844 #endif /* CK_F_PR_DEC_8_ZERO */ 845 846 #endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */ 847 848 #undef CK_PR_UNARY_Z_S 849 #undef CK_PR_UNARY_S 850 #undef CK_PR_UNARY_Z 851 #undef CK_PR_UNARY 852 853 #define CK_PR_N(K, S, M, T, P, C) \ 854 CK_CC_INLINE static void \ 855 ck_pr_##K##_##S(M *target) \ 856 { \ 857 T previous; \ 858 C punt; \ 859 punt = (C)ck_pr_md_load_##S(target); \ 860 previous = (T)punt; \ 861 while (ck_pr_cas_##S##_value(target, \ 862 (C)previous, \ 863 (C)(P previous), \ 864 &previous) == false) \ 865 ck_pr_stall(); \ 866 \ 867 return; \ 868 } 869 870 #define CK_PR_N_Z(S, M, T, C) \ 871 CK_CC_INLINE static void \ 872 ck_pr_neg_##S##_zero(M *target, bool *zero) \ 873 { \ 874 T previous; \ 875 C punt; \ 876 punt = (C)ck_pr_md_load_##S(target); \ 877 previous = (T)punt; \ 878 while (ck_pr_cas_##S##_value(target, \ 879 (C)previous, \ 880 (C)(-previous), \ 881 &previous) == false) \ 882 ck_pr_stall(); \ 883 \ 884 *zero = previous == 0; \ 885 return; \ 886 } 887 888 #define CK_PR_N_S(K, S, M, P) CK_PR_N(K, S, M, M, P, M) 889 #define CK_PR_N_Z_S(S, M) CK_PR_N_Z(S, M, M, M) 890 891 #if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE) 892 893 #ifndef CK_F_PR_NOT_CHAR 894 #define CK_F_PR_NOT_CHAR 895 CK_PR_N_S(not, char, char, ~) 896 #endif /* CK_F_PR_NOT_CHAR */ 897 898 #ifndef CK_F_PR_NEG_CHAR 899 #define CK_F_PR_NEG_CHAR 900 CK_PR_N_S(neg, char, char, -) 901 #endif /* CK_F_PR_NEG_CHAR */ 902 903 #ifndef CK_F_PR_NEG_CHAR_ZERO 904 #define CK_F_PR_NEG_CHAR_ZERO 905 CK_PR_N_Z_S(char, char) 906 #endif /* CK_F_PR_NEG_CHAR_ZERO */ 907 908 #endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */ 909 910 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) 911 912 #ifndef CK_F_PR_NOT_INT 913 #define CK_F_PR_NOT_INT 914 CK_PR_N_S(not, int, int, ~) 915 #endif /* CK_F_PR_NOT_INT */ 916 917 #ifndef CK_F_PR_NEG_INT 918 #define CK_F_PR_NEG_INT 919 CK_PR_N_S(neg, int, int, -) 920 #endif /* CK_F_PR_NEG_INT */ 921 922 #ifndef CK_F_PR_NEG_INT_ZERO 923 #define CK_F_PR_NEG_INT_ZERO 924 CK_PR_N_Z_S(int, int) 925 #endif /* CK_F_PR_NEG_INT_ZERO */ 926 927 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ 928 929 #if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \ 930 !defined(CK_PR_DISABLE_DOUBLE) 931 932 #ifndef CK_F_PR_NEG_DOUBLE 933 #define CK_F_PR_NEG_DOUBLE 934 CK_PR_N_S(neg, double, double, -) 935 #endif /* CK_F_PR_NEG_DOUBLE */ 936 937 #endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */ 938 939 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) 940 941 #ifndef CK_F_PR_NOT_UINT 942 #define CK_F_PR_NOT_UINT 943 CK_PR_N_S(not, uint, unsigned int, ~) 944 #endif /* CK_F_PR_NOT_UINT */ 945 946 #ifndef CK_F_PR_NEG_UINT 947 #define CK_F_PR_NEG_UINT 948 CK_PR_N_S(neg, uint, unsigned int, -) 949 #endif /* CK_F_PR_NEG_UINT */ 950 951 #ifndef CK_F_PR_NEG_UINT_ZERO 952 #define CK_F_PR_NEG_UINT_ZERO 953 CK_PR_N_Z_S(uint, unsigned int) 954 #endif /* CK_F_PR_NEG_UINT_ZERO */ 955 956 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ 957 958 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) 959 960 #ifndef CK_F_PR_NOT_PTR 961 #define CK_F_PR_NOT_PTR 962 CK_PR_N(not, ptr, void, uintptr_t, ~, void *) 963 #endif /* CK_F_PR_NOT_PTR */ 964 965 #ifndef CK_F_PR_NEG_PTR 966 #define CK_F_PR_NEG_PTR 967 CK_PR_N(neg, ptr, void, uintptr_t, -, void *) 968 #endif /* CK_F_PR_NEG_PTR */ 969 970 #ifndef CK_F_PR_NEG_PTR_ZERO 971 #define CK_F_PR_NEG_PTR_ZERO 972 CK_PR_N_Z(ptr, void, uintptr_t, void *) 973 #endif /* CK_F_PR_NEG_PTR_ZERO */ 974 975 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ 976 977 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) 978 979 #ifndef CK_F_PR_NOT_64 980 #define CK_F_PR_NOT_64 981 CK_PR_N_S(not, 64, uint64_t, ~) 982 #endif /* CK_F_PR_NOT_64 */ 983 984 #ifndef CK_F_PR_NEG_64 985 #define CK_F_PR_NEG_64 986 CK_PR_N_S(neg, 64, uint64_t, -) 987 #endif /* CK_F_PR_NEG_64 */ 988 989 #ifndef CK_F_PR_NEG_64_ZERO 990 #define CK_F_PR_NEG_64_ZERO 991 CK_PR_N_Z_S(64, uint64_t) 992 #endif /* CK_F_PR_NEG_64_ZERO */ 993 994 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ 995 996 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) 997 998 #ifndef CK_F_PR_NOT_32 999 #define CK_F_PR_NOT_32 1000 CK_PR_N_S(not, 32, uint32_t, ~) 1001 #endif /* CK_F_PR_NOT_32 */ 1002 1003 #ifndef CK_F_PR_NEG_32 1004 #define CK_F_PR_NEG_32 1005 CK_PR_N_S(neg, 32, uint32_t, -) 1006 #endif /* CK_F_PR_NEG_32 */ 1007 1008 #ifndef CK_F_PR_NEG_32_ZERO 1009 #define CK_F_PR_NEG_32_ZERO 1010 CK_PR_N_Z_S(32, uint32_t) 1011 #endif /* CK_F_PR_NEG_32_ZERO */ 1012 1013 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ 1014 1015 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) 1016 1017 #ifndef CK_F_PR_NOT_16 1018 #define CK_F_PR_NOT_16 1019 CK_PR_N_S(not, 16, uint16_t, ~) 1020 #endif /* CK_F_PR_NOT_16 */ 1021 1022 #ifndef CK_F_PR_NEG_16 1023 #define CK_F_PR_NEG_16 1024 CK_PR_N_S(neg, 16, uint16_t, -) 1025 #endif /* CK_F_PR_NEG_16 */ 1026 1027 #ifndef CK_F_PR_NEG_16_ZERO 1028 #define CK_F_PR_NEG_16_ZERO 1029 CK_PR_N_Z_S(16, uint16_t) 1030 #endif /* CK_F_PR_NEG_16_ZERO */ 1031 1032 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ 1033 1034 #if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE) 1035 1036 #ifndef CK_F_PR_NOT_8 1037 #define CK_F_PR_NOT_8 1038 CK_PR_N_S(not, 8, uint8_t, ~) 1039 #endif /* CK_F_PR_NOT_8 */ 1040 1041 #ifndef CK_F_PR_NEG_8 1042 #define CK_F_PR_NEG_8 1043 CK_PR_N_S(neg, 8, uint8_t, -) 1044 #endif /* CK_F_PR_NEG_8 */ 1045 1046 #ifndef CK_F_PR_NEG_8_ZERO 1047 #define CK_F_PR_NEG_8_ZERO 1048 CK_PR_N_Z_S(8, uint8_t) 1049 #endif /* CK_F_PR_NEG_8_ZERO */ 1050 1051 #endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */ 1052 1053 #undef CK_PR_N_Z_S 1054 #undef CK_PR_N_S 1055 #undef CK_PR_N_Z 1056 #undef CK_PR_N 1057 1058 #define CK_PR_FAA(S, M, T, C) \ 1059 CK_CC_INLINE static C \ 1060 ck_pr_faa_##S(M *target, T delta) \ 1061 { \ 1062 T previous; \ 1063 C punt; \ 1064 punt = (C)ck_pr_md_load_##S(target); \ 1065 previous = (T)punt; \ 1066 while (ck_pr_cas_##S##_value(target, \ 1067 (C)previous, \ 1068 (C)(previous + delta), \ 1069 &previous) == false) \ 1070 ck_pr_stall(); \ 1071 \ 1072 return ((C)previous); \ 1073 } 1074 1075 #define CK_PR_FAS(S, M, C) \ 1076 CK_CC_INLINE static C \ 1077 ck_pr_fas_##S(M *target, C update) \ 1078 { \ 1079 C previous; \ 1080 previous = ck_pr_md_load_##S(target); \ 1081 while (ck_pr_cas_##S##_value(target, \ 1082 previous, \ 1083 update, \ 1084 &previous) == false) \ 1085 ck_pr_stall(); \ 1086 \ 1087 return (previous); \ 1088 } 1089 1090 #define CK_PR_FAA_S(S, M) CK_PR_FAA(S, M, M, M) 1091 #define CK_PR_FAS_S(S, M) CK_PR_FAS(S, M, M) 1092 1093 #if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE) 1094 1095 #ifndef CK_F_PR_FAA_CHAR 1096 #define CK_F_PR_FAA_CHAR 1097 CK_PR_FAA_S(char, char) 1098 #endif /* CK_F_PR_FAA_CHAR */ 1099 1100 #ifndef CK_F_PR_FAS_CHAR 1101 #define CK_F_PR_FAS_CHAR 1102 CK_PR_FAS_S(char, char) 1103 #endif /* CK_F_PR_FAS_CHAR */ 1104 1105 #endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */ 1106 1107 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) 1108 1109 #ifndef CK_F_PR_FAA_INT 1110 #define CK_F_PR_FAA_INT 1111 CK_PR_FAA_S(int, int) 1112 #endif /* CK_F_PR_FAA_INT */ 1113 1114 #ifndef CK_F_PR_FAS_INT 1115 #define CK_F_PR_FAS_INT 1116 CK_PR_FAS_S(int, int) 1117 #endif /* CK_F_PR_FAS_INT */ 1118 1119 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ 1120 1121 #if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \ 1122 !defined(CK_PR_DISABLE_DOUBLE) 1123 1124 #ifndef CK_F_PR_FAA_DOUBLE 1125 #define CK_F_PR_FAA_DOUBLE 1126 CK_PR_FAA_S(double, double) 1127 #endif /* CK_F_PR_FAA_DOUBLE */ 1128 1129 #ifndef CK_F_PR_FAS_DOUBLE 1130 #define CK_F_PR_FAS_DOUBLE 1131 CK_PR_FAS_S(double, double) 1132 #endif /* CK_F_PR_FAS_DOUBLE */ 1133 1134 #endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */ 1135 1136 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) 1137 1138 #ifndef CK_F_PR_FAA_UINT 1139 #define CK_F_PR_FAA_UINT 1140 CK_PR_FAA_S(uint, unsigned int) 1141 #endif /* CK_F_PR_FAA_UINT */ 1142 1143 #ifndef CK_F_PR_FAS_UINT 1144 #define CK_F_PR_FAS_UINT 1145 CK_PR_FAS_S(uint, unsigned int) 1146 #endif /* CK_F_PR_FAS_UINT */ 1147 1148 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ 1149 1150 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) 1151 1152 #ifndef CK_F_PR_FAA_PTR 1153 #define CK_F_PR_FAA_PTR 1154 CK_PR_FAA(ptr, void, uintptr_t, void *) 1155 #endif /* CK_F_PR_FAA_PTR */ 1156 1157 #ifndef CK_F_PR_FAS_PTR 1158 #define CK_F_PR_FAS_PTR 1159 CK_PR_FAS(ptr, void, void *) 1160 #endif /* CK_F_PR_FAS_PTR */ 1161 1162 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ 1163 1164 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) 1165 1166 #ifndef CK_F_PR_FAA_64 1167 #define CK_F_PR_FAA_64 1168 CK_PR_FAA_S(64, uint64_t) 1169 #endif /* CK_F_PR_FAA_64 */ 1170 1171 #ifndef CK_F_PR_FAS_64 1172 #define CK_F_PR_FAS_64 1173 CK_PR_FAS_S(64, uint64_t) 1174 #endif /* CK_F_PR_FAS_64 */ 1175 1176 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ 1177 1178 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) 1179 1180 #ifndef CK_F_PR_FAA_32 1181 #define CK_F_PR_FAA_32 1182 CK_PR_FAA_S(32, uint32_t) 1183 #endif /* CK_F_PR_FAA_32 */ 1184 1185 #ifndef CK_F_PR_FAS_32 1186 #define CK_F_PR_FAS_32 1187 CK_PR_FAS_S(32, uint32_t) 1188 #endif /* CK_F_PR_FAS_32 */ 1189 1190 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ 1191 1192 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) 1193 1194 #ifndef CK_F_PR_FAA_16 1195 #define CK_F_PR_FAA_16 1196 CK_PR_FAA_S(16, uint16_t) 1197 #endif /* CK_F_PR_FAA_16 */ 1198 1199 #ifndef CK_F_PR_FAS_16 1200 #define CK_F_PR_FAS_16 1201 CK_PR_FAS_S(16, uint16_t) 1202 #endif /* CK_F_PR_FAS_16 */ 1203 1204 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ 1205 1206 #if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE) 1207 1208 #ifndef CK_F_PR_FAA_8 1209 #define CK_F_PR_FAA_8 1210 CK_PR_FAA_S(8, uint8_t) 1211 #endif /* CK_F_PR_FAA_8 */ 1212 1213 #ifndef CK_F_PR_FAS_8 1214 #define CK_F_PR_FAS_8 1215 CK_PR_FAS_S(8, uint8_t) 1216 #endif /* CK_F_PR_FAS_8 */ 1217 1218 #endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */ 1219 1220 #undef CK_PR_FAA_S 1221 #undef CK_PR_FAS_S 1222 #undef CK_PR_FAA 1223 #undef CK_PR_FAS 1224 1225 #endif /* CK_PR_H */ 1226