1 /* 2 * Copyright 2009-2015 Samy Al Bahra. 3 * Copyright 2011 David Joseph. 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #ifndef CK_PR_H 29 #define CK_PR_H 30 31 #include <ck_cc.h> 32 #include <ck_limits.h> 33 #include <ck_md.h> 34 #include <ck_stdint.h> 35 #include <ck_stdbool.h> 36 37 #ifndef CK_USE_CC_BUILTINS 38 #if defined(__x86_64__) 39 #include "gcc/x86_64/ck_pr.h" 40 #elif defined(__x86__) 41 #include "gcc/x86/ck_pr.h" 42 #elif defined(__sparcv9__) 43 #include "gcc/sparcv9/ck_pr.h" 44 #elif defined(__ppc64__) 45 #include "gcc/ppc64/ck_pr.h" 46 #elif defined(__ppc__) 47 #include "gcc/ppc/ck_pr.h" 48 #elif defined(__arm__) 49 #if __ARM_ARCH >= 6 50 #include "gcc/arm/ck_pr.h" 51 #else 52 #include "gcc/arm/ck_pr_armv4.h" 53 #endif 54 #elif defined(__aarch64__) 55 #include "gcc/aarch64/ck_pr.h" 56 #elif !defined(__GNUC__) 57 #error Your platform is unsupported 58 #endif 59 #endif /* !CK_USE_CC_BUILTINS */ 60 61 #if defined(__GNUC__) 62 #include "gcc/ck_pr.h" 63 #endif 64 65 #define CK_PR_FENCE_EMIT(T) \ 66 CK_CC_INLINE static void \ 67 ck_pr_fence_##T(void) \ 68 { \ 69 ck_pr_fence_strict_##T(); \ 70 return; \ 71 } 72 #define CK_PR_FENCE_NOOP(T) \ 73 CK_CC_INLINE static void \ 74 ck_pr_fence_##T(void) \ 75 { \ 76 ck_pr_barrier(); \ 77 return; \ 78 } 79 80 /* 81 * None of the currently supported platforms allow for data-dependent 82 * load ordering. 83 */ 84 CK_PR_FENCE_NOOP(load_depends) 85 #define ck_pr_fence_strict_load_depends ck_pr_fence_load_depends 86 87 /* 88 * In memory models where atomic operations do not have serializing 89 * effects, atomic read-modify-write operations are modeled as stores. 90 */ 91 #if defined(CK_MD_RMO) 92 /* 93 * Only stores to the same location have a global 94 * ordering. 95 */ 96 CK_PR_FENCE_EMIT(atomic) 97 CK_PR_FENCE_EMIT(atomic_load) 98 CK_PR_FENCE_EMIT(atomic_store) 99 CK_PR_FENCE_EMIT(store_atomic) 100 CK_PR_FENCE_EMIT(load_atomic) 101 CK_PR_FENCE_EMIT(load_store) 102 CK_PR_FENCE_EMIT(store_load) 103 CK_PR_FENCE_EMIT(load) 104 CK_PR_FENCE_EMIT(store) 105 CK_PR_FENCE_EMIT(memory) 106 CK_PR_FENCE_EMIT(acquire) 107 CK_PR_FENCE_EMIT(release) 108 CK_PR_FENCE_EMIT(acqrel) 109 CK_PR_FENCE_EMIT(lock) 110 CK_PR_FENCE_EMIT(unlock) 111 #elif defined(CK_MD_PSO) 112 /* 113 * Anything can be re-ordered with respect to stores. 114 * Otherwise, loads are executed in-order. 115 */ 116 CK_PR_FENCE_EMIT(atomic) 117 CK_PR_FENCE_NOOP(atomic_load) 118 CK_PR_FENCE_EMIT(atomic_store) 119 CK_PR_FENCE_EMIT(store_atomic) 120 CK_PR_FENCE_NOOP(load_atomic) 121 CK_PR_FENCE_EMIT(load_store) 122 CK_PR_FENCE_EMIT(store_load) 123 CK_PR_FENCE_NOOP(load) 124 CK_PR_FENCE_EMIT(store) 125 CK_PR_FENCE_EMIT(memory) 126 CK_PR_FENCE_EMIT(acquire) 127 CK_PR_FENCE_EMIT(release) 128 CK_PR_FENCE_EMIT(acqrel) 129 CK_PR_FENCE_EMIT(lock) 130 CK_PR_FENCE_EMIT(unlock) 131 #elif defined(CK_MD_TSO) 132 /* 133 * Only loads are re-ordered and only with respect to 134 * prior stores. Atomic operations are serializing. 135 */ 136 CK_PR_FENCE_NOOP(atomic) 137 CK_PR_FENCE_NOOP(atomic_load) 138 CK_PR_FENCE_NOOP(atomic_store) 139 CK_PR_FENCE_NOOP(store_atomic) 140 CK_PR_FENCE_NOOP(load_atomic) 141 CK_PR_FENCE_NOOP(load_store) 142 CK_PR_FENCE_EMIT(store_load) 143 CK_PR_FENCE_NOOP(load) 144 CK_PR_FENCE_NOOP(store) 145 CK_PR_FENCE_EMIT(memory) 146 CK_PR_FENCE_NOOP(acquire) 147 CK_PR_FENCE_NOOP(release) 148 CK_PR_FENCE_NOOP(acqrel) 149 CK_PR_FENCE_NOOP(lock) 150 CK_PR_FENCE_NOOP(unlock) 151 #else 152 #error "No memory model has been defined." 153 #endif /* CK_MD_TSO */ 154 155 #undef CK_PR_FENCE_EMIT 156 #undef CK_PR_FENCE_NOOP 157 158 #ifndef CK_F_PR_RFO 159 #define CK_F_PR_RFO 160 CK_CC_INLINE static void 161 ck_pr_rfo(const void *m) 162 { 163 164 (void)m; 165 return; 166 } 167 #endif /* CK_F_PR_RFO */ 168 169 #define CK_PR_STORE_SAFE(DST, VAL, TYPE) \ 170 ck_pr_md_store_##TYPE( \ 171 ((void)sizeof(*(DST) = (VAL)), (DST)), \ 172 (VAL)) 173 174 #define ck_pr_store_ptr(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), ptr) 175 #define ck_pr_store_char(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), char) 176 #ifndef CK_PR_DISABLE_DOUBLE 177 #define ck_pr_store_double(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), double) 178 #endif 179 #define ck_pr_store_uint(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), uint) 180 #define ck_pr_store_int(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), int) 181 #define ck_pr_store_32(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 32) 182 #define ck_pr_store_16(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 16) 183 #define ck_pr_store_8(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 8) 184 185 #define ck_pr_store_ptr_unsafe(DST, VAL) ck_pr_md_store_ptr((DST), (VAL)) 186 187 #ifdef CK_F_PR_LOAD_64 188 #define ck_pr_store_64(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 64) 189 #endif /* CK_F_PR_LOAD_64 */ 190 191 #define CK_PR_LOAD_PTR_SAFE(SRC) (CK_CC_TYPEOF(*(SRC), (void *)))ck_pr_md_load_ptr((SRC)) 192 #define ck_pr_load_ptr(SRC) CK_PR_LOAD_PTR_SAFE((SRC)) 193 194 #define CK_PR_LOAD_SAFE(SRC, TYPE) ck_pr_md_load_##TYPE((SRC)) 195 #define ck_pr_load_char(SRC) CK_PR_LOAD_SAFE((SRC), char) 196 #ifndef CK_PR_DISABLE_DOUBLE 197 #define ck_pr_load_double(SRC) CK_PR_LOAD_SAFE((SRC), double) 198 #endif 199 #define ck_pr_load_uint(SRC) CK_PR_LOAD_SAFE((SRC), uint) 200 #define ck_pr_load_int(SRC) CK_PR_LOAD_SAFE((SRC), int) 201 #define ck_pr_load_32(SRC) CK_PR_LOAD_SAFE((SRC), 32) 202 #define ck_pr_load_16(SRC) CK_PR_LOAD_SAFE((SRC), 16) 203 #define ck_pr_load_8(SRC) CK_PR_LOAD_SAFE((SRC), 8) 204 205 #ifdef CK_F_PR_LOAD_64 206 #define ck_pr_load_64(SRC) CK_PR_LOAD_SAFE((SRC), 64) 207 #endif /* CK_F_PR_LOAD_64 */ 208 209 #define CK_PR_BIN(K, S, M, T, P, C) \ 210 CK_CC_INLINE static void \ 211 ck_pr_##K##_##S(M *target, T value) \ 212 { \ 213 T previous; \ 214 C punt; \ 215 punt = ck_pr_md_load_##S(target); \ 216 previous = (T)punt; \ 217 while (ck_pr_cas_##S##_value(target, \ 218 (C)previous, \ 219 (C)(previous P value), \ 220 &previous) == false) \ 221 ck_pr_stall(); \ 222 \ 223 return; \ 224 } 225 226 #define CK_PR_BIN_S(K, S, T, P) CK_PR_BIN(K, S, T, T, P, T) 227 228 #if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE) 229 230 #ifndef CK_F_PR_ADD_CHAR 231 #define CK_F_PR_ADD_CHAR 232 CK_PR_BIN_S(add, char, char, +) 233 #endif /* CK_F_PR_ADD_CHAR */ 234 235 #ifndef CK_F_PR_SUB_CHAR 236 #define CK_F_PR_SUB_CHAR 237 CK_PR_BIN_S(sub, char, char, -) 238 #endif /* CK_F_PR_SUB_CHAR */ 239 240 #ifndef CK_F_PR_AND_CHAR 241 #define CK_F_PR_AND_CHAR 242 CK_PR_BIN_S(and, char, char, &) 243 #endif /* CK_F_PR_AND_CHAR */ 244 245 #ifndef CK_F_PR_XOR_CHAR 246 #define CK_F_PR_XOR_CHAR 247 CK_PR_BIN_S(xor, char, char, ^) 248 #endif /* CK_F_PR_XOR_CHAR */ 249 250 #ifndef CK_F_PR_OR_CHAR 251 #define CK_F_PR_OR_CHAR 252 CK_PR_BIN_S(or, char, char, |) 253 #endif /* CK_F_PR_OR_CHAR */ 254 255 #endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */ 256 257 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) 258 259 #ifndef CK_F_PR_ADD_INT 260 #define CK_F_PR_ADD_INT 261 CK_PR_BIN_S(add, int, int, +) 262 #endif /* CK_F_PR_ADD_INT */ 263 264 #ifndef CK_F_PR_SUB_INT 265 #define CK_F_PR_SUB_INT 266 CK_PR_BIN_S(sub, int, int, -) 267 #endif /* CK_F_PR_SUB_INT */ 268 269 #ifndef CK_F_PR_AND_INT 270 #define CK_F_PR_AND_INT 271 CK_PR_BIN_S(and, int, int, &) 272 #endif /* CK_F_PR_AND_INT */ 273 274 #ifndef CK_F_PR_XOR_INT 275 #define CK_F_PR_XOR_INT 276 CK_PR_BIN_S(xor, int, int, ^) 277 #endif /* CK_F_PR_XOR_INT */ 278 279 #ifndef CK_F_PR_OR_INT 280 #define CK_F_PR_OR_INT 281 CK_PR_BIN_S(or, int, int, |) 282 #endif /* CK_F_PR_OR_INT */ 283 284 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ 285 286 #if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \ 287 !defined(CK_PR_DISABLE_DOUBLE) 288 289 #ifndef CK_F_PR_ADD_DOUBLE 290 #define CK_F_PR_ADD_DOUBLE 291 CK_PR_BIN_S(add, double, double, +) 292 #endif /* CK_F_PR_ADD_DOUBLE */ 293 294 #ifndef CK_F_PR_SUB_DOUBLE 295 #define CK_F_PR_SUB_DOUBLE 296 CK_PR_BIN_S(sub, double, double, -) 297 #endif /* CK_F_PR_SUB_DOUBLE */ 298 299 #endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */ 300 301 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) 302 303 #ifndef CK_F_PR_ADD_UINT 304 #define CK_F_PR_ADD_UINT 305 CK_PR_BIN_S(add, uint, unsigned int, +) 306 #endif /* CK_F_PR_ADD_UINT */ 307 308 #ifndef CK_F_PR_SUB_UINT 309 #define CK_F_PR_SUB_UINT 310 CK_PR_BIN_S(sub, uint, unsigned int, -) 311 #endif /* CK_F_PR_SUB_UINT */ 312 313 #ifndef CK_F_PR_AND_UINT 314 #define CK_F_PR_AND_UINT 315 CK_PR_BIN_S(and, uint, unsigned int, &) 316 #endif /* CK_F_PR_AND_UINT */ 317 318 #ifndef CK_F_PR_XOR_UINT 319 #define CK_F_PR_XOR_UINT 320 CK_PR_BIN_S(xor, uint, unsigned int, ^) 321 #endif /* CK_F_PR_XOR_UINT */ 322 323 #ifndef CK_F_PR_OR_UINT 324 #define CK_F_PR_OR_UINT 325 CK_PR_BIN_S(or, uint, unsigned int, |) 326 #endif /* CK_F_PR_OR_UINT */ 327 328 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ 329 330 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) 331 332 #ifndef CK_F_PR_ADD_PTR 333 #define CK_F_PR_ADD_PTR 334 CK_PR_BIN(add, ptr, void, uintptr_t, +, void *) 335 #endif /* CK_F_PR_ADD_PTR */ 336 337 #ifndef CK_F_PR_SUB_PTR 338 #define CK_F_PR_SUB_PTR 339 CK_PR_BIN(sub, ptr, void, uintptr_t, -, void *) 340 #endif /* CK_F_PR_SUB_PTR */ 341 342 #ifndef CK_F_PR_AND_PTR 343 #define CK_F_PR_AND_PTR 344 CK_PR_BIN(and, ptr, void, uintptr_t, &, void *) 345 #endif /* CK_F_PR_AND_PTR */ 346 347 #ifndef CK_F_PR_XOR_PTR 348 #define CK_F_PR_XOR_PTR 349 CK_PR_BIN(xor, ptr, void, uintptr_t, ^, void *) 350 #endif /* CK_F_PR_XOR_PTR */ 351 352 #ifndef CK_F_PR_OR_PTR 353 #define CK_F_PR_OR_PTR 354 CK_PR_BIN(or, ptr, void, uintptr_t, |, void *) 355 #endif /* CK_F_PR_OR_PTR */ 356 357 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ 358 359 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) 360 361 #ifndef CK_F_PR_ADD_64 362 #define CK_F_PR_ADD_64 363 CK_PR_BIN_S(add, 64, uint64_t, +) 364 #endif /* CK_F_PR_ADD_64 */ 365 366 #ifndef CK_F_PR_SUB_64 367 #define CK_F_PR_SUB_64 368 CK_PR_BIN_S(sub, 64, uint64_t, -) 369 #endif /* CK_F_PR_SUB_64 */ 370 371 #ifndef CK_F_PR_AND_64 372 #define CK_F_PR_AND_64 373 CK_PR_BIN_S(and, 64, uint64_t, &) 374 #endif /* CK_F_PR_AND_64 */ 375 376 #ifndef CK_F_PR_XOR_64 377 #define CK_F_PR_XOR_64 378 CK_PR_BIN_S(xor, 64, uint64_t, ^) 379 #endif /* CK_F_PR_XOR_64 */ 380 381 #ifndef CK_F_PR_OR_64 382 #define CK_F_PR_OR_64 383 CK_PR_BIN_S(or, 64, uint64_t, |) 384 #endif /* CK_F_PR_OR_64 */ 385 386 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ 387 388 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) 389 390 #ifndef CK_F_PR_ADD_32 391 #define CK_F_PR_ADD_32 392 CK_PR_BIN_S(add, 32, uint32_t, +) 393 #endif /* CK_F_PR_ADD_32 */ 394 395 #ifndef CK_F_PR_SUB_32 396 #define CK_F_PR_SUB_32 397 CK_PR_BIN_S(sub, 32, uint32_t, -) 398 #endif /* CK_F_PR_SUB_32 */ 399 400 #ifndef CK_F_PR_AND_32 401 #define CK_F_PR_AND_32 402 CK_PR_BIN_S(and, 32, uint32_t, &) 403 #endif /* CK_F_PR_AND_32 */ 404 405 #ifndef CK_F_PR_XOR_32 406 #define CK_F_PR_XOR_32 407 CK_PR_BIN_S(xor, 32, uint32_t, ^) 408 #endif /* CK_F_PR_XOR_32 */ 409 410 #ifndef CK_F_PR_OR_32 411 #define CK_F_PR_OR_32 412 CK_PR_BIN_S(or, 32, uint32_t, |) 413 #endif /* CK_F_PR_OR_32 */ 414 415 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ 416 417 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) 418 419 #ifndef CK_F_PR_ADD_16 420 #define CK_F_PR_ADD_16 421 CK_PR_BIN_S(add, 16, uint16_t, +) 422 #endif /* CK_F_PR_ADD_16 */ 423 424 #ifndef CK_F_PR_SUB_16 425 #define CK_F_PR_SUB_16 426 CK_PR_BIN_S(sub, 16, uint16_t, -) 427 #endif /* CK_F_PR_SUB_16 */ 428 429 #ifndef CK_F_PR_AND_16 430 #define CK_F_PR_AND_16 431 CK_PR_BIN_S(and, 16, uint16_t, &) 432 #endif /* CK_F_PR_AND_16 */ 433 434 #ifndef CK_F_PR_XOR_16 435 #define CK_F_PR_XOR_16 436 CK_PR_BIN_S(xor, 16, uint16_t, ^) 437 #endif /* CK_F_PR_XOR_16 */ 438 439 #ifndef CK_F_PR_OR_16 440 #define CK_F_PR_OR_16 441 CK_PR_BIN_S(or, 16, uint16_t, |) 442 #endif /* CK_F_PR_OR_16 */ 443 444 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ 445 446 #if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE) 447 448 #ifndef CK_F_PR_ADD_8 449 #define CK_F_PR_ADD_8 450 CK_PR_BIN_S(add, 8, uint8_t, +) 451 #endif /* CK_F_PR_ADD_8 */ 452 453 #ifndef CK_F_PR_SUB_8 454 #define CK_F_PR_SUB_8 455 CK_PR_BIN_S(sub, 8, uint8_t, -) 456 #endif /* CK_F_PR_SUB_8 */ 457 458 #ifndef CK_F_PR_AND_8 459 #define CK_F_PR_AND_8 460 CK_PR_BIN_S(and, 8, uint8_t, &) 461 #endif /* CK_F_PR_AND_8 */ 462 463 #ifndef CK_F_PR_XOR_8 464 #define CK_F_PR_XOR_8 465 CK_PR_BIN_S(xor, 8, uint8_t, ^) 466 #endif /* CK_F_PR_XOR_8 */ 467 468 #ifndef CK_F_PR_OR_8 469 #define CK_F_PR_OR_8 470 CK_PR_BIN_S(or, 8, uint8_t, |) 471 #endif /* CK_F_PR_OR_8 */ 472 473 #endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */ 474 475 #undef CK_PR_BIN_S 476 #undef CK_PR_BIN 477 478 #define CK_PR_BTX(K, S, M, T, P, C, R) \ 479 CK_CC_INLINE static bool \ 480 ck_pr_##K##_##S(M *target, unsigned int offset) \ 481 { \ 482 T previous; \ 483 C punt; \ 484 punt = ck_pr_md_load_##S(target); \ 485 previous = (T)punt; \ 486 while (ck_pr_cas_##S##_value(target, (C)previous, \ 487 (C)(previous P (R ((T)1 << offset))), &previous) == false) \ 488 ck_pr_stall(); \ 489 return ((previous >> offset) & 1); \ 490 } 491 492 #define CK_PR_BTX_S(K, S, T, P, R) CK_PR_BTX(K, S, T, T, P, T, R) 493 494 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) 495 496 #ifndef CK_F_PR_BTC_INT 497 #define CK_F_PR_BTC_INT 498 CK_PR_BTX_S(btc, int, int, ^,) 499 #endif /* CK_F_PR_BTC_INT */ 500 501 #ifndef CK_F_PR_BTR_INT 502 #define CK_F_PR_BTR_INT 503 CK_PR_BTX_S(btr, int, int, &, ~) 504 #endif /* CK_F_PR_BTR_INT */ 505 506 #ifndef CK_F_PR_BTS_INT 507 #define CK_F_PR_BTS_INT 508 CK_PR_BTX_S(bts, int, int, |,) 509 #endif /* CK_F_PR_BTS_INT */ 510 511 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ 512 513 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) 514 515 #ifndef CK_F_PR_BTC_UINT 516 #define CK_F_PR_BTC_UINT 517 CK_PR_BTX_S(btc, uint, unsigned int, ^,) 518 #endif /* CK_F_PR_BTC_UINT */ 519 520 #ifndef CK_F_PR_BTR_UINT 521 #define CK_F_PR_BTR_UINT 522 CK_PR_BTX_S(btr, uint, unsigned int, &, ~) 523 #endif /* CK_F_PR_BTR_UINT */ 524 525 #ifndef CK_F_PR_BTS_UINT 526 #define CK_F_PR_BTS_UINT 527 CK_PR_BTX_S(bts, uint, unsigned int, |,) 528 #endif /* CK_F_PR_BTS_UINT */ 529 530 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ 531 532 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) 533 534 #ifndef CK_F_PR_BTC_PTR 535 #define CK_F_PR_BTC_PTR 536 CK_PR_BTX(btc, ptr, void, uintptr_t, ^, void *,) 537 #endif /* CK_F_PR_BTC_PTR */ 538 539 #ifndef CK_F_PR_BTR_PTR 540 #define CK_F_PR_BTR_PTR 541 CK_PR_BTX(btr, ptr, void, uintptr_t, &, void *, ~) 542 #endif /* CK_F_PR_BTR_PTR */ 543 544 #ifndef CK_F_PR_BTS_PTR 545 #define CK_F_PR_BTS_PTR 546 CK_PR_BTX(bts, ptr, void, uintptr_t, |, void *,) 547 #endif /* CK_F_PR_BTS_PTR */ 548 549 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ 550 551 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) 552 553 #ifndef CK_F_PR_BTC_64 554 #define CK_F_PR_BTC_64 555 CK_PR_BTX_S(btc, 64, uint64_t, ^,) 556 #endif /* CK_F_PR_BTC_64 */ 557 558 #ifndef CK_F_PR_BTR_64 559 #define CK_F_PR_BTR_64 560 CK_PR_BTX_S(btr, 64, uint64_t, &, ~) 561 #endif /* CK_F_PR_BTR_64 */ 562 563 #ifndef CK_F_PR_BTS_64 564 #define CK_F_PR_BTS_64 565 CK_PR_BTX_S(bts, 64, uint64_t, |,) 566 #endif /* CK_F_PR_BTS_64 */ 567 568 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ 569 570 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) 571 572 #ifndef CK_F_PR_BTC_32 573 #define CK_F_PR_BTC_32 574 CK_PR_BTX_S(btc, 32, uint32_t, ^,) 575 #endif /* CK_F_PR_BTC_32 */ 576 577 #ifndef CK_F_PR_BTR_32 578 #define CK_F_PR_BTR_32 579 CK_PR_BTX_S(btr, 32, uint32_t, &, ~) 580 #endif /* CK_F_PR_BTR_32 */ 581 582 #ifndef CK_F_PR_BTS_32 583 #define CK_F_PR_BTS_32 584 CK_PR_BTX_S(bts, 32, uint32_t, |,) 585 #endif /* CK_F_PR_BTS_32 */ 586 587 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ 588 589 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) 590 591 #ifndef CK_F_PR_BTC_16 592 #define CK_F_PR_BTC_16 593 CK_PR_BTX_S(btc, 16, uint16_t, ^,) 594 #endif /* CK_F_PR_BTC_16 */ 595 596 #ifndef CK_F_PR_BTR_16 597 #define CK_F_PR_BTR_16 598 CK_PR_BTX_S(btr, 16, uint16_t, &, ~) 599 #endif /* CK_F_PR_BTR_16 */ 600 601 #ifndef CK_F_PR_BTS_16 602 #define CK_F_PR_BTS_16 603 CK_PR_BTX_S(bts, 16, uint16_t, |,) 604 #endif /* CK_F_PR_BTS_16 */ 605 606 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ 607 608 #undef CK_PR_BTX_S 609 #undef CK_PR_BTX 610 611 #define CK_PR_UNARY(K, X, S, M, T) \ 612 CK_CC_INLINE static void \ 613 ck_pr_##K##_##S(M *target) \ 614 { \ 615 ck_pr_##X##_##S(target, (T)1); \ 616 return; \ 617 } 618 619 #define CK_PR_UNARY_Z(K, S, M, T, P, C, Z) \ 620 CK_CC_INLINE static void \ 621 ck_pr_##K##_##S##_zero(M *target, bool *zero) \ 622 { \ 623 T previous; \ 624 C punt; \ 625 punt = (C)ck_pr_md_load_##S(target); \ 626 previous = (T)punt; \ 627 while (ck_pr_cas_##S##_value(target, \ 628 (C)previous, \ 629 (C)(previous P 1), \ 630 &previous) == false) \ 631 ck_pr_stall(); \ 632 *zero = previous == (T)Z; \ 633 return; \ 634 } 635 636 #define CK_PR_UNARY_S(K, X, S, M) CK_PR_UNARY(K, X, S, M, M) 637 #define CK_PR_UNARY_Z_S(K, S, M, P, Z) CK_PR_UNARY_Z(K, S, M, M, P, M, Z) 638 639 #if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE) 640 641 #ifndef CK_F_PR_INC_CHAR 642 #define CK_F_PR_INC_CHAR 643 CK_PR_UNARY_S(inc, add, char, char) 644 #endif /* CK_F_PR_INC_CHAR */ 645 646 #ifndef CK_F_PR_INC_CHAR_ZERO 647 #define CK_F_PR_INC_CHAR_ZERO 648 CK_PR_UNARY_Z_S(inc, char, char, +, -1) 649 #endif /* CK_F_PR_INC_CHAR_ZERO */ 650 651 #ifndef CK_F_PR_DEC_CHAR 652 #define CK_F_PR_DEC_CHAR 653 CK_PR_UNARY_S(dec, sub, char, char) 654 #endif /* CK_F_PR_DEC_CHAR */ 655 656 #ifndef CK_F_PR_DEC_CHAR_ZERO 657 #define CK_F_PR_DEC_CHAR_ZERO 658 CK_PR_UNARY_Z_S(dec, char, char, -, 1) 659 #endif /* CK_F_PR_DEC_CHAR_ZERO */ 660 661 #endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */ 662 663 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) 664 665 #ifndef CK_F_PR_INC_INT 666 #define CK_F_PR_INC_INT 667 CK_PR_UNARY_S(inc, add, int, int) 668 #endif /* CK_F_PR_INC_INT */ 669 670 #ifndef CK_F_PR_INC_INT_ZERO 671 #define CK_F_PR_INC_INT_ZERO 672 CK_PR_UNARY_Z_S(inc, int, int, +, -1) 673 #endif /* CK_F_PR_INC_INT_ZERO */ 674 675 #ifndef CK_F_PR_DEC_INT 676 #define CK_F_PR_DEC_INT 677 CK_PR_UNARY_S(dec, sub, int, int) 678 #endif /* CK_F_PR_DEC_INT */ 679 680 #ifndef CK_F_PR_DEC_INT_ZERO 681 #define CK_F_PR_DEC_INT_ZERO 682 CK_PR_UNARY_Z_S(dec, int, int, -, 1) 683 #endif /* CK_F_PR_DEC_INT_ZERO */ 684 685 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ 686 687 #if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \ 688 !defined(CK_PR_DISABLE_DOUBLE) 689 690 #ifndef CK_F_PR_INC_DOUBLE 691 #define CK_F_PR_INC_DOUBLE 692 CK_PR_UNARY_S(inc, add, double, double) 693 #endif /* CK_F_PR_INC_DOUBLE */ 694 695 #ifndef CK_F_PR_DEC_DOUBLE 696 #define CK_F_PR_DEC_DOUBLE 697 CK_PR_UNARY_S(dec, sub, double, double) 698 #endif /* CK_F_PR_DEC_DOUBLE */ 699 700 #endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */ 701 702 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) 703 704 #ifndef CK_F_PR_INC_UINT 705 #define CK_F_PR_INC_UINT 706 CK_PR_UNARY_S(inc, add, uint, unsigned int) 707 #endif /* CK_F_PR_INC_UINT */ 708 709 #ifndef CK_F_PR_INC_UINT_ZERO 710 #define CK_F_PR_INC_UINT_ZERO 711 CK_PR_UNARY_Z_S(inc, uint, unsigned int, +, UINT_MAX) 712 #endif /* CK_F_PR_INC_UINT_ZERO */ 713 714 #ifndef CK_F_PR_DEC_UINT 715 #define CK_F_PR_DEC_UINT 716 CK_PR_UNARY_S(dec, sub, uint, unsigned int) 717 #endif /* CK_F_PR_DEC_UINT */ 718 719 #ifndef CK_F_PR_DEC_UINT_ZERO 720 #define CK_F_PR_DEC_UINT_ZERO 721 CK_PR_UNARY_Z_S(dec, uint, unsigned int, -, 1) 722 #endif /* CK_F_PR_DEC_UINT_ZERO */ 723 724 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ 725 726 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) 727 728 #ifndef CK_F_PR_INC_PTR 729 #define CK_F_PR_INC_PTR 730 CK_PR_UNARY(inc, add, ptr, void, uintptr_t) 731 #endif /* CK_F_PR_INC_PTR */ 732 733 #ifndef CK_F_PR_INC_PTR_ZERO 734 #define CK_F_PR_INC_PTR_ZERO 735 CK_PR_UNARY_Z(inc, ptr, void, uintptr_t, +, void *, UINT_MAX) 736 #endif /* CK_F_PR_INC_PTR_ZERO */ 737 738 #ifndef CK_F_PR_DEC_PTR 739 #define CK_F_PR_DEC_PTR 740 CK_PR_UNARY(dec, sub, ptr, void, uintptr_t) 741 #endif /* CK_F_PR_DEC_PTR */ 742 743 #ifndef CK_F_PR_DEC_PTR_ZERO 744 #define CK_F_PR_DEC_PTR_ZERO 745 CK_PR_UNARY_Z(dec, ptr, void, uintptr_t, -, void *, 1) 746 #endif /* CK_F_PR_DEC_PTR_ZERO */ 747 748 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ 749 750 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) 751 752 #ifndef CK_F_PR_INC_64 753 #define CK_F_PR_INC_64 754 CK_PR_UNARY_S(inc, add, 64, uint64_t) 755 #endif /* CK_F_PR_INC_64 */ 756 757 #ifndef CK_F_PR_INC_64_ZERO 758 #define CK_F_PR_INC_64_ZERO 759 CK_PR_UNARY_Z_S(inc, 64, uint64_t, +, UINT64_MAX) 760 #endif /* CK_F_PR_INC_64_ZERO */ 761 762 #ifndef CK_F_PR_DEC_64 763 #define CK_F_PR_DEC_64 764 CK_PR_UNARY_S(dec, sub, 64, uint64_t) 765 #endif /* CK_F_PR_DEC_64 */ 766 767 #ifndef CK_F_PR_DEC_64_ZERO 768 #define CK_F_PR_DEC_64_ZERO 769 CK_PR_UNARY_Z_S(dec, 64, uint64_t, -, 1) 770 #endif /* CK_F_PR_DEC_64_ZERO */ 771 772 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ 773 774 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) 775 776 #ifndef CK_F_PR_INC_32 777 #define CK_F_PR_INC_32 778 CK_PR_UNARY_S(inc, add, 32, uint32_t) 779 #endif /* CK_F_PR_INC_32 */ 780 781 #ifndef CK_F_PR_INC_32_ZERO 782 #define CK_F_PR_INC_32_ZERO 783 CK_PR_UNARY_Z_S(inc, 32, uint32_t, +, UINT32_MAX) 784 #endif /* CK_F_PR_INC_32_ZERO */ 785 786 #ifndef CK_F_PR_DEC_32 787 #define CK_F_PR_DEC_32 788 CK_PR_UNARY_S(dec, sub, 32, uint32_t) 789 #endif /* CK_F_PR_DEC_32 */ 790 791 #ifndef CK_F_PR_DEC_32_ZERO 792 #define CK_F_PR_DEC_32_ZERO 793 CK_PR_UNARY_Z_S(dec, 32, uint32_t, -, 1) 794 #endif /* CK_F_PR_DEC_32_ZERO */ 795 796 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ 797 798 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) 799 800 #ifndef CK_F_PR_INC_16 801 #define CK_F_PR_INC_16 802 CK_PR_UNARY_S(inc, add, 16, uint16_t) 803 #endif /* CK_F_PR_INC_16 */ 804 805 #ifndef CK_F_PR_INC_16_ZERO 806 #define CK_F_PR_INC_16_ZERO 807 CK_PR_UNARY_Z_S(inc, 16, uint16_t, +, UINT16_MAX) 808 #endif /* CK_F_PR_INC_16_ZERO */ 809 810 #ifndef CK_F_PR_DEC_16 811 #define CK_F_PR_DEC_16 812 CK_PR_UNARY_S(dec, sub, 16, uint16_t) 813 #endif /* CK_F_PR_DEC_16 */ 814 815 #ifndef CK_F_PR_DEC_16_ZERO 816 #define CK_F_PR_DEC_16_ZERO 817 CK_PR_UNARY_Z_S(dec, 16, uint16_t, -, 1) 818 #endif /* CK_F_PR_DEC_16_ZERO */ 819 820 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ 821 822 #if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE) 823 824 #ifndef CK_F_PR_INC_8 825 #define CK_F_PR_INC_8 826 CK_PR_UNARY_S(inc, add, 8, uint8_t) 827 #endif /* CK_F_PR_INC_8 */ 828 829 #ifndef CK_F_PR_INC_8_ZERO 830 #define CK_F_PR_INC_8_ZERO 831 CK_PR_UNARY_Z_S(inc, 8, uint8_t, +, UINT8_MAX) 832 #endif /* CK_F_PR_INC_8_ZERO */ 833 834 #ifndef CK_F_PR_DEC_8 835 #define CK_F_PR_DEC_8 836 CK_PR_UNARY_S(dec, sub, 8, uint8_t) 837 #endif /* CK_F_PR_DEC_8 */ 838 839 #ifndef CK_F_PR_DEC_8_ZERO 840 #define CK_F_PR_DEC_8_ZERO 841 CK_PR_UNARY_Z_S(dec, 8, uint8_t, -, 1) 842 #endif /* CK_F_PR_DEC_8_ZERO */ 843 844 #endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */ 845 846 #undef CK_PR_UNARY_Z_S 847 #undef CK_PR_UNARY_S 848 #undef CK_PR_UNARY_Z 849 #undef CK_PR_UNARY 850 851 #define CK_PR_N(K, S, M, T, P, C) \ 852 CK_CC_INLINE static void \ 853 ck_pr_##K##_##S(M *target) \ 854 { \ 855 T previous; \ 856 C punt; \ 857 punt = (C)ck_pr_md_load_##S(target); \ 858 previous = (T)punt; \ 859 while (ck_pr_cas_##S##_value(target, \ 860 (C)previous, \ 861 (C)(P previous), \ 862 &previous) == false) \ 863 ck_pr_stall(); \ 864 \ 865 return; \ 866 } 867 868 #define CK_PR_N_Z(S, M, T, C) \ 869 CK_CC_INLINE static void \ 870 ck_pr_neg_##S##_zero(M *target, bool *zero) \ 871 { \ 872 T previous; \ 873 C punt; \ 874 punt = (C)ck_pr_md_load_##S(target); \ 875 previous = (T)punt; \ 876 while (ck_pr_cas_##S##_value(target, \ 877 (C)previous, \ 878 (C)(-previous), \ 879 &previous) == false) \ 880 ck_pr_stall(); \ 881 \ 882 *zero = previous == 0; \ 883 return; \ 884 } 885 886 #define CK_PR_N_S(K, S, M, P) CK_PR_N(K, S, M, M, P, M) 887 #define CK_PR_N_Z_S(S, M) CK_PR_N_Z(S, M, M, M) 888 889 #if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE) 890 891 #ifndef CK_F_PR_NOT_CHAR 892 #define CK_F_PR_NOT_CHAR 893 CK_PR_N_S(not, char, char, ~) 894 #endif /* CK_F_PR_NOT_CHAR */ 895 896 #ifndef CK_F_PR_NEG_CHAR 897 #define CK_F_PR_NEG_CHAR 898 CK_PR_N_S(neg, char, char, -) 899 #endif /* CK_F_PR_NEG_CHAR */ 900 901 #ifndef CK_F_PR_NEG_CHAR_ZERO 902 #define CK_F_PR_NEG_CHAR_ZERO 903 CK_PR_N_Z_S(char, char) 904 #endif /* CK_F_PR_NEG_CHAR_ZERO */ 905 906 #endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */ 907 908 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) 909 910 #ifndef CK_F_PR_NOT_INT 911 #define CK_F_PR_NOT_INT 912 CK_PR_N_S(not, int, int, ~) 913 #endif /* CK_F_PR_NOT_INT */ 914 915 #ifndef CK_F_PR_NEG_INT 916 #define CK_F_PR_NEG_INT 917 CK_PR_N_S(neg, int, int, -) 918 #endif /* CK_F_PR_NEG_INT */ 919 920 #ifndef CK_F_PR_NEG_INT_ZERO 921 #define CK_F_PR_NEG_INT_ZERO 922 CK_PR_N_Z_S(int, int) 923 #endif /* CK_F_PR_NEG_INT_ZERO */ 924 925 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ 926 927 #if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \ 928 !defined(CK_PR_DISABLE_DOUBLE) 929 930 #ifndef CK_F_PR_NEG_DOUBLE 931 #define CK_F_PR_NEG_DOUBLE 932 CK_PR_N_S(neg, double, double, -) 933 #endif /* CK_F_PR_NEG_DOUBLE */ 934 935 #endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */ 936 937 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) 938 939 #ifndef CK_F_PR_NOT_UINT 940 #define CK_F_PR_NOT_UINT 941 CK_PR_N_S(not, uint, unsigned int, ~) 942 #endif /* CK_F_PR_NOT_UINT */ 943 944 #ifndef CK_F_PR_NEG_UINT 945 #define CK_F_PR_NEG_UINT 946 CK_PR_N_S(neg, uint, unsigned int, -) 947 #endif /* CK_F_PR_NEG_UINT */ 948 949 #ifndef CK_F_PR_NEG_UINT_ZERO 950 #define CK_F_PR_NEG_UINT_ZERO 951 CK_PR_N_Z_S(uint, unsigned int) 952 #endif /* CK_F_PR_NEG_UINT_ZERO */ 953 954 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ 955 956 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) 957 958 #ifndef CK_F_PR_NOT_PTR 959 #define CK_F_PR_NOT_PTR 960 CK_PR_N(not, ptr, void, uintptr_t, ~, void *) 961 #endif /* CK_F_PR_NOT_PTR */ 962 963 #ifndef CK_F_PR_NEG_PTR 964 #define CK_F_PR_NEG_PTR 965 CK_PR_N(neg, ptr, void, uintptr_t, -, void *) 966 #endif /* CK_F_PR_NEG_PTR */ 967 968 #ifndef CK_F_PR_NEG_PTR_ZERO 969 #define CK_F_PR_NEG_PTR_ZERO 970 CK_PR_N_Z(ptr, void, uintptr_t, void *) 971 #endif /* CK_F_PR_NEG_PTR_ZERO */ 972 973 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ 974 975 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) 976 977 #ifndef CK_F_PR_NOT_64 978 #define CK_F_PR_NOT_64 979 CK_PR_N_S(not, 64, uint64_t, ~) 980 #endif /* CK_F_PR_NOT_64 */ 981 982 #ifndef CK_F_PR_NEG_64 983 #define CK_F_PR_NEG_64 984 CK_PR_N_S(neg, 64, uint64_t, -) 985 #endif /* CK_F_PR_NEG_64 */ 986 987 #ifndef CK_F_PR_NEG_64_ZERO 988 #define CK_F_PR_NEG_64_ZERO 989 CK_PR_N_Z_S(64, uint64_t) 990 #endif /* CK_F_PR_NEG_64_ZERO */ 991 992 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ 993 994 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) 995 996 #ifndef CK_F_PR_NOT_32 997 #define CK_F_PR_NOT_32 998 CK_PR_N_S(not, 32, uint32_t, ~) 999 #endif /* CK_F_PR_NOT_32 */ 1000 1001 #ifndef CK_F_PR_NEG_32 1002 #define CK_F_PR_NEG_32 1003 CK_PR_N_S(neg, 32, uint32_t, -) 1004 #endif /* CK_F_PR_NEG_32 */ 1005 1006 #ifndef CK_F_PR_NEG_32_ZERO 1007 #define CK_F_PR_NEG_32_ZERO 1008 CK_PR_N_Z_S(32, uint32_t) 1009 #endif /* CK_F_PR_NEG_32_ZERO */ 1010 1011 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ 1012 1013 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) 1014 1015 #ifndef CK_F_PR_NOT_16 1016 #define CK_F_PR_NOT_16 1017 CK_PR_N_S(not, 16, uint16_t, ~) 1018 #endif /* CK_F_PR_NOT_16 */ 1019 1020 #ifndef CK_F_PR_NEG_16 1021 #define CK_F_PR_NEG_16 1022 CK_PR_N_S(neg, 16, uint16_t, -) 1023 #endif /* CK_F_PR_NEG_16 */ 1024 1025 #ifndef CK_F_PR_NEG_16_ZERO 1026 #define CK_F_PR_NEG_16_ZERO 1027 CK_PR_N_Z_S(16, uint16_t) 1028 #endif /* CK_F_PR_NEG_16_ZERO */ 1029 1030 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ 1031 1032 #if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE) 1033 1034 #ifndef CK_F_PR_NOT_8 1035 #define CK_F_PR_NOT_8 1036 CK_PR_N_S(not, 8, uint8_t, ~) 1037 #endif /* CK_F_PR_NOT_8 */ 1038 1039 #ifndef CK_F_PR_NEG_8 1040 #define CK_F_PR_NEG_8 1041 CK_PR_N_S(neg, 8, uint8_t, -) 1042 #endif /* CK_F_PR_NEG_8 */ 1043 1044 #ifndef CK_F_PR_NEG_8_ZERO 1045 #define CK_F_PR_NEG_8_ZERO 1046 CK_PR_N_Z_S(8, uint8_t) 1047 #endif /* CK_F_PR_NEG_8_ZERO */ 1048 1049 #endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */ 1050 1051 #undef CK_PR_N_Z_S 1052 #undef CK_PR_N_S 1053 #undef CK_PR_N_Z 1054 #undef CK_PR_N 1055 1056 #define CK_PR_FAA(S, M, T, C) \ 1057 CK_CC_INLINE static C \ 1058 ck_pr_faa_##S(M *target, T delta) \ 1059 { \ 1060 T previous; \ 1061 C punt; \ 1062 punt = (C)ck_pr_md_load_##S(target); \ 1063 previous = (T)punt; \ 1064 while (ck_pr_cas_##S##_value(target, \ 1065 (C)previous, \ 1066 (C)(previous + delta), \ 1067 &previous) == false) \ 1068 ck_pr_stall(); \ 1069 \ 1070 return ((C)previous); \ 1071 } 1072 1073 #define CK_PR_FAS(S, M, C) \ 1074 CK_CC_INLINE static C \ 1075 ck_pr_fas_##S(M *target, C update) \ 1076 { \ 1077 C previous; \ 1078 previous = ck_pr_md_load_##S(target); \ 1079 while (ck_pr_cas_##S##_value(target, \ 1080 previous, \ 1081 update, \ 1082 &previous) == false) \ 1083 ck_pr_stall(); \ 1084 \ 1085 return (previous); \ 1086 } 1087 1088 #define CK_PR_FAA_S(S, M) CK_PR_FAA(S, M, M, M) 1089 #define CK_PR_FAS_S(S, M) CK_PR_FAS(S, M, M) 1090 1091 #if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE) 1092 1093 #ifndef CK_F_PR_FAA_CHAR 1094 #define CK_F_PR_FAA_CHAR 1095 CK_PR_FAA_S(char, char) 1096 #endif /* CK_F_PR_FAA_CHAR */ 1097 1098 #ifndef CK_F_PR_FAS_CHAR 1099 #define CK_F_PR_FAS_CHAR 1100 CK_PR_FAS_S(char, char) 1101 #endif /* CK_F_PR_FAS_CHAR */ 1102 1103 #endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */ 1104 1105 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) 1106 1107 #ifndef CK_F_PR_FAA_INT 1108 #define CK_F_PR_FAA_INT 1109 CK_PR_FAA_S(int, int) 1110 #endif /* CK_F_PR_FAA_INT */ 1111 1112 #ifndef CK_F_PR_FAS_INT 1113 #define CK_F_PR_FAS_INT 1114 CK_PR_FAS_S(int, int) 1115 #endif /* CK_F_PR_FAS_INT */ 1116 1117 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ 1118 1119 #if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \ 1120 !defined(CK_PR_DISABLE_DOUBLE) 1121 1122 #ifndef CK_F_PR_FAA_DOUBLE 1123 #define CK_F_PR_FAA_DOUBLE 1124 CK_PR_FAA_S(double, double) 1125 #endif /* CK_F_PR_FAA_DOUBLE */ 1126 1127 #ifndef CK_F_PR_FAS_DOUBLE 1128 #define CK_F_PR_FAS_DOUBLE 1129 CK_PR_FAS_S(double, double) 1130 #endif /* CK_F_PR_FAS_DOUBLE */ 1131 1132 #endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */ 1133 1134 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) 1135 1136 #ifndef CK_F_PR_FAA_UINT 1137 #define CK_F_PR_FAA_UINT 1138 CK_PR_FAA_S(uint, unsigned int) 1139 #endif /* CK_F_PR_FAA_UINT */ 1140 1141 #ifndef CK_F_PR_FAS_UINT 1142 #define CK_F_PR_FAS_UINT 1143 CK_PR_FAS_S(uint, unsigned int) 1144 #endif /* CK_F_PR_FAS_UINT */ 1145 1146 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ 1147 1148 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) 1149 1150 #ifndef CK_F_PR_FAA_PTR 1151 #define CK_F_PR_FAA_PTR 1152 CK_PR_FAA(ptr, void, uintptr_t, void *) 1153 #endif /* CK_F_PR_FAA_PTR */ 1154 1155 #ifndef CK_F_PR_FAS_PTR 1156 #define CK_F_PR_FAS_PTR 1157 CK_PR_FAS(ptr, void, void *) 1158 #endif /* CK_F_PR_FAS_PTR */ 1159 1160 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ 1161 1162 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) 1163 1164 #ifndef CK_F_PR_FAA_64 1165 #define CK_F_PR_FAA_64 1166 CK_PR_FAA_S(64, uint64_t) 1167 #endif /* CK_F_PR_FAA_64 */ 1168 1169 #ifndef CK_F_PR_FAS_64 1170 #define CK_F_PR_FAS_64 1171 CK_PR_FAS_S(64, uint64_t) 1172 #endif /* CK_F_PR_FAS_64 */ 1173 1174 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ 1175 1176 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) 1177 1178 #ifndef CK_F_PR_FAA_32 1179 #define CK_F_PR_FAA_32 1180 CK_PR_FAA_S(32, uint32_t) 1181 #endif /* CK_F_PR_FAA_32 */ 1182 1183 #ifndef CK_F_PR_FAS_32 1184 #define CK_F_PR_FAS_32 1185 CK_PR_FAS_S(32, uint32_t) 1186 #endif /* CK_F_PR_FAS_32 */ 1187 1188 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ 1189 1190 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) 1191 1192 #ifndef CK_F_PR_FAA_16 1193 #define CK_F_PR_FAA_16 1194 CK_PR_FAA_S(16, uint16_t) 1195 #endif /* CK_F_PR_FAA_16 */ 1196 1197 #ifndef CK_F_PR_FAS_16 1198 #define CK_F_PR_FAS_16 1199 CK_PR_FAS_S(16, uint16_t) 1200 #endif /* CK_F_PR_FAS_16 */ 1201 1202 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ 1203 1204 #if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE) 1205 1206 #ifndef CK_F_PR_FAA_8 1207 #define CK_F_PR_FAA_8 1208 CK_PR_FAA_S(8, uint8_t) 1209 #endif /* CK_F_PR_FAA_8 */ 1210 1211 #ifndef CK_F_PR_FAS_8 1212 #define CK_F_PR_FAS_8 1213 CK_PR_FAS_S(8, uint8_t) 1214 #endif /* CK_F_PR_FAS_8 */ 1215 1216 #endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */ 1217 1218 #undef CK_PR_FAA_S 1219 #undef CK_PR_FAS_S 1220 #undef CK_PR_FAA 1221 #undef CK_PR_FAS 1222 1223 #endif /* CK_PR_H */ 1224