1 /* 2 * Copyright 2009-2015 Samy Al Bahra. 3 * Copyright 2011 David Joseph. 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #ifndef CK_PR_H 29 #define CK_PR_H 30 31 #include <ck_cc.h> 32 #include <ck_limits.h> 33 #include <ck_md.h> 34 #include <ck_stdint.h> 35 #include <ck_stdbool.h> 36 37 #ifndef CK_USE_CC_BUILTINS 38 #if defined(__x86_64__) 39 #include "gcc/x86_64/ck_pr.h" 40 #elif defined(__x86__) 41 #include "gcc/x86/ck_pr.h" 42 #elif defined(__sparcv9__) 43 #include "gcc/sparcv9/ck_pr.h" 44 #elif defined(__ppc64__) 45 #include "gcc/ppc64/ck_pr.h" 46 #elif defined(__s390x__) 47 #include "gcc/s390x/ck_pr.h" 48 #elif defined(__ppc__) 49 #include "gcc/ppc/ck_pr.h" 50 #elif defined(__arm__) 51 #include "gcc/arm/ck_pr.h" 52 #elif defined(__aarch64__) 53 #include "gcc/aarch64/ck_pr.h" 54 #elif !defined(__GNUC__) 55 #error Your platform is unsupported 56 #endif 57 #endif /* !CK_USE_CC_BUILTINS */ 58 59 #if defined(__GNUC__) 60 #include "gcc/ck_pr.h" 61 #endif 62 63 #define CK_PR_FENCE_EMIT(T) \ 64 CK_CC_INLINE static void \ 65 ck_pr_fence_##T(void) \ 66 { \ 67 ck_pr_fence_strict_##T(); \ 68 return; \ 69 } 70 #define CK_PR_FENCE_NOOP(T) \ 71 CK_CC_INLINE static void \ 72 ck_pr_fence_##T(void) \ 73 { \ 74 ck_pr_barrier(); \ 75 return; \ 76 } 77 78 /* 79 * None of the currently supported platforms allow for data-dependent 80 * load ordering. 81 */ 82 CK_PR_FENCE_NOOP(load_depends) 83 #define ck_pr_fence_strict_load_depends ck_pr_fence_load_depends 84 85 /* 86 * In memory models where atomic operations do not have serializing 87 * effects, atomic read-modify-write operations are modeled as stores. 88 */ 89 #if defined(CK_MD_RMO) 90 /* 91 * Only stores to the same location have a global 92 * ordering. 93 */ 94 CK_PR_FENCE_EMIT(atomic) 95 CK_PR_FENCE_EMIT(atomic_load) 96 CK_PR_FENCE_EMIT(atomic_store) 97 CK_PR_FENCE_EMIT(store_atomic) 98 CK_PR_FENCE_EMIT(load_atomic) 99 CK_PR_FENCE_EMIT(load_store) 100 CK_PR_FENCE_EMIT(store_load) 101 CK_PR_FENCE_EMIT(load) 102 CK_PR_FENCE_EMIT(store) 103 CK_PR_FENCE_EMIT(memory) 104 CK_PR_FENCE_EMIT(acquire) 105 CK_PR_FENCE_EMIT(release) 106 CK_PR_FENCE_EMIT(acqrel) 107 CK_PR_FENCE_EMIT(lock) 108 CK_PR_FENCE_EMIT(unlock) 109 #elif defined(CK_MD_PSO) 110 /* 111 * Anything can be re-ordered with respect to stores. 112 * Otherwise, loads are executed in-order. 113 */ 114 CK_PR_FENCE_EMIT(atomic) 115 CK_PR_FENCE_NOOP(atomic_load) 116 CK_PR_FENCE_EMIT(atomic_store) 117 CK_PR_FENCE_EMIT(store_atomic) 118 CK_PR_FENCE_NOOP(load_atomic) 119 CK_PR_FENCE_EMIT(load_store) 120 CK_PR_FENCE_EMIT(store_load) 121 CK_PR_FENCE_NOOP(load) 122 CK_PR_FENCE_EMIT(store) 123 CK_PR_FENCE_EMIT(memory) 124 CK_PR_FENCE_EMIT(acquire) 125 CK_PR_FENCE_EMIT(release) 126 CK_PR_FENCE_EMIT(acqrel) 127 CK_PR_FENCE_EMIT(lock) 128 CK_PR_FENCE_EMIT(unlock) 129 #elif defined(CK_MD_TSO) 130 /* 131 * Only loads are re-ordered and only with respect to 132 * prior stores. Atomic operations are serializing. 133 */ 134 CK_PR_FENCE_NOOP(atomic) 135 CK_PR_FENCE_NOOP(atomic_load) 136 CK_PR_FENCE_NOOP(atomic_store) 137 CK_PR_FENCE_NOOP(store_atomic) 138 CK_PR_FENCE_NOOP(load_atomic) 139 CK_PR_FENCE_NOOP(load_store) 140 CK_PR_FENCE_EMIT(store_load) 141 CK_PR_FENCE_NOOP(load) 142 CK_PR_FENCE_NOOP(store) 143 CK_PR_FENCE_EMIT(memory) 144 CK_PR_FENCE_NOOP(acquire) 145 CK_PR_FENCE_NOOP(release) 146 CK_PR_FENCE_NOOP(acqrel) 147 CK_PR_FENCE_NOOP(lock) 148 CK_PR_FENCE_NOOP(unlock) 149 #else 150 #error "No memory model has been defined." 151 #endif /* CK_MD_TSO */ 152 153 #undef CK_PR_FENCE_EMIT 154 #undef CK_PR_FENCE_NOOP 155 156 #ifndef CK_F_PR_RFO 157 #define CK_F_PR_RFO 158 CK_CC_INLINE static void 159 ck_pr_rfo(const void *m) 160 { 161 162 (void)m; 163 return; 164 } 165 #endif /* CK_F_PR_RFO */ 166 167 #define CK_PR_STORE_SAFE(DST, VAL, TYPE) \ 168 ck_pr_md_store_##TYPE( \ 169 ((void)sizeof(*(DST) = (VAL)), (DST)), \ 170 (VAL)) 171 172 #define ck_pr_store_ptr(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), ptr) 173 #define ck_pr_store_char(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), char) 174 #ifndef CK_PR_DISABLE_DOUBLE 175 #define ck_pr_store_double(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), double) 176 #endif 177 #define ck_pr_store_uint(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), uint) 178 #define ck_pr_store_int(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), int) 179 #define ck_pr_store_32(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 32) 180 #define ck_pr_store_16(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 16) 181 #define ck_pr_store_8(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 8) 182 183 #define ck_pr_store_ptr_unsafe(DST, VAL) ck_pr_md_store_ptr((DST), (VAL)) 184 185 #ifdef CK_F_PR_LOAD_64 186 #define ck_pr_store_64(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 64) 187 #endif /* CK_F_PR_LOAD_64 */ 188 189 #define CK_PR_LOAD_PTR_SAFE(SRC) (CK_CC_TYPEOF(*(SRC), (void *)))ck_pr_md_load_ptr((SRC)) 190 #define ck_pr_load_ptr(SRC) CK_PR_LOAD_PTR_SAFE((SRC)) 191 192 #define CK_PR_LOAD_SAFE(SRC, TYPE) ck_pr_md_load_##TYPE((SRC)) 193 #define ck_pr_load_char(SRC) CK_PR_LOAD_SAFE((SRC), char) 194 #ifndef CK_PR_DISABLE_DOUBLE 195 #define ck_pr_load_double(SRC) CK_PR_LOAD_SAFE((SRC), double) 196 #endif 197 #define ck_pr_load_uint(SRC) CK_PR_LOAD_SAFE((SRC), uint) 198 #define ck_pr_load_int(SRC) CK_PR_LOAD_SAFE((SRC), int) 199 #define ck_pr_load_32(SRC) CK_PR_LOAD_SAFE((SRC), 32) 200 #define ck_pr_load_16(SRC) CK_PR_LOAD_SAFE((SRC), 16) 201 #define ck_pr_load_8(SRC) CK_PR_LOAD_SAFE((SRC), 8) 202 203 #ifdef CK_F_PR_LOAD_64 204 #define ck_pr_load_64(SRC) CK_PR_LOAD_SAFE((SRC), 64) 205 #endif /* CK_F_PR_LOAD_64 */ 206 207 #define CK_PR_BIN(K, S, M, T, P, C) \ 208 CK_CC_INLINE static void \ 209 ck_pr_##K##_##S(M *target, T value) \ 210 { \ 211 T previous; \ 212 C punt; \ 213 punt = ck_pr_md_load_##S(target); \ 214 previous = (T)punt; \ 215 while (ck_pr_cas_##S##_value(target, \ 216 (C)previous, \ 217 (C)(previous P value), \ 218 &previous) == false) \ 219 ck_pr_stall(); \ 220 \ 221 return; \ 222 } 223 224 #define CK_PR_BIN_S(K, S, T, P) CK_PR_BIN(K, S, T, T, P, T) 225 226 #if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE) 227 228 #ifndef CK_F_PR_ADD_CHAR 229 #define CK_F_PR_ADD_CHAR 230 CK_PR_BIN_S(add, char, char, +) 231 #endif /* CK_F_PR_ADD_CHAR */ 232 233 #ifndef CK_F_PR_SUB_CHAR 234 #define CK_F_PR_SUB_CHAR 235 CK_PR_BIN_S(sub, char, char, -) 236 #endif /* CK_F_PR_SUB_CHAR */ 237 238 #ifndef CK_F_PR_AND_CHAR 239 #define CK_F_PR_AND_CHAR 240 CK_PR_BIN_S(and, char, char, &) 241 #endif /* CK_F_PR_AND_CHAR */ 242 243 #ifndef CK_F_PR_XOR_CHAR 244 #define CK_F_PR_XOR_CHAR 245 CK_PR_BIN_S(xor, char, char, ^) 246 #endif /* CK_F_PR_XOR_CHAR */ 247 248 #ifndef CK_F_PR_OR_CHAR 249 #define CK_F_PR_OR_CHAR 250 CK_PR_BIN_S(or, char, char, |) 251 #endif /* CK_F_PR_OR_CHAR */ 252 253 #endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */ 254 255 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) 256 257 #ifndef CK_F_PR_ADD_INT 258 #define CK_F_PR_ADD_INT 259 CK_PR_BIN_S(add, int, int, +) 260 #endif /* CK_F_PR_ADD_INT */ 261 262 #ifndef CK_F_PR_SUB_INT 263 #define CK_F_PR_SUB_INT 264 CK_PR_BIN_S(sub, int, int, -) 265 #endif /* CK_F_PR_SUB_INT */ 266 267 #ifndef CK_F_PR_AND_INT 268 #define CK_F_PR_AND_INT 269 CK_PR_BIN_S(and, int, int, &) 270 #endif /* CK_F_PR_AND_INT */ 271 272 #ifndef CK_F_PR_XOR_INT 273 #define CK_F_PR_XOR_INT 274 CK_PR_BIN_S(xor, int, int, ^) 275 #endif /* CK_F_PR_XOR_INT */ 276 277 #ifndef CK_F_PR_OR_INT 278 #define CK_F_PR_OR_INT 279 CK_PR_BIN_S(or, int, int, |) 280 #endif /* CK_F_PR_OR_INT */ 281 282 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ 283 284 #if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \ 285 !defined(CK_PR_DISABLE_DOUBLE) 286 287 #ifndef CK_F_PR_ADD_DOUBLE 288 #define CK_F_PR_ADD_DOUBLE 289 CK_PR_BIN_S(add, double, double, +) 290 #endif /* CK_F_PR_ADD_DOUBLE */ 291 292 #ifndef CK_F_PR_SUB_DOUBLE 293 #define CK_F_PR_SUB_DOUBLE 294 CK_PR_BIN_S(sub, double, double, -) 295 #endif /* CK_F_PR_SUB_DOUBLE */ 296 297 #endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */ 298 299 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) 300 301 #ifndef CK_F_PR_ADD_UINT 302 #define CK_F_PR_ADD_UINT 303 CK_PR_BIN_S(add, uint, unsigned int, +) 304 #endif /* CK_F_PR_ADD_UINT */ 305 306 #ifndef CK_F_PR_SUB_UINT 307 #define CK_F_PR_SUB_UINT 308 CK_PR_BIN_S(sub, uint, unsigned int, -) 309 #endif /* CK_F_PR_SUB_UINT */ 310 311 #ifndef CK_F_PR_AND_UINT 312 #define CK_F_PR_AND_UINT 313 CK_PR_BIN_S(and, uint, unsigned int, &) 314 #endif /* CK_F_PR_AND_UINT */ 315 316 #ifndef CK_F_PR_XOR_UINT 317 #define CK_F_PR_XOR_UINT 318 CK_PR_BIN_S(xor, uint, unsigned int, ^) 319 #endif /* CK_F_PR_XOR_UINT */ 320 321 #ifndef CK_F_PR_OR_UINT 322 #define CK_F_PR_OR_UINT 323 CK_PR_BIN_S(or, uint, unsigned int, |) 324 #endif /* CK_F_PR_OR_UINT */ 325 326 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ 327 328 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) 329 330 #ifndef CK_F_PR_ADD_PTR 331 #define CK_F_PR_ADD_PTR 332 CK_PR_BIN(add, ptr, void, uintptr_t, +, void *) 333 #endif /* CK_F_PR_ADD_PTR */ 334 335 #ifndef CK_F_PR_SUB_PTR 336 #define CK_F_PR_SUB_PTR 337 CK_PR_BIN(sub, ptr, void, uintptr_t, -, void *) 338 #endif /* CK_F_PR_SUB_PTR */ 339 340 #ifndef CK_F_PR_AND_PTR 341 #define CK_F_PR_AND_PTR 342 CK_PR_BIN(and, ptr, void, uintptr_t, &, void *) 343 #endif /* CK_F_PR_AND_PTR */ 344 345 #ifndef CK_F_PR_XOR_PTR 346 #define CK_F_PR_XOR_PTR 347 CK_PR_BIN(xor, ptr, void, uintptr_t, ^, void *) 348 #endif /* CK_F_PR_XOR_PTR */ 349 350 #ifndef CK_F_PR_OR_PTR 351 #define CK_F_PR_OR_PTR 352 CK_PR_BIN(or, ptr, void, uintptr_t, |, void *) 353 #endif /* CK_F_PR_OR_PTR */ 354 355 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ 356 357 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) 358 359 #ifndef CK_F_PR_ADD_64 360 #define CK_F_PR_ADD_64 361 CK_PR_BIN_S(add, 64, uint64_t, +) 362 #endif /* CK_F_PR_ADD_64 */ 363 364 #ifndef CK_F_PR_SUB_64 365 #define CK_F_PR_SUB_64 366 CK_PR_BIN_S(sub, 64, uint64_t, -) 367 #endif /* CK_F_PR_SUB_64 */ 368 369 #ifndef CK_F_PR_AND_64 370 #define CK_F_PR_AND_64 371 CK_PR_BIN_S(and, 64, uint64_t, &) 372 #endif /* CK_F_PR_AND_64 */ 373 374 #ifndef CK_F_PR_XOR_64 375 #define CK_F_PR_XOR_64 376 CK_PR_BIN_S(xor, 64, uint64_t, ^) 377 #endif /* CK_F_PR_XOR_64 */ 378 379 #ifndef CK_F_PR_OR_64 380 #define CK_F_PR_OR_64 381 CK_PR_BIN_S(or, 64, uint64_t, |) 382 #endif /* CK_F_PR_OR_64 */ 383 384 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ 385 386 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) 387 388 #ifndef CK_F_PR_ADD_32 389 #define CK_F_PR_ADD_32 390 CK_PR_BIN_S(add, 32, uint32_t, +) 391 #endif /* CK_F_PR_ADD_32 */ 392 393 #ifndef CK_F_PR_SUB_32 394 #define CK_F_PR_SUB_32 395 CK_PR_BIN_S(sub, 32, uint32_t, -) 396 #endif /* CK_F_PR_SUB_32 */ 397 398 #ifndef CK_F_PR_AND_32 399 #define CK_F_PR_AND_32 400 CK_PR_BIN_S(and, 32, uint32_t, &) 401 #endif /* CK_F_PR_AND_32 */ 402 403 #ifndef CK_F_PR_XOR_32 404 #define CK_F_PR_XOR_32 405 CK_PR_BIN_S(xor, 32, uint32_t, ^) 406 #endif /* CK_F_PR_XOR_32 */ 407 408 #ifndef CK_F_PR_OR_32 409 #define CK_F_PR_OR_32 410 CK_PR_BIN_S(or, 32, uint32_t, |) 411 #endif /* CK_F_PR_OR_32 */ 412 413 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ 414 415 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) 416 417 #ifndef CK_F_PR_ADD_16 418 #define CK_F_PR_ADD_16 419 CK_PR_BIN_S(add, 16, uint16_t, +) 420 #endif /* CK_F_PR_ADD_16 */ 421 422 #ifndef CK_F_PR_SUB_16 423 #define CK_F_PR_SUB_16 424 CK_PR_BIN_S(sub, 16, uint16_t, -) 425 #endif /* CK_F_PR_SUB_16 */ 426 427 #ifndef CK_F_PR_AND_16 428 #define CK_F_PR_AND_16 429 CK_PR_BIN_S(and, 16, uint16_t, &) 430 #endif /* CK_F_PR_AND_16 */ 431 432 #ifndef CK_F_PR_XOR_16 433 #define CK_F_PR_XOR_16 434 CK_PR_BIN_S(xor, 16, uint16_t, ^) 435 #endif /* CK_F_PR_XOR_16 */ 436 437 #ifndef CK_F_PR_OR_16 438 #define CK_F_PR_OR_16 439 CK_PR_BIN_S(or, 16, uint16_t, |) 440 #endif /* CK_F_PR_OR_16 */ 441 442 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ 443 444 #if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE) 445 446 #ifndef CK_F_PR_ADD_8 447 #define CK_F_PR_ADD_8 448 CK_PR_BIN_S(add, 8, uint8_t, +) 449 #endif /* CK_F_PR_ADD_8 */ 450 451 #ifndef CK_F_PR_SUB_8 452 #define CK_F_PR_SUB_8 453 CK_PR_BIN_S(sub, 8, uint8_t, -) 454 #endif /* CK_F_PR_SUB_8 */ 455 456 #ifndef CK_F_PR_AND_8 457 #define CK_F_PR_AND_8 458 CK_PR_BIN_S(and, 8, uint8_t, &) 459 #endif /* CK_F_PR_AND_8 */ 460 461 #ifndef CK_F_PR_XOR_8 462 #define CK_F_PR_XOR_8 463 CK_PR_BIN_S(xor, 8, uint8_t, ^) 464 #endif /* CK_F_PR_XOR_8 */ 465 466 #ifndef CK_F_PR_OR_8 467 #define CK_F_PR_OR_8 468 CK_PR_BIN_S(or, 8, uint8_t, |) 469 #endif /* CK_F_PR_OR_8 */ 470 471 #endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */ 472 473 #undef CK_PR_BIN_S 474 #undef CK_PR_BIN 475 476 #define CK_PR_BTX(K, S, M, T, P, C, R) \ 477 CK_CC_INLINE static bool \ 478 ck_pr_##K##_##S(M *target, unsigned int offset) \ 479 { \ 480 T previous; \ 481 C punt; \ 482 punt = ck_pr_md_load_##S(target); \ 483 previous = (T)punt; \ 484 while (ck_pr_cas_##S##_value(target, (C)previous, \ 485 (C)(previous P (R ((T)1 << offset))), &previous) == false) \ 486 ck_pr_stall(); \ 487 return ((previous >> offset) & 1); \ 488 } 489 490 #define CK_PR_BTX_S(K, S, T, P, R) CK_PR_BTX(K, S, T, T, P, T, R) 491 492 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) 493 494 #ifndef CK_F_PR_BTC_INT 495 #define CK_F_PR_BTC_INT 496 CK_PR_BTX_S(btc, int, int, ^,) 497 #endif /* CK_F_PR_BTC_INT */ 498 499 #ifndef CK_F_PR_BTR_INT 500 #define CK_F_PR_BTR_INT 501 CK_PR_BTX_S(btr, int, int, &, ~) 502 #endif /* CK_F_PR_BTR_INT */ 503 504 #ifndef CK_F_PR_BTS_INT 505 #define CK_F_PR_BTS_INT 506 CK_PR_BTX_S(bts, int, int, |,) 507 #endif /* CK_F_PR_BTS_INT */ 508 509 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ 510 511 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) 512 513 #ifndef CK_F_PR_BTC_UINT 514 #define CK_F_PR_BTC_UINT 515 CK_PR_BTX_S(btc, uint, unsigned int, ^,) 516 #endif /* CK_F_PR_BTC_UINT */ 517 518 #ifndef CK_F_PR_BTR_UINT 519 #define CK_F_PR_BTR_UINT 520 CK_PR_BTX_S(btr, uint, unsigned int, &, ~) 521 #endif /* CK_F_PR_BTR_UINT */ 522 523 #ifndef CK_F_PR_BTS_UINT 524 #define CK_F_PR_BTS_UINT 525 CK_PR_BTX_S(bts, uint, unsigned int, |,) 526 #endif /* CK_F_PR_BTS_UINT */ 527 528 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ 529 530 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) 531 532 #ifndef CK_F_PR_BTC_PTR 533 #define CK_F_PR_BTC_PTR 534 CK_PR_BTX(btc, ptr, void, uintptr_t, ^, void *,) 535 #endif /* CK_F_PR_BTC_PTR */ 536 537 #ifndef CK_F_PR_BTR_PTR 538 #define CK_F_PR_BTR_PTR 539 CK_PR_BTX(btr, ptr, void, uintptr_t, &, void *, ~) 540 #endif /* CK_F_PR_BTR_PTR */ 541 542 #ifndef CK_F_PR_BTS_PTR 543 #define CK_F_PR_BTS_PTR 544 CK_PR_BTX(bts, ptr, void, uintptr_t, |, void *,) 545 #endif /* CK_F_PR_BTS_PTR */ 546 547 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ 548 549 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) 550 551 #ifndef CK_F_PR_BTC_64 552 #define CK_F_PR_BTC_64 553 CK_PR_BTX_S(btc, 64, uint64_t, ^,) 554 #endif /* CK_F_PR_BTC_64 */ 555 556 #ifndef CK_F_PR_BTR_64 557 #define CK_F_PR_BTR_64 558 CK_PR_BTX_S(btr, 64, uint64_t, &, ~) 559 #endif /* CK_F_PR_BTR_64 */ 560 561 #ifndef CK_F_PR_BTS_64 562 #define CK_F_PR_BTS_64 563 CK_PR_BTX_S(bts, 64, uint64_t, |,) 564 #endif /* CK_F_PR_BTS_64 */ 565 566 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ 567 568 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) 569 570 #ifndef CK_F_PR_BTC_32 571 #define CK_F_PR_BTC_32 572 CK_PR_BTX_S(btc, 32, uint32_t, ^,) 573 #endif /* CK_F_PR_BTC_32 */ 574 575 #ifndef CK_F_PR_BTR_32 576 #define CK_F_PR_BTR_32 577 CK_PR_BTX_S(btr, 32, uint32_t, &, ~) 578 #endif /* CK_F_PR_BTR_32 */ 579 580 #ifndef CK_F_PR_BTS_32 581 #define CK_F_PR_BTS_32 582 CK_PR_BTX_S(bts, 32, uint32_t, |,) 583 #endif /* CK_F_PR_BTS_32 */ 584 585 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ 586 587 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) 588 589 #ifndef CK_F_PR_BTC_16 590 #define CK_F_PR_BTC_16 591 CK_PR_BTX_S(btc, 16, uint16_t, ^,) 592 #endif /* CK_F_PR_BTC_16 */ 593 594 #ifndef CK_F_PR_BTR_16 595 #define CK_F_PR_BTR_16 596 CK_PR_BTX_S(btr, 16, uint16_t, &, ~) 597 #endif /* CK_F_PR_BTR_16 */ 598 599 #ifndef CK_F_PR_BTS_16 600 #define CK_F_PR_BTS_16 601 CK_PR_BTX_S(bts, 16, uint16_t, |,) 602 #endif /* CK_F_PR_BTS_16 */ 603 604 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ 605 606 #undef CK_PR_BTX_S 607 #undef CK_PR_BTX 608 609 #define CK_PR_UNARY(K, X, S, M, T) \ 610 CK_CC_INLINE static void \ 611 ck_pr_##K##_##S(M *target) \ 612 { \ 613 ck_pr_##X##_##S(target, (T)1); \ 614 return; \ 615 } 616 617 #define CK_PR_UNARY_Z(K, S, M, T, P, C, Z) \ 618 CK_CC_INLINE static bool \ 619 ck_pr_##K##_##S##_is_zero(M *target) \ 620 { \ 621 T previous; \ 622 C punt; \ 623 punt = (C)ck_pr_md_load_##S(target); \ 624 previous = (T)punt; \ 625 while (ck_pr_cas_##S##_value(target, \ 626 (C)previous, \ 627 (C)(previous P 1), \ 628 &previous) == false) \ 629 ck_pr_stall(); \ 630 return previous == (T)Z; \ 631 } 632 633 #define CK_PR_UNARY_Z_STUB(K, S, M) \ 634 CK_CC_INLINE static void \ 635 ck_pr_##K##_##S##_zero(M *target, bool *zero) \ 636 { \ 637 *zero = ck_pr_##K##_##S##_is_zero(target); \ 638 return; \ 639 } 640 641 #define CK_PR_UNARY_S(K, X, S, M) CK_PR_UNARY(K, X, S, M, M) 642 #define CK_PR_UNARY_Z_S(K, S, M, P, Z) \ 643 CK_PR_UNARY_Z(K, S, M, M, P, M, Z) \ 644 CK_PR_UNARY_Z_STUB(K, S, M) 645 646 #if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE) 647 648 #ifndef CK_F_PR_INC_CHAR 649 #define CK_F_PR_INC_CHAR 650 CK_PR_UNARY_S(inc, add, char, char) 651 #endif /* CK_F_PR_INC_CHAR */ 652 653 #ifndef CK_F_PR_INC_CHAR_ZERO 654 #define CK_F_PR_INC_CHAR_ZERO 655 CK_PR_UNARY_Z_S(inc, char, char, +, -1) 656 #else 657 CK_PR_UNARY_Z_STUB(inc, char, char) 658 #endif /* CK_F_PR_INC_CHAR_ZERO */ 659 660 #ifndef CK_F_PR_DEC_CHAR 661 #define CK_F_PR_DEC_CHAR 662 CK_PR_UNARY_S(dec, sub, char, char) 663 #endif /* CK_F_PR_DEC_CHAR */ 664 665 #ifndef CK_F_PR_DEC_CHAR_ZERO 666 #define CK_F_PR_DEC_CHAR_ZERO 667 CK_PR_UNARY_Z_S(dec, char, char, -, 1) 668 #else 669 CK_PR_UNARY_Z_STUB(dec, char, char) 670 #endif /* CK_F_PR_DEC_CHAR_ZERO */ 671 672 #endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */ 673 674 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) 675 676 #ifndef CK_F_PR_INC_INT 677 #define CK_F_PR_INC_INT 678 CK_PR_UNARY_S(inc, add, int, int) 679 #endif /* CK_F_PR_INC_INT */ 680 681 #ifndef CK_F_PR_INC_INT_ZERO 682 #define CK_F_PR_INC_INT_ZERO 683 CK_PR_UNARY_Z_S(inc, int, int, +, -1) 684 #else 685 CK_PR_UNARY_Z_STUB(inc, int, int) 686 #endif /* CK_F_PR_INC_INT_ZERO */ 687 688 #ifndef CK_F_PR_DEC_INT 689 #define CK_F_PR_DEC_INT 690 CK_PR_UNARY_S(dec, sub, int, int) 691 #endif /* CK_F_PR_DEC_INT */ 692 693 #ifndef CK_F_PR_DEC_INT_ZERO 694 #define CK_F_PR_DEC_INT_ZERO 695 CK_PR_UNARY_Z_S(dec, int, int, -, 1) 696 #else 697 CK_PR_UNARY_Z_STUB(dec, int, int) 698 #endif /* CK_F_PR_DEC_INT_ZERO */ 699 700 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ 701 702 #if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \ 703 !defined(CK_PR_DISABLE_DOUBLE) 704 705 #ifndef CK_F_PR_INC_DOUBLE 706 #define CK_F_PR_INC_DOUBLE 707 CK_PR_UNARY_S(inc, add, double, double) 708 #endif /* CK_F_PR_INC_DOUBLE */ 709 710 #ifndef CK_F_PR_DEC_DOUBLE 711 #define CK_F_PR_DEC_DOUBLE 712 CK_PR_UNARY_S(dec, sub, double, double) 713 #endif /* CK_F_PR_DEC_DOUBLE */ 714 715 #endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */ 716 717 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) 718 719 #ifndef CK_F_PR_INC_UINT 720 #define CK_F_PR_INC_UINT 721 CK_PR_UNARY_S(inc, add, uint, unsigned int) 722 #endif /* CK_F_PR_INC_UINT */ 723 724 #ifndef CK_F_PR_INC_UINT_ZERO 725 #define CK_F_PR_INC_UINT_ZERO 726 CK_PR_UNARY_Z_S(inc, uint, unsigned int, +, UINT_MAX) 727 #else 728 CK_PR_UNARY_Z_STUB(inc, uint, unsigned int) 729 #endif /* CK_F_PR_INC_UINT_ZERO */ 730 731 #ifndef CK_F_PR_DEC_UINT 732 #define CK_F_PR_DEC_UINT 733 CK_PR_UNARY_S(dec, sub, uint, unsigned int) 734 #endif /* CK_F_PR_DEC_UINT */ 735 736 #ifndef CK_F_PR_DEC_UINT_ZERO 737 #define CK_F_PR_DEC_UINT_ZERO 738 CK_PR_UNARY_Z_S(dec, uint, unsigned int, -, 1) 739 #else 740 CK_PR_UNARY_Z_STUB(dec, uint, unsigned int) 741 #endif /* CK_F_PR_DEC_UINT_ZERO */ 742 743 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ 744 745 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) 746 747 #ifndef CK_F_PR_INC_PTR 748 #define CK_F_PR_INC_PTR 749 CK_PR_UNARY(inc, add, ptr, void, uintptr_t) 750 #endif /* CK_F_PR_INC_PTR */ 751 752 #ifndef CK_F_PR_INC_PTR_ZERO 753 #define CK_F_PR_INC_PTR_ZERO 754 CK_PR_UNARY_Z(inc, ptr, void, uintptr_t, +, void *, UINT_MAX) 755 #else 756 CK_PR_UNARY_Z_STUB(inc, ptr, void) 757 #endif /* CK_F_PR_INC_PTR_ZERO */ 758 759 #ifndef CK_F_PR_DEC_PTR 760 #define CK_F_PR_DEC_PTR 761 CK_PR_UNARY(dec, sub, ptr, void, uintptr_t) 762 #endif /* CK_F_PR_DEC_PTR */ 763 764 #ifndef CK_F_PR_DEC_PTR_ZERO 765 #define CK_F_PR_DEC_PTR_ZERO 766 CK_PR_UNARY_Z(dec, ptr, void, uintptr_t, -, void *, 1) 767 #else 768 CK_PR_UNARY_Z_STUB(dec, ptr, void) 769 #endif /* CK_F_PR_DEC_PTR_ZERO */ 770 771 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ 772 773 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) 774 775 #ifndef CK_F_PR_INC_64 776 #define CK_F_PR_INC_64 777 CK_PR_UNARY_S(inc, add, 64, uint64_t) 778 #endif /* CK_F_PR_INC_64 */ 779 780 #ifndef CK_F_PR_INC_64_ZERO 781 #define CK_F_PR_INC_64_ZERO 782 CK_PR_UNARY_Z_S(inc, 64, uint64_t, +, UINT64_MAX) 783 #else 784 CK_PR_UNARY_Z_STUB(inc, 64, uint64_t) 785 #endif /* CK_F_PR_INC_64_ZERO */ 786 787 #ifndef CK_F_PR_DEC_64 788 #define CK_F_PR_DEC_64 789 CK_PR_UNARY_S(dec, sub, 64, uint64_t) 790 #endif /* CK_F_PR_DEC_64 */ 791 792 #ifndef CK_F_PR_DEC_64_ZERO 793 #define CK_F_PR_DEC_64_ZERO 794 CK_PR_UNARY_Z_S(dec, 64, uint64_t, -, 1) 795 #else 796 CK_PR_UNARY_Z_STUB(dec, 64, uint64_t) 797 #endif /* CK_F_PR_DEC_64_ZERO */ 798 799 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ 800 801 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) 802 803 #ifndef CK_F_PR_INC_32 804 #define CK_F_PR_INC_32 805 CK_PR_UNARY_S(inc, add, 32, uint32_t) 806 #endif /* CK_F_PR_INC_32 */ 807 808 #ifndef CK_F_PR_INC_32_ZERO 809 #define CK_F_PR_INC_32_ZERO 810 CK_PR_UNARY_Z_S(inc, 32, uint32_t, +, UINT32_MAX) 811 #else 812 CK_PR_UNARY_Z_STUB(inc, 32, uint32_t) 813 #endif /* CK_F_PR_INC_32_ZERO */ 814 815 #ifndef CK_F_PR_DEC_32 816 #define CK_F_PR_DEC_32 817 CK_PR_UNARY_S(dec, sub, 32, uint32_t) 818 #endif /* CK_F_PR_DEC_32 */ 819 820 #ifndef CK_F_PR_DEC_32_ZERO 821 #define CK_F_PR_DEC_32_ZERO 822 CK_PR_UNARY_Z_S(dec, 32, uint32_t, -, 1) 823 #else 824 CK_PR_UNARY_Z_STUB(dec, 32, uint32_t) 825 #endif /* CK_F_PR_DEC_32_ZERO */ 826 827 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ 828 829 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) 830 831 #ifndef CK_F_PR_INC_16 832 #define CK_F_PR_INC_16 833 CK_PR_UNARY_S(inc, add, 16, uint16_t) 834 #endif /* CK_F_PR_INC_16 */ 835 836 #ifndef CK_F_PR_INC_16_ZERO 837 #define CK_F_PR_INC_16_ZERO 838 CK_PR_UNARY_Z_S(inc, 16, uint16_t, +, UINT16_MAX) 839 #else 840 CK_PR_UNARY_Z_STUB(inc, 16, uint16_t) 841 #endif /* CK_F_PR_INC_16_ZERO */ 842 843 #ifndef CK_F_PR_DEC_16 844 #define CK_F_PR_DEC_16 845 CK_PR_UNARY_S(dec, sub, 16, uint16_t) 846 #endif /* CK_F_PR_DEC_16 */ 847 848 #ifndef CK_F_PR_DEC_16_ZERO 849 #define CK_F_PR_DEC_16_ZERO 850 CK_PR_UNARY_Z_S(dec, 16, uint16_t, -, 1) 851 #else 852 CK_PR_UNARY_Z_STUB(dec, 16, uint16_t) 853 #endif /* CK_F_PR_DEC_16_ZERO */ 854 855 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ 856 857 #if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE) 858 859 #ifndef CK_F_PR_INC_8 860 #define CK_F_PR_INC_8 861 CK_PR_UNARY_S(inc, add, 8, uint8_t) 862 #endif /* CK_F_PR_INC_8 */ 863 864 #ifndef CK_F_PR_INC_8_ZERO 865 #define CK_F_PR_INC_8_ZERO 866 CK_PR_UNARY_Z_S(inc, 8, uint8_t, +, UINT8_MAX) 867 #else 868 CK_PR_UNARY_Z_STUB(inc, 8, uint8_t) 869 #endif /* CK_F_PR_INC_8_ZERO */ 870 871 #ifndef CK_F_PR_DEC_8 872 #define CK_F_PR_DEC_8 873 CK_PR_UNARY_S(dec, sub, 8, uint8_t) 874 #endif /* CK_F_PR_DEC_8 */ 875 876 #ifndef CK_F_PR_DEC_8_ZERO 877 #define CK_F_PR_DEC_8_ZERO 878 CK_PR_UNARY_Z_S(dec, 8, uint8_t, -, 1) 879 #else 880 CK_PR_UNARY_Z_STUB(dec, 8, uint8_t) 881 #endif /* CK_F_PR_DEC_8_ZERO */ 882 883 #endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */ 884 885 #undef CK_PR_UNARY_Z_S 886 #undef CK_PR_UNARY_S 887 #undef CK_PR_UNARY_Z 888 #undef CK_PR_UNARY 889 890 #define CK_PR_N(K, S, M, T, P, C) \ 891 CK_CC_INLINE static void \ 892 ck_pr_##K##_##S(M *target) \ 893 { \ 894 T previous; \ 895 C punt; \ 896 punt = (C)ck_pr_md_load_##S(target); \ 897 previous = (T)punt; \ 898 while (ck_pr_cas_##S##_value(target, \ 899 (C)previous, \ 900 (C)(P previous), \ 901 &previous) == false) \ 902 ck_pr_stall(); \ 903 \ 904 return; \ 905 } 906 907 #define CK_PR_N_Z(S, M, T, C) \ 908 CK_CC_INLINE static void \ 909 ck_pr_neg_##S##_zero(M *target, bool *zero) \ 910 { \ 911 T previous; \ 912 C punt; \ 913 punt = (C)ck_pr_md_load_##S(target); \ 914 previous = (T)punt; \ 915 while (ck_pr_cas_##S##_value(target, \ 916 (C)previous, \ 917 (C)(-previous), \ 918 &previous) == false) \ 919 ck_pr_stall(); \ 920 \ 921 *zero = previous == 0; \ 922 return; \ 923 } 924 925 #define CK_PR_N_S(K, S, M, P) CK_PR_N(K, S, M, M, P, M) 926 #define CK_PR_N_Z_S(S, M) CK_PR_N_Z(S, M, M, M) 927 928 #if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE) 929 930 #ifndef CK_F_PR_NOT_CHAR 931 #define CK_F_PR_NOT_CHAR 932 CK_PR_N_S(not, char, char, ~) 933 #endif /* CK_F_PR_NOT_CHAR */ 934 935 #ifndef CK_F_PR_NEG_CHAR 936 #define CK_F_PR_NEG_CHAR 937 CK_PR_N_S(neg, char, char, -) 938 #endif /* CK_F_PR_NEG_CHAR */ 939 940 #ifndef CK_F_PR_NEG_CHAR_ZERO 941 #define CK_F_PR_NEG_CHAR_ZERO 942 CK_PR_N_Z_S(char, char) 943 #endif /* CK_F_PR_NEG_CHAR_ZERO */ 944 945 #endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */ 946 947 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) 948 949 #ifndef CK_F_PR_NOT_INT 950 #define CK_F_PR_NOT_INT 951 CK_PR_N_S(not, int, int, ~) 952 #endif /* CK_F_PR_NOT_INT */ 953 954 #ifndef CK_F_PR_NEG_INT 955 #define CK_F_PR_NEG_INT 956 CK_PR_N_S(neg, int, int, -) 957 #endif /* CK_F_PR_NEG_INT */ 958 959 #ifndef CK_F_PR_NEG_INT_ZERO 960 #define CK_F_PR_NEG_INT_ZERO 961 CK_PR_N_Z_S(int, int) 962 #endif /* CK_F_PR_NEG_INT_ZERO */ 963 964 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ 965 966 #if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \ 967 !defined(CK_PR_DISABLE_DOUBLE) 968 969 #ifndef CK_F_PR_NEG_DOUBLE 970 #define CK_F_PR_NEG_DOUBLE 971 CK_PR_N_S(neg, double, double, -) 972 #endif /* CK_F_PR_NEG_DOUBLE */ 973 974 #endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */ 975 976 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) 977 978 #ifndef CK_F_PR_NOT_UINT 979 #define CK_F_PR_NOT_UINT 980 CK_PR_N_S(not, uint, unsigned int, ~) 981 #endif /* CK_F_PR_NOT_UINT */ 982 983 #ifndef CK_F_PR_NEG_UINT 984 #define CK_F_PR_NEG_UINT 985 CK_PR_N_S(neg, uint, unsigned int, -) 986 #endif /* CK_F_PR_NEG_UINT */ 987 988 #ifndef CK_F_PR_NEG_UINT_ZERO 989 #define CK_F_PR_NEG_UINT_ZERO 990 CK_PR_N_Z_S(uint, unsigned int) 991 #endif /* CK_F_PR_NEG_UINT_ZERO */ 992 993 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ 994 995 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) 996 997 #ifndef CK_F_PR_NOT_PTR 998 #define CK_F_PR_NOT_PTR 999 CK_PR_N(not, ptr, void, uintptr_t, ~, void *) 1000 #endif /* CK_F_PR_NOT_PTR */ 1001 1002 #ifndef CK_F_PR_NEG_PTR 1003 #define CK_F_PR_NEG_PTR 1004 CK_PR_N(neg, ptr, void, uintptr_t, -, void *) 1005 #endif /* CK_F_PR_NEG_PTR */ 1006 1007 #ifndef CK_F_PR_NEG_PTR_ZERO 1008 #define CK_F_PR_NEG_PTR_ZERO 1009 CK_PR_N_Z(ptr, void, uintptr_t, void *) 1010 #endif /* CK_F_PR_NEG_PTR_ZERO */ 1011 1012 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ 1013 1014 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) 1015 1016 #ifndef CK_F_PR_NOT_64 1017 #define CK_F_PR_NOT_64 1018 CK_PR_N_S(not, 64, uint64_t, ~) 1019 #endif /* CK_F_PR_NOT_64 */ 1020 1021 #ifndef CK_F_PR_NEG_64 1022 #define CK_F_PR_NEG_64 1023 CK_PR_N_S(neg, 64, uint64_t, -) 1024 #endif /* CK_F_PR_NEG_64 */ 1025 1026 #ifndef CK_F_PR_NEG_64_ZERO 1027 #define CK_F_PR_NEG_64_ZERO 1028 CK_PR_N_Z_S(64, uint64_t) 1029 #endif /* CK_F_PR_NEG_64_ZERO */ 1030 1031 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ 1032 1033 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) 1034 1035 #ifndef CK_F_PR_NOT_32 1036 #define CK_F_PR_NOT_32 1037 CK_PR_N_S(not, 32, uint32_t, ~) 1038 #endif /* CK_F_PR_NOT_32 */ 1039 1040 #ifndef CK_F_PR_NEG_32 1041 #define CK_F_PR_NEG_32 1042 CK_PR_N_S(neg, 32, uint32_t, -) 1043 #endif /* CK_F_PR_NEG_32 */ 1044 1045 #ifndef CK_F_PR_NEG_32_ZERO 1046 #define CK_F_PR_NEG_32_ZERO 1047 CK_PR_N_Z_S(32, uint32_t) 1048 #endif /* CK_F_PR_NEG_32_ZERO */ 1049 1050 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ 1051 1052 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) 1053 1054 #ifndef CK_F_PR_NOT_16 1055 #define CK_F_PR_NOT_16 1056 CK_PR_N_S(not, 16, uint16_t, ~) 1057 #endif /* CK_F_PR_NOT_16 */ 1058 1059 #ifndef CK_F_PR_NEG_16 1060 #define CK_F_PR_NEG_16 1061 CK_PR_N_S(neg, 16, uint16_t, -) 1062 #endif /* CK_F_PR_NEG_16 */ 1063 1064 #ifndef CK_F_PR_NEG_16_ZERO 1065 #define CK_F_PR_NEG_16_ZERO 1066 CK_PR_N_Z_S(16, uint16_t) 1067 #endif /* CK_F_PR_NEG_16_ZERO */ 1068 1069 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ 1070 1071 #if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE) 1072 1073 #ifndef CK_F_PR_NOT_8 1074 #define CK_F_PR_NOT_8 1075 CK_PR_N_S(not, 8, uint8_t, ~) 1076 #endif /* CK_F_PR_NOT_8 */ 1077 1078 #ifndef CK_F_PR_NEG_8 1079 #define CK_F_PR_NEG_8 1080 CK_PR_N_S(neg, 8, uint8_t, -) 1081 #endif /* CK_F_PR_NEG_8 */ 1082 1083 #ifndef CK_F_PR_NEG_8_ZERO 1084 #define CK_F_PR_NEG_8_ZERO 1085 CK_PR_N_Z_S(8, uint8_t) 1086 #endif /* CK_F_PR_NEG_8_ZERO */ 1087 1088 #endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */ 1089 1090 #undef CK_PR_N_Z_S 1091 #undef CK_PR_N_S 1092 #undef CK_PR_N_Z 1093 #undef CK_PR_N 1094 1095 #define CK_PR_FAA(S, M, T, C) \ 1096 CK_CC_INLINE static C \ 1097 ck_pr_faa_##S(M *target, T delta) \ 1098 { \ 1099 T previous; \ 1100 C punt; \ 1101 punt = (C)ck_pr_md_load_##S(target); \ 1102 previous = (T)punt; \ 1103 while (ck_pr_cas_##S##_value(target, \ 1104 (C)previous, \ 1105 (C)(previous + delta), \ 1106 &previous) == false) \ 1107 ck_pr_stall(); \ 1108 \ 1109 return ((C)previous); \ 1110 } 1111 1112 #define CK_PR_FAS(S, M, C) \ 1113 CK_CC_INLINE static C \ 1114 ck_pr_fas_##S(M *target, C update) \ 1115 { \ 1116 C previous; \ 1117 previous = ck_pr_md_load_##S(target); \ 1118 while (ck_pr_cas_##S##_value(target, \ 1119 previous, \ 1120 update, \ 1121 &previous) == false) \ 1122 ck_pr_stall(); \ 1123 \ 1124 return (previous); \ 1125 } 1126 1127 #define CK_PR_FAA_S(S, M) CK_PR_FAA(S, M, M, M) 1128 #define CK_PR_FAS_S(S, M) CK_PR_FAS(S, M, M) 1129 1130 #if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE) 1131 1132 #ifndef CK_F_PR_FAA_CHAR 1133 #define CK_F_PR_FAA_CHAR 1134 CK_PR_FAA_S(char, char) 1135 #endif /* CK_F_PR_FAA_CHAR */ 1136 1137 #ifndef CK_F_PR_FAS_CHAR 1138 #define CK_F_PR_FAS_CHAR 1139 CK_PR_FAS_S(char, char) 1140 #endif /* CK_F_PR_FAS_CHAR */ 1141 1142 #endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */ 1143 1144 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) 1145 1146 #ifndef CK_F_PR_FAA_INT 1147 #define CK_F_PR_FAA_INT 1148 CK_PR_FAA_S(int, int) 1149 #endif /* CK_F_PR_FAA_INT */ 1150 1151 #ifndef CK_F_PR_FAS_INT 1152 #define CK_F_PR_FAS_INT 1153 CK_PR_FAS_S(int, int) 1154 #endif /* CK_F_PR_FAS_INT */ 1155 1156 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ 1157 1158 #if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \ 1159 !defined(CK_PR_DISABLE_DOUBLE) 1160 1161 #ifndef CK_F_PR_FAA_DOUBLE 1162 #define CK_F_PR_FAA_DOUBLE 1163 CK_PR_FAA_S(double, double) 1164 #endif /* CK_F_PR_FAA_DOUBLE */ 1165 1166 #ifndef CK_F_PR_FAS_DOUBLE 1167 #define CK_F_PR_FAS_DOUBLE 1168 CK_PR_FAS_S(double, double) 1169 #endif /* CK_F_PR_FAS_DOUBLE */ 1170 1171 #endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */ 1172 1173 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) 1174 1175 #ifndef CK_F_PR_FAA_UINT 1176 #define CK_F_PR_FAA_UINT 1177 CK_PR_FAA_S(uint, unsigned int) 1178 #endif /* CK_F_PR_FAA_UINT */ 1179 1180 #ifndef CK_F_PR_FAS_UINT 1181 #define CK_F_PR_FAS_UINT 1182 CK_PR_FAS_S(uint, unsigned int) 1183 #endif /* CK_F_PR_FAS_UINT */ 1184 1185 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ 1186 1187 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) 1188 1189 #ifndef CK_F_PR_FAA_PTR 1190 #define CK_F_PR_FAA_PTR 1191 CK_PR_FAA(ptr, void, uintptr_t, void *) 1192 #endif /* CK_F_PR_FAA_PTR */ 1193 1194 #ifndef CK_F_PR_FAS_PTR 1195 #define CK_F_PR_FAS_PTR 1196 CK_PR_FAS(ptr, void, void *) 1197 #endif /* CK_F_PR_FAS_PTR */ 1198 1199 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ 1200 1201 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) 1202 1203 #ifndef CK_F_PR_FAA_64 1204 #define CK_F_PR_FAA_64 1205 CK_PR_FAA_S(64, uint64_t) 1206 #endif /* CK_F_PR_FAA_64 */ 1207 1208 #ifndef CK_F_PR_FAS_64 1209 #define CK_F_PR_FAS_64 1210 CK_PR_FAS_S(64, uint64_t) 1211 #endif /* CK_F_PR_FAS_64 */ 1212 1213 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ 1214 1215 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) 1216 1217 #ifndef CK_F_PR_FAA_32 1218 #define CK_F_PR_FAA_32 1219 CK_PR_FAA_S(32, uint32_t) 1220 #endif /* CK_F_PR_FAA_32 */ 1221 1222 #ifndef CK_F_PR_FAS_32 1223 #define CK_F_PR_FAS_32 1224 CK_PR_FAS_S(32, uint32_t) 1225 #endif /* CK_F_PR_FAS_32 */ 1226 1227 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ 1228 1229 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) 1230 1231 #ifndef CK_F_PR_FAA_16 1232 #define CK_F_PR_FAA_16 1233 CK_PR_FAA_S(16, uint16_t) 1234 #endif /* CK_F_PR_FAA_16 */ 1235 1236 #ifndef CK_F_PR_FAS_16 1237 #define CK_F_PR_FAS_16 1238 CK_PR_FAS_S(16, uint16_t) 1239 #endif /* CK_F_PR_FAS_16 */ 1240 1241 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ 1242 1243 #if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE) 1244 1245 #ifndef CK_F_PR_FAA_8 1246 #define CK_F_PR_FAA_8 1247 CK_PR_FAA_S(8, uint8_t) 1248 #endif /* CK_F_PR_FAA_8 */ 1249 1250 #ifndef CK_F_PR_FAS_8 1251 #define CK_F_PR_FAS_8 1252 CK_PR_FAS_S(8, uint8_t) 1253 #endif /* CK_F_PR_FAS_8 */ 1254 1255 #endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */ 1256 1257 #undef CK_PR_FAA_S 1258 #undef CK_PR_FAS_S 1259 #undef CK_PR_FAA 1260 #undef CK_PR_FAS 1261 1262 #endif /* CK_PR_H */ 1263