1 /* 2 * Copyright 2009-2015 Samy Al Bahra. 3 * Copyright 2011 David Joseph. 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #ifndef CK_PR_H 29 #define CK_PR_H 30 31 #include <ck_cc.h> 32 #include <ck_limits.h> 33 #include <ck_md.h> 34 #include <ck_stdint.h> 35 #include <ck_stdbool.h> 36 37 #ifndef CK_USE_CC_BUILTINS 38 #if defined(__x86_64__) 39 #include "gcc/x86_64/ck_pr.h" 40 #elif defined(__x86__) 41 #include "gcc/x86/ck_pr.h" 42 #elif defined(__sparcv9__) 43 #include "gcc/sparcv9/ck_pr.h" 44 #elif defined(__ppc64__) 45 #include "gcc/ppc64/ck_pr.h" 46 #elif defined(__ppc__) 47 #include "gcc/ppc/ck_pr.h" 48 #elif defined(__arm__) 49 #if __ARM_ARCH >= 6 50 #include "gcc/arm/ck_pr.h" 51 #else 52 #include "gcc/arm/ck_pr_armv4.h" 53 #endif 54 #elif defined(__aarch64__) 55 #include "gcc/aarch64/ck_pr.h" 56 #elif !defined(__GNUC__) 57 #error Your platform is unsupported 58 #endif 59 #endif /* !CK_USE_CC_BUILTINS */ 60 61 #if defined(__GNUC__) 62 #include "gcc/ck_pr.h" 63 #endif 64 65 #define CK_PR_FENCE_EMIT(T) \ 66 CK_CC_INLINE static void \ 67 ck_pr_fence_##T(void) \ 68 { \ 69 ck_pr_fence_strict_##T(); \ 70 return; \ 71 } 72 #define CK_PR_FENCE_NOOP(T) \ 73 CK_CC_INLINE static void \ 74 ck_pr_fence_##T(void) \ 75 { \ 76 ck_pr_barrier(); \ 77 return; \ 78 } 79 80 /* 81 * None of the currently supported platforms allow for data-dependent 82 * load ordering. 83 */ 84 CK_PR_FENCE_NOOP(load_depends) 85 #define ck_pr_fence_strict_load_depends ck_pr_fence_load_depends 86 87 /* 88 * In memory models where atomic operations do not have serializing 89 * effects, atomic read-modify-write operations are modeled as stores. 90 */ 91 #if defined(CK_MD_RMO) 92 /* 93 * Only stores to the same location have a global 94 * ordering. 95 */ 96 CK_PR_FENCE_EMIT(atomic) 97 CK_PR_FENCE_EMIT(atomic_load) 98 CK_PR_FENCE_EMIT(atomic_store) 99 CK_PR_FENCE_EMIT(store_atomic) 100 CK_PR_FENCE_EMIT(load_atomic) 101 CK_PR_FENCE_EMIT(load_store) 102 CK_PR_FENCE_EMIT(store_load) 103 CK_PR_FENCE_EMIT(load) 104 CK_PR_FENCE_EMIT(store) 105 CK_PR_FENCE_EMIT(memory) 106 CK_PR_FENCE_EMIT(acquire) 107 CK_PR_FENCE_EMIT(release) 108 CK_PR_FENCE_EMIT(acqrel) 109 CK_PR_FENCE_EMIT(lock) 110 CK_PR_FENCE_EMIT(unlock) 111 #elif defined(CK_MD_PSO) 112 /* 113 * Anything can be re-ordered with respect to stores. 114 * Otherwise, loads are executed in-order. 115 */ 116 CK_PR_FENCE_EMIT(atomic) 117 CK_PR_FENCE_NOOP(atomic_load) 118 CK_PR_FENCE_EMIT(atomic_store) 119 CK_PR_FENCE_EMIT(store_atomic) 120 CK_PR_FENCE_NOOP(load_atomic) 121 CK_PR_FENCE_EMIT(load_store) 122 CK_PR_FENCE_EMIT(store_load) 123 CK_PR_FENCE_NOOP(load) 124 CK_PR_FENCE_EMIT(store) 125 CK_PR_FENCE_EMIT(memory) 126 CK_PR_FENCE_EMIT(acquire) 127 CK_PR_FENCE_EMIT(release) 128 CK_PR_FENCE_EMIT(acqrel) 129 CK_PR_FENCE_EMIT(lock) 130 CK_PR_FENCE_EMIT(unlock) 131 #elif defined(CK_MD_TSO) 132 /* 133 * Only loads are re-ordered and only with respect to 134 * prior stores. Atomic operations are serializing. 135 */ 136 CK_PR_FENCE_NOOP(atomic) 137 CK_PR_FENCE_NOOP(atomic_load) 138 CK_PR_FENCE_NOOP(atomic_store) 139 CK_PR_FENCE_NOOP(store_atomic) 140 CK_PR_FENCE_NOOP(load_atomic) 141 CK_PR_FENCE_NOOP(load_store) 142 CK_PR_FENCE_EMIT(store_load) 143 CK_PR_FENCE_NOOP(load) 144 CK_PR_FENCE_NOOP(store) 145 CK_PR_FENCE_EMIT(memory) 146 CK_PR_FENCE_NOOP(acquire) 147 CK_PR_FENCE_NOOP(release) 148 CK_PR_FENCE_NOOP(acqrel) 149 CK_PR_FENCE_NOOP(lock) 150 CK_PR_FENCE_NOOP(unlock) 151 #else 152 #error "No memory model has been defined." 153 #endif /* CK_MD_TSO */ 154 155 #undef CK_PR_FENCE_EMIT 156 #undef CK_PR_FENCE_NOOP 157 158 #ifndef CK_F_PR_RFO 159 #define CK_F_PR_RFO 160 CK_CC_INLINE static void 161 ck_pr_rfo(const void *m) 162 { 163 164 (void)m; 165 return; 166 } 167 #endif /* CK_F_PR_RFO */ 168 169 #define CK_PR_STORE_SAFE(DST, VAL, TYPE) \ 170 ck_pr_md_store_##TYPE( \ 171 ((void)sizeof(*(DST) = (VAL)), (DST)), \ 172 (VAL)) 173 174 #define ck_pr_store_ptr(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), ptr) 175 #define ck_pr_store_char(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), char) 176 #define ck_pr_store_double(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), double) 177 #define ck_pr_store_uint(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), uint) 178 #define ck_pr_store_int(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), int) 179 #define ck_pr_store_32(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 32) 180 #define ck_pr_store_16(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 16) 181 #define ck_pr_store_8(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 8) 182 183 #define ck_pr_store_ptr_unsafe(DST, VAL) ck_pr_md_store_ptr((DST), (VAL)) 184 185 #ifdef CK_F_PR_LOAD_64 186 #define ck_pr_store_64(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 64) 187 #endif /* CK_F_PR_LOAD_64 */ 188 189 #define CK_PR_LOAD_PTR_SAFE(SRC) (CK_CC_TYPEOF(*(SRC), (void *)))ck_pr_md_load_ptr((SRC)) 190 #define ck_pr_load_ptr(SRC) CK_PR_LOAD_PTR_SAFE((SRC)) 191 192 #define CK_PR_LOAD_SAFE(SRC, TYPE) ck_pr_md_load_##TYPE((SRC)) 193 #define ck_pr_load_char(SRC) CK_PR_LOAD_SAFE((SRC), char) 194 #define ck_pr_load_double(SRC) CK_PR_LOAD_SAFE((SRC), double) 195 #define ck_pr_load_uint(SRC) CK_PR_LOAD_SAFE((SRC), uint) 196 #define ck_pr_load_int(SRC) CK_PR_LOAD_SAFE((SRC), int) 197 #define ck_pr_load_32(SRC) CK_PR_LOAD_SAFE((SRC), 32) 198 #define ck_pr_load_16(SRC) CK_PR_LOAD_SAFE((SRC), 16) 199 #define ck_pr_load_8(SRC) CK_PR_LOAD_SAFE((SRC), 8) 200 201 #ifdef CK_F_PR_LOAD_64 202 #define ck_pr_load_64(SRC) CK_PR_LOAD_SAFE((SRC), 64) 203 #endif /* CK_F_PR_LOAD_64 */ 204 205 #define CK_PR_BIN(K, S, M, T, P, C) \ 206 CK_CC_INLINE static void \ 207 ck_pr_##K##_##S(M *target, T value) \ 208 { \ 209 T previous; \ 210 C punt; \ 211 punt = ck_pr_md_load_##S(target); \ 212 previous = (T)punt; \ 213 while (ck_pr_cas_##S##_value(target, \ 214 (C)previous, \ 215 (C)(previous P value), \ 216 &previous) == false) \ 217 ck_pr_stall(); \ 218 \ 219 return; \ 220 } 221 222 #define CK_PR_BIN_S(K, S, T, P) CK_PR_BIN(K, S, T, T, P, T) 223 224 #if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE) 225 226 #ifndef CK_F_PR_ADD_CHAR 227 #define CK_F_PR_ADD_CHAR 228 CK_PR_BIN_S(add, char, char, +) 229 #endif /* CK_F_PR_ADD_CHAR */ 230 231 #ifndef CK_F_PR_SUB_CHAR 232 #define CK_F_PR_SUB_CHAR 233 CK_PR_BIN_S(sub, char, char, -) 234 #endif /* CK_F_PR_SUB_CHAR */ 235 236 #ifndef CK_F_PR_AND_CHAR 237 #define CK_F_PR_AND_CHAR 238 CK_PR_BIN_S(and, char, char, &) 239 #endif /* CK_F_PR_AND_CHAR */ 240 241 #ifndef CK_F_PR_XOR_CHAR 242 #define CK_F_PR_XOR_CHAR 243 CK_PR_BIN_S(xor, char, char, ^) 244 #endif /* CK_F_PR_XOR_CHAR */ 245 246 #ifndef CK_F_PR_OR_CHAR 247 #define CK_F_PR_OR_CHAR 248 CK_PR_BIN_S(or, char, char, |) 249 #endif /* CK_F_PR_OR_CHAR */ 250 251 #endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */ 252 253 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) 254 255 #ifndef CK_F_PR_ADD_INT 256 #define CK_F_PR_ADD_INT 257 CK_PR_BIN_S(add, int, int, +) 258 #endif /* CK_F_PR_ADD_INT */ 259 260 #ifndef CK_F_PR_SUB_INT 261 #define CK_F_PR_SUB_INT 262 CK_PR_BIN_S(sub, int, int, -) 263 #endif /* CK_F_PR_SUB_INT */ 264 265 #ifndef CK_F_PR_AND_INT 266 #define CK_F_PR_AND_INT 267 CK_PR_BIN_S(and, int, int, &) 268 #endif /* CK_F_PR_AND_INT */ 269 270 #ifndef CK_F_PR_XOR_INT 271 #define CK_F_PR_XOR_INT 272 CK_PR_BIN_S(xor, int, int, ^) 273 #endif /* CK_F_PR_XOR_INT */ 274 275 #ifndef CK_F_PR_OR_INT 276 #define CK_F_PR_OR_INT 277 CK_PR_BIN_S(or, int, int, |) 278 #endif /* CK_F_PR_OR_INT */ 279 280 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ 281 282 #if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) 283 284 #ifndef CK_F_PR_ADD_DOUBLE 285 #define CK_F_PR_ADD_DOUBLE 286 CK_PR_BIN_S(add, double, double, +) 287 #endif /* CK_F_PR_ADD_DOUBLE */ 288 289 #ifndef CK_F_PR_SUB_DOUBLE 290 #define CK_F_PR_SUB_DOUBLE 291 CK_PR_BIN_S(sub, double, double, -) 292 #endif /* CK_F_PR_SUB_DOUBLE */ 293 294 #endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE */ 295 296 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) 297 298 #ifndef CK_F_PR_ADD_UINT 299 #define CK_F_PR_ADD_UINT 300 CK_PR_BIN_S(add, uint, unsigned int, +) 301 #endif /* CK_F_PR_ADD_UINT */ 302 303 #ifndef CK_F_PR_SUB_UINT 304 #define CK_F_PR_SUB_UINT 305 CK_PR_BIN_S(sub, uint, unsigned int, -) 306 #endif /* CK_F_PR_SUB_UINT */ 307 308 #ifndef CK_F_PR_AND_UINT 309 #define CK_F_PR_AND_UINT 310 CK_PR_BIN_S(and, uint, unsigned int, &) 311 #endif /* CK_F_PR_AND_UINT */ 312 313 #ifndef CK_F_PR_XOR_UINT 314 #define CK_F_PR_XOR_UINT 315 CK_PR_BIN_S(xor, uint, unsigned int, ^) 316 #endif /* CK_F_PR_XOR_UINT */ 317 318 #ifndef CK_F_PR_OR_UINT 319 #define CK_F_PR_OR_UINT 320 CK_PR_BIN_S(or, uint, unsigned int, |) 321 #endif /* CK_F_PR_OR_UINT */ 322 323 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ 324 325 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) 326 327 #ifndef CK_F_PR_ADD_PTR 328 #define CK_F_PR_ADD_PTR 329 CK_PR_BIN(add, ptr, void, uintptr_t, +, void *) 330 #endif /* CK_F_PR_ADD_PTR */ 331 332 #ifndef CK_F_PR_SUB_PTR 333 #define CK_F_PR_SUB_PTR 334 CK_PR_BIN(sub, ptr, void, uintptr_t, -, void *) 335 #endif /* CK_F_PR_SUB_PTR */ 336 337 #ifndef CK_F_PR_AND_PTR 338 #define CK_F_PR_AND_PTR 339 CK_PR_BIN(and, ptr, void, uintptr_t, &, void *) 340 #endif /* CK_F_PR_AND_PTR */ 341 342 #ifndef CK_F_PR_XOR_PTR 343 #define CK_F_PR_XOR_PTR 344 CK_PR_BIN(xor, ptr, void, uintptr_t, ^, void *) 345 #endif /* CK_F_PR_XOR_PTR */ 346 347 #ifndef CK_F_PR_OR_PTR 348 #define CK_F_PR_OR_PTR 349 CK_PR_BIN(or, ptr, void, uintptr_t, |, void *) 350 #endif /* CK_F_PR_OR_PTR */ 351 352 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ 353 354 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) 355 356 #ifndef CK_F_PR_ADD_64 357 #define CK_F_PR_ADD_64 358 CK_PR_BIN_S(add, 64, uint64_t, +) 359 #endif /* CK_F_PR_ADD_64 */ 360 361 #ifndef CK_F_PR_SUB_64 362 #define CK_F_PR_SUB_64 363 CK_PR_BIN_S(sub, 64, uint64_t, -) 364 #endif /* CK_F_PR_SUB_64 */ 365 366 #ifndef CK_F_PR_AND_64 367 #define CK_F_PR_AND_64 368 CK_PR_BIN_S(and, 64, uint64_t, &) 369 #endif /* CK_F_PR_AND_64 */ 370 371 #ifndef CK_F_PR_XOR_64 372 #define CK_F_PR_XOR_64 373 CK_PR_BIN_S(xor, 64, uint64_t, ^) 374 #endif /* CK_F_PR_XOR_64 */ 375 376 #ifndef CK_F_PR_OR_64 377 #define CK_F_PR_OR_64 378 CK_PR_BIN_S(or, 64, uint64_t, |) 379 #endif /* CK_F_PR_OR_64 */ 380 381 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ 382 383 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) 384 385 #ifndef CK_F_PR_ADD_32 386 #define CK_F_PR_ADD_32 387 CK_PR_BIN_S(add, 32, uint32_t, +) 388 #endif /* CK_F_PR_ADD_32 */ 389 390 #ifndef CK_F_PR_SUB_32 391 #define CK_F_PR_SUB_32 392 CK_PR_BIN_S(sub, 32, uint32_t, -) 393 #endif /* CK_F_PR_SUB_32 */ 394 395 #ifndef CK_F_PR_AND_32 396 #define CK_F_PR_AND_32 397 CK_PR_BIN_S(and, 32, uint32_t, &) 398 #endif /* CK_F_PR_AND_32 */ 399 400 #ifndef CK_F_PR_XOR_32 401 #define CK_F_PR_XOR_32 402 CK_PR_BIN_S(xor, 32, uint32_t, ^) 403 #endif /* CK_F_PR_XOR_32 */ 404 405 #ifndef CK_F_PR_OR_32 406 #define CK_F_PR_OR_32 407 CK_PR_BIN_S(or, 32, uint32_t, |) 408 #endif /* CK_F_PR_OR_32 */ 409 410 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ 411 412 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) 413 414 #ifndef CK_F_PR_ADD_16 415 #define CK_F_PR_ADD_16 416 CK_PR_BIN_S(add, 16, uint16_t, +) 417 #endif /* CK_F_PR_ADD_16 */ 418 419 #ifndef CK_F_PR_SUB_16 420 #define CK_F_PR_SUB_16 421 CK_PR_BIN_S(sub, 16, uint16_t, -) 422 #endif /* CK_F_PR_SUB_16 */ 423 424 #ifndef CK_F_PR_AND_16 425 #define CK_F_PR_AND_16 426 CK_PR_BIN_S(and, 16, uint16_t, &) 427 #endif /* CK_F_PR_AND_16 */ 428 429 #ifndef CK_F_PR_XOR_16 430 #define CK_F_PR_XOR_16 431 CK_PR_BIN_S(xor, 16, uint16_t, ^) 432 #endif /* CK_F_PR_XOR_16 */ 433 434 #ifndef CK_F_PR_OR_16 435 #define CK_F_PR_OR_16 436 CK_PR_BIN_S(or, 16, uint16_t, |) 437 #endif /* CK_F_PR_OR_16 */ 438 439 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ 440 441 #if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE) 442 443 #ifndef CK_F_PR_ADD_8 444 #define CK_F_PR_ADD_8 445 CK_PR_BIN_S(add, 8, uint8_t, +) 446 #endif /* CK_F_PR_ADD_8 */ 447 448 #ifndef CK_F_PR_SUB_8 449 #define CK_F_PR_SUB_8 450 CK_PR_BIN_S(sub, 8, uint8_t, -) 451 #endif /* CK_F_PR_SUB_8 */ 452 453 #ifndef CK_F_PR_AND_8 454 #define CK_F_PR_AND_8 455 CK_PR_BIN_S(and, 8, uint8_t, &) 456 #endif /* CK_F_PR_AND_8 */ 457 458 #ifndef CK_F_PR_XOR_8 459 #define CK_F_PR_XOR_8 460 CK_PR_BIN_S(xor, 8, uint8_t, ^) 461 #endif /* CK_F_PR_XOR_8 */ 462 463 #ifndef CK_F_PR_OR_8 464 #define CK_F_PR_OR_8 465 CK_PR_BIN_S(or, 8, uint8_t, |) 466 #endif /* CK_F_PR_OR_8 */ 467 468 #endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */ 469 470 #undef CK_PR_BIN_S 471 #undef CK_PR_BIN 472 473 #define CK_PR_BTX(K, S, M, T, P, C, R) \ 474 CK_CC_INLINE static bool \ 475 ck_pr_##K##_##S(M *target, unsigned int offset) \ 476 { \ 477 T previous; \ 478 C punt; \ 479 punt = ck_pr_md_load_##S(target); \ 480 previous = (T)punt; \ 481 while (ck_pr_cas_##S##_value(target, (C)previous, \ 482 (C)(previous P (R ((T)1 << offset))), &previous) == false) \ 483 ck_pr_stall(); \ 484 return ((previous >> offset) & 1); \ 485 } 486 487 #define CK_PR_BTX_S(K, S, T, P, R) CK_PR_BTX(K, S, T, T, P, T, R) 488 489 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) 490 491 #ifndef CK_F_PR_BTC_INT 492 #define CK_F_PR_BTC_INT 493 CK_PR_BTX_S(btc, int, int, ^,) 494 #endif /* CK_F_PR_BTC_INT */ 495 496 #ifndef CK_F_PR_BTR_INT 497 #define CK_F_PR_BTR_INT 498 CK_PR_BTX_S(btr, int, int, &, ~) 499 #endif /* CK_F_PR_BTR_INT */ 500 501 #ifndef CK_F_PR_BTS_INT 502 #define CK_F_PR_BTS_INT 503 CK_PR_BTX_S(bts, int, int, |,) 504 #endif /* CK_F_PR_BTS_INT */ 505 506 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ 507 508 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) 509 510 #ifndef CK_F_PR_BTC_UINT 511 #define CK_F_PR_BTC_UINT 512 CK_PR_BTX_S(btc, uint, unsigned int, ^,) 513 #endif /* CK_F_PR_BTC_UINT */ 514 515 #ifndef CK_F_PR_BTR_UINT 516 #define CK_F_PR_BTR_UINT 517 CK_PR_BTX_S(btr, uint, unsigned int, &, ~) 518 #endif /* CK_F_PR_BTR_UINT */ 519 520 #ifndef CK_F_PR_BTS_UINT 521 #define CK_F_PR_BTS_UINT 522 CK_PR_BTX_S(bts, uint, unsigned int, |,) 523 #endif /* CK_F_PR_BTS_UINT */ 524 525 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ 526 527 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) 528 529 #ifndef CK_F_PR_BTC_PTR 530 #define CK_F_PR_BTC_PTR 531 CK_PR_BTX(btc, ptr, void, uintptr_t, ^, void *,) 532 #endif /* CK_F_PR_BTC_PTR */ 533 534 #ifndef CK_F_PR_BTR_PTR 535 #define CK_F_PR_BTR_PTR 536 CK_PR_BTX(btr, ptr, void, uintptr_t, &, void *, ~) 537 #endif /* CK_F_PR_BTR_PTR */ 538 539 #ifndef CK_F_PR_BTS_PTR 540 #define CK_F_PR_BTS_PTR 541 CK_PR_BTX(bts, ptr, void, uintptr_t, |, void *,) 542 #endif /* CK_F_PR_BTS_PTR */ 543 544 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ 545 546 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) 547 548 #ifndef CK_F_PR_BTC_64 549 #define CK_F_PR_BTC_64 550 CK_PR_BTX_S(btc, 64, uint64_t, ^,) 551 #endif /* CK_F_PR_BTC_64 */ 552 553 #ifndef CK_F_PR_BTR_64 554 #define CK_F_PR_BTR_64 555 CK_PR_BTX_S(btr, 64, uint64_t, &, ~) 556 #endif /* CK_F_PR_BTR_64 */ 557 558 #ifndef CK_F_PR_BTS_64 559 #define CK_F_PR_BTS_64 560 CK_PR_BTX_S(bts, 64, uint64_t, |,) 561 #endif /* CK_F_PR_BTS_64 */ 562 563 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ 564 565 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) 566 567 #ifndef CK_F_PR_BTC_32 568 #define CK_F_PR_BTC_32 569 CK_PR_BTX_S(btc, 32, uint32_t, ^,) 570 #endif /* CK_F_PR_BTC_32 */ 571 572 #ifndef CK_F_PR_BTR_32 573 #define CK_F_PR_BTR_32 574 CK_PR_BTX_S(btr, 32, uint32_t, &, ~) 575 #endif /* CK_F_PR_BTR_32 */ 576 577 #ifndef CK_F_PR_BTS_32 578 #define CK_F_PR_BTS_32 579 CK_PR_BTX_S(bts, 32, uint32_t, |,) 580 #endif /* CK_F_PR_BTS_32 */ 581 582 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ 583 584 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) 585 586 #ifndef CK_F_PR_BTC_16 587 #define CK_F_PR_BTC_16 588 CK_PR_BTX_S(btc, 16, uint16_t, ^,) 589 #endif /* CK_F_PR_BTC_16 */ 590 591 #ifndef CK_F_PR_BTR_16 592 #define CK_F_PR_BTR_16 593 CK_PR_BTX_S(btr, 16, uint16_t, &, ~) 594 #endif /* CK_F_PR_BTR_16 */ 595 596 #ifndef CK_F_PR_BTS_16 597 #define CK_F_PR_BTS_16 598 CK_PR_BTX_S(bts, 16, uint16_t, |,) 599 #endif /* CK_F_PR_BTS_16 */ 600 601 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ 602 603 #undef CK_PR_BTX_S 604 #undef CK_PR_BTX 605 606 #define CK_PR_UNARY(K, X, S, M, T) \ 607 CK_CC_INLINE static void \ 608 ck_pr_##K##_##S(M *target) \ 609 { \ 610 ck_pr_##X##_##S(target, (T)1); \ 611 return; \ 612 } 613 614 #define CK_PR_UNARY_Z(K, S, M, T, P, C, Z) \ 615 CK_CC_INLINE static void \ 616 ck_pr_##K##_##S##_zero(M *target, bool *zero) \ 617 { \ 618 T previous; \ 619 C punt; \ 620 punt = (C)ck_pr_md_load_##S(target); \ 621 previous = (T)punt; \ 622 while (ck_pr_cas_##S##_value(target, \ 623 (C)previous, \ 624 (C)(previous P 1), \ 625 &previous) == false) \ 626 ck_pr_stall(); \ 627 *zero = previous == (T)Z; \ 628 return; \ 629 } 630 631 #define CK_PR_UNARY_S(K, X, S, M) CK_PR_UNARY(K, X, S, M, M) 632 #define CK_PR_UNARY_Z_S(K, S, M, P, Z) CK_PR_UNARY_Z(K, S, M, M, P, M, Z) 633 634 #if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE) 635 636 #ifndef CK_F_PR_INC_CHAR 637 #define CK_F_PR_INC_CHAR 638 CK_PR_UNARY_S(inc, add, char, char) 639 #endif /* CK_F_PR_INC_CHAR */ 640 641 #ifndef CK_F_PR_INC_CHAR_ZERO 642 #define CK_F_PR_INC_CHAR_ZERO 643 CK_PR_UNARY_Z_S(inc, char, char, +, -1) 644 #endif /* CK_F_PR_INC_CHAR_ZERO */ 645 646 #ifndef CK_F_PR_DEC_CHAR 647 #define CK_F_PR_DEC_CHAR 648 CK_PR_UNARY_S(dec, sub, char, char) 649 #endif /* CK_F_PR_DEC_CHAR */ 650 651 #ifndef CK_F_PR_DEC_CHAR_ZERO 652 #define CK_F_PR_DEC_CHAR_ZERO 653 CK_PR_UNARY_Z_S(dec, char, char, -, 1) 654 #endif /* CK_F_PR_DEC_CHAR_ZERO */ 655 656 #endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */ 657 658 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) 659 660 #ifndef CK_F_PR_INC_INT 661 #define CK_F_PR_INC_INT 662 CK_PR_UNARY_S(inc, add, int, int) 663 #endif /* CK_F_PR_INC_INT */ 664 665 #ifndef CK_F_PR_INC_INT_ZERO 666 #define CK_F_PR_INC_INT_ZERO 667 CK_PR_UNARY_Z_S(inc, int, int, +, -1) 668 #endif /* CK_F_PR_INC_INT_ZERO */ 669 670 #ifndef CK_F_PR_DEC_INT 671 #define CK_F_PR_DEC_INT 672 CK_PR_UNARY_S(dec, sub, int, int) 673 #endif /* CK_F_PR_DEC_INT */ 674 675 #ifndef CK_F_PR_DEC_INT_ZERO 676 #define CK_F_PR_DEC_INT_ZERO 677 CK_PR_UNARY_Z_S(dec, int, int, -, 1) 678 #endif /* CK_F_PR_DEC_INT_ZERO */ 679 680 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ 681 682 #if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) 683 684 #ifndef CK_F_PR_INC_DOUBLE 685 #define CK_F_PR_INC_DOUBLE 686 CK_PR_UNARY_S(inc, add, double, double) 687 #endif /* CK_F_PR_INC_DOUBLE */ 688 689 #ifndef CK_F_PR_DEC_DOUBLE 690 #define CK_F_PR_DEC_DOUBLE 691 CK_PR_UNARY_S(dec, sub, double, double) 692 #endif /* CK_F_PR_DEC_DOUBLE */ 693 694 #endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE */ 695 696 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) 697 698 #ifndef CK_F_PR_INC_UINT 699 #define CK_F_PR_INC_UINT 700 CK_PR_UNARY_S(inc, add, uint, unsigned int) 701 #endif /* CK_F_PR_INC_UINT */ 702 703 #ifndef CK_F_PR_INC_UINT_ZERO 704 #define CK_F_PR_INC_UINT_ZERO 705 CK_PR_UNARY_Z_S(inc, uint, unsigned int, +, UINT_MAX) 706 #endif /* CK_F_PR_INC_UINT_ZERO */ 707 708 #ifndef CK_F_PR_DEC_UINT 709 #define CK_F_PR_DEC_UINT 710 CK_PR_UNARY_S(dec, sub, uint, unsigned int) 711 #endif /* CK_F_PR_DEC_UINT */ 712 713 #ifndef CK_F_PR_DEC_UINT_ZERO 714 #define CK_F_PR_DEC_UINT_ZERO 715 CK_PR_UNARY_Z_S(dec, uint, unsigned int, -, 1) 716 #endif /* CK_F_PR_DEC_UINT_ZERO */ 717 718 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ 719 720 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) 721 722 #ifndef CK_F_PR_INC_PTR 723 #define CK_F_PR_INC_PTR 724 CK_PR_UNARY(inc, add, ptr, void, uintptr_t) 725 #endif /* CK_F_PR_INC_PTR */ 726 727 #ifndef CK_F_PR_INC_PTR_ZERO 728 #define CK_F_PR_INC_PTR_ZERO 729 CK_PR_UNARY_Z(inc, ptr, void, uintptr_t, +, void *, UINT_MAX) 730 #endif /* CK_F_PR_INC_PTR_ZERO */ 731 732 #ifndef CK_F_PR_DEC_PTR 733 #define CK_F_PR_DEC_PTR 734 CK_PR_UNARY(dec, sub, ptr, void, uintptr_t) 735 #endif /* CK_F_PR_DEC_PTR */ 736 737 #ifndef CK_F_PR_DEC_PTR_ZERO 738 #define CK_F_PR_DEC_PTR_ZERO 739 CK_PR_UNARY_Z(dec, ptr, void, uintptr_t, -, void *, 1) 740 #endif /* CK_F_PR_DEC_PTR_ZERO */ 741 742 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ 743 744 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) 745 746 #ifndef CK_F_PR_INC_64 747 #define CK_F_PR_INC_64 748 CK_PR_UNARY_S(inc, add, 64, uint64_t) 749 #endif /* CK_F_PR_INC_64 */ 750 751 #ifndef CK_F_PR_INC_64_ZERO 752 #define CK_F_PR_INC_64_ZERO 753 CK_PR_UNARY_Z_S(inc, 64, uint64_t, +, UINT64_MAX) 754 #endif /* CK_F_PR_INC_64_ZERO */ 755 756 #ifndef CK_F_PR_DEC_64 757 #define CK_F_PR_DEC_64 758 CK_PR_UNARY_S(dec, sub, 64, uint64_t) 759 #endif /* CK_F_PR_DEC_64 */ 760 761 #ifndef CK_F_PR_DEC_64_ZERO 762 #define CK_F_PR_DEC_64_ZERO 763 CK_PR_UNARY_Z_S(dec, 64, uint64_t, -, 1) 764 #endif /* CK_F_PR_DEC_64_ZERO */ 765 766 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ 767 768 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) 769 770 #ifndef CK_F_PR_INC_32 771 #define CK_F_PR_INC_32 772 CK_PR_UNARY_S(inc, add, 32, uint32_t) 773 #endif /* CK_F_PR_INC_32 */ 774 775 #ifndef CK_F_PR_INC_32_ZERO 776 #define CK_F_PR_INC_32_ZERO 777 CK_PR_UNARY_Z_S(inc, 32, uint32_t, +, UINT32_MAX) 778 #endif /* CK_F_PR_INC_32_ZERO */ 779 780 #ifndef CK_F_PR_DEC_32 781 #define CK_F_PR_DEC_32 782 CK_PR_UNARY_S(dec, sub, 32, uint32_t) 783 #endif /* CK_F_PR_DEC_32 */ 784 785 #ifndef CK_F_PR_DEC_32_ZERO 786 #define CK_F_PR_DEC_32_ZERO 787 CK_PR_UNARY_Z_S(dec, 32, uint32_t, -, 1) 788 #endif /* CK_F_PR_DEC_32_ZERO */ 789 790 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ 791 792 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) 793 794 #ifndef CK_F_PR_INC_16 795 #define CK_F_PR_INC_16 796 CK_PR_UNARY_S(inc, add, 16, uint16_t) 797 #endif /* CK_F_PR_INC_16 */ 798 799 #ifndef CK_F_PR_INC_16_ZERO 800 #define CK_F_PR_INC_16_ZERO 801 CK_PR_UNARY_Z_S(inc, 16, uint16_t, +, UINT16_MAX) 802 #endif /* CK_F_PR_INC_16_ZERO */ 803 804 #ifndef CK_F_PR_DEC_16 805 #define CK_F_PR_DEC_16 806 CK_PR_UNARY_S(dec, sub, 16, uint16_t) 807 #endif /* CK_F_PR_DEC_16 */ 808 809 #ifndef CK_F_PR_DEC_16_ZERO 810 #define CK_F_PR_DEC_16_ZERO 811 CK_PR_UNARY_Z_S(dec, 16, uint16_t, -, 1) 812 #endif /* CK_F_PR_DEC_16_ZERO */ 813 814 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ 815 816 #if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE) 817 818 #ifndef CK_F_PR_INC_8 819 #define CK_F_PR_INC_8 820 CK_PR_UNARY_S(inc, add, 8, uint8_t) 821 #endif /* CK_F_PR_INC_8 */ 822 823 #ifndef CK_F_PR_INC_8_ZERO 824 #define CK_F_PR_INC_8_ZERO 825 CK_PR_UNARY_Z_S(inc, 8, uint8_t, +, UINT8_MAX) 826 #endif /* CK_F_PR_INC_8_ZERO */ 827 828 #ifndef CK_F_PR_DEC_8 829 #define CK_F_PR_DEC_8 830 CK_PR_UNARY_S(dec, sub, 8, uint8_t) 831 #endif /* CK_F_PR_DEC_8 */ 832 833 #ifndef CK_F_PR_DEC_8_ZERO 834 #define CK_F_PR_DEC_8_ZERO 835 CK_PR_UNARY_Z_S(dec, 8, uint8_t, -, 1) 836 #endif /* CK_F_PR_DEC_8_ZERO */ 837 838 #endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */ 839 840 #undef CK_PR_UNARY_Z_S 841 #undef CK_PR_UNARY_S 842 #undef CK_PR_UNARY_Z 843 #undef CK_PR_UNARY 844 845 #define CK_PR_N(K, S, M, T, P, C) \ 846 CK_CC_INLINE static void \ 847 ck_pr_##K##_##S(M *target) \ 848 { \ 849 T previous; \ 850 C punt; \ 851 punt = (C)ck_pr_md_load_##S(target); \ 852 previous = (T)punt; \ 853 while (ck_pr_cas_##S##_value(target, \ 854 (C)previous, \ 855 (C)(P previous), \ 856 &previous) == false) \ 857 ck_pr_stall(); \ 858 \ 859 return; \ 860 } 861 862 #define CK_PR_N_Z(S, M, T, C) \ 863 CK_CC_INLINE static void \ 864 ck_pr_neg_##S##_zero(M *target, bool *zero) \ 865 { \ 866 T previous; \ 867 C punt; \ 868 punt = (C)ck_pr_md_load_##S(target); \ 869 previous = (T)punt; \ 870 while (ck_pr_cas_##S##_value(target, \ 871 (C)previous, \ 872 (C)(-previous), \ 873 &previous) == false) \ 874 ck_pr_stall(); \ 875 \ 876 *zero = previous == 0; \ 877 return; \ 878 } 879 880 #define CK_PR_N_S(K, S, M, P) CK_PR_N(K, S, M, M, P, M) 881 #define CK_PR_N_Z_S(S, M) CK_PR_N_Z(S, M, M, M) 882 883 #if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE) 884 885 #ifndef CK_F_PR_NOT_CHAR 886 #define CK_F_PR_NOT_CHAR 887 CK_PR_N_S(not, char, char, ~) 888 #endif /* CK_F_PR_NOT_CHAR */ 889 890 #ifndef CK_F_PR_NEG_CHAR 891 #define CK_F_PR_NEG_CHAR 892 CK_PR_N_S(neg, char, char, -) 893 #endif /* CK_F_PR_NEG_CHAR */ 894 895 #ifndef CK_F_PR_NEG_CHAR_ZERO 896 #define CK_F_PR_NEG_CHAR_ZERO 897 CK_PR_N_Z_S(char, char) 898 #endif /* CK_F_PR_NEG_CHAR_ZERO */ 899 900 #endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */ 901 902 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) 903 904 #ifndef CK_F_PR_NOT_INT 905 #define CK_F_PR_NOT_INT 906 CK_PR_N_S(not, int, int, ~) 907 #endif /* CK_F_PR_NOT_INT */ 908 909 #ifndef CK_F_PR_NEG_INT 910 #define CK_F_PR_NEG_INT 911 CK_PR_N_S(neg, int, int, -) 912 #endif /* CK_F_PR_NEG_INT */ 913 914 #ifndef CK_F_PR_NEG_INT_ZERO 915 #define CK_F_PR_NEG_INT_ZERO 916 CK_PR_N_Z_S(int, int) 917 #endif /* CK_F_PR_NEG_INT_ZERO */ 918 919 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ 920 921 #if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) 922 923 #ifndef CK_F_PR_NEG_DOUBLE 924 #define CK_F_PR_NEG_DOUBLE 925 CK_PR_N_S(neg, double, double, -) 926 #endif /* CK_F_PR_NEG_DOUBLE */ 927 928 #endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE */ 929 930 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) 931 932 #ifndef CK_F_PR_NOT_UINT 933 #define CK_F_PR_NOT_UINT 934 CK_PR_N_S(not, uint, unsigned int, ~) 935 #endif /* CK_F_PR_NOT_UINT */ 936 937 #ifndef CK_F_PR_NEG_UINT 938 #define CK_F_PR_NEG_UINT 939 CK_PR_N_S(neg, uint, unsigned int, -) 940 #endif /* CK_F_PR_NEG_UINT */ 941 942 #ifndef CK_F_PR_NEG_UINT_ZERO 943 #define CK_F_PR_NEG_UINT_ZERO 944 CK_PR_N_Z_S(uint, unsigned int) 945 #endif /* CK_F_PR_NEG_UINT_ZERO */ 946 947 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ 948 949 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) 950 951 #ifndef CK_F_PR_NOT_PTR 952 #define CK_F_PR_NOT_PTR 953 CK_PR_N(not, ptr, void, uintptr_t, ~, void *) 954 #endif /* CK_F_PR_NOT_PTR */ 955 956 #ifndef CK_F_PR_NEG_PTR 957 #define CK_F_PR_NEG_PTR 958 CK_PR_N(neg, ptr, void, uintptr_t, -, void *) 959 #endif /* CK_F_PR_NEG_PTR */ 960 961 #ifndef CK_F_PR_NEG_PTR_ZERO 962 #define CK_F_PR_NEG_PTR_ZERO 963 CK_PR_N_Z(ptr, void, uintptr_t, void *) 964 #endif /* CK_F_PR_NEG_PTR_ZERO */ 965 966 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ 967 968 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) 969 970 #ifndef CK_F_PR_NOT_64 971 #define CK_F_PR_NOT_64 972 CK_PR_N_S(not, 64, uint64_t, ~) 973 #endif /* CK_F_PR_NOT_64 */ 974 975 #ifndef CK_F_PR_NEG_64 976 #define CK_F_PR_NEG_64 977 CK_PR_N_S(neg, 64, uint64_t, -) 978 #endif /* CK_F_PR_NEG_64 */ 979 980 #ifndef CK_F_PR_NEG_64_ZERO 981 #define CK_F_PR_NEG_64_ZERO 982 CK_PR_N_Z_S(64, uint64_t) 983 #endif /* CK_F_PR_NEG_64_ZERO */ 984 985 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ 986 987 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) 988 989 #ifndef CK_F_PR_NOT_32 990 #define CK_F_PR_NOT_32 991 CK_PR_N_S(not, 32, uint32_t, ~) 992 #endif /* CK_F_PR_NOT_32 */ 993 994 #ifndef CK_F_PR_NEG_32 995 #define CK_F_PR_NEG_32 996 CK_PR_N_S(neg, 32, uint32_t, -) 997 #endif /* CK_F_PR_NEG_32 */ 998 999 #ifndef CK_F_PR_NEG_32_ZERO 1000 #define CK_F_PR_NEG_32_ZERO 1001 CK_PR_N_Z_S(32, uint32_t) 1002 #endif /* CK_F_PR_NEG_32_ZERO */ 1003 1004 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ 1005 1006 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) 1007 1008 #ifndef CK_F_PR_NOT_16 1009 #define CK_F_PR_NOT_16 1010 CK_PR_N_S(not, 16, uint16_t, ~) 1011 #endif /* CK_F_PR_NOT_16 */ 1012 1013 #ifndef CK_F_PR_NEG_16 1014 #define CK_F_PR_NEG_16 1015 CK_PR_N_S(neg, 16, uint16_t, -) 1016 #endif /* CK_F_PR_NEG_16 */ 1017 1018 #ifndef CK_F_PR_NEG_16_ZERO 1019 #define CK_F_PR_NEG_16_ZERO 1020 CK_PR_N_Z_S(16, uint16_t) 1021 #endif /* CK_F_PR_NEG_16_ZERO */ 1022 1023 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ 1024 1025 #if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE) 1026 1027 #ifndef CK_F_PR_NOT_8 1028 #define CK_F_PR_NOT_8 1029 CK_PR_N_S(not, 8, uint8_t, ~) 1030 #endif /* CK_F_PR_NOT_8 */ 1031 1032 #ifndef CK_F_PR_NEG_8 1033 #define CK_F_PR_NEG_8 1034 CK_PR_N_S(neg, 8, uint8_t, -) 1035 #endif /* CK_F_PR_NEG_8 */ 1036 1037 #ifndef CK_F_PR_NEG_8_ZERO 1038 #define CK_F_PR_NEG_8_ZERO 1039 CK_PR_N_Z_S(8, uint8_t) 1040 #endif /* CK_F_PR_NEG_8_ZERO */ 1041 1042 #endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */ 1043 1044 #undef CK_PR_N_Z_S 1045 #undef CK_PR_N_S 1046 #undef CK_PR_N_Z 1047 #undef CK_PR_N 1048 1049 #define CK_PR_FAA(S, M, T, C) \ 1050 CK_CC_INLINE static C \ 1051 ck_pr_faa_##S(M *target, T delta) \ 1052 { \ 1053 T previous; \ 1054 C punt; \ 1055 punt = (C)ck_pr_md_load_##S(target); \ 1056 previous = (T)punt; \ 1057 while (ck_pr_cas_##S##_value(target, \ 1058 (C)previous, \ 1059 (C)(previous + delta), \ 1060 &previous) == false) \ 1061 ck_pr_stall(); \ 1062 \ 1063 return ((C)previous); \ 1064 } 1065 1066 #define CK_PR_FAS(S, M, C) \ 1067 CK_CC_INLINE static C \ 1068 ck_pr_fas_##S(M *target, C update) \ 1069 { \ 1070 C previous; \ 1071 previous = ck_pr_md_load_##S(target); \ 1072 while (ck_pr_cas_##S##_value(target, \ 1073 previous, \ 1074 update, \ 1075 &previous) == false) \ 1076 ck_pr_stall(); \ 1077 \ 1078 return (previous); \ 1079 } 1080 1081 #define CK_PR_FAA_S(S, M) CK_PR_FAA(S, M, M, M) 1082 #define CK_PR_FAS_S(S, M) CK_PR_FAS(S, M, M) 1083 1084 #if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE) 1085 1086 #ifndef CK_F_PR_FAA_CHAR 1087 #define CK_F_PR_FAA_CHAR 1088 CK_PR_FAA_S(char, char) 1089 #endif /* CK_F_PR_FAA_CHAR */ 1090 1091 #ifndef CK_F_PR_FAS_CHAR 1092 #define CK_F_PR_FAS_CHAR 1093 CK_PR_FAS_S(char, char) 1094 #endif /* CK_F_PR_FAS_CHAR */ 1095 1096 #endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */ 1097 1098 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) 1099 1100 #ifndef CK_F_PR_FAA_INT 1101 #define CK_F_PR_FAA_INT 1102 CK_PR_FAA_S(int, int) 1103 #endif /* CK_F_PR_FAA_INT */ 1104 1105 #ifndef CK_F_PR_FAS_INT 1106 #define CK_F_PR_FAS_INT 1107 CK_PR_FAS_S(int, int) 1108 #endif /* CK_F_PR_FAS_INT */ 1109 1110 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ 1111 1112 #if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) 1113 1114 #ifndef CK_F_PR_FAA_DOUBLE 1115 #define CK_F_PR_FAA_DOUBLE 1116 CK_PR_FAA_S(double, double) 1117 #endif /* CK_F_PR_FAA_DOUBLE */ 1118 1119 #ifndef CK_F_PR_FAS_DOUBLE 1120 #define CK_F_PR_FAS_DOUBLE 1121 CK_PR_FAS_S(double, double) 1122 #endif /* CK_F_PR_FAS_DOUBLE */ 1123 1124 #endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE */ 1125 1126 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) 1127 1128 #ifndef CK_F_PR_FAA_UINT 1129 #define CK_F_PR_FAA_UINT 1130 CK_PR_FAA_S(uint, unsigned int) 1131 #endif /* CK_F_PR_FAA_UINT */ 1132 1133 #ifndef CK_F_PR_FAS_UINT 1134 #define CK_F_PR_FAS_UINT 1135 CK_PR_FAS_S(uint, unsigned int) 1136 #endif /* CK_F_PR_FAS_UINT */ 1137 1138 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ 1139 1140 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) 1141 1142 #ifndef CK_F_PR_FAA_PTR 1143 #define CK_F_PR_FAA_PTR 1144 CK_PR_FAA(ptr, void, uintptr_t, void *) 1145 #endif /* CK_F_PR_FAA_PTR */ 1146 1147 #ifndef CK_F_PR_FAS_PTR 1148 #define CK_F_PR_FAS_PTR 1149 CK_PR_FAS(ptr, void, void *) 1150 #endif /* CK_F_PR_FAS_PTR */ 1151 1152 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ 1153 1154 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) 1155 1156 #ifndef CK_F_PR_FAA_64 1157 #define CK_F_PR_FAA_64 1158 CK_PR_FAA_S(64, uint64_t) 1159 #endif /* CK_F_PR_FAA_64 */ 1160 1161 #ifndef CK_F_PR_FAS_64 1162 #define CK_F_PR_FAS_64 1163 CK_PR_FAS_S(64, uint64_t) 1164 #endif /* CK_F_PR_FAS_64 */ 1165 1166 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ 1167 1168 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) 1169 1170 #ifndef CK_F_PR_FAA_32 1171 #define CK_F_PR_FAA_32 1172 CK_PR_FAA_S(32, uint32_t) 1173 #endif /* CK_F_PR_FAA_32 */ 1174 1175 #ifndef CK_F_PR_FAS_32 1176 #define CK_F_PR_FAS_32 1177 CK_PR_FAS_S(32, uint32_t) 1178 #endif /* CK_F_PR_FAS_32 */ 1179 1180 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ 1181 1182 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) 1183 1184 #ifndef CK_F_PR_FAA_16 1185 #define CK_F_PR_FAA_16 1186 CK_PR_FAA_S(16, uint16_t) 1187 #endif /* CK_F_PR_FAA_16 */ 1188 1189 #ifndef CK_F_PR_FAS_16 1190 #define CK_F_PR_FAS_16 1191 CK_PR_FAS_S(16, uint16_t) 1192 #endif /* CK_F_PR_FAS_16 */ 1193 1194 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ 1195 1196 #if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE) 1197 1198 #ifndef CK_F_PR_FAA_8 1199 #define CK_F_PR_FAA_8 1200 CK_PR_FAA_S(8, uint8_t) 1201 #endif /* CK_F_PR_FAA_8 */ 1202 1203 #ifndef CK_F_PR_FAS_8 1204 #define CK_F_PR_FAS_8 1205 CK_PR_FAS_S(8, uint8_t) 1206 #endif /* CK_F_PR_FAS_8 */ 1207 1208 #endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */ 1209 1210 #undef CK_PR_FAA_S 1211 #undef CK_PR_FAS_S 1212 #undef CK_PR_FAA 1213 #undef CK_PR_FAS 1214 1215 #endif /* CK_PR_H */ 1216