1 /* 2 * Copyright 2009-2015 Samy Al Bahra. 3 * Copyright 2011 David Joseph. 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #ifndef CK_PR_H 29 #define CK_PR_H 30 31 #include <ck_cc.h> 32 #include <ck_limits.h> 33 #include <ck_md.h> 34 #include <ck_stdint.h> 35 #include <ck_stdbool.h> 36 37 #ifndef CK_USE_CC_BUILTINS 38 #if defined(__x86_64__) 39 #include "gcc/x86_64/ck_pr.h" 40 #elif defined(__x86__) 41 #include "gcc/x86/ck_pr.h" 42 #elif defined(__sparcv9__) 43 #include "gcc/sparcv9/ck_pr.h" 44 #elif defined(__ppc64__) 45 #include "gcc/ppc64/ck_pr.h" 46 #elif defined(__ppc__) 47 #include "gcc/ppc/ck_pr.h" 48 #elif defined(__arm__) 49 #include "gcc/arm/ck_pr.h" 50 #elif defined(__aarch64__) 51 #include "gcc/aarch64/ck_pr.h" 52 #elif !defined(__GNUC__) 53 #error Your platform is unsupported 54 #endif 55 #endif /* !CK_USE_CC_BUILTINS */ 56 57 #if defined(__GNUC__) 58 #include "gcc/ck_pr.h" 59 #endif 60 61 #define CK_PR_FENCE_EMIT(T) \ 62 CK_CC_INLINE static void \ 63 ck_pr_fence_##T(void) \ 64 { \ 65 ck_pr_fence_strict_##T(); \ 66 return; \ 67 } 68 #define CK_PR_FENCE_NOOP(T) \ 69 CK_CC_INLINE static void \ 70 ck_pr_fence_##T(void) \ 71 { \ 72 ck_pr_barrier(); \ 73 return; \ 74 } 75 76 /* 77 * None of the currently supported platforms allow for data-dependent 78 * load ordering. 79 */ 80 CK_PR_FENCE_NOOP(load_depends) 81 #define ck_pr_fence_strict_load_depends ck_pr_fence_load_depends 82 83 /* 84 * In memory models where atomic operations do not have serializing 85 * effects, atomic read-modify-write operations are modeled as stores. 86 */ 87 #if defined(CK_MD_RMO) 88 /* 89 * Only stores to the same location have a global 90 * ordering. 91 */ 92 CK_PR_FENCE_EMIT(atomic) 93 CK_PR_FENCE_EMIT(atomic_load) 94 CK_PR_FENCE_EMIT(atomic_store) 95 CK_PR_FENCE_EMIT(store_atomic) 96 CK_PR_FENCE_EMIT(load_atomic) 97 CK_PR_FENCE_EMIT(load_store) 98 CK_PR_FENCE_EMIT(store_load) 99 CK_PR_FENCE_EMIT(load) 100 CK_PR_FENCE_EMIT(store) 101 CK_PR_FENCE_EMIT(memory) 102 CK_PR_FENCE_EMIT(acquire) 103 CK_PR_FENCE_EMIT(release) 104 CK_PR_FENCE_EMIT(acqrel) 105 CK_PR_FENCE_EMIT(lock) 106 CK_PR_FENCE_EMIT(unlock) 107 #elif defined(CK_MD_PSO) 108 /* 109 * Anything can be re-ordered with respect to stores. 110 * Otherwise, loads are executed in-order. 111 */ 112 CK_PR_FENCE_EMIT(atomic) 113 CK_PR_FENCE_NOOP(atomic_load) 114 CK_PR_FENCE_EMIT(atomic_store) 115 CK_PR_FENCE_EMIT(store_atomic) 116 CK_PR_FENCE_NOOP(load_atomic) 117 CK_PR_FENCE_EMIT(load_store) 118 CK_PR_FENCE_EMIT(store_load) 119 CK_PR_FENCE_NOOP(load) 120 CK_PR_FENCE_EMIT(store) 121 CK_PR_FENCE_EMIT(memory) 122 CK_PR_FENCE_EMIT(acquire) 123 CK_PR_FENCE_EMIT(release) 124 CK_PR_FENCE_EMIT(acqrel) 125 CK_PR_FENCE_EMIT(lock) 126 CK_PR_FENCE_EMIT(unlock) 127 #elif defined(CK_MD_TSO) 128 /* 129 * Only loads are re-ordered and only with respect to 130 * prior stores. Atomic operations are serializing. 131 */ 132 CK_PR_FENCE_NOOP(atomic) 133 CK_PR_FENCE_NOOP(atomic_load) 134 CK_PR_FENCE_NOOP(atomic_store) 135 CK_PR_FENCE_NOOP(store_atomic) 136 CK_PR_FENCE_NOOP(load_atomic) 137 CK_PR_FENCE_NOOP(load_store) 138 CK_PR_FENCE_EMIT(store_load) 139 CK_PR_FENCE_NOOP(load) 140 CK_PR_FENCE_NOOP(store) 141 CK_PR_FENCE_EMIT(memory) 142 CK_PR_FENCE_NOOP(acquire) 143 CK_PR_FENCE_NOOP(release) 144 CK_PR_FENCE_NOOP(acqrel) 145 CK_PR_FENCE_NOOP(lock) 146 CK_PR_FENCE_NOOP(unlock) 147 #else 148 #error "No memory model has been defined." 149 #endif /* CK_MD_TSO */ 150 151 #undef CK_PR_FENCE_EMIT 152 #undef CK_PR_FENCE_NOOP 153 154 #ifndef CK_F_PR_RFO 155 #define CK_F_PR_RFO 156 CK_CC_INLINE static void 157 ck_pr_rfo(const void *m) 158 { 159 160 (void)m; 161 return; 162 } 163 #endif /* CK_F_PR_RFO */ 164 165 #define CK_PR_STORE_SAFE(DST, VAL, TYPE) \ 166 ck_pr_md_store_##TYPE( \ 167 ((void)sizeof(*(DST) = (VAL)), (DST)), \ 168 (VAL)) 169 170 #define ck_pr_store_ptr(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), ptr) 171 #define ck_pr_store_char(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), char) 172 #define ck_pr_store_double(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), double) 173 #define ck_pr_store_uint(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), uint) 174 #define ck_pr_store_int(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), int) 175 #define ck_pr_store_32(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 32) 176 #define ck_pr_store_16(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 16) 177 #define ck_pr_store_8(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 8) 178 179 #define ck_pr_store_ptr_unsafe(DST, VAL) ck_pr_md_store_ptr((DST), (VAL)) 180 181 #ifdef CK_F_PR_LOAD_64 182 #define ck_pr_store_64(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 64) 183 #endif /* CK_F_PR_LOAD_64 */ 184 185 #define CK_PR_LOAD_PTR_SAFE(SRC) (CK_CC_TYPEOF(*(SRC), (void *)))ck_pr_md_load_ptr((SRC)) 186 #define ck_pr_load_ptr(SRC) CK_PR_LOAD_PTR_SAFE((SRC)) 187 188 #define CK_PR_LOAD_SAFE(SRC, TYPE) ck_pr_md_load_##TYPE((SRC)) 189 #define ck_pr_load_char(SRC) CK_PR_LOAD_SAFE((SRC), char) 190 #define ck_pr_load_double(SRC) CK_PR_LOAD_SAFE((SRC), double) 191 #define ck_pr_load_uint(SRC) CK_PR_LOAD_SAFE((SRC), uint) 192 #define ck_pr_load_int(SRC) CK_PR_LOAD_SAFE((SRC), int) 193 #define ck_pr_load_32(SRC) CK_PR_LOAD_SAFE((SRC), 32) 194 #define ck_pr_load_16(SRC) CK_PR_LOAD_SAFE((SRC), 16) 195 #define ck_pr_load_8(SRC) CK_PR_LOAD_SAFE((SRC), 8) 196 197 #ifdef CK_F_PR_LOAD_64 198 #define ck_pr_load_64(SRC) CK_PR_LOAD_SAFE((SRC), 64) 199 #endif /* CK_F_PR_LOAD_64 */ 200 201 #define CK_PR_BIN(K, S, M, T, P, C) \ 202 CK_CC_INLINE static void \ 203 ck_pr_##K##_##S(M *target, T value) \ 204 { \ 205 T previous; \ 206 C punt; \ 207 punt = ck_pr_md_load_##S(target); \ 208 previous = (T)punt; \ 209 while (ck_pr_cas_##S##_value(target, \ 210 (C)previous, \ 211 (C)(previous P value), \ 212 &previous) == false) \ 213 ck_pr_stall(); \ 214 \ 215 return; \ 216 } 217 218 #define CK_PR_BIN_S(K, S, T, P) CK_PR_BIN(K, S, T, T, P, T) 219 220 #if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE) 221 222 #ifndef CK_F_PR_ADD_CHAR 223 #define CK_F_PR_ADD_CHAR 224 CK_PR_BIN_S(add, char, char, +) 225 #endif /* CK_F_PR_ADD_CHAR */ 226 227 #ifndef CK_F_PR_SUB_CHAR 228 #define CK_F_PR_SUB_CHAR 229 CK_PR_BIN_S(sub, char, char, -) 230 #endif /* CK_F_PR_SUB_CHAR */ 231 232 #ifndef CK_F_PR_AND_CHAR 233 #define CK_F_PR_AND_CHAR 234 CK_PR_BIN_S(and, char, char, &) 235 #endif /* CK_F_PR_AND_CHAR */ 236 237 #ifndef CK_F_PR_XOR_CHAR 238 #define CK_F_PR_XOR_CHAR 239 CK_PR_BIN_S(xor, char, char, ^) 240 #endif /* CK_F_PR_XOR_CHAR */ 241 242 #ifndef CK_F_PR_OR_CHAR 243 #define CK_F_PR_OR_CHAR 244 CK_PR_BIN_S(or, char, char, |) 245 #endif /* CK_F_PR_OR_CHAR */ 246 247 #endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */ 248 249 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) 250 251 #ifndef CK_F_PR_ADD_INT 252 #define CK_F_PR_ADD_INT 253 CK_PR_BIN_S(add, int, int, +) 254 #endif /* CK_F_PR_ADD_INT */ 255 256 #ifndef CK_F_PR_SUB_INT 257 #define CK_F_PR_SUB_INT 258 CK_PR_BIN_S(sub, int, int, -) 259 #endif /* CK_F_PR_SUB_INT */ 260 261 #ifndef CK_F_PR_AND_INT 262 #define CK_F_PR_AND_INT 263 CK_PR_BIN_S(and, int, int, &) 264 #endif /* CK_F_PR_AND_INT */ 265 266 #ifndef CK_F_PR_XOR_INT 267 #define CK_F_PR_XOR_INT 268 CK_PR_BIN_S(xor, int, int, ^) 269 #endif /* CK_F_PR_XOR_INT */ 270 271 #ifndef CK_F_PR_OR_INT 272 #define CK_F_PR_OR_INT 273 CK_PR_BIN_S(or, int, int, |) 274 #endif /* CK_F_PR_OR_INT */ 275 276 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ 277 278 #if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) 279 280 #ifndef CK_F_PR_ADD_DOUBLE 281 #define CK_F_PR_ADD_DOUBLE 282 CK_PR_BIN_S(add, double, double, +) 283 #endif /* CK_F_PR_ADD_DOUBLE */ 284 285 #ifndef CK_F_PR_SUB_DOUBLE 286 #define CK_F_PR_SUB_DOUBLE 287 CK_PR_BIN_S(sub, double, double, -) 288 #endif /* CK_F_PR_SUB_DOUBLE */ 289 290 #endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE */ 291 292 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) 293 294 #ifndef CK_F_PR_ADD_UINT 295 #define CK_F_PR_ADD_UINT 296 CK_PR_BIN_S(add, uint, unsigned int, +) 297 #endif /* CK_F_PR_ADD_UINT */ 298 299 #ifndef CK_F_PR_SUB_UINT 300 #define CK_F_PR_SUB_UINT 301 CK_PR_BIN_S(sub, uint, unsigned int, -) 302 #endif /* CK_F_PR_SUB_UINT */ 303 304 #ifndef CK_F_PR_AND_UINT 305 #define CK_F_PR_AND_UINT 306 CK_PR_BIN_S(and, uint, unsigned int, &) 307 #endif /* CK_F_PR_AND_UINT */ 308 309 #ifndef CK_F_PR_XOR_UINT 310 #define CK_F_PR_XOR_UINT 311 CK_PR_BIN_S(xor, uint, unsigned int, ^) 312 #endif /* CK_F_PR_XOR_UINT */ 313 314 #ifndef CK_F_PR_OR_UINT 315 #define CK_F_PR_OR_UINT 316 CK_PR_BIN_S(or, uint, unsigned int, |) 317 #endif /* CK_F_PR_OR_UINT */ 318 319 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ 320 321 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) 322 323 #ifndef CK_F_PR_ADD_PTR 324 #define CK_F_PR_ADD_PTR 325 CK_PR_BIN(add, ptr, void, uintptr_t, +, void *) 326 #endif /* CK_F_PR_ADD_PTR */ 327 328 #ifndef CK_F_PR_SUB_PTR 329 #define CK_F_PR_SUB_PTR 330 CK_PR_BIN(sub, ptr, void, uintptr_t, -, void *) 331 #endif /* CK_F_PR_SUB_PTR */ 332 333 #ifndef CK_F_PR_AND_PTR 334 #define CK_F_PR_AND_PTR 335 CK_PR_BIN(and, ptr, void, uintptr_t, &, void *) 336 #endif /* CK_F_PR_AND_PTR */ 337 338 #ifndef CK_F_PR_XOR_PTR 339 #define CK_F_PR_XOR_PTR 340 CK_PR_BIN(xor, ptr, void, uintptr_t, ^, void *) 341 #endif /* CK_F_PR_XOR_PTR */ 342 343 #ifndef CK_F_PR_OR_PTR 344 #define CK_F_PR_OR_PTR 345 CK_PR_BIN(or, ptr, void, uintptr_t, |, void *) 346 #endif /* CK_F_PR_OR_PTR */ 347 348 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ 349 350 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) 351 352 #ifndef CK_F_PR_ADD_64 353 #define CK_F_PR_ADD_64 354 CK_PR_BIN_S(add, 64, uint64_t, +) 355 #endif /* CK_F_PR_ADD_64 */ 356 357 #ifndef CK_F_PR_SUB_64 358 #define CK_F_PR_SUB_64 359 CK_PR_BIN_S(sub, 64, uint64_t, -) 360 #endif /* CK_F_PR_SUB_64 */ 361 362 #ifndef CK_F_PR_AND_64 363 #define CK_F_PR_AND_64 364 CK_PR_BIN_S(and, 64, uint64_t, &) 365 #endif /* CK_F_PR_AND_64 */ 366 367 #ifndef CK_F_PR_XOR_64 368 #define CK_F_PR_XOR_64 369 CK_PR_BIN_S(xor, 64, uint64_t, ^) 370 #endif /* CK_F_PR_XOR_64 */ 371 372 #ifndef CK_F_PR_OR_64 373 #define CK_F_PR_OR_64 374 CK_PR_BIN_S(or, 64, uint64_t, |) 375 #endif /* CK_F_PR_OR_64 */ 376 377 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ 378 379 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) 380 381 #ifndef CK_F_PR_ADD_32 382 #define CK_F_PR_ADD_32 383 CK_PR_BIN_S(add, 32, uint32_t, +) 384 #endif /* CK_F_PR_ADD_32 */ 385 386 #ifndef CK_F_PR_SUB_32 387 #define CK_F_PR_SUB_32 388 CK_PR_BIN_S(sub, 32, uint32_t, -) 389 #endif /* CK_F_PR_SUB_32 */ 390 391 #ifndef CK_F_PR_AND_32 392 #define CK_F_PR_AND_32 393 CK_PR_BIN_S(and, 32, uint32_t, &) 394 #endif /* CK_F_PR_AND_32 */ 395 396 #ifndef CK_F_PR_XOR_32 397 #define CK_F_PR_XOR_32 398 CK_PR_BIN_S(xor, 32, uint32_t, ^) 399 #endif /* CK_F_PR_XOR_32 */ 400 401 #ifndef CK_F_PR_OR_32 402 #define CK_F_PR_OR_32 403 CK_PR_BIN_S(or, 32, uint32_t, |) 404 #endif /* CK_F_PR_OR_32 */ 405 406 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ 407 408 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) 409 410 #ifndef CK_F_PR_ADD_16 411 #define CK_F_PR_ADD_16 412 CK_PR_BIN_S(add, 16, uint16_t, +) 413 #endif /* CK_F_PR_ADD_16 */ 414 415 #ifndef CK_F_PR_SUB_16 416 #define CK_F_PR_SUB_16 417 CK_PR_BIN_S(sub, 16, uint16_t, -) 418 #endif /* CK_F_PR_SUB_16 */ 419 420 #ifndef CK_F_PR_AND_16 421 #define CK_F_PR_AND_16 422 CK_PR_BIN_S(and, 16, uint16_t, &) 423 #endif /* CK_F_PR_AND_16 */ 424 425 #ifndef CK_F_PR_XOR_16 426 #define CK_F_PR_XOR_16 427 CK_PR_BIN_S(xor, 16, uint16_t, ^) 428 #endif /* CK_F_PR_XOR_16 */ 429 430 #ifndef CK_F_PR_OR_16 431 #define CK_F_PR_OR_16 432 CK_PR_BIN_S(or, 16, uint16_t, |) 433 #endif /* CK_F_PR_OR_16 */ 434 435 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ 436 437 #if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE) 438 439 #ifndef CK_F_PR_ADD_8 440 #define CK_F_PR_ADD_8 441 CK_PR_BIN_S(add, 8, uint8_t, +) 442 #endif /* CK_F_PR_ADD_8 */ 443 444 #ifndef CK_F_PR_SUB_8 445 #define CK_F_PR_SUB_8 446 CK_PR_BIN_S(sub, 8, uint8_t, -) 447 #endif /* CK_F_PR_SUB_8 */ 448 449 #ifndef CK_F_PR_AND_8 450 #define CK_F_PR_AND_8 451 CK_PR_BIN_S(and, 8, uint8_t, &) 452 #endif /* CK_F_PR_AND_8 */ 453 454 #ifndef CK_F_PR_XOR_8 455 #define CK_F_PR_XOR_8 456 CK_PR_BIN_S(xor, 8, uint8_t, ^) 457 #endif /* CK_F_PR_XOR_8 */ 458 459 #ifndef CK_F_PR_OR_8 460 #define CK_F_PR_OR_8 461 CK_PR_BIN_S(or, 8, uint8_t, |) 462 #endif /* CK_F_PR_OR_8 */ 463 464 #endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */ 465 466 #undef CK_PR_BIN_S 467 #undef CK_PR_BIN 468 469 #define CK_PR_BTX(K, S, M, T, P, C, R) \ 470 CK_CC_INLINE static bool \ 471 ck_pr_##K##_##S(M *target, unsigned int offset) \ 472 { \ 473 T previous; \ 474 C punt; \ 475 punt = ck_pr_md_load_##S(target); \ 476 previous = (T)punt; \ 477 while (ck_pr_cas_##S##_value(target, (C)previous, \ 478 (C)(previous P (R ((T)1 << offset))), &previous) == false) \ 479 ck_pr_stall(); \ 480 return ((previous >> offset) & 1); \ 481 } 482 483 #define CK_PR_BTX_S(K, S, T, P, R) CK_PR_BTX(K, S, T, T, P, T, R) 484 485 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) 486 487 #ifndef CK_F_PR_BTC_INT 488 #define CK_F_PR_BTC_INT 489 CK_PR_BTX_S(btc, int, int, ^,) 490 #endif /* CK_F_PR_BTC_INT */ 491 492 #ifndef CK_F_PR_BTR_INT 493 #define CK_F_PR_BTR_INT 494 CK_PR_BTX_S(btr, int, int, &, ~) 495 #endif /* CK_F_PR_BTR_INT */ 496 497 #ifndef CK_F_PR_BTS_INT 498 #define CK_F_PR_BTS_INT 499 CK_PR_BTX_S(bts, int, int, |,) 500 #endif /* CK_F_PR_BTS_INT */ 501 502 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ 503 504 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) 505 506 #ifndef CK_F_PR_BTC_UINT 507 #define CK_F_PR_BTC_UINT 508 CK_PR_BTX_S(btc, uint, unsigned int, ^,) 509 #endif /* CK_F_PR_BTC_UINT */ 510 511 #ifndef CK_F_PR_BTR_UINT 512 #define CK_F_PR_BTR_UINT 513 CK_PR_BTX_S(btr, uint, unsigned int, &, ~) 514 #endif /* CK_F_PR_BTR_UINT */ 515 516 #ifndef CK_F_PR_BTS_UINT 517 #define CK_F_PR_BTS_UINT 518 CK_PR_BTX_S(bts, uint, unsigned int, |,) 519 #endif /* CK_F_PR_BTS_UINT */ 520 521 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ 522 523 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) 524 525 #ifndef CK_F_PR_BTC_PTR 526 #define CK_F_PR_BTC_PTR 527 CK_PR_BTX(btc, ptr, void, uintptr_t, ^, void *,) 528 #endif /* CK_F_PR_BTC_PTR */ 529 530 #ifndef CK_F_PR_BTR_PTR 531 #define CK_F_PR_BTR_PTR 532 CK_PR_BTX(btr, ptr, void, uintptr_t, &, void *, ~) 533 #endif /* CK_F_PR_BTR_PTR */ 534 535 #ifndef CK_F_PR_BTS_PTR 536 #define CK_F_PR_BTS_PTR 537 CK_PR_BTX(bts, ptr, void, uintptr_t, |, void *,) 538 #endif /* CK_F_PR_BTS_PTR */ 539 540 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ 541 542 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) 543 544 #ifndef CK_F_PR_BTC_64 545 #define CK_F_PR_BTC_64 546 CK_PR_BTX_S(btc, 64, uint64_t, ^,) 547 #endif /* CK_F_PR_BTC_64 */ 548 549 #ifndef CK_F_PR_BTR_64 550 #define CK_F_PR_BTR_64 551 CK_PR_BTX_S(btr, 64, uint64_t, &, ~) 552 #endif /* CK_F_PR_BTR_64 */ 553 554 #ifndef CK_F_PR_BTS_64 555 #define CK_F_PR_BTS_64 556 CK_PR_BTX_S(bts, 64, uint64_t, |,) 557 #endif /* CK_F_PR_BTS_64 */ 558 559 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ 560 561 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) 562 563 #ifndef CK_F_PR_BTC_32 564 #define CK_F_PR_BTC_32 565 CK_PR_BTX_S(btc, 32, uint32_t, ^,) 566 #endif /* CK_F_PR_BTC_32 */ 567 568 #ifndef CK_F_PR_BTR_32 569 #define CK_F_PR_BTR_32 570 CK_PR_BTX_S(btr, 32, uint32_t, &, ~) 571 #endif /* CK_F_PR_BTR_32 */ 572 573 #ifndef CK_F_PR_BTS_32 574 #define CK_F_PR_BTS_32 575 CK_PR_BTX_S(bts, 32, uint32_t, |,) 576 #endif /* CK_F_PR_BTS_32 */ 577 578 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ 579 580 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) 581 582 #ifndef CK_F_PR_BTC_16 583 #define CK_F_PR_BTC_16 584 CK_PR_BTX_S(btc, 16, uint16_t, ^,) 585 #endif /* CK_F_PR_BTC_16 */ 586 587 #ifndef CK_F_PR_BTR_16 588 #define CK_F_PR_BTR_16 589 CK_PR_BTX_S(btr, 16, uint16_t, &, ~) 590 #endif /* CK_F_PR_BTR_16 */ 591 592 #ifndef CK_F_PR_BTS_16 593 #define CK_F_PR_BTS_16 594 CK_PR_BTX_S(bts, 16, uint16_t, |,) 595 #endif /* CK_F_PR_BTS_16 */ 596 597 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ 598 599 #undef CK_PR_BTX_S 600 #undef CK_PR_BTX 601 602 #define CK_PR_UNARY(K, X, S, M, T) \ 603 CK_CC_INLINE static void \ 604 ck_pr_##K##_##S(M *target) \ 605 { \ 606 ck_pr_##X##_##S(target, (T)1); \ 607 return; \ 608 } 609 610 #define CK_PR_UNARY_Z(K, S, M, T, P, C, Z) \ 611 CK_CC_INLINE static void \ 612 ck_pr_##K##_##S##_zero(M *target, bool *zero) \ 613 { \ 614 T previous; \ 615 C punt; \ 616 punt = (C)ck_pr_md_load_##S(target); \ 617 previous = (T)punt; \ 618 while (ck_pr_cas_##S##_value(target, \ 619 (C)previous, \ 620 (C)(previous P 1), \ 621 &previous) == false) \ 622 ck_pr_stall(); \ 623 *zero = previous == (T)Z; \ 624 return; \ 625 } 626 627 #define CK_PR_UNARY_S(K, X, S, M) CK_PR_UNARY(K, X, S, M, M) 628 #define CK_PR_UNARY_Z_S(K, S, M, P, Z) CK_PR_UNARY_Z(K, S, M, M, P, M, Z) 629 630 #if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE) 631 632 #ifndef CK_F_PR_INC_CHAR 633 #define CK_F_PR_INC_CHAR 634 CK_PR_UNARY_S(inc, add, char, char) 635 #endif /* CK_F_PR_INC_CHAR */ 636 637 #ifndef CK_F_PR_INC_CHAR_ZERO 638 #define CK_F_PR_INC_CHAR_ZERO 639 CK_PR_UNARY_Z_S(inc, char, char, +, -1) 640 #endif /* CK_F_PR_INC_CHAR_ZERO */ 641 642 #ifndef CK_F_PR_DEC_CHAR 643 #define CK_F_PR_DEC_CHAR 644 CK_PR_UNARY_S(dec, sub, char, char) 645 #endif /* CK_F_PR_DEC_CHAR */ 646 647 #ifndef CK_F_PR_DEC_CHAR_ZERO 648 #define CK_F_PR_DEC_CHAR_ZERO 649 CK_PR_UNARY_Z_S(dec, char, char, -, 1) 650 #endif /* CK_F_PR_DEC_CHAR_ZERO */ 651 652 #endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */ 653 654 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) 655 656 #ifndef CK_F_PR_INC_INT 657 #define CK_F_PR_INC_INT 658 CK_PR_UNARY_S(inc, add, int, int) 659 #endif /* CK_F_PR_INC_INT */ 660 661 #ifndef CK_F_PR_INC_INT_ZERO 662 #define CK_F_PR_INC_INT_ZERO 663 CK_PR_UNARY_Z_S(inc, int, int, +, -1) 664 #endif /* CK_F_PR_INC_INT_ZERO */ 665 666 #ifndef CK_F_PR_DEC_INT 667 #define CK_F_PR_DEC_INT 668 CK_PR_UNARY_S(dec, sub, int, int) 669 #endif /* CK_F_PR_DEC_INT */ 670 671 #ifndef CK_F_PR_DEC_INT_ZERO 672 #define CK_F_PR_DEC_INT_ZERO 673 CK_PR_UNARY_Z_S(dec, int, int, -, 1) 674 #endif /* CK_F_PR_DEC_INT_ZERO */ 675 676 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ 677 678 #if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) 679 680 #ifndef CK_F_PR_INC_DOUBLE 681 #define CK_F_PR_INC_DOUBLE 682 CK_PR_UNARY_S(inc, add, double, double) 683 #endif /* CK_F_PR_INC_DOUBLE */ 684 685 #ifndef CK_F_PR_DEC_DOUBLE 686 #define CK_F_PR_DEC_DOUBLE 687 CK_PR_UNARY_S(dec, sub, double, double) 688 #endif /* CK_F_PR_DEC_DOUBLE */ 689 690 #endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE */ 691 692 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) 693 694 #ifndef CK_F_PR_INC_UINT 695 #define CK_F_PR_INC_UINT 696 CK_PR_UNARY_S(inc, add, uint, unsigned int) 697 #endif /* CK_F_PR_INC_UINT */ 698 699 #ifndef CK_F_PR_INC_UINT_ZERO 700 #define CK_F_PR_INC_UINT_ZERO 701 CK_PR_UNARY_Z_S(inc, uint, unsigned int, +, UINT_MAX) 702 #endif /* CK_F_PR_INC_UINT_ZERO */ 703 704 #ifndef CK_F_PR_DEC_UINT 705 #define CK_F_PR_DEC_UINT 706 CK_PR_UNARY_S(dec, sub, uint, unsigned int) 707 #endif /* CK_F_PR_DEC_UINT */ 708 709 #ifndef CK_F_PR_DEC_UINT_ZERO 710 #define CK_F_PR_DEC_UINT_ZERO 711 CK_PR_UNARY_Z_S(dec, uint, unsigned int, -, 1) 712 #endif /* CK_F_PR_DEC_UINT_ZERO */ 713 714 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ 715 716 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) 717 718 #ifndef CK_F_PR_INC_PTR 719 #define CK_F_PR_INC_PTR 720 CK_PR_UNARY(inc, add, ptr, void, uintptr_t) 721 #endif /* CK_F_PR_INC_PTR */ 722 723 #ifndef CK_F_PR_INC_PTR_ZERO 724 #define CK_F_PR_INC_PTR_ZERO 725 CK_PR_UNARY_Z(inc, ptr, void, uintptr_t, +, void *, UINT_MAX) 726 #endif /* CK_F_PR_INC_PTR_ZERO */ 727 728 #ifndef CK_F_PR_DEC_PTR 729 #define CK_F_PR_DEC_PTR 730 CK_PR_UNARY(dec, sub, ptr, void, uintptr_t) 731 #endif /* CK_F_PR_DEC_PTR */ 732 733 #ifndef CK_F_PR_DEC_PTR_ZERO 734 #define CK_F_PR_DEC_PTR_ZERO 735 CK_PR_UNARY_Z(dec, ptr, void, uintptr_t, -, void *, 1) 736 #endif /* CK_F_PR_DEC_PTR_ZERO */ 737 738 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ 739 740 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) 741 742 #ifndef CK_F_PR_INC_64 743 #define CK_F_PR_INC_64 744 CK_PR_UNARY_S(inc, add, 64, uint64_t) 745 #endif /* CK_F_PR_INC_64 */ 746 747 #ifndef CK_F_PR_INC_64_ZERO 748 #define CK_F_PR_INC_64_ZERO 749 CK_PR_UNARY_Z_S(inc, 64, uint64_t, +, UINT64_MAX) 750 #endif /* CK_F_PR_INC_64_ZERO */ 751 752 #ifndef CK_F_PR_DEC_64 753 #define CK_F_PR_DEC_64 754 CK_PR_UNARY_S(dec, sub, 64, uint64_t) 755 #endif /* CK_F_PR_DEC_64 */ 756 757 #ifndef CK_F_PR_DEC_64_ZERO 758 #define CK_F_PR_DEC_64_ZERO 759 CK_PR_UNARY_Z_S(dec, 64, uint64_t, -, 1) 760 #endif /* CK_F_PR_DEC_64_ZERO */ 761 762 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ 763 764 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) 765 766 #ifndef CK_F_PR_INC_32 767 #define CK_F_PR_INC_32 768 CK_PR_UNARY_S(inc, add, 32, uint32_t) 769 #endif /* CK_F_PR_INC_32 */ 770 771 #ifndef CK_F_PR_INC_32_ZERO 772 #define CK_F_PR_INC_32_ZERO 773 CK_PR_UNARY_Z_S(inc, 32, uint32_t, +, UINT32_MAX) 774 #endif /* CK_F_PR_INC_32_ZERO */ 775 776 #ifndef CK_F_PR_DEC_32 777 #define CK_F_PR_DEC_32 778 CK_PR_UNARY_S(dec, sub, 32, uint32_t) 779 #endif /* CK_F_PR_DEC_32 */ 780 781 #ifndef CK_F_PR_DEC_32_ZERO 782 #define CK_F_PR_DEC_32_ZERO 783 CK_PR_UNARY_Z_S(dec, 32, uint32_t, -, 1) 784 #endif /* CK_F_PR_DEC_32_ZERO */ 785 786 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ 787 788 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) 789 790 #ifndef CK_F_PR_INC_16 791 #define CK_F_PR_INC_16 792 CK_PR_UNARY_S(inc, add, 16, uint16_t) 793 #endif /* CK_F_PR_INC_16 */ 794 795 #ifndef CK_F_PR_INC_16_ZERO 796 #define CK_F_PR_INC_16_ZERO 797 CK_PR_UNARY_Z_S(inc, 16, uint16_t, +, UINT16_MAX) 798 #endif /* CK_F_PR_INC_16_ZERO */ 799 800 #ifndef CK_F_PR_DEC_16 801 #define CK_F_PR_DEC_16 802 CK_PR_UNARY_S(dec, sub, 16, uint16_t) 803 #endif /* CK_F_PR_DEC_16 */ 804 805 #ifndef CK_F_PR_DEC_16_ZERO 806 #define CK_F_PR_DEC_16_ZERO 807 CK_PR_UNARY_Z_S(dec, 16, uint16_t, -, 1) 808 #endif /* CK_F_PR_DEC_16_ZERO */ 809 810 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ 811 812 #if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE) 813 814 #ifndef CK_F_PR_INC_8 815 #define CK_F_PR_INC_8 816 CK_PR_UNARY_S(inc, add, 8, uint8_t) 817 #endif /* CK_F_PR_INC_8 */ 818 819 #ifndef CK_F_PR_INC_8_ZERO 820 #define CK_F_PR_INC_8_ZERO 821 CK_PR_UNARY_Z_S(inc, 8, uint8_t, +, UINT8_MAX) 822 #endif /* CK_F_PR_INC_8_ZERO */ 823 824 #ifndef CK_F_PR_DEC_8 825 #define CK_F_PR_DEC_8 826 CK_PR_UNARY_S(dec, sub, 8, uint8_t) 827 #endif /* CK_F_PR_DEC_8 */ 828 829 #ifndef CK_F_PR_DEC_8_ZERO 830 #define CK_F_PR_DEC_8_ZERO 831 CK_PR_UNARY_Z_S(dec, 8, uint8_t, -, 1) 832 #endif /* CK_F_PR_DEC_8_ZERO */ 833 834 #endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */ 835 836 #undef CK_PR_UNARY_Z_S 837 #undef CK_PR_UNARY_S 838 #undef CK_PR_UNARY_Z 839 #undef CK_PR_UNARY 840 841 #define CK_PR_N(K, S, M, T, P, C) \ 842 CK_CC_INLINE static void \ 843 ck_pr_##K##_##S(M *target) \ 844 { \ 845 T previous; \ 846 C punt; \ 847 punt = (C)ck_pr_md_load_##S(target); \ 848 previous = (T)punt; \ 849 while (ck_pr_cas_##S##_value(target, \ 850 (C)previous, \ 851 (C)(P previous), \ 852 &previous) == false) \ 853 ck_pr_stall(); \ 854 \ 855 return; \ 856 } 857 858 #define CK_PR_N_Z(S, M, T, C) \ 859 CK_CC_INLINE static void \ 860 ck_pr_neg_##S##_zero(M *target, bool *zero) \ 861 { \ 862 T previous; \ 863 C punt; \ 864 punt = (C)ck_pr_md_load_##S(target); \ 865 previous = (T)punt; \ 866 while (ck_pr_cas_##S##_value(target, \ 867 (C)previous, \ 868 (C)(-previous), \ 869 &previous) == false) \ 870 ck_pr_stall(); \ 871 \ 872 *zero = previous == 0; \ 873 return; \ 874 } 875 876 #define CK_PR_N_S(K, S, M, P) CK_PR_N(K, S, M, M, P, M) 877 #define CK_PR_N_Z_S(S, M) CK_PR_N_Z(S, M, M, M) 878 879 #if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE) 880 881 #ifndef CK_F_PR_NOT_CHAR 882 #define CK_F_PR_NOT_CHAR 883 CK_PR_N_S(not, char, char, ~) 884 #endif /* CK_F_PR_NOT_CHAR */ 885 886 #ifndef CK_F_PR_NEG_CHAR 887 #define CK_F_PR_NEG_CHAR 888 CK_PR_N_S(neg, char, char, -) 889 #endif /* CK_F_PR_NEG_CHAR */ 890 891 #ifndef CK_F_PR_NEG_CHAR_ZERO 892 #define CK_F_PR_NEG_CHAR_ZERO 893 CK_PR_N_Z_S(char, char) 894 #endif /* CK_F_PR_NEG_CHAR_ZERO */ 895 896 #endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */ 897 898 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) 899 900 #ifndef CK_F_PR_NOT_INT 901 #define CK_F_PR_NOT_INT 902 CK_PR_N_S(not, int, int, ~) 903 #endif /* CK_F_PR_NOT_INT */ 904 905 #ifndef CK_F_PR_NEG_INT 906 #define CK_F_PR_NEG_INT 907 CK_PR_N_S(neg, int, int, -) 908 #endif /* CK_F_PR_NEG_INT */ 909 910 #ifndef CK_F_PR_NEG_INT_ZERO 911 #define CK_F_PR_NEG_INT_ZERO 912 CK_PR_N_Z_S(int, int) 913 #endif /* CK_F_PR_NEG_INT_ZERO */ 914 915 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ 916 917 #if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) 918 919 #ifndef CK_F_PR_NEG_DOUBLE 920 #define CK_F_PR_NEG_DOUBLE 921 CK_PR_N_S(neg, double, double, -) 922 #endif /* CK_F_PR_NEG_DOUBLE */ 923 924 #endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE */ 925 926 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) 927 928 #ifndef CK_F_PR_NOT_UINT 929 #define CK_F_PR_NOT_UINT 930 CK_PR_N_S(not, uint, unsigned int, ~) 931 #endif /* CK_F_PR_NOT_UINT */ 932 933 #ifndef CK_F_PR_NEG_UINT 934 #define CK_F_PR_NEG_UINT 935 CK_PR_N_S(neg, uint, unsigned int, -) 936 #endif /* CK_F_PR_NEG_UINT */ 937 938 #ifndef CK_F_PR_NEG_UINT_ZERO 939 #define CK_F_PR_NEG_UINT_ZERO 940 CK_PR_N_Z_S(uint, unsigned int) 941 #endif /* CK_F_PR_NEG_UINT_ZERO */ 942 943 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ 944 945 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) 946 947 #ifndef CK_F_PR_NOT_PTR 948 #define CK_F_PR_NOT_PTR 949 CK_PR_N(not, ptr, void, uintptr_t, ~, void *) 950 #endif /* CK_F_PR_NOT_PTR */ 951 952 #ifndef CK_F_PR_NEG_PTR 953 #define CK_F_PR_NEG_PTR 954 CK_PR_N(neg, ptr, void, uintptr_t, -, void *) 955 #endif /* CK_F_PR_NEG_PTR */ 956 957 #ifndef CK_F_PR_NEG_PTR_ZERO 958 #define CK_F_PR_NEG_PTR_ZERO 959 CK_PR_N_Z(ptr, void, uintptr_t, void *) 960 #endif /* CK_F_PR_NEG_PTR_ZERO */ 961 962 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ 963 964 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) 965 966 #ifndef CK_F_PR_NOT_64 967 #define CK_F_PR_NOT_64 968 CK_PR_N_S(not, 64, uint64_t, ~) 969 #endif /* CK_F_PR_NOT_64 */ 970 971 #ifndef CK_F_PR_NEG_64 972 #define CK_F_PR_NEG_64 973 CK_PR_N_S(neg, 64, uint64_t, -) 974 #endif /* CK_F_PR_NEG_64 */ 975 976 #ifndef CK_F_PR_NEG_64_ZERO 977 #define CK_F_PR_NEG_64_ZERO 978 CK_PR_N_Z_S(64, uint64_t) 979 #endif /* CK_F_PR_NEG_64_ZERO */ 980 981 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ 982 983 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) 984 985 #ifndef CK_F_PR_NOT_32 986 #define CK_F_PR_NOT_32 987 CK_PR_N_S(not, 32, uint32_t, ~) 988 #endif /* CK_F_PR_NOT_32 */ 989 990 #ifndef CK_F_PR_NEG_32 991 #define CK_F_PR_NEG_32 992 CK_PR_N_S(neg, 32, uint32_t, -) 993 #endif /* CK_F_PR_NEG_32 */ 994 995 #ifndef CK_F_PR_NEG_32_ZERO 996 #define CK_F_PR_NEG_32_ZERO 997 CK_PR_N_Z_S(32, uint32_t) 998 #endif /* CK_F_PR_NEG_32_ZERO */ 999 1000 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ 1001 1002 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) 1003 1004 #ifndef CK_F_PR_NOT_16 1005 #define CK_F_PR_NOT_16 1006 CK_PR_N_S(not, 16, uint16_t, ~) 1007 #endif /* CK_F_PR_NOT_16 */ 1008 1009 #ifndef CK_F_PR_NEG_16 1010 #define CK_F_PR_NEG_16 1011 CK_PR_N_S(neg, 16, uint16_t, -) 1012 #endif /* CK_F_PR_NEG_16 */ 1013 1014 #ifndef CK_F_PR_NEG_16_ZERO 1015 #define CK_F_PR_NEG_16_ZERO 1016 CK_PR_N_Z_S(16, uint16_t) 1017 #endif /* CK_F_PR_NEG_16_ZERO */ 1018 1019 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ 1020 1021 #if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE) 1022 1023 #ifndef CK_F_PR_NOT_8 1024 #define CK_F_PR_NOT_8 1025 CK_PR_N_S(not, 8, uint8_t, ~) 1026 #endif /* CK_F_PR_NOT_8 */ 1027 1028 #ifndef CK_F_PR_NEG_8 1029 #define CK_F_PR_NEG_8 1030 CK_PR_N_S(neg, 8, uint8_t, -) 1031 #endif /* CK_F_PR_NEG_8 */ 1032 1033 #ifndef CK_F_PR_NEG_8_ZERO 1034 #define CK_F_PR_NEG_8_ZERO 1035 CK_PR_N_Z_S(8, uint8_t) 1036 #endif /* CK_F_PR_NEG_8_ZERO */ 1037 1038 #endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */ 1039 1040 #undef CK_PR_N_Z_S 1041 #undef CK_PR_N_S 1042 #undef CK_PR_N_Z 1043 #undef CK_PR_N 1044 1045 #define CK_PR_FAA(S, M, T, C) \ 1046 CK_CC_INLINE static C \ 1047 ck_pr_faa_##S(M *target, T delta) \ 1048 { \ 1049 T previous; \ 1050 C punt; \ 1051 punt = (C)ck_pr_md_load_##S(target); \ 1052 previous = (T)punt; \ 1053 while (ck_pr_cas_##S##_value(target, \ 1054 (C)previous, \ 1055 (C)(previous + delta), \ 1056 &previous) == false) \ 1057 ck_pr_stall(); \ 1058 \ 1059 return ((C)previous); \ 1060 } 1061 1062 #define CK_PR_FAS(S, M, C) \ 1063 CK_CC_INLINE static C \ 1064 ck_pr_fas_##S(M *target, C update) \ 1065 { \ 1066 C previous; \ 1067 previous = ck_pr_md_load_##S(target); \ 1068 while (ck_pr_cas_##S##_value(target, \ 1069 previous, \ 1070 update, \ 1071 &previous) == false) \ 1072 ck_pr_stall(); \ 1073 \ 1074 return (previous); \ 1075 } 1076 1077 #define CK_PR_FAA_S(S, M) CK_PR_FAA(S, M, M, M) 1078 #define CK_PR_FAS_S(S, M) CK_PR_FAS(S, M, M) 1079 1080 #if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE) 1081 1082 #ifndef CK_F_PR_FAA_CHAR 1083 #define CK_F_PR_FAA_CHAR 1084 CK_PR_FAA_S(char, char) 1085 #endif /* CK_F_PR_FAA_CHAR */ 1086 1087 #ifndef CK_F_PR_FAS_CHAR 1088 #define CK_F_PR_FAS_CHAR 1089 CK_PR_FAS_S(char, char) 1090 #endif /* CK_F_PR_FAS_CHAR */ 1091 1092 #endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */ 1093 1094 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) 1095 1096 #ifndef CK_F_PR_FAA_INT 1097 #define CK_F_PR_FAA_INT 1098 CK_PR_FAA_S(int, int) 1099 #endif /* CK_F_PR_FAA_INT */ 1100 1101 #ifndef CK_F_PR_FAS_INT 1102 #define CK_F_PR_FAS_INT 1103 CK_PR_FAS_S(int, int) 1104 #endif /* CK_F_PR_FAS_INT */ 1105 1106 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ 1107 1108 #if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) 1109 1110 #ifndef CK_F_PR_FAA_DOUBLE 1111 #define CK_F_PR_FAA_DOUBLE 1112 CK_PR_FAA_S(double, double) 1113 #endif /* CK_F_PR_FAA_DOUBLE */ 1114 1115 #ifndef CK_F_PR_FAS_DOUBLE 1116 #define CK_F_PR_FAS_DOUBLE 1117 CK_PR_FAS_S(double, double) 1118 #endif /* CK_F_PR_FAS_DOUBLE */ 1119 1120 #endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE */ 1121 1122 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) 1123 1124 #ifndef CK_F_PR_FAA_UINT 1125 #define CK_F_PR_FAA_UINT 1126 CK_PR_FAA_S(uint, unsigned int) 1127 #endif /* CK_F_PR_FAA_UINT */ 1128 1129 #ifndef CK_F_PR_FAS_UINT 1130 #define CK_F_PR_FAS_UINT 1131 CK_PR_FAS_S(uint, unsigned int) 1132 #endif /* CK_F_PR_FAS_UINT */ 1133 1134 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ 1135 1136 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) 1137 1138 #ifndef CK_F_PR_FAA_PTR 1139 #define CK_F_PR_FAA_PTR 1140 CK_PR_FAA(ptr, void, uintptr_t, void *) 1141 #endif /* CK_F_PR_FAA_PTR */ 1142 1143 #ifndef CK_F_PR_FAS_PTR 1144 #define CK_F_PR_FAS_PTR 1145 CK_PR_FAS(ptr, void, void *) 1146 #endif /* CK_F_PR_FAS_PTR */ 1147 1148 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ 1149 1150 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) 1151 1152 #ifndef CK_F_PR_FAA_64 1153 #define CK_F_PR_FAA_64 1154 CK_PR_FAA_S(64, uint64_t) 1155 #endif /* CK_F_PR_FAA_64 */ 1156 1157 #ifndef CK_F_PR_FAS_64 1158 #define CK_F_PR_FAS_64 1159 CK_PR_FAS_S(64, uint64_t) 1160 #endif /* CK_F_PR_FAS_64 */ 1161 1162 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ 1163 1164 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) 1165 1166 #ifndef CK_F_PR_FAA_32 1167 #define CK_F_PR_FAA_32 1168 CK_PR_FAA_S(32, uint32_t) 1169 #endif /* CK_F_PR_FAA_32 */ 1170 1171 #ifndef CK_F_PR_FAS_32 1172 #define CK_F_PR_FAS_32 1173 CK_PR_FAS_S(32, uint32_t) 1174 #endif /* CK_F_PR_FAS_32 */ 1175 1176 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ 1177 1178 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) 1179 1180 #ifndef CK_F_PR_FAA_16 1181 #define CK_F_PR_FAA_16 1182 CK_PR_FAA_S(16, uint16_t) 1183 #endif /* CK_F_PR_FAA_16 */ 1184 1185 #ifndef CK_F_PR_FAS_16 1186 #define CK_F_PR_FAS_16 1187 CK_PR_FAS_S(16, uint16_t) 1188 #endif /* CK_F_PR_FAS_16 */ 1189 1190 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ 1191 1192 #if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE) 1193 1194 #ifndef CK_F_PR_FAA_8 1195 #define CK_F_PR_FAA_8 1196 CK_PR_FAA_S(8, uint8_t) 1197 #endif /* CK_F_PR_FAA_8 */ 1198 1199 #ifndef CK_F_PR_FAS_8 1200 #define CK_F_PR_FAS_8 1201 CK_PR_FAS_S(8, uint8_t) 1202 #endif /* CK_F_PR_FAS_8 */ 1203 1204 #endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */ 1205 1206 #undef CK_PR_FAA_S 1207 #undef CK_PR_FAS_S 1208 #undef CK_PR_FAA 1209 #undef CK_PR_FAS 1210 1211 #endif /* CK_PR_H */ 1212