1 /* 2 * Copyright 2016-2026 The OpenSSL Project Authors. All Rights Reserved. 3 * 4 * Licensed under the Apache License 2.0 (the "License"). You may not use 5 * this file except in compliance with the License. You can obtain a copy 6 * in the file LICENSE in the source distribution or at 7 * https://www.openssl.org/source/license.html 8 */ 9 10 /* We need to use the OPENSSL_fork_*() deprecated APIs */ 11 #define OPENSSL_SUPPRESS_DEPRECATED 12 13 #include <openssl/crypto.h> 14 #include <crypto/cryptlib.h> 15 #include "internal/cryptlib.h" 16 #include "internal/rcu.h" 17 #include "rcu_internal.h" 18 19 #if defined(__clang__) && defined(__has_feature) 20 #if __has_feature(thread_sanitizer) 21 #define __SANITIZE_THREAD__ 22 #endif 23 #endif 24 25 #if defined(__SANITIZE_THREAD__) 26 #include <sanitizer/tsan_interface.h> 27 #define TSAN_FAKE_UNLOCK(x) \ 28 __tsan_mutex_pre_unlock((x), 0); \ 29 __tsan_mutex_post_unlock((x), 0) 30 31 #define TSAN_FAKE_LOCK(x) \ 32 __tsan_mutex_pre_lock((x), 0); \ 33 __tsan_mutex_post_lock((x), 0, 0) 34 #else 35 #define TSAN_FAKE_UNLOCK(x) 36 #define TSAN_FAKE_LOCK(x) 37 #endif 38 39 #if defined(__sun) 40 #include <atomic.h> 41 #endif 42 43 #if defined(__apple_build_version__) && __apple_build_version__ < 6000000 44 /* 45 * OS/X 10.7 and 10.8 had a weird version of clang which has __ATOMIC_ACQUIRE and 46 * __ATOMIC_ACQ_REL but which expects only one parameter for __atomic_is_lock_free() 47 * rather than two which has signature __atomic_is_lock_free(sizeof(_Atomic(T))). 48 * All of this makes impossible to use __atomic_is_lock_free here. 49 * 50 * See: https://github.com/llvm/llvm-project/commit/a4c2602b714e6c6edb98164550a5ae829b2de760 51 */ 52 #define BROKEN_CLANG_ATOMICS 53 #endif 54 55 #if defined(OPENSSL_THREADS) && !defined(CRYPTO_TDEBUG) && !defined(OPENSSL_SYS_WINDOWS) 56 57 #if defined(OPENSSL_SYS_UNIX) 58 #include <sys/types.h> 59 #include <unistd.h> 60 #endif 61 62 #include <assert.h> 63 64 /* 65 * The Non-Stop KLT thread model currently seems broken in its rwlock 66 * implementation 67 * Likewise is there a problem with the glibc implementation on riscv. 68 */ 69 #if defined(PTHREAD_RWLOCK_INITIALIZER) && !defined(_KLT_MODEL_) \ 70 && !defined(__riscv) 71 #define USE_RWLOCK 72 #endif 73 74 /* 75 * For all GNU/clang atomic builtins, we also need fallbacks, to cover all 76 * other compilers. 77 78 * Unfortunately, we can't do that with some "generic type", because there's no 79 * guarantee that the chosen generic type is large enough to cover all cases. 80 * Therefore, we implement fallbacks for each applicable type, with composed 81 * names that include the type they handle. 82 * 83 * (an anecdote: we previously tried to use |void *| as the generic type, with 84 * the thought that the pointer itself is the largest type. However, this is 85 * not true on 32-bit pointer platforms, as a |uint64_t| is twice as large) 86 * 87 * All applicable ATOMIC_ macros take the intended type as first parameter, so 88 * they can map to the correct fallback function. In the GNU/clang case, that 89 * parameter is simply ignored. 90 */ 91 92 /* 93 * Internal types used with the ATOMIC_ macros, to make it possible to compose 94 * fallback function names. 95 */ 96 typedef void *pvoid; 97 98 #if defined(__GNUC__) && defined(__ATOMIC_ACQUIRE) && !defined(BROKEN_CLANG_ATOMICS) \ 99 && !defined(USE_ATOMIC_FALLBACKS) 100 #define ATOMIC_LOAD_N(t, p, o) __atomic_load_n(p, o) 101 #define ATOMIC_STORE_N(t, p, v, o) __atomic_store_n(p, v, o) 102 #define ATOMIC_STORE(t, p, v, o) __atomic_store(p, v, o) 103 #define ATOMIC_ADD_FETCH(p, v, o) __atomic_add_fetch(p, v, o) 104 #define ATOMIC_SUB_FETCH(p, v, o) __atomic_sub_fetch(p, v, o) 105 #else 106 static pthread_mutex_t atomic_sim_lock = PTHREAD_MUTEX_INITIALIZER; 107 108 #define IMPL_fallback_atomic_load_n(t) \ 109 static ossl_inline t fallback_atomic_load_n_##t(t *p) \ 110 { \ 111 t ret; \ 112 \ 113 pthread_mutex_lock(&atomic_sim_lock); \ 114 ret = *p; \ 115 pthread_mutex_unlock(&atomic_sim_lock); \ 116 return ret; \ 117 } 118 IMPL_fallback_atomic_load_n(uint32_t) 119 IMPL_fallback_atomic_load_n(uint64_t) 120 IMPL_fallback_atomic_load_n(pvoid) 121 122 #define ATOMIC_LOAD_N(t, p, o) fallback_atomic_load_n_##t(p) 123 124 #define IMPL_fallback_atomic_store_n(t) \ 125 static ossl_inline t fallback_atomic_store_n_##t(t *p, t v) \ 126 { \ 127 t ret; \ 128 \ 129 pthread_mutex_lock(&atomic_sim_lock); \ 130 ret = *p; \ 131 *p = v; \ 132 pthread_mutex_unlock(&atomic_sim_lock); \ 133 return ret; \ 134 } 135 IMPL_fallback_atomic_store_n(uint32_t) 136 137 #define ATOMIC_STORE_N(t, p, v, o) fallback_atomic_store_n_##t(p, v) 138 139 #define IMPL_fallback_atomic_store(t) \ 140 static ossl_inline void fallback_atomic_store_##t(t *p, t *v) \ 141 { \ 142 pthread_mutex_lock(&atomic_sim_lock); \ 143 *p = *v; \ 144 pthread_mutex_unlock(&atomic_sim_lock); \ 145 } 146 IMPL_fallback_atomic_store(pvoid) 147 148 #define ATOMIC_STORE(t, p, v, o) fallback_atomic_store_##t(p, v) 149 150 /* 151 * The fallbacks that follow don't need any per type implementation, as 152 * they are designed for uint64_t only. If there comes a time when multiple 153 * types need to be covered, it's relatively easy to refactor them the same 154 * way as the fallbacks above. 155 */ 156 157 static ossl_inline uint64_t fallback_atomic_add_fetch(uint64_t *p, uint64_t v) 158 { 159 uint64_t ret; 160 161 pthread_mutex_lock(&atomic_sim_lock); 162 *p += v; 163 ret = *p; 164 pthread_mutex_unlock(&atomic_sim_lock); 165 return ret; 166 } 167 168 #define ATOMIC_ADD_FETCH(p, v, o) fallback_atomic_add_fetch(p, v) 169 170 static ossl_inline uint64_t fallback_atomic_sub_fetch(uint64_t *p, uint64_t v) 171 { 172 uint64_t ret; 173 174 pthread_mutex_lock(&atomic_sim_lock); 175 *p -= v; 176 ret = *p; 177 pthread_mutex_unlock(&atomic_sim_lock); 178 return ret; 179 } 180 181 #define ATOMIC_SUB_FETCH(p, v, o) fallback_atomic_sub_fetch(p, v) 182 #endif 183 184 /* 185 * This is the core of an rcu lock. It tracks the readers and writers for the 186 * current quiescence point for a given lock. Users is the 64 bit value that 187 * stores the READERS/ID as defined above 188 * 189 */ 190 struct rcu_qp { 191 uint64_t users; 192 }; 193 194 struct thread_qp { 195 struct rcu_qp *qp; 196 unsigned int depth; 197 CRYPTO_RCU_LOCK *lock; 198 }; 199 200 #define MAX_QPS 10 201 /* 202 * This is the per thread tracking data 203 * that is assigned to each thread participating 204 * in an rcu qp 205 * 206 * qp points to the qp that it last acquired 207 * 208 */ 209 struct rcu_thr_data { 210 struct thread_qp thread_qps[MAX_QPS]; 211 }; 212 213 /* 214 * This is the internal version of a CRYPTO_RCU_LOCK 215 * it is cast from CRYPTO_RCU_LOCK 216 */ 217 struct rcu_lock_st { 218 /* Callbacks to call for next ossl_synchronize_rcu */ 219 struct rcu_cb_item *cb_items; 220 221 /* The context we are being created against */ 222 OSSL_LIB_CTX *ctx; 223 224 /* Array of quiescent points for synchronization */ 225 struct rcu_qp *qp_group; 226 227 /* rcu generation counter for in-order retirement */ 228 uint32_t id_ctr; 229 230 /* Number of elements in qp_group array */ 231 uint32_t group_count; 232 233 /* Index of the current qp in the qp_group array */ 234 uint32_t reader_idx; 235 236 /* value of the next id_ctr value to be retired */ 237 uint32_t next_to_retire; 238 239 /* index of the next free rcu_qp in the qp_group */ 240 uint32_t current_alloc_idx; 241 242 /* number of qp's in qp_group array currently being retired */ 243 uint32_t writers_alloced; 244 245 /* lock protecting write side operations */ 246 pthread_mutex_t write_lock; 247 248 /* lock protecting updates to writers_alloced/current_alloc_idx */ 249 pthread_mutex_t alloc_lock; 250 251 /* signal to wake threads waiting on alloc_lock */ 252 pthread_cond_t alloc_signal; 253 254 /* lock to enforce in-order retirement */ 255 pthread_mutex_t prior_lock; 256 257 /* signal to wake threads waiting on prior_lock */ 258 pthread_cond_t prior_signal; 259 }; 260 261 /* Read side acquisition of the current qp */ 262 static struct rcu_qp *get_hold_current_qp(struct rcu_lock_st *lock) 263 { 264 uint32_t qp_idx; 265 266 /* get the current qp index */ 267 for (;;) { 268 qp_idx = ATOMIC_LOAD_N(uint32_t, &lock->reader_idx, __ATOMIC_RELAXED); 269 270 /* 271 * Notes on use of __ATOMIC_ACQUIRE 272 * We need to ensure the following: 273 * 1) That subsequent operations aren't optimized by hoisting them above 274 * this operation. Specifically, we don't want the below re-load of 275 * qp_idx to get optimized away 276 * 2) We want to ensure that any updating of reader_idx on the write side 277 * of the lock is flushed from a local cpu cache so that we see any 278 * updates prior to the load. This is a non-issue on cache coherent 279 * systems like x86, but is relevant on other arches 280 */ 281 ATOMIC_ADD_FETCH(&lock->qp_group[qp_idx].users, (uint64_t)1, 282 __ATOMIC_ACQUIRE); 283 284 /* if the idx hasn't changed, we're good, else try again */ 285 if (qp_idx == ATOMIC_LOAD_N(uint32_t, &lock->reader_idx, __ATOMIC_ACQUIRE)) 286 break; 287 288 ATOMIC_SUB_FETCH(&lock->qp_group[qp_idx].users, (uint64_t)1, 289 __ATOMIC_RELAXED); 290 } 291 292 return &lock->qp_group[qp_idx]; 293 } 294 295 static void ossl_rcu_free_local_data(void *arg) 296 { 297 OSSL_LIB_CTX *ctx = arg; 298 CRYPTO_THREAD_LOCAL *lkey = ossl_lib_ctx_get_rcukey(ctx); 299 struct rcu_thr_data *data = CRYPTO_THREAD_get_local(lkey); 300 301 OPENSSL_free(data); 302 CRYPTO_THREAD_set_local(lkey, NULL); 303 } 304 305 void ossl_rcu_read_lock(CRYPTO_RCU_LOCK *lock) 306 { 307 struct rcu_thr_data *data; 308 int i, available_qp = -1; 309 CRYPTO_THREAD_LOCAL *lkey = ossl_lib_ctx_get_rcukey(lock->ctx); 310 311 /* 312 * we're going to access current_qp here so ask the 313 * processor to fetch it 314 */ 315 data = CRYPTO_THREAD_get_local(lkey); 316 317 if (data == NULL) { 318 data = OPENSSL_zalloc(sizeof(*data)); 319 OPENSSL_assert(data != NULL); 320 CRYPTO_THREAD_set_local(lkey, data); 321 ossl_init_thread_start(NULL, lock->ctx, ossl_rcu_free_local_data); 322 } 323 324 for (i = 0; i < MAX_QPS; i++) { 325 if (data->thread_qps[i].qp == NULL && available_qp == -1) 326 available_qp = i; 327 /* If we have a hold on this lock already, we're good */ 328 if (data->thread_qps[i].lock == lock) { 329 data->thread_qps[i].depth++; 330 return; 331 } 332 } 333 334 /* 335 * if we get here, then we don't have a hold on this lock yet 336 */ 337 assert(available_qp != -1); 338 339 data->thread_qps[available_qp].qp = get_hold_current_qp(lock); 340 data->thread_qps[available_qp].depth = 1; 341 data->thread_qps[available_qp].lock = lock; 342 } 343 344 void ossl_rcu_read_unlock(CRYPTO_RCU_LOCK *lock) 345 { 346 int i; 347 CRYPTO_THREAD_LOCAL *lkey = ossl_lib_ctx_get_rcukey(lock->ctx); 348 struct rcu_thr_data *data = CRYPTO_THREAD_get_local(lkey); 349 uint64_t ret; 350 351 assert(data != NULL); 352 353 for (i = 0; i < MAX_QPS; i++) { 354 if (data->thread_qps[i].lock == lock) { 355 /* 356 * we have to use __ATOMIC_RELEASE here 357 * to ensure that all preceding read instructions complete 358 * before the decrement is visible to ossl_synchronize_rcu 359 */ 360 data->thread_qps[i].depth--; 361 if (data->thread_qps[i].depth == 0) { 362 ret = ATOMIC_SUB_FETCH(&data->thread_qps[i].qp->users, 363 (uint64_t)1, __ATOMIC_RELEASE); 364 OPENSSL_assert(ret != UINT64_MAX); 365 data->thread_qps[i].qp = NULL; 366 data->thread_qps[i].lock = NULL; 367 } 368 return; 369 } 370 } 371 /* 372 * If we get here, we're trying to unlock a lock that we never acquired - 373 * that's fatal. 374 */ 375 assert(0); 376 } 377 378 /* 379 * Write side allocation routine to get the current qp 380 * and replace it with a new one 381 */ 382 static struct rcu_qp *update_qp(CRYPTO_RCU_LOCK *lock, uint32_t *curr_id) 383 { 384 uint32_t current_idx; 385 386 pthread_mutex_lock(&lock->alloc_lock); 387 388 /* 389 * we need at least one qp to be available with one 390 * left over, so that readers can start working on 391 * one that isn't yet being waited on 392 */ 393 while (lock->group_count - lock->writers_alloced < 2) 394 /* we have to wait for one to be free */ 395 pthread_cond_wait(&lock->alloc_signal, &lock->alloc_lock); 396 397 current_idx = lock->current_alloc_idx; 398 399 /* Allocate the qp */ 400 lock->writers_alloced++; 401 402 /* increment the allocation index */ 403 lock->current_alloc_idx = (lock->current_alloc_idx + 1) % lock->group_count; 404 405 *curr_id = lock->id_ctr; 406 lock->id_ctr++; 407 408 /* 409 * make the current state of everything visible by this release 410 * when get_hold_current_qp acquires the next qp 411 */ 412 ATOMIC_STORE_N(uint32_t, &lock->reader_idx, lock->current_alloc_idx, 413 __ATOMIC_RELEASE); 414 415 /* 416 * this should make sure that the new value of reader_idx is visible in 417 * get_hold_current_qp, directly after incrementing the users count 418 */ 419 ATOMIC_ADD_FETCH(&lock->qp_group[current_idx].users, (uint64_t)0, 420 __ATOMIC_RELEASE); 421 422 /* wake up any waiters */ 423 pthread_cond_signal(&lock->alloc_signal); 424 pthread_mutex_unlock(&lock->alloc_lock); 425 return &lock->qp_group[current_idx]; 426 } 427 428 static void retire_qp(CRYPTO_RCU_LOCK *lock, struct rcu_qp *qp) 429 { 430 pthread_mutex_lock(&lock->alloc_lock); 431 lock->writers_alloced--; 432 pthread_cond_signal(&lock->alloc_signal); 433 pthread_mutex_unlock(&lock->alloc_lock); 434 } 435 436 static struct rcu_qp *allocate_new_qp_group(CRYPTO_RCU_LOCK *lock, 437 uint32_t count) 438 { 439 struct rcu_qp *new = OPENSSL_zalloc(sizeof(*new) * count); 440 441 lock->group_count = count; 442 return new; 443 } 444 445 void ossl_rcu_write_lock(CRYPTO_RCU_LOCK *lock) 446 { 447 pthread_mutex_lock(&lock->write_lock); 448 TSAN_FAKE_UNLOCK(&lock->write_lock); 449 } 450 451 void ossl_rcu_write_unlock(CRYPTO_RCU_LOCK *lock) 452 { 453 TSAN_FAKE_LOCK(&lock->write_lock); 454 pthread_mutex_unlock(&lock->write_lock); 455 } 456 457 void ossl_synchronize_rcu(CRYPTO_RCU_LOCK *lock) 458 { 459 struct rcu_qp *qp; 460 uint64_t count; 461 uint32_t curr_id; 462 struct rcu_cb_item *cb_items, *tmpcb; 463 464 pthread_mutex_lock(&lock->write_lock); 465 cb_items = lock->cb_items; 466 lock->cb_items = NULL; 467 pthread_mutex_unlock(&lock->write_lock); 468 469 qp = update_qp(lock, &curr_id); 470 471 /* retire in order */ 472 pthread_mutex_lock(&lock->prior_lock); 473 while (lock->next_to_retire != curr_id) 474 pthread_cond_wait(&lock->prior_signal, &lock->prior_lock); 475 476 /* 477 * wait for the reader count to reach zero 478 * Note the use of __ATOMIC_ACQUIRE here to ensure that any 479 * prior __ATOMIC_RELEASE write operation in ossl_rcu_read_unlock 480 * is visible prior to our read 481 * however this is likely just necessary to silence a tsan warning 482 * because the read side should not do any write operation 483 * outside the atomic itself 484 */ 485 do { 486 count = ATOMIC_LOAD_N(uint64_t, &qp->users, __ATOMIC_ACQUIRE); 487 } while (count != (uint64_t)0); 488 489 lock->next_to_retire++; 490 pthread_cond_broadcast(&lock->prior_signal); 491 pthread_mutex_unlock(&lock->prior_lock); 492 493 retire_qp(lock, qp); 494 495 /* handle any callbacks that we have */ 496 while (cb_items != NULL) { 497 tmpcb = cb_items; 498 cb_items = cb_items->next; 499 tmpcb->fn(tmpcb->data); 500 OPENSSL_free(tmpcb); 501 } 502 } 503 504 CRYPTO_RCU_CB_ITEM *ossl_rcu_cb_item_new(void) 505 { 506 return OPENSSL_zalloc(sizeof(CRYPTO_RCU_CB_ITEM)); 507 } 508 509 void ossl_rcu_cb_item_free(CRYPTO_RCU_CB_ITEM *item) 510 { 511 OPENSSL_free(item); 512 } 513 514 /* 515 * Note: This call assumes its made under the protection of 516 * ossl_rcu_write_lock 517 */ 518 void ossl_rcu_call(CRYPTO_RCU_LOCK *lock, CRYPTO_RCU_CB_ITEM *item, 519 rcu_cb_fn cb, void *data) 520 { 521 item->fn = cb; 522 item->data = data; 523 item->next = lock->cb_items; 524 lock->cb_items = item; 525 } 526 527 void *ossl_rcu_uptr_deref(void **p) 528 { 529 return ATOMIC_LOAD_N(pvoid, p, __ATOMIC_ACQUIRE); 530 } 531 532 void ossl_rcu_assign_uptr(void **p, void **v) 533 { 534 ATOMIC_STORE(pvoid, p, v, __ATOMIC_RELEASE); 535 } 536 537 CRYPTO_RCU_LOCK *ossl_rcu_lock_new(int num_writers, OSSL_LIB_CTX *ctx) 538 { 539 struct rcu_lock_st *new; 540 pthread_mutex_t *mutexes[3] = { NULL }; 541 pthread_cond_t *conds[2] = { NULL }; 542 int i; 543 544 /* 545 * We need a minimum of 2 qp's 546 */ 547 if (num_writers < 2) 548 num_writers = 2; 549 550 ctx = ossl_lib_ctx_get_concrete(ctx); 551 if (ctx == NULL) 552 return 0; 553 554 new = OPENSSL_zalloc(sizeof(*new)); 555 if (new == NULL) 556 return NULL; 557 558 new->ctx = ctx; 559 i = 0; 560 mutexes[i] = pthread_mutex_init(&new->write_lock, NULL) == 0 ? &new->write_lock : NULL; 561 if (mutexes[i++] == NULL) 562 goto err; 563 mutexes[i] = pthread_mutex_init(&new->prior_lock, NULL) == 0 ? &new->prior_lock : NULL; 564 if (mutexes[i++] == NULL) 565 goto err; 566 mutexes[i] = pthread_mutex_init(&new->alloc_lock, NULL) == 0 ? &new->alloc_lock : NULL; 567 if (mutexes[i++] == NULL) 568 goto err; 569 conds[i - 3] = pthread_cond_init(&new->prior_signal, NULL) == 0 ? &new->prior_signal : NULL; 570 if (conds[i - 3] == NULL) 571 goto err; 572 i++; 573 conds[i - 3] = pthread_cond_init(&new->alloc_signal, NULL) == 0 ? &new->alloc_signal : NULL; 574 if (conds[i - 3] == NULL) 575 goto err; 576 i++; 577 new->qp_group = allocate_new_qp_group(new, num_writers); 578 if (new->qp_group == NULL) 579 goto err; 580 581 return new; 582 583 err: 584 for (i = 0; i < 3; i++) 585 if (mutexes[i] != NULL) 586 pthread_mutex_destroy(mutexes[i]); 587 for (i = 0; i < 2; i++) 588 if (conds[i] != NULL) 589 pthread_cond_destroy(conds[i]); 590 OPENSSL_free(new->qp_group); 591 OPENSSL_free(new); 592 return NULL; 593 } 594 595 void ossl_rcu_lock_free(CRYPTO_RCU_LOCK *lock) 596 { 597 struct rcu_lock_st *rlock = (struct rcu_lock_st *)lock; 598 599 if (lock == NULL) 600 return; 601 602 /* make sure we're synchronized */ 603 ossl_synchronize_rcu(rlock); 604 605 OPENSSL_free(rlock->qp_group); 606 /* 607 * Some targets (BSD) allocate heap when initializing 608 * a mutex or condition, to prevent leaks, those need 609 * to be destroyed here 610 */ 611 pthread_mutex_destroy(&rlock->write_lock); 612 pthread_mutex_destroy(&rlock->prior_lock); 613 pthread_mutex_destroy(&rlock->alloc_lock); 614 pthread_cond_destroy(&rlock->prior_signal); 615 pthread_cond_destroy(&rlock->alloc_signal); 616 617 /* There should only be a single qp left now */ 618 OPENSSL_free(rlock); 619 } 620 621 CRYPTO_RWLOCK *CRYPTO_THREAD_lock_new(void) 622 { 623 #ifdef USE_RWLOCK 624 CRYPTO_RWLOCK *lock; 625 626 if ((lock = OPENSSL_zalloc(sizeof(pthread_rwlock_t))) == NULL) 627 /* Don't set error, to avoid recursion blowup. */ 628 return NULL; 629 630 if (pthread_rwlock_init(lock, NULL) != 0) { 631 OPENSSL_free(lock); 632 return NULL; 633 } 634 #else 635 pthread_mutexattr_t attr; 636 CRYPTO_RWLOCK *lock; 637 638 if ((lock = OPENSSL_zalloc(sizeof(pthread_mutex_t))) == NULL) 639 /* Don't set error, to avoid recursion blowup. */ 640 return NULL; 641 642 /* 643 * We don't use recursive mutexes, but try to catch errors if we do. 644 */ 645 pthread_mutexattr_init(&attr); 646 #if !defined(__TANDEM) && !defined(_SPT_MODEL_) 647 #if !defined(NDEBUG) && !defined(OPENSSL_NO_MUTEX_ERRORCHECK) 648 pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ERRORCHECK); 649 #endif 650 #else 651 /* The SPT Thread Library does not define MUTEX attributes. */ 652 #endif 653 654 if (pthread_mutex_init(lock, &attr) != 0) { 655 pthread_mutexattr_destroy(&attr); 656 OPENSSL_free(lock); 657 return NULL; 658 } 659 660 pthread_mutexattr_destroy(&attr); 661 #endif 662 663 return lock; 664 } 665 666 __owur int CRYPTO_THREAD_read_lock(CRYPTO_RWLOCK *lock) 667 { 668 #ifdef USE_RWLOCK 669 if (!ossl_assert(pthread_rwlock_rdlock(lock) == 0)) 670 return 0; 671 #else 672 if (pthread_mutex_lock(lock) != 0) { 673 assert(errno != EDEADLK && errno != EBUSY); 674 return 0; 675 } 676 #endif 677 678 return 1; 679 } 680 681 __owur int CRYPTO_THREAD_write_lock(CRYPTO_RWLOCK *lock) 682 { 683 #ifdef USE_RWLOCK 684 if (!ossl_assert(pthread_rwlock_wrlock(lock) == 0)) 685 return 0; 686 #else 687 if (pthread_mutex_lock(lock) != 0) { 688 assert(errno != EDEADLK && errno != EBUSY); 689 return 0; 690 } 691 #endif 692 693 return 1; 694 } 695 696 int CRYPTO_THREAD_unlock(CRYPTO_RWLOCK *lock) 697 { 698 #ifdef USE_RWLOCK 699 if (pthread_rwlock_unlock(lock) != 0) 700 return 0; 701 #else 702 if (pthread_mutex_unlock(lock) != 0) { 703 assert(errno != EPERM); 704 return 0; 705 } 706 #endif 707 708 return 1; 709 } 710 711 void CRYPTO_THREAD_lock_free(CRYPTO_RWLOCK *lock) 712 { 713 if (lock == NULL) 714 return; 715 716 #ifdef USE_RWLOCK 717 pthread_rwlock_destroy(lock); 718 #else 719 pthread_mutex_destroy(lock); 720 #endif 721 OPENSSL_free(lock); 722 723 return; 724 } 725 726 int CRYPTO_THREAD_run_once(CRYPTO_ONCE *once, void (*init)(void)) 727 { 728 if (pthread_once(once, init) != 0) 729 return 0; 730 731 return 1; 732 } 733 734 int ossl_thread_init_local(CRYPTO_THREAD_LOCAL *key, void (*cleanup)(void *)) 735 { 736 737 if (pthread_key_create(key, cleanup) != 0) 738 return 0; 739 740 return 1; 741 } 742 743 void *CRYPTO_THREAD_get_local(CRYPTO_THREAD_LOCAL *key) 744 { 745 return pthread_getspecific(*key); 746 } 747 748 int CRYPTO_THREAD_set_local(CRYPTO_THREAD_LOCAL *key, void *val) 749 { 750 if (pthread_setspecific(*key, val) != 0) 751 return 0; 752 753 return 1; 754 } 755 756 int CRYPTO_THREAD_cleanup_local(CRYPTO_THREAD_LOCAL *key) 757 { 758 if (pthread_key_delete(*key) != 0) 759 return 0; 760 761 return 1; 762 } 763 764 CRYPTO_THREAD_ID CRYPTO_THREAD_get_current_id(void) 765 { 766 return pthread_self(); 767 } 768 769 int CRYPTO_THREAD_compare_id(CRYPTO_THREAD_ID a, CRYPTO_THREAD_ID b) 770 { 771 return pthread_equal(a, b); 772 } 773 774 int CRYPTO_atomic_add(int *val, int amount, int *ret, CRYPTO_RWLOCK *lock) 775 { 776 #if defined(__GNUC__) && defined(__ATOMIC_ACQ_REL) && !defined(BROKEN_CLANG_ATOMICS) 777 if (__atomic_is_lock_free(sizeof(*val), val)) { 778 *ret = __atomic_add_fetch(val, amount, __ATOMIC_ACQ_REL); 779 return 1; 780 } 781 #elif defined(__sun) && (defined(__SunOS_5_10) || defined(__SunOS_5_11)) 782 /* This will work for all future Solaris versions. */ 783 if (ret != NULL) { 784 *ret = atomic_add_int_nv((volatile unsigned int *)val, amount); 785 return 1; 786 } 787 #endif 788 if (lock == NULL || !CRYPTO_THREAD_write_lock(lock)) 789 return 0; 790 791 *val += amount; 792 *ret = *val; 793 794 if (!CRYPTO_THREAD_unlock(lock)) 795 return 0; 796 797 return 1; 798 } 799 800 int CRYPTO_atomic_add64(uint64_t *val, uint64_t op, uint64_t *ret, 801 CRYPTO_RWLOCK *lock) 802 { 803 #if defined(__GNUC__) && defined(__ATOMIC_ACQ_REL) && !defined(BROKEN_CLANG_ATOMICS) 804 if (__atomic_is_lock_free(sizeof(*val), val)) { 805 *ret = __atomic_add_fetch(val, op, __ATOMIC_ACQ_REL); 806 return 1; 807 } 808 #elif defined(__sun) && (defined(__SunOS_5_10) || defined(__SunOS_5_11)) 809 /* This will work for all future Solaris versions. */ 810 if (ret != NULL) { 811 *ret = atomic_add_64_nv(val, op); 812 return 1; 813 } 814 #endif 815 if (lock == NULL || !CRYPTO_THREAD_write_lock(lock)) 816 return 0; 817 *val += op; 818 *ret = *val; 819 820 if (!CRYPTO_THREAD_unlock(lock)) 821 return 0; 822 823 return 1; 824 } 825 826 int CRYPTO_atomic_and(uint64_t *val, uint64_t op, uint64_t *ret, 827 CRYPTO_RWLOCK *lock) 828 { 829 #if defined(__GNUC__) && defined(__ATOMIC_ACQ_REL) && !defined(BROKEN_CLANG_ATOMICS) 830 if (__atomic_is_lock_free(sizeof(*val), val)) { 831 *ret = __atomic_and_fetch(val, op, __ATOMIC_ACQ_REL); 832 return 1; 833 } 834 #elif defined(__sun) && (defined(__SunOS_5_10) || defined(__SunOS_5_11)) 835 /* This will work for all future Solaris versions. */ 836 if (ret != NULL) { 837 *ret = atomic_and_64_nv(val, op); 838 return 1; 839 } 840 #endif 841 if (lock == NULL || !CRYPTO_THREAD_write_lock(lock)) 842 return 0; 843 *val &= op; 844 *ret = *val; 845 846 if (!CRYPTO_THREAD_unlock(lock)) 847 return 0; 848 849 return 1; 850 } 851 852 int CRYPTO_atomic_or(uint64_t *val, uint64_t op, uint64_t *ret, 853 CRYPTO_RWLOCK *lock) 854 { 855 #if defined(__GNUC__) && defined(__ATOMIC_ACQ_REL) && !defined(BROKEN_CLANG_ATOMICS) 856 if (__atomic_is_lock_free(sizeof(*val), val)) { 857 *ret = __atomic_or_fetch(val, op, __ATOMIC_ACQ_REL); 858 return 1; 859 } 860 #elif defined(__sun) && (defined(__SunOS_5_10) || defined(__SunOS_5_11)) 861 /* This will work for all future Solaris versions. */ 862 if (ret != NULL) { 863 *ret = atomic_or_64_nv(val, op); 864 return 1; 865 } 866 #endif 867 if (lock == NULL || !CRYPTO_THREAD_write_lock(lock)) 868 return 0; 869 *val |= op; 870 *ret = *val; 871 872 if (!CRYPTO_THREAD_unlock(lock)) 873 return 0; 874 875 return 1; 876 } 877 878 int CRYPTO_atomic_load(uint64_t *val, uint64_t *ret, CRYPTO_RWLOCK *lock) 879 { 880 #if defined(__GNUC__) && defined(__ATOMIC_ACQ_REL) && !defined(BROKEN_CLANG_ATOMICS) 881 if (__atomic_is_lock_free(sizeof(*val), val)) { 882 __atomic_load(val, ret, __ATOMIC_ACQUIRE); 883 return 1; 884 } 885 #elif defined(__sun) && (defined(__SunOS_5_10) || defined(__SunOS_5_11)) 886 /* This will work for all future Solaris versions. */ 887 if (ret != NULL) { 888 *ret = atomic_or_64_nv(val, 0); 889 return 1; 890 } 891 #endif 892 if (lock == NULL || !CRYPTO_THREAD_read_lock(lock)) 893 return 0; 894 *ret = *val; 895 if (!CRYPTO_THREAD_unlock(lock)) 896 return 0; 897 898 return 1; 899 } 900 901 int CRYPTO_atomic_store(uint64_t *dst, uint64_t val, CRYPTO_RWLOCK *lock) 902 { 903 #if defined(__GNUC__) && defined(__ATOMIC_ACQ_REL) && !defined(BROKEN_CLANG_ATOMICS) 904 if (__atomic_is_lock_free(sizeof(*dst), dst)) { 905 __atomic_store(dst, &val, __ATOMIC_RELEASE); 906 return 1; 907 } 908 #elif defined(__sun) && (defined(__SunOS_5_10) || defined(__SunOS_5_11)) 909 /* This will work for all future Solaris versions. */ 910 if (dst != NULL) { 911 atomic_swap_64(dst, val); 912 return 1; 913 } 914 #endif 915 if (lock == NULL || !CRYPTO_THREAD_write_lock(lock)) 916 return 0; 917 *dst = val; 918 if (!CRYPTO_THREAD_unlock(lock)) 919 return 0; 920 921 return 1; 922 } 923 924 int CRYPTO_atomic_load_int(int *val, int *ret, CRYPTO_RWLOCK *lock) 925 { 926 #if defined(__GNUC__) && defined(__ATOMIC_ACQ_REL) && !defined(BROKEN_CLANG_ATOMICS) 927 if (__atomic_is_lock_free(sizeof(*val), val)) { 928 __atomic_load(val, ret, __ATOMIC_ACQUIRE); 929 return 1; 930 } 931 #elif defined(__sun) && (defined(__SunOS_5_10) || defined(__SunOS_5_11)) 932 /* This will work for all future Solaris versions. */ 933 if (ret != NULL) { 934 *ret = (int)atomic_or_uint_nv((unsigned int *)val, 0); 935 return 1; 936 } 937 #endif 938 if (lock == NULL || !CRYPTO_THREAD_read_lock(lock)) 939 return 0; 940 *ret = *val; 941 if (!CRYPTO_THREAD_unlock(lock)) 942 return 0; 943 944 return 1; 945 } 946 947 #ifndef FIPS_MODULE 948 int openssl_init_fork_handlers(void) 949 { 950 return 1; 951 } 952 #endif /* FIPS_MODULE */ 953 954 int openssl_get_fork_id(void) 955 { 956 return getpid(); 957 } 958 #endif 959