1349cc55cSDimitry Andric //===----------------------------------------------------------------------===// 25ffd83dbSDimitry Andric // 35ffd83dbSDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 45ffd83dbSDimitry Andric // See https://llvm.org/LICENSE.txt for license information. 55ffd83dbSDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 65ffd83dbSDimitry Andric // 75ffd83dbSDimitry Andric //===----------------------------------------------------------------------===// 85ffd83dbSDimitry Andric 95ffd83dbSDimitry Andric #include <__config> 105ffd83dbSDimitry Andric 115ffd83dbSDimitry Andric #ifndef _LIBCPP_HAS_NO_THREADS 125ffd83dbSDimitry Andric 135ffd83dbSDimitry Andric #include <barrier> 145ffd83dbSDimitry Andric #include <thread> 155ffd83dbSDimitry Andric 165ffd83dbSDimitry Andric _LIBCPP_BEGIN_NAMESPACE_STD 175ffd83dbSDimitry Andric 18*0eae32dcSDimitry Andric #if !defined(_LIBCPP_HAS_NO_TREE_BARRIER) 195ffd83dbSDimitry Andric 205ffd83dbSDimitry Andric class __barrier_algorithm_base { 215ffd83dbSDimitry Andric public: 225ffd83dbSDimitry Andric struct alignas(64) /* naturally-align the heap state */ __state_t 235ffd83dbSDimitry Andric { 245ffd83dbSDimitry Andric struct { 25*0eae32dcSDimitry Andric __atomic_base<__barrier_phase_t> __phase{0}; 265ffd83dbSDimitry Andric } __tickets[64]; 275ffd83dbSDimitry Andric }; 285ffd83dbSDimitry Andric 295ffd83dbSDimitry Andric ptrdiff_t& __expected; 30e8d8bef9SDimitry Andric unique_ptr<__state_t[]> __state; 315ffd83dbSDimitry Andric 325ffd83dbSDimitry Andric _LIBCPP_HIDDEN 335ffd83dbSDimitry Andric __barrier_algorithm_base(ptrdiff_t& __expected) 345ffd83dbSDimitry Andric : __expected(__expected) 355ffd83dbSDimitry Andric { 365ffd83dbSDimitry Andric size_t const __count = (__expected + 1) >> 1; 37e8d8bef9SDimitry Andric __state = unique_ptr<__state_t[]>(new __state_t[__count]); 385ffd83dbSDimitry Andric } 395ffd83dbSDimitry Andric _LIBCPP_HIDDEN 405ffd83dbSDimitry Andric bool __arrive(__barrier_phase_t __old_phase) 415ffd83dbSDimitry Andric { 425ffd83dbSDimitry Andric __barrier_phase_t const __half_step = __old_phase + 1, 435ffd83dbSDimitry Andric __full_step = __old_phase + 2; 445ffd83dbSDimitry Andric size_t __current_expected = __expected, 455ffd83dbSDimitry Andric __current = hash<thread::id>()(this_thread::get_id()) % ((__expected + 1) >> 1); 465ffd83dbSDimitry Andric for(int __round = 0;; ++__round) { 475ffd83dbSDimitry Andric if(__current_expected <= 1) 485ffd83dbSDimitry Andric return true; 495ffd83dbSDimitry Andric size_t const __end_node = ((__current_expected + 1) >> 1), 505ffd83dbSDimitry Andric __last_node = __end_node - 1; 515ffd83dbSDimitry Andric for(;;++__current) { 525ffd83dbSDimitry Andric if(__current == __end_node) 535ffd83dbSDimitry Andric __current = 0; 545ffd83dbSDimitry Andric __barrier_phase_t expect = __old_phase; 555ffd83dbSDimitry Andric if(__current == __last_node && (__current_expected & 1)) 565ffd83dbSDimitry Andric { 575ffd83dbSDimitry Andric if(__state[__current].__tickets[__round].__phase.compare_exchange_strong(expect, __full_step, memory_order_acq_rel)) 585ffd83dbSDimitry Andric break; // I'm 1 in 1, go to next __round 595ffd83dbSDimitry Andric } 605ffd83dbSDimitry Andric else if(__state[__current].__tickets[__round].__phase.compare_exchange_strong(expect, __half_step, memory_order_acq_rel)) 615ffd83dbSDimitry Andric { 625ffd83dbSDimitry Andric return false; // I'm 1 in 2, done with arrival 635ffd83dbSDimitry Andric } 645ffd83dbSDimitry Andric else if(expect == __half_step) 655ffd83dbSDimitry Andric { 665ffd83dbSDimitry Andric if(__state[__current].__tickets[__round].__phase.compare_exchange_strong(expect, __full_step, memory_order_acq_rel)) 675ffd83dbSDimitry Andric break; // I'm 2 in 2, go to next __round 685ffd83dbSDimitry Andric } 695ffd83dbSDimitry Andric } 705ffd83dbSDimitry Andric __current_expected = __last_node + 1; 715ffd83dbSDimitry Andric __current >>= 1; 725ffd83dbSDimitry Andric } 735ffd83dbSDimitry Andric } 745ffd83dbSDimitry Andric }; 755ffd83dbSDimitry Andric 765ffd83dbSDimitry Andric _LIBCPP_EXPORTED_FROM_ABI 775ffd83dbSDimitry Andric __barrier_algorithm_base * __construct_barrier_algorithm_base(ptrdiff_t& __expected) 785ffd83dbSDimitry Andric { 795ffd83dbSDimitry Andric return new __barrier_algorithm_base(__expected); 805ffd83dbSDimitry Andric } 815ffd83dbSDimitry Andric _LIBCPP_EXPORTED_FROM_ABI 825ffd83dbSDimitry Andric bool __arrive_barrier_algorithm_base(__barrier_algorithm_base* __barrier, 835ffd83dbSDimitry Andric __barrier_phase_t __old_phase) 845ffd83dbSDimitry Andric { 855ffd83dbSDimitry Andric return __barrier->__arrive(__old_phase); 865ffd83dbSDimitry Andric } 875ffd83dbSDimitry Andric _LIBCPP_EXPORTED_FROM_ABI 885ffd83dbSDimitry Andric void __destroy_barrier_algorithm_base(__barrier_algorithm_base* __barrier) 895ffd83dbSDimitry Andric { 905ffd83dbSDimitry Andric delete __barrier; 915ffd83dbSDimitry Andric } 925ffd83dbSDimitry Andric 93*0eae32dcSDimitry Andric #endif //!defined(_LIBCPP_HAS_NO_TREE_BARRIER) 945ffd83dbSDimitry Andric 955ffd83dbSDimitry Andric _LIBCPP_END_NAMESPACE_STD 965ffd83dbSDimitry Andric 975ffd83dbSDimitry Andric #endif //_LIBCPP_HAS_NO_THREADS 98