1 /* 2 * kmp_wait_release.h -- Wait/Release implementation 3 */ 4 5 //===----------------------------------------------------------------------===// 6 // 7 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 8 // See https://llvm.org/LICENSE.txt for license information. 9 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 10 // 11 //===----------------------------------------------------------------------===// 12 13 #ifndef KMP_WAIT_RELEASE_H 14 #define KMP_WAIT_RELEASE_H 15 16 #include "kmp.h" 17 #include "kmp_itt.h" 18 #include "kmp_stats.h" 19 #if OMPT_SUPPORT 20 #include "ompt-specific.h" 21 #endif 22 23 /*! 24 @defgroup WAIT_RELEASE Wait/Release operations 25 26 The definitions and functions here implement the lowest level thread 27 synchronizations of suspending a thread and awaking it. They are used to build 28 higher level operations such as barriers and fork/join. 29 */ 30 31 /*! 32 @ingroup WAIT_RELEASE 33 @{ 34 */ 35 36 struct flag_properties { 37 unsigned int type : 16; 38 unsigned int reserved : 16; 39 }; 40 41 template <enum flag_type FlagType> struct flag_traits {}; 42 43 template <> struct flag_traits<flag32> { 44 typedef kmp_uint32 flag_t; 45 static const flag_type t = flag32; 46 static inline flag_t tcr(flag_t f) { return TCR_4(f); } 47 static inline flag_t test_then_add4(volatile flag_t *f) { 48 return KMP_TEST_THEN_ADD4_32(RCAST(volatile kmp_int32 *, f)); 49 } 50 static inline flag_t test_then_or(volatile flag_t *f, flag_t v) { 51 return KMP_TEST_THEN_OR32(f, v); 52 } 53 static inline flag_t test_then_and(volatile flag_t *f, flag_t v) { 54 return KMP_TEST_THEN_AND32(f, v); 55 } 56 }; 57 58 template <> struct flag_traits<atomic_flag64> { 59 typedef kmp_uint64 flag_t; 60 static const flag_type t = atomic_flag64; 61 static inline flag_t tcr(flag_t f) { return TCR_8(f); } 62 static inline flag_t test_then_add4(volatile flag_t *f) { 63 return KMP_TEST_THEN_ADD4_64(RCAST(volatile kmp_int64 *, f)); 64 } 65 static inline flag_t test_then_or(volatile flag_t *f, flag_t v) { 66 return KMP_TEST_THEN_OR64(f, v); 67 } 68 static inline flag_t test_then_and(volatile flag_t *f, flag_t v) { 69 return KMP_TEST_THEN_AND64(f, v); 70 } 71 }; 72 73 template <> struct flag_traits<flag64> { 74 typedef kmp_uint64 flag_t; 75 static const flag_type t = flag64; 76 static inline flag_t tcr(flag_t f) { return TCR_8(f); } 77 static inline flag_t test_then_add4(volatile flag_t *f) { 78 return KMP_TEST_THEN_ADD4_64(RCAST(volatile kmp_int64 *, f)); 79 } 80 static inline flag_t test_then_or(volatile flag_t *f, flag_t v) { 81 return KMP_TEST_THEN_OR64(f, v); 82 } 83 static inline flag_t test_then_and(volatile flag_t *f, flag_t v) { 84 return KMP_TEST_THEN_AND64(f, v); 85 } 86 }; 87 88 template <> struct flag_traits<flag_oncore> { 89 typedef kmp_uint64 flag_t; 90 static const flag_type t = flag_oncore; 91 static inline flag_t tcr(flag_t f) { return TCR_8(f); } 92 static inline flag_t test_then_add4(volatile flag_t *f) { 93 return KMP_TEST_THEN_ADD4_64(RCAST(volatile kmp_int64 *, f)); 94 } 95 static inline flag_t test_then_or(volatile flag_t *f, flag_t v) { 96 return KMP_TEST_THEN_OR64(f, v); 97 } 98 static inline flag_t test_then_and(volatile flag_t *f, flag_t v) { 99 return KMP_TEST_THEN_AND64(f, v); 100 } 101 }; 102 103 /*! Base class for all flags */ 104 template <flag_type FlagType> class kmp_flag { 105 protected: 106 flag_properties t; /**< "Type" of the flag in loc */ 107 kmp_info_t *waiting_threads[1]; /**< Threads sleeping on this thread. */ 108 kmp_uint32 num_waiting_threads; /**< Num threads sleeping on this thread. */ 109 std::atomic<bool> *sleepLoc; 110 111 public: 112 typedef flag_traits<FlagType> traits_type; 113 kmp_flag() : t({FlagType, 0U}), num_waiting_threads(0), sleepLoc(nullptr) {} 114 kmp_flag(int nwaiters) 115 : t({FlagType, 0U}), num_waiting_threads(nwaiters), sleepLoc(nullptr) {} 116 kmp_flag(std::atomic<bool> *sloc) 117 : t({FlagType, 0U}), num_waiting_threads(0), sleepLoc(sloc) {} 118 /*! @result the flag_type */ 119 flag_type get_type() { return (flag_type)(t.type); } 120 121 /*! param i in index into waiting_threads 122 * @result the thread that is waiting at index i */ 123 kmp_info_t *get_waiter(kmp_uint32 i) { 124 KMP_DEBUG_ASSERT(i < num_waiting_threads); 125 return waiting_threads[i]; 126 } 127 /*! @result num_waiting_threads */ 128 kmp_uint32 get_num_waiters() { return num_waiting_threads; } 129 /*! @param thr in the thread which is now waiting 130 * Insert a waiting thread at index 0. */ 131 void set_waiter(kmp_info_t *thr) { 132 waiting_threads[0] = thr; 133 num_waiting_threads = 1; 134 } 135 enum barrier_type get_bt() { return bs_last_barrier; } 136 }; 137 138 /*! Base class for wait/release volatile flag */ 139 template <typename PtrType, flag_type FlagType, bool Sleepable> 140 class kmp_flag_native : public kmp_flag<FlagType> { 141 protected: 142 volatile PtrType *loc; 143 PtrType checker; /**< When flag==checker, it has been released. */ 144 typedef flag_traits<FlagType> traits_type; 145 146 public: 147 typedef PtrType flag_t; 148 kmp_flag_native(volatile PtrType *p) : kmp_flag<FlagType>(), loc(p) {} 149 kmp_flag_native(volatile PtrType *p, kmp_info_t *thr) 150 : kmp_flag<FlagType>(1), loc(p) { 151 this->waiting_threads[0] = thr; 152 } 153 kmp_flag_native(volatile PtrType *p, PtrType c) 154 : kmp_flag<FlagType>(), loc(p), checker(c) {} 155 kmp_flag_native(volatile PtrType *p, PtrType c, std::atomic<bool> *sloc) 156 : kmp_flag<FlagType>(sloc), loc(p), checker(c) {} 157 virtual ~kmp_flag_native() {} 158 void *operator new(size_t size) { return __kmp_allocate(size); } 159 void operator delete(void *p) { __kmp_free(p); } 160 volatile PtrType *get() { return loc; } 161 void *get_void_p() { return RCAST(void *, CCAST(PtrType *, loc)); } 162 void set(volatile PtrType *new_loc) { loc = new_loc; } 163 PtrType load() { return *loc; } 164 void store(PtrType val) { *loc = val; } 165 /*! @result true if the flag object has been released. */ 166 virtual bool done_check() { 167 if (Sleepable && !(this->sleepLoc)) 168 return (traits_type::tcr(*(this->get())) & ~KMP_BARRIER_SLEEP_STATE) == 169 checker; 170 else 171 return traits_type::tcr(*(this->get())) == checker; 172 } 173 /*! @param old_loc in old value of flag 174 * @result true if the flag's old value indicates it was released. */ 175 virtual bool done_check_val(PtrType old_loc) { return old_loc == checker; } 176 /*! @result true if the flag object is not yet released. 177 * Used in __kmp_wait_template like: 178 * @code 179 * while (flag.notdone_check()) { pause(); } 180 * @endcode */ 181 virtual bool notdone_check() { 182 return traits_type::tcr(*(this->get())) != checker; 183 } 184 /*! @result Actual flag value before release was applied. 185 * Trigger all waiting threads to run by modifying flag to release state. */ 186 void internal_release() { 187 (void)traits_type::test_then_add4((volatile PtrType *)this->get()); 188 } 189 /*! @result Actual flag value before sleep bit(s) set. 190 * Notes that there is at least one thread sleeping on the flag by setting 191 * sleep bit(s). */ 192 PtrType set_sleeping() { 193 if (this->sleepLoc) { 194 this->sleepLoc->store(true); 195 return *(this->get()); 196 } 197 return traits_type::test_then_or((volatile PtrType *)this->get(), 198 KMP_BARRIER_SLEEP_STATE); 199 } 200 /*! @result Actual flag value before sleep bit(s) cleared. 201 * Notes that there are no longer threads sleeping on the flag by clearing 202 * sleep bit(s). */ 203 void unset_sleeping() { 204 if (this->sleepLoc) { 205 this->sleepLoc->store(false); 206 return; 207 } 208 traits_type::test_then_and((volatile PtrType *)this->get(), 209 ~KMP_BARRIER_SLEEP_STATE); 210 } 211 /*! @param old_loc in old value of flag 212 * Test if there are threads sleeping on the flag's old value in old_loc. */ 213 bool is_sleeping_val(PtrType old_loc) { 214 if (this->sleepLoc) 215 return this->sleepLoc->load(); 216 return old_loc & KMP_BARRIER_SLEEP_STATE; 217 } 218 /*! Test whether there are threads sleeping on the flag. */ 219 bool is_sleeping() { 220 if (this->sleepLoc) 221 return this->sleepLoc->load(); 222 return is_sleeping_val(*(this->get())); 223 } 224 bool is_any_sleeping() { 225 if (this->sleepLoc) 226 return this->sleepLoc->load(); 227 return is_sleeping_val(*(this->get())); 228 } 229 kmp_uint8 *get_stolen() { return NULL; } 230 }; 231 232 /*! Base class for wait/release atomic flag */ 233 template <typename PtrType, flag_type FlagType, bool Sleepable> 234 class kmp_flag_atomic : public kmp_flag<FlagType> { 235 protected: 236 std::atomic<PtrType> *loc; /**< Pointer to flag location to wait on */ 237 PtrType checker; /**< Flag == checker means it has been released. */ 238 public: 239 typedef flag_traits<FlagType> traits_type; 240 typedef PtrType flag_t; 241 kmp_flag_atomic(std::atomic<PtrType> *p) : kmp_flag<FlagType>(), loc(p) {} 242 kmp_flag_atomic(std::atomic<PtrType> *p, kmp_info_t *thr) 243 : kmp_flag<FlagType>(1), loc(p) { 244 this->waiting_threads[0] = thr; 245 } 246 kmp_flag_atomic(std::atomic<PtrType> *p, PtrType c) 247 : kmp_flag<FlagType>(), loc(p), checker(c) {} 248 kmp_flag_atomic(std::atomic<PtrType> *p, PtrType c, std::atomic<bool> *sloc) 249 : kmp_flag<FlagType>(sloc), loc(p), checker(c) {} 250 /*! @result the pointer to the actual flag */ 251 std::atomic<PtrType> *get() { return loc; } 252 /*! @result void* pointer to the actual flag */ 253 void *get_void_p() { return RCAST(void *, loc); } 254 /*! @param new_loc in set loc to point at new_loc */ 255 void set(std::atomic<PtrType> *new_loc) { loc = new_loc; } 256 /*! @result flag value */ 257 PtrType load() { return loc->load(std::memory_order_acquire); } 258 /*! @param val the new flag value to be stored */ 259 void store(PtrType val) { loc->store(val, std::memory_order_release); } 260 /*! @result true if the flag object has been released. */ 261 bool done_check() { 262 if (Sleepable && !(this->sleepLoc)) 263 return (this->load() & ~KMP_BARRIER_SLEEP_STATE) == checker; 264 else 265 return this->load() == checker; 266 } 267 /*! @param old_loc in old value of flag 268 * @result true if the flag's old value indicates it was released. */ 269 bool done_check_val(PtrType old_loc) { return old_loc == checker; } 270 /*! @result true if the flag object is not yet released. 271 * Used in __kmp_wait_template like: 272 * @code 273 * while (flag.notdone_check()) { pause(); } 274 * @endcode */ 275 bool notdone_check() { return this->load() != checker; } 276 /*! @result Actual flag value before release was applied. 277 * Trigger all waiting threads to run by modifying flag to release state. */ 278 void internal_release() { KMP_ATOMIC_ADD(this->get(), 4); } 279 /*! @result Actual flag value before sleep bit(s) set. 280 * Notes that there is at least one thread sleeping on the flag by setting 281 * sleep bit(s). */ 282 PtrType set_sleeping() { 283 if (this->sleepLoc) { 284 this->sleepLoc->store(true); 285 return *(this->get()); 286 } 287 return KMP_ATOMIC_OR(this->get(), KMP_BARRIER_SLEEP_STATE); 288 } 289 /*! @result Actual flag value before sleep bit(s) cleared. 290 * Notes that there are no longer threads sleeping on the flag by clearing 291 * sleep bit(s). */ 292 void unset_sleeping() { 293 if (this->sleepLoc) { 294 this->sleepLoc->store(false); 295 return; 296 } 297 KMP_ATOMIC_AND(this->get(), ~KMP_BARRIER_SLEEP_STATE); 298 } 299 /*! @param old_loc in old value of flag 300 * Test whether there are threads sleeping on flag's old value in old_loc. */ 301 bool is_sleeping_val(PtrType old_loc) { 302 if (this->sleepLoc) 303 return this->sleepLoc->load(); 304 return old_loc & KMP_BARRIER_SLEEP_STATE; 305 } 306 /*! Test whether there are threads sleeping on the flag. */ 307 bool is_sleeping() { 308 if (this->sleepLoc) 309 return this->sleepLoc->load(); 310 return is_sleeping_val(this->load()); 311 } 312 bool is_any_sleeping() { 313 if (this->sleepLoc) 314 return this->sleepLoc->load(); 315 return is_sleeping_val(this->load()); 316 } 317 kmp_uint8 *get_stolen() { return NULL; } 318 }; 319 320 #if OMPT_SUPPORT 321 OMPT_NOINLINE 322 static void __ompt_implicit_task_end(kmp_info_t *this_thr, 323 ompt_state_t ompt_state, 324 ompt_data_t *tId) { 325 int ds_tid = this_thr->th.th_info.ds.ds_tid; 326 if (ompt_state == ompt_state_wait_barrier_implicit_parallel || 327 ompt_state == ompt_state_wait_barrier_teams) { 328 this_thr->th.ompt_thread_info.state = ompt_state_overhead; 329 #if OMPT_OPTIONAL 330 void *codeptr = NULL; 331 ompt_sync_region_t sync_kind = ompt_sync_region_barrier_implicit_parallel; 332 if (this_thr->th.ompt_thread_info.parallel_flags & ompt_parallel_league) 333 sync_kind = ompt_sync_region_barrier_teams; 334 if (ompt_enabled.ompt_callback_sync_region_wait) { 335 ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)( 336 sync_kind, ompt_scope_end, NULL, tId, codeptr); 337 } 338 if (ompt_enabled.ompt_callback_sync_region) { 339 ompt_callbacks.ompt_callback(ompt_callback_sync_region)( 340 sync_kind, ompt_scope_end, NULL, tId, codeptr); 341 } 342 #endif 343 if (!KMP_MASTER_TID(ds_tid)) { 344 if (ompt_enabled.ompt_callback_implicit_task) { 345 int flags = this_thr->th.ompt_thread_info.parallel_flags; 346 flags = (flags & ompt_parallel_league) ? ompt_task_initial 347 : ompt_task_implicit; 348 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)( 349 ompt_scope_end, NULL, tId, 0, ds_tid, flags); 350 } 351 // return to idle state 352 this_thr->th.ompt_thread_info.state = ompt_state_idle; 353 } else { 354 this_thr->th.ompt_thread_info.state = ompt_state_overhead; 355 } 356 } 357 } 358 #endif 359 360 /* Spin wait loop that first does pause/yield, then sleep. A thread that calls 361 __kmp_wait_* must make certain that another thread calls __kmp_release 362 to wake it back up to prevent deadlocks! 363 364 NOTE: We may not belong to a team at this point. */ 365 template <class C, bool final_spin, bool Cancellable = false, 366 bool Sleepable = true> 367 static inline bool 368 __kmp_wait_template(kmp_info_t *this_thr, 369 C *flag USE_ITT_BUILD_ARG(void *itt_sync_obj)) { 370 #if USE_ITT_BUILD && USE_ITT_NOTIFY 371 volatile void *spin = flag->get(); 372 #endif 373 kmp_uint32 spins; 374 int th_gtid; 375 int tasks_completed = FALSE; 376 #if !KMP_USE_MONITOR 377 kmp_uint64 poll_count; 378 kmp_uint64 hibernate_goal; 379 #else 380 kmp_uint32 hibernate; 381 #endif 382 kmp_uint64 time; 383 384 KMP_FSYNC_SPIN_INIT(spin, NULL); 385 if (flag->done_check()) { 386 KMP_FSYNC_SPIN_ACQUIRED(CCAST(void *, spin)); 387 return false; 388 } 389 th_gtid = this_thr->th.th_info.ds.ds_gtid; 390 if (Cancellable) { 391 kmp_team_t *team = this_thr->th.th_team; 392 if (team && team->t.t_cancel_request == cancel_parallel) 393 return true; 394 } 395 #if KMP_OS_UNIX 396 if (final_spin) 397 KMP_ATOMIC_ST_REL(&this_thr->th.th_blocking, true); 398 #endif 399 KA_TRACE(20, 400 ("__kmp_wait_sleep: T#%d waiting for flag(%p)\n", th_gtid, flag)); 401 #if KMP_STATS_ENABLED 402 stats_state_e thread_state = KMP_GET_THREAD_STATE(); 403 #endif 404 405 /* OMPT Behavior: 406 THIS function is called from 407 __kmp_barrier (2 times) (implicit or explicit barrier in parallel regions) 408 these have join / fork behavior 409 410 In these cases, we don't change the state or trigger events in THIS 411 function. 412 Events are triggered in the calling code (__kmp_barrier): 413 414 state := ompt_state_overhead 415 barrier-begin 416 barrier-wait-begin 417 state := ompt_state_wait_barrier 418 call join-barrier-implementation (finally arrive here) 419 {} 420 call fork-barrier-implementation (finally arrive here) 421 {} 422 state := ompt_state_overhead 423 barrier-wait-end 424 barrier-end 425 state := ompt_state_work_parallel 426 427 428 __kmp_fork_barrier (after thread creation, before executing implicit task) 429 call fork-barrier-implementation (finally arrive here) 430 {} // worker arrive here with state = ompt_state_idle 431 432 433 __kmp_join_barrier (implicit barrier at end of parallel region) 434 state := ompt_state_barrier_implicit 435 barrier-begin 436 barrier-wait-begin 437 call join-barrier-implementation (finally arrive here 438 final_spin=FALSE) 439 { 440 } 441 __kmp_fork_barrier (implicit barrier at end of parallel region) 442 call fork-barrier-implementation (finally arrive here final_spin=TRUE) 443 444 Worker after task-team is finished: 445 barrier-wait-end 446 barrier-end 447 implicit-task-end 448 idle-begin 449 state := ompt_state_idle 450 451 Before leaving, if state = ompt_state_idle 452 idle-end 453 state := ompt_state_overhead 454 */ 455 #if OMPT_SUPPORT 456 ompt_state_t ompt_entry_state; 457 ompt_data_t *tId; 458 if (ompt_enabled.enabled) { 459 ompt_entry_state = this_thr->th.ompt_thread_info.state; 460 if (!final_spin || 461 (ompt_entry_state != ompt_state_wait_barrier_implicit_parallel && 462 ompt_entry_state != ompt_state_wait_barrier_teams) || 463 KMP_MASTER_TID(this_thr->th.th_info.ds.ds_tid)) { 464 ompt_lw_taskteam_t *team = NULL; 465 if (this_thr->th.th_team) 466 team = this_thr->th.th_team->t.ompt_serialized_team_info; 467 if (team) { 468 tId = &(team->ompt_task_info.task_data); 469 } else { 470 tId = OMPT_CUR_TASK_DATA(this_thr); 471 } 472 } else { 473 tId = &(this_thr->th.ompt_thread_info.task_data); 474 } 475 if (final_spin && (__kmp_tasking_mode == tskm_immediate_exec || 476 this_thr->th.th_task_team == NULL)) { 477 // implicit task is done. Either no taskqueue, or task-team finished 478 __ompt_implicit_task_end(this_thr, ompt_entry_state, tId); 479 } 480 } 481 #endif 482 483 KMP_INIT_YIELD(spins); // Setup for waiting 484 KMP_INIT_BACKOFF(time); 485 486 if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME || 487 __kmp_pause_status == kmp_soft_paused) { 488 #if KMP_USE_MONITOR 489 // The worker threads cannot rely on the team struct existing at this point. 490 // Use the bt values cached in the thread struct instead. 491 #ifdef KMP_ADJUST_BLOCKTIME 492 if (__kmp_pause_status == kmp_soft_paused || 493 (__kmp_zero_bt && !this_thr->th.th_team_bt_set)) 494 // Force immediate suspend if not set by user and more threads than 495 // available procs 496 hibernate = 0; 497 else 498 hibernate = this_thr->th.th_team_bt_intervals; 499 #else 500 hibernate = this_thr->th.th_team_bt_intervals; 501 #endif /* KMP_ADJUST_BLOCKTIME */ 502 503 /* If the blocktime is nonzero, we want to make sure that we spin wait for 504 the entirety of the specified #intervals, plus up to one interval more. 505 This increment make certain that this thread doesn't go to sleep too 506 soon. */ 507 if (hibernate != 0) 508 hibernate++; 509 510 // Add in the current time value. 511 hibernate += TCR_4(__kmp_global.g.g_time.dt.t_value); 512 KF_TRACE(20, ("__kmp_wait_sleep: T#%d now=%d, hibernate=%d, intervals=%d\n", 513 th_gtid, __kmp_global.g.g_time.dt.t_value, hibernate, 514 hibernate - __kmp_global.g.g_time.dt.t_value)); 515 #else 516 if (__kmp_pause_status == kmp_soft_paused) { 517 // Force immediate suspend 518 hibernate_goal = KMP_NOW(); 519 } else 520 hibernate_goal = KMP_NOW() + this_thr->th.th_team_bt_intervals; 521 poll_count = 0; 522 (void)poll_count; 523 #endif // KMP_USE_MONITOR 524 } 525 526 KMP_MB(); 527 528 // Main wait spin loop 529 while (flag->notdone_check()) { 530 kmp_task_team_t *task_team = NULL; 531 if (__kmp_tasking_mode != tskm_immediate_exec) { 532 task_team = this_thr->th.th_task_team; 533 /* If the thread's task team pointer is NULL, it means one of 3 things: 534 1) A newly-created thread is first being released by 535 __kmp_fork_barrier(), and its task team has not been set up yet. 536 2) All tasks have been executed to completion. 537 3) Tasking is off for this region. This could be because we are in a 538 serialized region (perhaps the outer one), or else tasking was manually 539 disabled (KMP_TASKING=0). */ 540 if (task_team != NULL) { 541 if (TCR_SYNC_4(task_team->tt.tt_active)) { 542 if (KMP_TASKING_ENABLED(task_team)) { 543 flag->execute_tasks( 544 this_thr, th_gtid, final_spin, 545 &tasks_completed USE_ITT_BUILD_ARG(itt_sync_obj), 0); 546 } else 547 this_thr->th.th_reap_state = KMP_SAFE_TO_REAP; 548 } else { 549 KMP_DEBUG_ASSERT(!KMP_MASTER_TID(this_thr->th.th_info.ds.ds_tid)); 550 #if OMPT_SUPPORT 551 // task-team is done now, other cases should be catched above 552 if (final_spin && ompt_enabled.enabled) 553 __ompt_implicit_task_end(this_thr, ompt_entry_state, tId); 554 #endif 555 this_thr->th.th_task_team = NULL; 556 this_thr->th.th_reap_state = KMP_SAFE_TO_REAP; 557 } 558 } else { 559 this_thr->th.th_reap_state = KMP_SAFE_TO_REAP; 560 } // if 561 } // if 562 563 KMP_FSYNC_SPIN_PREPARE(CCAST(void *, spin)); 564 if (TCR_4(__kmp_global.g.g_done)) { 565 if (__kmp_global.g.g_abort) 566 __kmp_abort_thread(); 567 break; 568 } 569 570 // If we are oversubscribed, or have waited a bit (and 571 // KMP_LIBRARY=throughput), then yield 572 KMP_YIELD_OVERSUB_ELSE_SPIN(spins, time); 573 574 #if KMP_STATS_ENABLED 575 // Check if thread has been signalled to idle state 576 // This indicates that the logical "join-barrier" has finished 577 if (this_thr->th.th_stats->isIdle() && 578 KMP_GET_THREAD_STATE() == FORK_JOIN_BARRIER) { 579 KMP_SET_THREAD_STATE(IDLE); 580 KMP_PUSH_PARTITIONED_TIMER(OMP_idle); 581 } 582 #endif 583 // Check if the barrier surrounding this wait loop has been cancelled 584 if (Cancellable) { 585 kmp_team_t *team = this_thr->th.th_team; 586 if (team && team->t.t_cancel_request == cancel_parallel) 587 break; 588 } 589 590 // For hidden helper thread, if task_team is nullptr, it means the main 591 // thread has not released the barrier. We cannot wait here because once the 592 // main thread releases all children barriers, all hidden helper threads are 593 // still sleeping. This leads to a problem that following configuration, 594 // such as task team sync, will not be performed such that this thread does 595 // not have task team. Usually it is not bad. However, a corner case is, 596 // when the first task encountered is an untied task, the check in 597 // __kmp_task_alloc will crash because it uses the task team pointer without 598 // checking whether it is nullptr. It is probably under some kind of 599 // assumption. 600 if (task_team && KMP_HIDDEN_HELPER_WORKER_THREAD(th_gtid) && 601 !TCR_4(__kmp_hidden_helper_team_done)) { 602 // If there is still hidden helper tasks to be executed, the hidden helper 603 // thread will not enter a waiting status. 604 if (KMP_ATOMIC_LD_ACQ(&__kmp_unexecuted_hidden_helper_tasks) == 0) { 605 __kmp_hidden_helper_worker_thread_wait(); 606 } 607 continue; 608 } 609 610 // Don't suspend if KMP_BLOCKTIME is set to "infinite" 611 if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME && 612 __kmp_pause_status != kmp_soft_paused) 613 continue; 614 615 // Don't suspend if there is a likelihood of new tasks being spawned. 616 if (task_team != NULL && TCR_4(task_team->tt.tt_found_tasks) && 617 !__kmp_wpolicy_passive) 618 continue; 619 620 #if KMP_USE_MONITOR 621 // If we have waited a bit more, fall asleep 622 if (TCR_4(__kmp_global.g.g_time.dt.t_value) < hibernate) 623 continue; 624 #else 625 if (KMP_BLOCKING(hibernate_goal, poll_count++)) 626 continue; 627 #endif 628 // Don't suspend if wait loop designated non-sleepable 629 // in template parameters 630 if (!Sleepable) 631 continue; 632 633 #if KMP_HAVE_MWAIT || KMP_HAVE_UMWAIT 634 if (__kmp_mwait_enabled || __kmp_umwait_enabled) { 635 KF_TRACE(50, ("__kmp_wait_sleep: T#%d using monitor/mwait\n", th_gtid)); 636 flag->mwait(th_gtid); 637 } else { 638 #endif 639 KF_TRACE(50, ("__kmp_wait_sleep: T#%d suspend time reached\n", th_gtid)); 640 #if KMP_OS_UNIX 641 if (final_spin) 642 KMP_ATOMIC_ST_REL(&this_thr->th.th_blocking, false); 643 #endif 644 flag->suspend(th_gtid); 645 #if KMP_OS_UNIX 646 if (final_spin) 647 KMP_ATOMIC_ST_REL(&this_thr->th.th_blocking, true); 648 #endif 649 #if KMP_HAVE_MWAIT || KMP_HAVE_UMWAIT 650 } 651 #endif 652 653 if (TCR_4(__kmp_global.g.g_done)) { 654 if (__kmp_global.g.g_abort) 655 __kmp_abort_thread(); 656 break; 657 } else if (__kmp_tasking_mode != tskm_immediate_exec && 658 this_thr->th.th_reap_state == KMP_SAFE_TO_REAP) { 659 this_thr->th.th_reap_state = KMP_NOT_SAFE_TO_REAP; 660 } 661 // TODO: If thread is done with work and times out, disband/free 662 } 663 664 #if OMPT_SUPPORT 665 ompt_state_t ompt_exit_state = this_thr->th.ompt_thread_info.state; 666 if (ompt_enabled.enabled && ompt_exit_state != ompt_state_undefined) { 667 #if OMPT_OPTIONAL 668 if (final_spin) { 669 __ompt_implicit_task_end(this_thr, ompt_exit_state, tId); 670 ompt_exit_state = this_thr->th.ompt_thread_info.state; 671 } 672 #endif 673 if (ompt_exit_state == ompt_state_idle) { 674 this_thr->th.ompt_thread_info.state = ompt_state_overhead; 675 } 676 } 677 #endif 678 #if KMP_STATS_ENABLED 679 // If we were put into idle state, pop that off the state stack 680 if (KMP_GET_THREAD_STATE() == IDLE) { 681 KMP_POP_PARTITIONED_TIMER(); 682 KMP_SET_THREAD_STATE(thread_state); 683 this_thr->th.th_stats->resetIdleFlag(); 684 } 685 #endif 686 687 #if KMP_OS_UNIX 688 if (final_spin) 689 KMP_ATOMIC_ST_REL(&this_thr->th.th_blocking, false); 690 #endif 691 KMP_FSYNC_SPIN_ACQUIRED(CCAST(void *, spin)); 692 if (Cancellable) { 693 kmp_team_t *team = this_thr->th.th_team; 694 if (team && team->t.t_cancel_request == cancel_parallel) { 695 if (tasks_completed) { 696 // undo the previous decrement of unfinished_threads so that the 697 // thread can decrement at the join barrier with no problem 698 kmp_task_team_t *task_team = this_thr->th.th_task_team; 699 std::atomic<kmp_int32> *unfinished_threads = 700 &(task_team->tt.tt_unfinished_threads); 701 KMP_ATOMIC_INC(unfinished_threads); 702 } 703 return true; 704 } 705 } 706 return false; 707 } 708 709 #if KMP_HAVE_MWAIT || KMP_HAVE_UMWAIT 710 // Set up a monitor on the flag variable causing the calling thread to wait in 711 // a less active state until the flag variable is modified. 712 template <class C> 713 static inline void __kmp_mwait_template(int th_gtid, C *flag) { 714 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_mwait); 715 kmp_info_t *th = __kmp_threads[th_gtid]; 716 717 KF_TRACE(30, ("__kmp_mwait_template: T#%d enter for flag = %p\n", th_gtid, 718 flag->get())); 719 720 // User-level mwait is available 721 KMP_DEBUG_ASSERT(__kmp_mwait_enabled || __kmp_umwait_enabled); 722 723 __kmp_suspend_initialize_thread(th); 724 __kmp_lock_suspend_mx(th); 725 726 volatile void *spin = flag->get(); 727 void *cacheline = (void *)(kmp_uintptr_t(spin) & ~(CACHE_LINE - 1)); 728 729 if (!flag->done_check()) { 730 // Mark thread as no longer active 731 th->th.th_active = FALSE; 732 if (th->th.th_active_in_pool) { 733 th->th.th_active_in_pool = FALSE; 734 KMP_ATOMIC_DEC(&__kmp_thread_pool_active_nth); 735 KMP_DEBUG_ASSERT(TCR_4(__kmp_thread_pool_active_nth) >= 0); 736 } 737 flag->set_sleeping(); 738 KF_TRACE(50, ("__kmp_mwait_template: T#%d calling monitor\n", th_gtid)); 739 #if KMP_HAVE_UMWAIT 740 if (__kmp_umwait_enabled) { 741 __kmp_umonitor(cacheline); 742 } 743 #elif KMP_HAVE_MWAIT 744 if (__kmp_mwait_enabled) { 745 __kmp_mm_monitor(cacheline, 0, 0); 746 } 747 #endif 748 // To avoid a race, check flag between 'monitor' and 'mwait'. A write to 749 // the address could happen after the last time we checked and before 750 // monitoring started, in which case monitor can't detect the change. 751 if (flag->done_check()) 752 flag->unset_sleeping(); 753 else { 754 // if flag changes here, wake-up happens immediately 755 TCW_PTR(th->th.th_sleep_loc, (void *)flag); 756 th->th.th_sleep_loc_type = flag->get_type(); 757 __kmp_unlock_suspend_mx(th); 758 KF_TRACE(50, ("__kmp_mwait_template: T#%d calling mwait\n", th_gtid)); 759 #if KMP_HAVE_UMWAIT 760 if (__kmp_umwait_enabled) { 761 __kmp_umwait(1, 100); // to do: enable ctrl via hints, backoff counter 762 } 763 #elif KMP_HAVE_MWAIT 764 if (__kmp_mwait_enabled) { 765 __kmp_mm_mwait(0, __kmp_mwait_hints); 766 } 767 #endif 768 KF_TRACE(50, ("__kmp_mwait_template: T#%d mwait done\n", th_gtid)); 769 __kmp_lock_suspend_mx(th); 770 // Clean up sleep info; doesn't matter how/why this thread stopped waiting 771 if (flag->is_sleeping()) 772 flag->unset_sleeping(); 773 TCW_PTR(th->th.th_sleep_loc, NULL); 774 th->th.th_sleep_loc_type = flag_unset; 775 } 776 // Mark thread as active again 777 th->th.th_active = TRUE; 778 if (TCR_4(th->th.th_in_pool)) { 779 KMP_ATOMIC_INC(&__kmp_thread_pool_active_nth); 780 th->th.th_active_in_pool = TRUE; 781 } 782 } // Drop out to main wait loop to check flag, handle tasks, etc. 783 __kmp_unlock_suspend_mx(th); 784 KF_TRACE(30, ("__kmp_mwait_template: T#%d exit\n", th_gtid)); 785 } 786 #endif // KMP_HAVE_MWAIT || KMP_HAVE_UMWAIT 787 788 /* Release any threads specified as waiting on the flag by releasing the flag 789 and resume the waiting thread if indicated by the sleep bit(s). A thread that 790 calls __kmp_wait_template must call this function to wake up the potentially 791 sleeping thread and prevent deadlocks! */ 792 template <class C> static inline void __kmp_release_template(C *flag) { 793 #ifdef KMP_DEBUG 794 int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1; 795 #endif 796 KF_TRACE(20, ("__kmp_release: T#%d releasing flag(%x)\n", gtid, flag->get())); 797 KMP_DEBUG_ASSERT(flag->get()); 798 KMP_FSYNC_RELEASING(flag->get_void_p()); 799 800 flag->internal_release(); 801 802 KF_TRACE(100, ("__kmp_release: T#%d set new spin=%d\n", gtid, flag->get(), 803 flag->load())); 804 805 if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) { 806 // Only need to check sleep stuff if infinite block time not set. 807 // Are *any* threads waiting on flag sleeping? 808 if (flag->is_any_sleeping()) { 809 for (unsigned int i = 0; i < flag->get_num_waiters(); ++i) { 810 // if sleeping waiter exists at i, sets current_waiter to i inside flag 811 kmp_info_t *waiter = flag->get_waiter(i); 812 if (waiter) { 813 int wait_gtid = waiter->th.th_info.ds.ds_gtid; 814 // Wake up thread if needed 815 KF_TRACE(50, ("__kmp_release: T#%d waking up thread T#%d since sleep " 816 "flag(%p) set\n", 817 gtid, wait_gtid, flag->get())); 818 flag->resume(wait_gtid); // unsets flag's current_waiter when done 819 } 820 } 821 } 822 } 823 } 824 825 template <bool Cancellable, bool Sleepable> 826 class kmp_flag_32 : public kmp_flag_atomic<kmp_uint32, flag32, Sleepable> { 827 public: 828 kmp_flag_32(std::atomic<kmp_uint32> *p) 829 : kmp_flag_atomic<kmp_uint32, flag32, Sleepable>(p) {} 830 kmp_flag_32(std::atomic<kmp_uint32> *p, kmp_info_t *thr) 831 : kmp_flag_atomic<kmp_uint32, flag32, Sleepable>(p, thr) {} 832 kmp_flag_32(std::atomic<kmp_uint32> *p, kmp_uint32 c) 833 : kmp_flag_atomic<kmp_uint32, flag32, Sleepable>(p, c) {} 834 void suspend(int th_gtid) { __kmp_suspend_32(th_gtid, this); } 835 #if KMP_HAVE_MWAIT || KMP_HAVE_UMWAIT 836 void mwait(int th_gtid) { __kmp_mwait_32(th_gtid, this); } 837 #endif 838 void resume(int th_gtid) { __kmp_resume_32(th_gtid, this); } 839 int execute_tasks(kmp_info_t *this_thr, kmp_int32 gtid, int final_spin, 840 int *thread_finished USE_ITT_BUILD_ARG(void *itt_sync_obj), 841 kmp_int32 is_constrained) { 842 return __kmp_execute_tasks_32( 843 this_thr, gtid, this, final_spin, 844 thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), is_constrained); 845 } 846 bool wait(kmp_info_t *this_thr, 847 int final_spin USE_ITT_BUILD_ARG(void *itt_sync_obj)) { 848 if (final_spin) 849 return __kmp_wait_template<kmp_flag_32, TRUE, Cancellable, Sleepable>( 850 this_thr, this USE_ITT_BUILD_ARG(itt_sync_obj)); 851 else 852 return __kmp_wait_template<kmp_flag_32, FALSE, Cancellable, Sleepable>( 853 this_thr, this USE_ITT_BUILD_ARG(itt_sync_obj)); 854 } 855 void release() { __kmp_release_template(this); } 856 flag_type get_ptr_type() { return flag32; } 857 }; 858 859 template <bool Cancellable, bool Sleepable> 860 class kmp_flag_64 : public kmp_flag_native<kmp_uint64, flag64, Sleepable> { 861 public: 862 kmp_flag_64(volatile kmp_uint64 *p) 863 : kmp_flag_native<kmp_uint64, flag64, Sleepable>(p) {} 864 kmp_flag_64(volatile kmp_uint64 *p, kmp_info_t *thr) 865 : kmp_flag_native<kmp_uint64, flag64, Sleepable>(p, thr) {} 866 kmp_flag_64(volatile kmp_uint64 *p, kmp_uint64 c) 867 : kmp_flag_native<kmp_uint64, flag64, Sleepable>(p, c) {} 868 kmp_flag_64(volatile kmp_uint64 *p, kmp_uint64 c, std::atomic<bool> *loc) 869 : kmp_flag_native<kmp_uint64, flag64, Sleepable>(p, c, loc) {} 870 void suspend(int th_gtid) { __kmp_suspend_64(th_gtid, this); } 871 #if KMP_HAVE_MWAIT || KMP_HAVE_UMWAIT 872 void mwait(int th_gtid) { __kmp_mwait_64(th_gtid, this); } 873 #endif 874 void resume(int th_gtid) { __kmp_resume_64(th_gtid, this); } 875 int execute_tasks(kmp_info_t *this_thr, kmp_int32 gtid, int final_spin, 876 int *thread_finished USE_ITT_BUILD_ARG(void *itt_sync_obj), 877 kmp_int32 is_constrained) { 878 return __kmp_execute_tasks_64( 879 this_thr, gtid, this, final_spin, 880 thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), is_constrained); 881 } 882 bool wait(kmp_info_t *this_thr, 883 int final_spin USE_ITT_BUILD_ARG(void *itt_sync_obj)) { 884 if (final_spin) 885 return __kmp_wait_template<kmp_flag_64, TRUE, Cancellable, Sleepable>( 886 this_thr, this USE_ITT_BUILD_ARG(itt_sync_obj)); 887 else 888 return __kmp_wait_template<kmp_flag_64, FALSE, Cancellable, Sleepable>( 889 this_thr, this USE_ITT_BUILD_ARG(itt_sync_obj)); 890 } 891 void release() { __kmp_release_template(this); } 892 flag_type get_ptr_type() { return flag64; } 893 }; 894 895 template <bool Cancellable, bool Sleepable> 896 class kmp_atomic_flag_64 897 : public kmp_flag_atomic<kmp_uint64, atomic_flag64, Sleepable> { 898 public: 899 kmp_atomic_flag_64(std::atomic<kmp_uint64> *p) 900 : kmp_flag_atomic<kmp_uint64, atomic_flag64, Sleepable>(p) {} 901 kmp_atomic_flag_64(std::atomic<kmp_uint64> *p, kmp_info_t *thr) 902 : kmp_flag_atomic<kmp_uint64, atomic_flag64, Sleepable>(p, thr) {} 903 kmp_atomic_flag_64(std::atomic<kmp_uint64> *p, kmp_uint64 c) 904 : kmp_flag_atomic<kmp_uint64, atomic_flag64, Sleepable>(p, c) {} 905 kmp_atomic_flag_64(std::atomic<kmp_uint64> *p, kmp_uint64 c, 906 std::atomic<bool> *loc) 907 : kmp_flag_atomic<kmp_uint64, atomic_flag64, Sleepable>(p, c, loc) {} 908 void suspend(int th_gtid) { __kmp_atomic_suspend_64(th_gtid, this); } 909 void mwait(int th_gtid) { __kmp_atomic_mwait_64(th_gtid, this); } 910 void resume(int th_gtid) { __kmp_atomic_resume_64(th_gtid, this); } 911 int execute_tasks(kmp_info_t *this_thr, kmp_int32 gtid, int final_spin, 912 int *thread_finished USE_ITT_BUILD_ARG(void *itt_sync_obj), 913 kmp_int32 is_constrained) { 914 return __kmp_atomic_execute_tasks_64( 915 this_thr, gtid, this, final_spin, 916 thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), is_constrained); 917 } 918 bool wait(kmp_info_t *this_thr, 919 int final_spin USE_ITT_BUILD_ARG(void *itt_sync_obj)) { 920 if (final_spin) 921 return __kmp_wait_template<kmp_atomic_flag_64, TRUE, Cancellable, 922 Sleepable>( 923 this_thr, this USE_ITT_BUILD_ARG(itt_sync_obj)); 924 else 925 return __kmp_wait_template<kmp_atomic_flag_64, FALSE, Cancellable, 926 Sleepable>( 927 this_thr, this USE_ITT_BUILD_ARG(itt_sync_obj)); 928 } 929 void release() { __kmp_release_template(this); } 930 flag_type get_ptr_type() { return atomic_flag64; } 931 }; 932 933 // Hierarchical 64-bit on-core barrier instantiation 934 class kmp_flag_oncore : public kmp_flag_native<kmp_uint64, flag_oncore, false> { 935 kmp_uint32 offset; /**< Portion of flag of interest for an operation. */ 936 bool flag_switch; /**< Indicates a switch in flag location. */ 937 enum barrier_type bt; /**< Barrier type. */ 938 kmp_info_t *this_thr; /**< Thread to redirect to different flag location. */ 939 #if USE_ITT_BUILD 940 void *itt_sync_obj; /**< ITT object to pass to new flag location. */ 941 #endif 942 unsigned char &byteref(volatile kmp_uint64 *loc, size_t offset) { 943 return (RCAST(unsigned char *, CCAST(kmp_uint64 *, loc)))[offset]; 944 } 945 946 public: 947 kmp_flag_oncore(volatile kmp_uint64 *p) 948 : kmp_flag_native<kmp_uint64, flag_oncore, false>(p), flag_switch(false) { 949 } 950 kmp_flag_oncore(volatile kmp_uint64 *p, kmp_uint32 idx) 951 : kmp_flag_native<kmp_uint64, flag_oncore, false>(p), offset(idx), 952 flag_switch(false), 953 bt(bs_last_barrier) USE_ITT_BUILD_ARG(itt_sync_obj(nullptr)) {} 954 kmp_flag_oncore(volatile kmp_uint64 *p, kmp_uint64 c, kmp_uint32 idx, 955 enum barrier_type bar_t, 956 kmp_info_t *thr USE_ITT_BUILD_ARG(void *itt)) 957 : kmp_flag_native<kmp_uint64, flag_oncore, false>(p, c), offset(idx), 958 flag_switch(false), bt(bar_t), 959 this_thr(thr) USE_ITT_BUILD_ARG(itt_sync_obj(itt)) {} 960 virtual ~kmp_flag_oncore() override {} 961 void *operator new(size_t size) { return __kmp_allocate(size); } 962 void operator delete(void *p) { __kmp_free(p); } 963 bool done_check_val(kmp_uint64 old_loc) override { 964 return byteref(&old_loc, offset) == checker; 965 } 966 bool done_check() override { return done_check_val(*get()); } 967 bool notdone_check() override { 968 // Calculate flag_switch 969 if (this_thr->th.th_bar[bt].bb.wait_flag == KMP_BARRIER_SWITCH_TO_OWN_FLAG) 970 flag_switch = true; 971 if (byteref(get(), offset) != 1 && !flag_switch) 972 return true; 973 else if (flag_switch) { 974 this_thr->th.th_bar[bt].bb.wait_flag = KMP_BARRIER_SWITCHING; 975 kmp_flag_64<> flag(&this_thr->th.th_bar[bt].bb.b_go, 976 (kmp_uint64)KMP_BARRIER_STATE_BUMP); 977 __kmp_wait_64(this_thr, &flag, TRUE USE_ITT_BUILD_ARG(itt_sync_obj)); 978 } 979 return false; 980 } 981 void internal_release() { 982 // Other threads can write their own bytes simultaneously. 983 if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME) { 984 byteref(get(), offset) = 1; 985 } else { 986 kmp_uint64 mask = 0; 987 byteref(&mask, offset) = 1; 988 KMP_TEST_THEN_OR64(get(), mask); 989 } 990 } 991 void wait(kmp_info_t *this_thr, int final_spin) { 992 if (final_spin) 993 __kmp_wait_template<kmp_flag_oncore, TRUE>( 994 this_thr, this USE_ITT_BUILD_ARG(itt_sync_obj)); 995 else 996 __kmp_wait_template<kmp_flag_oncore, FALSE>( 997 this_thr, this USE_ITT_BUILD_ARG(itt_sync_obj)); 998 } 999 void release() { __kmp_release_template(this); } 1000 void suspend(int th_gtid) { __kmp_suspend_oncore(th_gtid, this); } 1001 #if KMP_HAVE_MWAIT || KMP_HAVE_UMWAIT 1002 void mwait(int th_gtid) { __kmp_mwait_oncore(th_gtid, this); } 1003 #endif 1004 void resume(int th_gtid) { __kmp_resume_oncore(th_gtid, this); } 1005 int execute_tasks(kmp_info_t *this_thr, kmp_int32 gtid, int final_spin, 1006 int *thread_finished USE_ITT_BUILD_ARG(void *itt_sync_obj), 1007 kmp_int32 is_constrained) { 1008 #if OMPD_SUPPORT 1009 int ret = __kmp_execute_tasks_oncore( 1010 this_thr, gtid, this, final_spin, 1011 thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), is_constrained); 1012 if (ompd_state & OMPD_ENABLE_BP) 1013 ompd_bp_task_end(); 1014 return ret; 1015 #else 1016 return __kmp_execute_tasks_oncore( 1017 this_thr, gtid, this, final_spin, 1018 thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), is_constrained); 1019 #endif 1020 } 1021 enum barrier_type get_bt() { return bt; } 1022 flag_type get_ptr_type() { return flag_oncore; } 1023 }; 1024 1025 static inline void __kmp_null_resume_wrapper(kmp_info_t *thr) { 1026 int gtid = __kmp_gtid_from_thread(thr); 1027 void *flag = CCAST(void *, thr->th.th_sleep_loc); 1028 flag_type type = thr->th.th_sleep_loc_type; 1029 if (!flag) 1030 return; 1031 // Attempt to wake up a thread: examine its type and call appropriate template 1032 switch (type) { 1033 case flag32: 1034 __kmp_resume_32(gtid, RCAST(kmp_flag_32<> *, flag)); 1035 break; 1036 case flag64: 1037 __kmp_resume_64(gtid, RCAST(kmp_flag_64<> *, flag)); 1038 break; 1039 case atomic_flag64: 1040 __kmp_atomic_resume_64(gtid, RCAST(kmp_atomic_flag_64<> *, flag)); 1041 break; 1042 case flag_oncore: 1043 __kmp_resume_oncore(gtid, RCAST(kmp_flag_oncore *, flag)); 1044 break; 1045 case flag_unset: 1046 KF_TRACE(100, ("__kmp_null_resume_wrapper: flag type %d is unset\n", type)); 1047 break; 1048 } 1049 } 1050 1051 /*! 1052 @} 1053 */ 1054 1055 #endif // KMP_WAIT_RELEASE_H 1056