xref: /freebsd/contrib/llvm-project/openmp/runtime/src/z_Windows_NT_util.cpp (revision f126890ac5386406dadf7c4cfa9566cbb56537c5)
1 /*
2  * z_Windows_NT_util.cpp -- platform specific routines.
3  */
4 
5 //===----------------------------------------------------------------------===//
6 //
7 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
8 // See https://llvm.org/LICENSE.txt for license information.
9 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "kmp.h"
14 #include "kmp_affinity.h"
15 #include "kmp_i18n.h"
16 #include "kmp_io.h"
17 #include "kmp_itt.h"
18 #include "kmp_wait_release.h"
19 
20 /* This code is related to NtQuerySystemInformation() function. This function
21    is used in the Load balance algorithm for OMP_DYNAMIC=true to find the
22    number of running threads in the system. */
23 
24 #include <ntsecapi.h> // UNICODE_STRING
25 #include <ntstatus.h>
26 #include <psapi.h>
27 #ifdef _MSC_VER
28 #pragma comment(lib, "psapi.lib")
29 #endif
30 
31 enum SYSTEM_INFORMATION_CLASS {
32   SystemProcessInformation = 5
33 }; // SYSTEM_INFORMATION_CLASS
34 
35 struct CLIENT_ID {
36   HANDLE UniqueProcess;
37   HANDLE UniqueThread;
38 }; // struct CLIENT_ID
39 
40 enum THREAD_STATE {
41   StateInitialized,
42   StateReady,
43   StateRunning,
44   StateStandby,
45   StateTerminated,
46   StateWait,
47   StateTransition,
48   StateUnknown
49 }; // enum THREAD_STATE
50 
51 struct VM_COUNTERS {
52   SIZE_T PeakVirtualSize;
53   SIZE_T VirtualSize;
54   ULONG PageFaultCount;
55   SIZE_T PeakWorkingSetSize;
56   SIZE_T WorkingSetSize;
57   SIZE_T QuotaPeakPagedPoolUsage;
58   SIZE_T QuotaPagedPoolUsage;
59   SIZE_T QuotaPeakNonPagedPoolUsage;
60   SIZE_T QuotaNonPagedPoolUsage;
61   SIZE_T PagefileUsage;
62   SIZE_T PeakPagefileUsage;
63   SIZE_T PrivatePageCount;
64 }; // struct VM_COUNTERS
65 
66 struct SYSTEM_THREAD {
67   LARGE_INTEGER KernelTime;
68   LARGE_INTEGER UserTime;
69   LARGE_INTEGER CreateTime;
70   ULONG WaitTime;
71   LPVOID StartAddress;
72   CLIENT_ID ClientId;
73   DWORD Priority;
74   LONG BasePriority;
75   ULONG ContextSwitchCount;
76   THREAD_STATE State;
77   ULONG WaitReason;
78 }; // SYSTEM_THREAD
79 
80 KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, KernelTime) == 0);
81 #if KMP_ARCH_X86 || KMP_ARCH_ARM
82 KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, StartAddress) == 28);
83 KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, State) == 52);
84 #else
85 KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, StartAddress) == 32);
86 KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, State) == 68);
87 #endif
88 
89 struct SYSTEM_PROCESS_INFORMATION {
90   ULONG NextEntryOffset;
91   ULONG NumberOfThreads;
92   LARGE_INTEGER Reserved[3];
93   LARGE_INTEGER CreateTime;
94   LARGE_INTEGER UserTime;
95   LARGE_INTEGER KernelTime;
96   UNICODE_STRING ImageName;
97   DWORD BasePriority;
98   HANDLE ProcessId;
99   HANDLE ParentProcessId;
100   ULONG HandleCount;
101   ULONG Reserved2[2];
102   VM_COUNTERS VMCounters;
103   IO_COUNTERS IOCounters;
104   SYSTEM_THREAD Threads[1];
105 }; // SYSTEM_PROCESS_INFORMATION
106 typedef SYSTEM_PROCESS_INFORMATION *PSYSTEM_PROCESS_INFORMATION;
107 
108 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, NextEntryOffset) == 0);
109 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, CreateTime) == 32);
110 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, ImageName) == 56);
111 #if KMP_ARCH_X86 || KMP_ARCH_ARM
112 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, ProcessId) == 68);
113 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, HandleCount) == 76);
114 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, VMCounters) == 88);
115 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, IOCounters) == 136);
116 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, Threads) == 184);
117 #else
118 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, ProcessId) == 80);
119 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, HandleCount) == 96);
120 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, VMCounters) == 112);
121 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, IOCounters) == 208);
122 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, Threads) == 256);
123 #endif
124 
125 typedef NTSTATUS(NTAPI *NtQuerySystemInformation_t)(SYSTEM_INFORMATION_CLASS,
126                                                     PVOID, ULONG, PULONG);
127 NtQuerySystemInformation_t NtQuerySystemInformation = NULL;
128 
129 HMODULE ntdll = NULL;
130 
131 /* End of NtQuerySystemInformation()-related code */
132 
133 static HMODULE kernel32 = NULL;
134 
135 #if KMP_HANDLE_SIGNALS
136 typedef void (*sig_func_t)(int);
137 static sig_func_t __kmp_sighldrs[NSIG];
138 static int __kmp_siginstalled[NSIG];
139 #endif
140 
141 #if KMP_USE_MONITOR
142 static HANDLE __kmp_monitor_ev;
143 #endif
144 static kmp_int64 __kmp_win32_time;
145 double __kmp_win32_tick;
146 
147 int __kmp_init_runtime = FALSE;
148 CRITICAL_SECTION __kmp_win32_section;
149 
150 void __kmp_win32_mutex_init(kmp_win32_mutex_t *mx) {
151   InitializeCriticalSection(&mx->cs);
152 #if USE_ITT_BUILD
153   __kmp_itt_system_object_created(&mx->cs, "Critical Section");
154 #endif /* USE_ITT_BUILD */
155 }
156 
157 void __kmp_win32_mutex_destroy(kmp_win32_mutex_t *mx) {
158   DeleteCriticalSection(&mx->cs);
159 }
160 
161 void __kmp_win32_mutex_lock(kmp_win32_mutex_t *mx) {
162   EnterCriticalSection(&mx->cs);
163 }
164 
165 int __kmp_win32_mutex_trylock(kmp_win32_mutex_t *mx) {
166   return TryEnterCriticalSection(&mx->cs);
167 }
168 
169 void __kmp_win32_mutex_unlock(kmp_win32_mutex_t *mx) {
170   LeaveCriticalSection(&mx->cs);
171 }
172 
173 void __kmp_win32_cond_init(kmp_win32_cond_t *cv) {
174   cv->waiters_count_ = 0;
175   cv->wait_generation_count_ = 0;
176   cv->release_count_ = 0;
177 
178   /* Initialize the critical section */
179   __kmp_win32_mutex_init(&cv->waiters_count_lock_);
180 
181   /* Create a manual-reset event. */
182   cv->event_ = CreateEvent(NULL, // no security
183                            TRUE, // manual-reset
184                            FALSE, // non-signaled initially
185                            NULL); // unnamed
186 #if USE_ITT_BUILD
187   __kmp_itt_system_object_created(cv->event_, "Event");
188 #endif /* USE_ITT_BUILD */
189 }
190 
191 void __kmp_win32_cond_destroy(kmp_win32_cond_t *cv) {
192   __kmp_win32_mutex_destroy(&cv->waiters_count_lock_);
193   __kmp_free_handle(cv->event_);
194   memset(cv, '\0', sizeof(*cv));
195 }
196 
197 /* TODO associate cv with a team instead of a thread so as to optimize
198    the case where we wake up a whole team */
199 
200 template <class C>
201 static void __kmp_win32_cond_wait(kmp_win32_cond_t *cv, kmp_win32_mutex_t *mx,
202                                   kmp_info_t *th, C *flag) {
203   int my_generation;
204   int last_waiter;
205 
206   /* Avoid race conditions */
207   __kmp_win32_mutex_lock(&cv->waiters_count_lock_);
208 
209   /* Increment count of waiters */
210   cv->waiters_count_++;
211 
212   /* Store current generation in our activation record. */
213   my_generation = cv->wait_generation_count_;
214 
215   __kmp_win32_mutex_unlock(&cv->waiters_count_lock_);
216   __kmp_win32_mutex_unlock(mx);
217 
218   for (;;) {
219     int wait_done = 0;
220     DWORD res, timeout = 5000; // just tried to quess an appropriate number
221     /* Wait until the event is signaled */
222     res = WaitForSingleObject(cv->event_, timeout);
223 
224     if (res == WAIT_OBJECT_0) {
225       // event signaled
226       __kmp_win32_mutex_lock(&cv->waiters_count_lock_);
227       /* Exit the loop when the <cv->event_> is signaled and there are still
228          waiting threads from this <wait_generation> that haven't been released
229          from this wait yet. */
230       wait_done = (cv->release_count_ > 0) &&
231                   (cv->wait_generation_count_ != my_generation);
232       __kmp_win32_mutex_unlock(&cv->waiters_count_lock_);
233     } else if (res == WAIT_TIMEOUT || res == WAIT_FAILED) {
234       // check if the flag and cv counters are in consistent state
235       // as MS sent us debug dump whith inconsistent state of data
236       __kmp_win32_mutex_lock(mx);
237       typename C::flag_t old_f = flag->set_sleeping();
238       if (!flag->done_check_val(old_f & ~KMP_BARRIER_SLEEP_STATE)) {
239         __kmp_win32_mutex_unlock(mx);
240         continue;
241       }
242       // condition fulfilled, exiting
243       flag->unset_sleeping();
244       TCW_PTR(th->th.th_sleep_loc, NULL);
245       th->th.th_sleep_loc_type = flag_unset;
246       KF_TRACE(50, ("__kmp_win32_cond_wait: exiting, condition "
247                     "fulfilled: flag's loc(%p): %u\n",
248                     flag->get(), (unsigned int)flag->load()));
249 
250       __kmp_win32_mutex_lock(&cv->waiters_count_lock_);
251       KMP_DEBUG_ASSERT(cv->waiters_count_ > 0);
252       cv->release_count_ = cv->waiters_count_;
253       cv->wait_generation_count_++;
254       wait_done = 1;
255       __kmp_win32_mutex_unlock(&cv->waiters_count_lock_);
256 
257       __kmp_win32_mutex_unlock(mx);
258     }
259     /* there used to be a semicolon after the if statement, it looked like a
260        bug, so i removed it */
261     if (wait_done)
262       break;
263   }
264 
265   __kmp_win32_mutex_lock(mx);
266   __kmp_win32_mutex_lock(&cv->waiters_count_lock_);
267 
268   cv->waiters_count_--;
269   cv->release_count_--;
270 
271   last_waiter = (cv->release_count_ == 0);
272 
273   __kmp_win32_mutex_unlock(&cv->waiters_count_lock_);
274 
275   if (last_waiter) {
276     /* We're the last waiter to be notified, so reset the manual event. */
277     ResetEvent(cv->event_);
278   }
279 }
280 
281 void __kmp_win32_cond_broadcast(kmp_win32_cond_t *cv) {
282   __kmp_win32_mutex_lock(&cv->waiters_count_lock_);
283 
284   if (cv->waiters_count_ > 0) {
285     SetEvent(cv->event_);
286     /* Release all the threads in this generation. */
287 
288     cv->release_count_ = cv->waiters_count_;
289 
290     /* Start a new generation. */
291     cv->wait_generation_count_++;
292   }
293 
294   __kmp_win32_mutex_unlock(&cv->waiters_count_lock_);
295 }
296 
297 void __kmp_win32_cond_signal(kmp_win32_cond_t *cv) {
298   __kmp_win32_cond_broadcast(cv);
299 }
300 
301 void __kmp_enable(int new_state) {
302   if (__kmp_init_runtime)
303     LeaveCriticalSection(&__kmp_win32_section);
304 }
305 
306 void __kmp_disable(int *old_state) {
307   *old_state = 0;
308 
309   if (__kmp_init_runtime)
310     EnterCriticalSection(&__kmp_win32_section);
311 }
312 
313 void __kmp_suspend_initialize(void) { /* do nothing */
314 }
315 
316 void __kmp_suspend_initialize_thread(kmp_info_t *th) {
317   int old_value = KMP_ATOMIC_LD_RLX(&th->th.th_suspend_init);
318   int new_value = TRUE;
319   // Return if already initialized
320   if (old_value == new_value)
321     return;
322   // Wait, then return if being initialized
323   if (old_value == -1 ||
324       !__kmp_atomic_compare_store(&th->th.th_suspend_init, old_value, -1)) {
325     while (KMP_ATOMIC_LD_ACQ(&th->th.th_suspend_init) != new_value) {
326       KMP_CPU_PAUSE();
327     }
328   } else {
329     // Claim to be the initializer and do initializations
330     __kmp_win32_cond_init(&th->th.th_suspend_cv);
331     __kmp_win32_mutex_init(&th->th.th_suspend_mx);
332     KMP_ATOMIC_ST_REL(&th->th.th_suspend_init, new_value);
333   }
334 }
335 
336 void __kmp_suspend_uninitialize_thread(kmp_info_t *th) {
337   if (KMP_ATOMIC_LD_ACQ(&th->th.th_suspend_init)) {
338     /* this means we have initialize the suspension pthread objects for this
339        thread in this instance of the process */
340     __kmp_win32_cond_destroy(&th->th.th_suspend_cv);
341     __kmp_win32_mutex_destroy(&th->th.th_suspend_mx);
342     KMP_ATOMIC_ST_REL(&th->th.th_suspend_init, FALSE);
343   }
344 }
345 
346 int __kmp_try_suspend_mx(kmp_info_t *th) {
347   return __kmp_win32_mutex_trylock(&th->th.th_suspend_mx);
348 }
349 
350 void __kmp_lock_suspend_mx(kmp_info_t *th) {
351   __kmp_win32_mutex_lock(&th->th.th_suspend_mx);
352 }
353 
354 void __kmp_unlock_suspend_mx(kmp_info_t *th) {
355   __kmp_win32_mutex_unlock(&th->th.th_suspend_mx);
356 }
357 
358 /* This routine puts the calling thread to sleep after setting the
359    sleep bit for the indicated flag variable to true. */
360 template <class C>
361 static inline void __kmp_suspend_template(int th_gtid, C *flag) {
362   kmp_info_t *th = __kmp_threads[th_gtid];
363   typename C::flag_t old_spin;
364 
365   KF_TRACE(30, ("__kmp_suspend_template: T#%d enter for flag's loc(%p)\n",
366                 th_gtid, flag->get()));
367 
368   __kmp_suspend_initialize_thread(th);
369   __kmp_lock_suspend_mx(th);
370 
371   KF_TRACE(10, ("__kmp_suspend_template: T#%d setting sleep bit for flag's"
372                 " loc(%p)\n",
373                 th_gtid, flag->get()));
374 
375   /* TODO: shouldn't this use release semantics to ensure that
376      __kmp_suspend_initialize_thread gets called first? */
377   old_spin = flag->set_sleeping();
378   TCW_PTR(th->th.th_sleep_loc, (void *)flag);
379   th->th.th_sleep_loc_type = flag->get_type();
380   if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME &&
381       __kmp_pause_status != kmp_soft_paused) {
382     flag->unset_sleeping();
383     TCW_PTR(th->th.th_sleep_loc, NULL);
384     th->th.th_sleep_loc_type = flag_unset;
385     __kmp_unlock_suspend_mx(th);
386     return;
387   }
388 
389   KF_TRACE(5, ("__kmp_suspend_template: T#%d set sleep bit for flag's"
390                " loc(%p)==%u\n",
391                th_gtid, flag->get(), (unsigned int)flag->load()));
392 
393   if (flag->done_check_val(old_spin) || flag->done_check()) {
394     flag->unset_sleeping();
395     TCW_PTR(th->th.th_sleep_loc, NULL);
396     th->th.th_sleep_loc_type = flag_unset;
397     KF_TRACE(5, ("__kmp_suspend_template: T#%d false alarm, reset sleep bit "
398                  "for flag's loc(%p)\n",
399                  th_gtid, flag->get()));
400   } else {
401 #ifdef DEBUG_SUSPEND
402     __kmp_suspend_count++;
403 #endif
404     /* Encapsulate in a loop as the documentation states that this may "with
405        low probability" return when the condition variable has not been signaled
406        or broadcast */
407     int deactivated = FALSE;
408 
409     while (flag->is_sleeping()) {
410       KF_TRACE(15, ("__kmp_suspend_template: T#%d about to perform "
411                     "kmp_win32_cond_wait()\n",
412                     th_gtid));
413       // Mark the thread as no longer active (only in the first iteration of the
414       // loop).
415       if (!deactivated) {
416         th->th.th_active = FALSE;
417         if (th->th.th_active_in_pool) {
418           th->th.th_active_in_pool = FALSE;
419           KMP_ATOMIC_DEC(&__kmp_thread_pool_active_nth);
420           KMP_DEBUG_ASSERT(TCR_4(__kmp_thread_pool_active_nth) >= 0);
421         }
422         deactivated = TRUE;
423       }
424 
425       KMP_DEBUG_ASSERT(th->th.th_sleep_loc);
426       KMP_DEBUG_ASSERT(th->th.th_sleep_loc_type == flag->get_type());
427 
428       __kmp_win32_cond_wait(&th->th.th_suspend_cv, &th->th.th_suspend_mx, th,
429                             flag);
430 
431 #ifdef KMP_DEBUG
432       if (flag->is_sleeping()) {
433         KF_TRACE(100,
434                  ("__kmp_suspend_template: T#%d spurious wakeup\n", th_gtid));
435       }
436 #endif /* KMP_DEBUG */
437 
438     } // while
439 
440     // We may have had the loop variable set before entering the loop body;
441     // so we need to reset sleep_loc.
442     TCW_PTR(th->th.th_sleep_loc, NULL);
443     th->th.th_sleep_loc_type = flag_unset;
444 
445     KMP_DEBUG_ASSERT(!flag->is_sleeping());
446     KMP_DEBUG_ASSERT(!th->th.th_sleep_loc);
447 
448     // Mark the thread as active again (if it was previous marked as inactive)
449     if (deactivated) {
450       th->th.th_active = TRUE;
451       if (TCR_4(th->th.th_in_pool)) {
452         KMP_ATOMIC_INC(&__kmp_thread_pool_active_nth);
453         th->th.th_active_in_pool = TRUE;
454       }
455     }
456   }
457 
458   __kmp_unlock_suspend_mx(th);
459   KF_TRACE(30, ("__kmp_suspend_template: T#%d exit\n", th_gtid));
460 }
461 
462 template <bool C, bool S>
463 void __kmp_suspend_32(int th_gtid, kmp_flag_32<C, S> *flag) {
464   __kmp_suspend_template(th_gtid, flag);
465 }
466 template <bool C, bool S>
467 void __kmp_suspend_64(int th_gtid, kmp_flag_64<C, S> *flag) {
468   __kmp_suspend_template(th_gtid, flag);
469 }
470 template <bool C, bool S>
471 void __kmp_atomic_suspend_64(int th_gtid, kmp_atomic_flag_64<C, S> *flag) {
472   __kmp_suspend_template(th_gtid, flag);
473 }
474 void __kmp_suspend_oncore(int th_gtid, kmp_flag_oncore *flag) {
475   __kmp_suspend_template(th_gtid, flag);
476 }
477 
478 template void __kmp_suspend_32<false, false>(int, kmp_flag_32<false, false> *);
479 template void __kmp_suspend_64<false, true>(int, kmp_flag_64<false, true> *);
480 template void __kmp_suspend_64<true, false>(int, kmp_flag_64<true, false> *);
481 template void
482 __kmp_atomic_suspend_64<false, true>(int, kmp_atomic_flag_64<false, true> *);
483 template void
484 __kmp_atomic_suspend_64<true, false>(int, kmp_atomic_flag_64<true, false> *);
485 
486 /* This routine signals the thread specified by target_gtid to wake up
487    after setting the sleep bit indicated by the flag argument to FALSE */
488 template <class C>
489 static inline void __kmp_resume_template(int target_gtid, C *flag) {
490   kmp_info_t *th = __kmp_threads[target_gtid];
491 
492 #ifdef KMP_DEBUG
493   int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
494 #endif
495 
496   KF_TRACE(30, ("__kmp_resume_template: T#%d wants to wakeup T#%d enter\n",
497                 gtid, target_gtid));
498 
499   __kmp_suspend_initialize_thread(th);
500   __kmp_lock_suspend_mx(th);
501 
502   if (!flag || flag != th->th.th_sleep_loc) {
503     // coming from __kmp_null_resume_wrapper, or thread is now sleeping on a
504     // different location; wake up at new location
505     flag = (C *)th->th.th_sleep_loc;
506   }
507 
508   // First, check if the flag is null or its type has changed. If so, someone
509   // else woke it up.
510   if (!flag || flag->get_type() != th->th.th_sleep_loc_type) {
511     // simply shows what flag was cast to
512     KF_TRACE(5, ("__kmp_resume_template: T#%d exiting, thread T#%d already "
513                  "awake: flag's loc(%p)\n",
514                  gtid, target_gtid, NULL));
515     __kmp_unlock_suspend_mx(th);
516     return;
517   } else {
518     if (!flag->is_sleeping()) {
519       KF_TRACE(5, ("__kmp_resume_template: T#%d exiting, thread T#%d already "
520                    "awake: flag's loc(%p): %u\n",
521                    gtid, target_gtid, flag->get(), (unsigned int)flag->load()));
522       __kmp_unlock_suspend_mx(th);
523       return;
524     }
525   }
526   KMP_DEBUG_ASSERT(flag);
527   flag->unset_sleeping();
528   TCW_PTR(th->th.th_sleep_loc, NULL);
529   th->th.th_sleep_loc_type = flag_unset;
530 
531   KF_TRACE(5, ("__kmp_resume_template: T#%d about to wakeup T#%d, reset sleep "
532                "bit for flag's loc(%p)\n",
533                gtid, target_gtid, flag->get()));
534 
535   __kmp_win32_cond_signal(&th->th.th_suspend_cv);
536   __kmp_unlock_suspend_mx(th);
537 
538   KF_TRACE(30, ("__kmp_resume_template: T#%d exiting after signaling wake up"
539                 " for T#%d\n",
540                 gtid, target_gtid));
541 }
542 
543 template <bool C, bool S>
544 void __kmp_resume_32(int target_gtid, kmp_flag_32<C, S> *flag) {
545   __kmp_resume_template(target_gtid, flag);
546 }
547 template <bool C, bool S>
548 void __kmp_resume_64(int target_gtid, kmp_flag_64<C, S> *flag) {
549   __kmp_resume_template(target_gtid, flag);
550 }
551 template <bool C, bool S>
552 void __kmp_atomic_resume_64(int target_gtid, kmp_atomic_flag_64<C, S> *flag) {
553   __kmp_resume_template(target_gtid, flag);
554 }
555 void __kmp_resume_oncore(int target_gtid, kmp_flag_oncore *flag) {
556   __kmp_resume_template(target_gtid, flag);
557 }
558 
559 template void __kmp_resume_32<false, true>(int, kmp_flag_32<false, true> *);
560 template void __kmp_resume_32<false, false>(int, kmp_flag_32<false, false> *);
561 template void __kmp_resume_64<false, true>(int, kmp_flag_64<false, true> *);
562 template void
563 __kmp_atomic_resume_64<false, true>(int, kmp_atomic_flag_64<false, true> *);
564 
565 void __kmp_yield() { Sleep(0); }
566 
567 void __kmp_gtid_set_specific(int gtid) {
568   if (__kmp_init_gtid) {
569     KA_TRACE(50, ("__kmp_gtid_set_specific: T#%d key:%d\n", gtid,
570                   __kmp_gtid_threadprivate_key));
571     kmp_intptr_t g = (kmp_intptr_t)gtid;
572     if (!TlsSetValue(__kmp_gtid_threadprivate_key, (LPVOID)(g + 1)))
573       KMP_FATAL(TLSSetValueFailed);
574   } else {
575     KA_TRACE(50, ("__kmp_gtid_set_specific: runtime shutdown, returning\n"));
576   }
577 }
578 
579 int __kmp_gtid_get_specific() {
580   int gtid;
581   if (!__kmp_init_gtid) {
582     KA_TRACE(50, ("__kmp_gtid_get_specific: runtime shutdown, returning "
583                   "KMP_GTID_SHUTDOWN\n"));
584     return KMP_GTID_SHUTDOWN;
585   }
586   gtid = (int)(kmp_intptr_t)TlsGetValue(__kmp_gtid_threadprivate_key);
587   if (gtid == 0) {
588     gtid = KMP_GTID_DNE;
589   } else {
590     gtid--;
591   }
592   KA_TRACE(50, ("__kmp_gtid_get_specific: key:%d gtid:%d\n",
593                 __kmp_gtid_threadprivate_key, gtid));
594   return gtid;
595 }
596 
597 void __kmp_affinity_bind_thread(int proc) {
598   if (__kmp_num_proc_groups > 1) {
599     // Form the GROUP_AFFINITY struct directly, rather than filling
600     // out a bit vector and calling __kmp_set_system_affinity().
601     GROUP_AFFINITY ga;
602     KMP_DEBUG_ASSERT((proc >= 0) && (proc < (__kmp_num_proc_groups * CHAR_BIT *
603                                              sizeof(DWORD_PTR))));
604     ga.Group = proc / (CHAR_BIT * sizeof(DWORD_PTR));
605     ga.Mask = (unsigned long long)1 << (proc % (CHAR_BIT * sizeof(DWORD_PTR)));
606     ga.Reserved[0] = ga.Reserved[1] = ga.Reserved[2] = 0;
607 
608     KMP_DEBUG_ASSERT(__kmp_SetThreadGroupAffinity != NULL);
609     if (__kmp_SetThreadGroupAffinity(GetCurrentThread(), &ga, NULL) == 0) {
610       DWORD error = GetLastError();
611       // AC: continue silently if not verbose
612       if (__kmp_affinity.flags.verbose) {
613         kmp_msg_t err_code = KMP_ERR(error);
614         __kmp_msg(kmp_ms_warning, KMP_MSG(CantSetThreadAffMask), err_code,
615                   __kmp_msg_null);
616         if (__kmp_generate_warnings == kmp_warnings_off) {
617           __kmp_str_free(&err_code.str);
618         }
619       }
620     }
621   } else {
622     kmp_affin_mask_t *mask;
623     KMP_CPU_ALLOC_ON_STACK(mask);
624     KMP_CPU_ZERO(mask);
625     KMP_CPU_SET(proc, mask);
626     __kmp_set_system_affinity(mask, TRUE);
627     KMP_CPU_FREE_FROM_STACK(mask);
628   }
629 }
630 
631 void __kmp_affinity_determine_capable(const char *env_var) {
632   // All versions of Windows* OS (since Win '95) support
633   // SetThreadAffinityMask().
634 
635 #if KMP_GROUP_AFFINITY
636   KMP_AFFINITY_ENABLE(__kmp_num_proc_groups * sizeof(DWORD_PTR));
637 #else
638   KMP_AFFINITY_ENABLE(sizeof(DWORD_PTR));
639 #endif
640 
641   KA_TRACE(10, ("__kmp_affinity_determine_capable: "
642                 "Windows* OS affinity interface functional (mask size = "
643                 "%" KMP_SIZE_T_SPEC ").\n",
644                 __kmp_affin_mask_size));
645 }
646 
647 double __kmp_read_cpu_time(void) {
648   FILETIME CreationTime, ExitTime, KernelTime, UserTime;
649   int status;
650   double cpu_time;
651 
652   cpu_time = 0;
653 
654   status = GetProcessTimes(GetCurrentProcess(), &CreationTime, &ExitTime,
655                            &KernelTime, &UserTime);
656 
657   if (status) {
658     double sec = 0;
659 
660     sec += KernelTime.dwHighDateTime;
661     sec += UserTime.dwHighDateTime;
662 
663     /* Shift left by 32 bits */
664     sec *= (double)(1 << 16) * (double)(1 << 16);
665 
666     sec += KernelTime.dwLowDateTime;
667     sec += UserTime.dwLowDateTime;
668 
669     cpu_time += (sec * 100.0) / KMP_NSEC_PER_SEC;
670   }
671 
672   return cpu_time;
673 }
674 
675 int __kmp_read_system_info(struct kmp_sys_info *info) {
676   info->maxrss = 0; /* the maximum resident set size utilized (in kilobytes) */
677   info->minflt = 0; /* the number of page faults serviced without any I/O */
678   info->majflt = 0; /* the number of page faults serviced that required I/O */
679   info->nswap = 0; // the number of times a process was "swapped" out of memory
680   info->inblock = 0; // the number of times the file system had to perform input
681   info->oublock = 0; // number of times the file system had to perform output
682   info->nvcsw = 0; /* the number of times a context switch was voluntarily */
683   info->nivcsw = 0; /* the number of times a context switch was forced */
684 
685   return 1;
686 }
687 
688 void __kmp_runtime_initialize(void) {
689   SYSTEM_INFO info;
690   kmp_str_buf_t path;
691   UINT path_size;
692 
693   if (__kmp_init_runtime) {
694     return;
695   }
696 
697 #if KMP_DYNAMIC_LIB
698   /* Pin dynamic library for the lifetime of application */
699   {
700     // First, turn off error message boxes
701     UINT err_mode = SetErrorMode(SEM_FAILCRITICALERRORS);
702     HMODULE h;
703     BOOL ret = GetModuleHandleEx(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS |
704                                      GET_MODULE_HANDLE_EX_FLAG_PIN,
705                                  (LPCTSTR)&__kmp_serial_initialize, &h);
706     (void)ret;
707     KMP_DEBUG_ASSERT2(h && ret, "OpenMP RTL cannot find itself loaded");
708     SetErrorMode(err_mode); // Restore error mode
709     KA_TRACE(10, ("__kmp_runtime_initialize: dynamic library pinned\n"));
710   }
711 #endif
712 
713   InitializeCriticalSection(&__kmp_win32_section);
714 #if USE_ITT_BUILD
715   __kmp_itt_system_object_created(&__kmp_win32_section, "Critical Section");
716 #endif /* USE_ITT_BUILD */
717   __kmp_initialize_system_tick();
718 
719 #if (KMP_ARCH_X86 || KMP_ARCH_X86_64)
720   if (!__kmp_cpuinfo.initialized) {
721     __kmp_query_cpuid(&__kmp_cpuinfo);
722   }
723 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
724 
725 /* Set up minimum number of threads to switch to TLS gtid */
726 #if KMP_OS_WINDOWS && !KMP_DYNAMIC_LIB
727   // Windows* OS, static library.
728   /* New thread may use stack space previously used by another thread,
729      currently terminated. On Windows* OS, in case of static linking, we do not
730      know the moment of thread termination, and our structures (__kmp_threads
731      and __kmp_root arrays) are still keep info about dead threads. This leads
732      to problem in __kmp_get_global_thread_id() function: it wrongly finds gtid
733      (by searching through stack addresses of all known threads) for
734      unregistered foreign tread.
735 
736      Setting __kmp_tls_gtid_min to 0 workarounds this problem:
737      __kmp_get_global_thread_id() does not search through stacks, but get gtid
738      from TLS immediately.
739       --ln
740   */
741   __kmp_tls_gtid_min = 0;
742 #else
743   __kmp_tls_gtid_min = KMP_TLS_GTID_MIN;
744 #endif
745 
746   /* for the static library */
747   if (!__kmp_gtid_threadprivate_key) {
748     __kmp_gtid_threadprivate_key = TlsAlloc();
749     if (__kmp_gtid_threadprivate_key == TLS_OUT_OF_INDEXES) {
750       KMP_FATAL(TLSOutOfIndexes);
751     }
752   }
753 
754   // Load ntdll.dll.
755   /* Simple GetModuleHandle( "ntdll.dl" ) is not suitable due to security issue
756      (see http://www.microsoft.com/technet/security/advisory/2269637.mspx). We
757      have to specify full path to the library. */
758   __kmp_str_buf_init(&path);
759   path_size = GetSystemDirectory(path.str, path.size);
760   KMP_DEBUG_ASSERT(path_size > 0);
761   if (path_size >= path.size) {
762     // Buffer is too short.  Expand the buffer and try again.
763     __kmp_str_buf_reserve(&path, path_size);
764     path_size = GetSystemDirectory(path.str, path.size);
765     KMP_DEBUG_ASSERT(path_size > 0);
766   }
767   if (path_size > 0 && path_size < path.size) {
768     // Now we have system directory name in the buffer.
769     // Append backslash and name of dll to form full path,
770     path.used = path_size;
771     __kmp_str_buf_print(&path, "\\%s", "ntdll.dll");
772 
773     // Now load ntdll using full path.
774     ntdll = GetModuleHandle(path.str);
775   }
776 
777   KMP_DEBUG_ASSERT(ntdll != NULL);
778   if (ntdll != NULL) {
779     NtQuerySystemInformation = (NtQuerySystemInformation_t)GetProcAddress(
780         ntdll, "NtQuerySystemInformation");
781   }
782   KMP_DEBUG_ASSERT(NtQuerySystemInformation != NULL);
783 
784 #if KMP_GROUP_AFFINITY
785   // Load kernel32.dll.
786   // Same caveat - must use full system path name.
787   if (path_size > 0 && path_size < path.size) {
788     // Truncate the buffer back to just the system path length,
789     // discarding "\\ntdll.dll", and replacing it with "kernel32.dll".
790     path.used = path_size;
791     __kmp_str_buf_print(&path, "\\%s", "kernel32.dll");
792 
793     // Load kernel32.dll using full path.
794     kernel32 = GetModuleHandle(path.str);
795     KA_TRACE(10, ("__kmp_runtime_initialize: kernel32.dll = %s\n", path.str));
796 
797     // Load the function pointers to kernel32.dll routines
798     // that may or may not exist on this system.
799     if (kernel32 != NULL) {
800       __kmp_GetActiveProcessorCount =
801           (kmp_GetActiveProcessorCount_t)GetProcAddress(
802               kernel32, "GetActiveProcessorCount");
803       __kmp_GetActiveProcessorGroupCount =
804           (kmp_GetActiveProcessorGroupCount_t)GetProcAddress(
805               kernel32, "GetActiveProcessorGroupCount");
806       __kmp_GetThreadGroupAffinity =
807           (kmp_GetThreadGroupAffinity_t)GetProcAddress(
808               kernel32, "GetThreadGroupAffinity");
809       __kmp_SetThreadGroupAffinity =
810           (kmp_SetThreadGroupAffinity_t)GetProcAddress(
811               kernel32, "SetThreadGroupAffinity");
812 
813       KA_TRACE(10, ("__kmp_runtime_initialize: __kmp_GetActiveProcessorCount"
814                     " = %p\n",
815                     __kmp_GetActiveProcessorCount));
816       KA_TRACE(10, ("__kmp_runtime_initialize: "
817                     "__kmp_GetActiveProcessorGroupCount = %p\n",
818                     __kmp_GetActiveProcessorGroupCount));
819       KA_TRACE(10, ("__kmp_runtime_initialize:__kmp_GetThreadGroupAffinity"
820                     " = %p\n",
821                     __kmp_GetThreadGroupAffinity));
822       KA_TRACE(10, ("__kmp_runtime_initialize: __kmp_SetThreadGroupAffinity"
823                     " = %p\n",
824                     __kmp_SetThreadGroupAffinity));
825       KA_TRACE(10, ("__kmp_runtime_initialize: sizeof(kmp_affin_mask_t) = %d\n",
826                     sizeof(kmp_affin_mask_t)));
827 
828       // See if group affinity is supported on this system.
829       // If so, calculate the #groups and #procs.
830       //
831       // Group affinity was introduced with Windows* 7 OS and
832       // Windows* Server 2008 R2 OS.
833       if ((__kmp_GetActiveProcessorCount != NULL) &&
834           (__kmp_GetActiveProcessorGroupCount != NULL) &&
835           (__kmp_GetThreadGroupAffinity != NULL) &&
836           (__kmp_SetThreadGroupAffinity != NULL) &&
837           ((__kmp_num_proc_groups = __kmp_GetActiveProcessorGroupCount()) >
838            1)) {
839         // Calculate the total number of active OS procs.
840         int i;
841 
842         KA_TRACE(10, ("__kmp_runtime_initialize: %d processor groups"
843                       " detected\n",
844                       __kmp_num_proc_groups));
845 
846         __kmp_xproc = 0;
847 
848         for (i = 0; i < __kmp_num_proc_groups; i++) {
849           DWORD size = __kmp_GetActiveProcessorCount(i);
850           __kmp_xproc += size;
851           KA_TRACE(10, ("__kmp_runtime_initialize: proc group %d size = %d\n",
852                         i, size));
853         }
854       } else {
855         KA_TRACE(10, ("__kmp_runtime_initialize: %d processor groups"
856                       " detected\n",
857                       __kmp_num_proc_groups));
858       }
859     }
860   }
861   if (__kmp_num_proc_groups <= 1) {
862     GetSystemInfo(&info);
863     __kmp_xproc = info.dwNumberOfProcessors;
864   }
865 #else
866   (void)kernel32;
867   GetSystemInfo(&info);
868   __kmp_xproc = info.dwNumberOfProcessors;
869 #endif /* KMP_GROUP_AFFINITY */
870 
871   // If the OS said there were 0 procs, take a guess and use a value of 2.
872   // This is done for Linux* OS, also.  Do we need error / warning?
873   if (__kmp_xproc <= 0) {
874     __kmp_xproc = 2;
875   }
876 
877   KA_TRACE(5,
878            ("__kmp_runtime_initialize: total processors = %d\n", __kmp_xproc));
879 
880   __kmp_str_buf_free(&path);
881 
882 #if USE_ITT_BUILD
883   __kmp_itt_initialize();
884 #endif /* USE_ITT_BUILD */
885 
886   __kmp_init_runtime = TRUE;
887 } // __kmp_runtime_initialize
888 
889 void __kmp_runtime_destroy(void) {
890   if (!__kmp_init_runtime) {
891     return;
892   }
893 
894 #if USE_ITT_BUILD
895   __kmp_itt_destroy();
896 #endif /* USE_ITT_BUILD */
897 
898   /* we can't DeleteCriticalsection( & __kmp_win32_section ); */
899   /* due to the KX_TRACE() commands */
900   KA_TRACE(40, ("__kmp_runtime_destroy\n"));
901 
902   if (__kmp_gtid_threadprivate_key) {
903     TlsFree(__kmp_gtid_threadprivate_key);
904     __kmp_gtid_threadprivate_key = 0;
905   }
906 
907   __kmp_affinity_uninitialize();
908   DeleteCriticalSection(&__kmp_win32_section);
909 
910   ntdll = NULL;
911   NtQuerySystemInformation = NULL;
912 
913 #if KMP_ARCH_X86_64
914   kernel32 = NULL;
915   __kmp_GetActiveProcessorCount = NULL;
916   __kmp_GetActiveProcessorGroupCount = NULL;
917   __kmp_GetThreadGroupAffinity = NULL;
918   __kmp_SetThreadGroupAffinity = NULL;
919 #endif // KMP_ARCH_X86_64
920 
921   __kmp_init_runtime = FALSE;
922 }
923 
924 void __kmp_terminate_thread(int gtid) {
925   kmp_info_t *th = __kmp_threads[gtid];
926 
927   if (!th)
928     return;
929 
930   KA_TRACE(10, ("__kmp_terminate_thread: kill (%d)\n", gtid));
931 
932   if (TerminateThread(th->th.th_info.ds.ds_thread, (DWORD)-1) == FALSE) {
933     /* It's OK, the thread may have exited already */
934   }
935   __kmp_free_handle(th->th.th_info.ds.ds_thread);
936 }
937 
938 void __kmp_clear_system_time(void) {
939   LARGE_INTEGER time;
940   QueryPerformanceCounter(&time);
941   __kmp_win32_time = (kmp_int64)time.QuadPart;
942 }
943 
944 void __kmp_initialize_system_tick(void) {
945   {
946     BOOL status;
947     LARGE_INTEGER freq;
948 
949     status = QueryPerformanceFrequency(&freq);
950     if (!status) {
951       DWORD error = GetLastError();
952       __kmp_fatal(KMP_MSG(FunctionError, "QueryPerformanceFrequency()"),
953                   KMP_ERR(error), __kmp_msg_null);
954 
955     } else {
956       __kmp_win32_tick = ((double)1.0) / (double)freq.QuadPart;
957     }
958   }
959 }
960 
961 /* Calculate the elapsed wall clock time for the user */
962 
963 void __kmp_elapsed(double *t) {
964   LARGE_INTEGER now;
965   QueryPerformanceCounter(&now);
966   *t = ((double)now.QuadPart) * __kmp_win32_tick;
967 }
968 
969 /* Calculate the elapsed wall clock tick for the user */
970 
971 void __kmp_elapsed_tick(double *t) { *t = __kmp_win32_tick; }
972 
973 void __kmp_read_system_time(double *delta) {
974   if (delta != NULL) {
975     LARGE_INTEGER now;
976     QueryPerformanceCounter(&now);
977     *delta = ((double)(((kmp_int64)now.QuadPart) - __kmp_win32_time)) *
978              __kmp_win32_tick;
979   }
980 }
981 
982 /* Return the current time stamp in nsec */
983 kmp_uint64 __kmp_now_nsec() {
984   LARGE_INTEGER now;
985   QueryPerformanceCounter(&now);
986   return 1e9 * __kmp_win32_tick * now.QuadPart;
987 }
988 
989 extern "C" void *__stdcall __kmp_launch_worker(void *arg) {
990   volatile void *stack_data;
991   void *exit_val;
992   void *padding = 0;
993   kmp_info_t *this_thr = (kmp_info_t *)arg;
994   int gtid;
995 
996   gtid = this_thr->th.th_info.ds.ds_gtid;
997   __kmp_gtid_set_specific(gtid);
998 #ifdef KMP_TDATA_GTID
999 #error "This define causes problems with LoadLibrary() + declspec(thread) " \
1000         "on Windows* OS.  See CQ50564, tests kmp_load_library*.c and this MSDN " \
1001         "reference: http://support.microsoft.com/kb/118816"
1002 //__kmp_gtid = gtid;
1003 #endif
1004 
1005 #if USE_ITT_BUILD
1006   __kmp_itt_thread_name(gtid);
1007 #endif /* USE_ITT_BUILD */
1008 
1009   __kmp_affinity_set_init_mask(gtid, FALSE);
1010 
1011 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1012   // Set FP control regs to be a copy of the parallel initialization thread's.
1013   __kmp_clear_x87_fpu_status_word();
1014   __kmp_load_x87_fpu_control_word(&__kmp_init_x87_fpu_control_word);
1015   __kmp_load_mxcsr(&__kmp_init_mxcsr);
1016 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
1017 
1018   if (__kmp_stkoffset > 0 && gtid > 0) {
1019     padding = KMP_ALLOCA(gtid * __kmp_stkoffset);
1020     (void)padding;
1021   }
1022 
1023   KMP_FSYNC_RELEASING(&this_thr->th.th_info.ds.ds_alive);
1024   this_thr->th.th_info.ds.ds_thread_id = GetCurrentThreadId();
1025   TCW_4(this_thr->th.th_info.ds.ds_alive, TRUE);
1026 
1027   if (TCR_4(__kmp_gtid_mode) <
1028       2) { // check stack only if it is used to get gtid
1029     TCW_PTR(this_thr->th.th_info.ds.ds_stackbase, &stack_data);
1030     KMP_ASSERT(this_thr->th.th_info.ds.ds_stackgrow == FALSE);
1031     __kmp_check_stack_overlap(this_thr);
1032   }
1033   KMP_MB();
1034   exit_val = __kmp_launch_thread(this_thr);
1035   KMP_FSYNC_RELEASING(&this_thr->th.th_info.ds.ds_alive);
1036   TCW_4(this_thr->th.th_info.ds.ds_alive, FALSE);
1037   KMP_MB();
1038   return exit_val;
1039 }
1040 
1041 #if KMP_USE_MONITOR
1042 /* The monitor thread controls all of the threads in the complex */
1043 
1044 void *__stdcall __kmp_launch_monitor(void *arg) {
1045   DWORD wait_status;
1046   kmp_thread_t monitor;
1047   int status;
1048   int interval;
1049   kmp_info_t *this_thr = (kmp_info_t *)arg;
1050 
1051   KMP_DEBUG_ASSERT(__kmp_init_monitor);
1052   TCW_4(__kmp_init_monitor, 2); // AC: Signal library that monitor has started
1053   // TODO: hide "2" in enum (like {true,false,started})
1054   this_thr->th.th_info.ds.ds_thread_id = GetCurrentThreadId();
1055   TCW_4(this_thr->th.th_info.ds.ds_alive, TRUE);
1056 
1057   KMP_MB(); /* Flush all pending memory write invalidates.  */
1058   KA_TRACE(10, ("__kmp_launch_monitor: launched\n"));
1059 
1060   monitor = GetCurrentThread();
1061 
1062   /* set thread priority */
1063   status = SetThreadPriority(monitor, THREAD_PRIORITY_HIGHEST);
1064   if (!status) {
1065     DWORD error = GetLastError();
1066     __kmp_fatal(KMP_MSG(CantSetThreadPriority), KMP_ERR(error), __kmp_msg_null);
1067   }
1068 
1069   /* register us as monitor */
1070   __kmp_gtid_set_specific(KMP_GTID_MONITOR);
1071 #ifdef KMP_TDATA_GTID
1072 #error "This define causes problems with LoadLibrary() + declspec(thread) " \
1073         "on Windows* OS.  See CQ50564, tests kmp_load_library*.c and this MSDN " \
1074         "reference: http://support.microsoft.com/kb/118816"
1075 //__kmp_gtid = KMP_GTID_MONITOR;
1076 #endif
1077 
1078 #if USE_ITT_BUILD
1079   __kmp_itt_thread_ignore(); // Instruct Intel(R) Threading Tools to ignore
1080 // monitor thread.
1081 #endif /* USE_ITT_BUILD */
1082 
1083   KMP_MB(); /* Flush all pending memory write invalidates.  */
1084 
1085   interval = (1000 / __kmp_monitor_wakeups); /* in milliseconds */
1086 
1087   while (!TCR_4(__kmp_global.g.g_done)) {
1088     /*  This thread monitors the state of the system */
1089 
1090     KA_TRACE(15, ("__kmp_launch_monitor: update\n"));
1091 
1092     wait_status = WaitForSingleObject(__kmp_monitor_ev, interval);
1093 
1094     if (wait_status == WAIT_TIMEOUT) {
1095       TCW_4(__kmp_global.g.g_time.dt.t_value,
1096             TCR_4(__kmp_global.g.g_time.dt.t_value) + 1);
1097     }
1098 
1099     KMP_MB(); /* Flush all pending memory write invalidates.  */
1100   }
1101 
1102   KA_TRACE(10, ("__kmp_launch_monitor: finished\n"));
1103 
1104   status = SetThreadPriority(monitor, THREAD_PRIORITY_NORMAL);
1105   if (!status) {
1106     DWORD error = GetLastError();
1107     __kmp_fatal(KMP_MSG(CantSetThreadPriority), KMP_ERR(error), __kmp_msg_null);
1108   }
1109 
1110   if (__kmp_global.g.g_abort != 0) {
1111     /* now we need to terminate the worker threads   */
1112     /* the value of t_abort is the signal we caught */
1113     int gtid;
1114 
1115     KA_TRACE(10, ("__kmp_launch_monitor: terminate sig=%d\n",
1116                   (__kmp_global.g.g_abort)));
1117 
1118     /* terminate the OpenMP worker threads */
1119     /* TODO this is not valid for sibling threads!!
1120      * the uber master might not be 0 anymore.. */
1121     for (gtid = 1; gtid < __kmp_threads_capacity; ++gtid)
1122       __kmp_terminate_thread(gtid);
1123 
1124     __kmp_cleanup();
1125 
1126     Sleep(0);
1127 
1128     KA_TRACE(10,
1129              ("__kmp_launch_monitor: raise sig=%d\n", __kmp_global.g.g_abort));
1130 
1131     if (__kmp_global.g.g_abort > 0) {
1132       raise(__kmp_global.g.g_abort);
1133     }
1134   }
1135 
1136   TCW_4(this_thr->th.th_info.ds.ds_alive, FALSE);
1137 
1138   KMP_MB();
1139   return arg;
1140 }
1141 #endif
1142 
1143 void __kmp_create_worker(int gtid, kmp_info_t *th, size_t stack_size) {
1144   kmp_thread_t handle;
1145   DWORD idThread;
1146 
1147   KA_TRACE(10, ("__kmp_create_worker: try to create thread (%d)\n", gtid));
1148 
1149   th->th.th_info.ds.ds_gtid = gtid;
1150 
1151   if (KMP_UBER_GTID(gtid)) {
1152     int stack_data;
1153 
1154     /* TODO: GetCurrentThread() returns a pseudo-handle that is unsuitable for
1155        other threads to use. Is it appropriate to just use GetCurrentThread?
1156        When should we close this handle?  When unregistering the root? */
1157     {
1158       BOOL rc;
1159       rc = DuplicateHandle(GetCurrentProcess(), GetCurrentThread(),
1160                            GetCurrentProcess(), &th->th.th_info.ds.ds_thread, 0,
1161                            FALSE, DUPLICATE_SAME_ACCESS);
1162       KMP_ASSERT(rc);
1163       KA_TRACE(10, (" __kmp_create_worker: ROOT Handle duplicated, th = %p, "
1164                     "handle = %" KMP_UINTPTR_SPEC "\n",
1165                     (LPVOID)th, th->th.th_info.ds.ds_thread));
1166       th->th.th_info.ds.ds_thread_id = GetCurrentThreadId();
1167     }
1168     if (TCR_4(__kmp_gtid_mode) < 2) { // check stack only if used to get gtid
1169       /* we will dynamically update the stack range if gtid_mode == 1 */
1170       TCW_PTR(th->th.th_info.ds.ds_stackbase, &stack_data);
1171       TCW_PTR(th->th.th_info.ds.ds_stacksize, 0);
1172       TCW_4(th->th.th_info.ds.ds_stackgrow, TRUE);
1173       __kmp_check_stack_overlap(th);
1174     }
1175   } else {
1176     KMP_MB(); /* Flush all pending memory write invalidates.  */
1177 
1178     /* Set stack size for this thread now. */
1179     KA_TRACE(10,
1180              ("__kmp_create_worker: stack_size = %" KMP_SIZE_T_SPEC " bytes\n",
1181               stack_size));
1182 
1183     stack_size += gtid * __kmp_stkoffset;
1184 
1185     TCW_PTR(th->th.th_info.ds.ds_stacksize, stack_size);
1186     TCW_4(th->th.th_info.ds.ds_stackgrow, FALSE);
1187 
1188     KA_TRACE(10,
1189              ("__kmp_create_worker: (before) stack_size = %" KMP_SIZE_T_SPEC
1190               " bytes, &__kmp_launch_worker = %p, th = %p, &idThread = %p\n",
1191               (SIZE_T)stack_size, (LPTHREAD_START_ROUTINE)&__kmp_launch_worker,
1192               (LPVOID)th, &idThread));
1193 
1194     handle = CreateThread(
1195         NULL, (SIZE_T)stack_size, (LPTHREAD_START_ROUTINE)__kmp_launch_worker,
1196         (LPVOID)th, STACK_SIZE_PARAM_IS_A_RESERVATION, &idThread);
1197 
1198     KA_TRACE(10,
1199              ("__kmp_create_worker: (after) stack_size = %" KMP_SIZE_T_SPEC
1200               " bytes, &__kmp_launch_worker = %p, th = %p, "
1201               "idThread = %u, handle = %" KMP_UINTPTR_SPEC "\n",
1202               (SIZE_T)stack_size, (LPTHREAD_START_ROUTINE)&__kmp_launch_worker,
1203               (LPVOID)th, idThread, handle));
1204 
1205     if (handle == 0) {
1206       DWORD error = GetLastError();
1207       __kmp_fatal(KMP_MSG(CantCreateThread), KMP_ERR(error), __kmp_msg_null);
1208     } else {
1209       th->th.th_info.ds.ds_thread = handle;
1210     }
1211 
1212     KMP_MB(); /* Flush all pending memory write invalidates.  */
1213   }
1214 
1215   KA_TRACE(10, ("__kmp_create_worker: done creating thread (%d)\n", gtid));
1216 }
1217 
1218 int __kmp_still_running(kmp_info_t *th) {
1219   return (WAIT_TIMEOUT == WaitForSingleObject(th->th.th_info.ds.ds_thread, 0));
1220 }
1221 
1222 #if KMP_USE_MONITOR
1223 void __kmp_create_monitor(kmp_info_t *th) {
1224   kmp_thread_t handle;
1225   DWORD idThread;
1226   int ideal, new_ideal;
1227 
1228   if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME) {
1229     // We don't need monitor thread in case of MAX_BLOCKTIME
1230     KA_TRACE(10, ("__kmp_create_monitor: skipping monitor thread because of "
1231                   "MAX blocktime\n"));
1232     th->th.th_info.ds.ds_tid = 0; // this makes reap_monitor no-op
1233     th->th.th_info.ds.ds_gtid = 0;
1234     TCW_4(__kmp_init_monitor, 2); // Signal to stop waiting for monitor creation
1235     return;
1236   }
1237   KA_TRACE(10, ("__kmp_create_monitor: try to create monitor\n"));
1238 
1239   KMP_MB(); /* Flush all pending memory write invalidates.  */
1240 
1241   __kmp_monitor_ev = CreateEvent(NULL, TRUE, FALSE, NULL);
1242   if (__kmp_monitor_ev == NULL) {
1243     DWORD error = GetLastError();
1244     __kmp_fatal(KMP_MSG(CantCreateEvent), KMP_ERR(error), __kmp_msg_null);
1245   }
1246 #if USE_ITT_BUILD
1247   __kmp_itt_system_object_created(__kmp_monitor_ev, "Event");
1248 #endif /* USE_ITT_BUILD */
1249 
1250   th->th.th_info.ds.ds_tid = KMP_GTID_MONITOR;
1251   th->th.th_info.ds.ds_gtid = KMP_GTID_MONITOR;
1252 
1253   // FIXME - on Windows* OS, if __kmp_monitor_stksize = 0, figure out how
1254   // to automatically expand stacksize based on CreateThread error code.
1255   if (__kmp_monitor_stksize == 0) {
1256     __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE;
1257   }
1258   if (__kmp_monitor_stksize < __kmp_sys_min_stksize) {
1259     __kmp_monitor_stksize = __kmp_sys_min_stksize;
1260   }
1261 
1262   KA_TRACE(10, ("__kmp_create_monitor: requested stacksize = %d bytes\n",
1263                 (int)__kmp_monitor_stksize));
1264 
1265   TCW_4(__kmp_global.g.g_time.dt.t_value, 0);
1266 
1267   handle =
1268       CreateThread(NULL, (SIZE_T)__kmp_monitor_stksize,
1269                    (LPTHREAD_START_ROUTINE)__kmp_launch_monitor, (LPVOID)th,
1270                    STACK_SIZE_PARAM_IS_A_RESERVATION, &idThread);
1271   if (handle == 0) {
1272     DWORD error = GetLastError();
1273     __kmp_fatal(KMP_MSG(CantCreateThread), KMP_ERR(error), __kmp_msg_null);
1274   } else
1275     th->th.th_info.ds.ds_thread = handle;
1276 
1277   KMP_MB(); /* Flush all pending memory write invalidates.  */
1278 
1279   KA_TRACE(10, ("__kmp_create_monitor: monitor created %p\n",
1280                 (void *)th->th.th_info.ds.ds_thread));
1281 }
1282 #endif
1283 
1284 /* Check to see if thread is still alive.
1285    NOTE:  The ExitProcess(code) system call causes all threads to Terminate
1286    with a exit_val = code.  Because of this we can not rely on exit_val having
1287    any particular value.  So this routine may return STILL_ALIVE in exit_val
1288    even after the thread is dead. */
1289 
1290 int __kmp_is_thread_alive(kmp_info_t *th, DWORD *exit_val) {
1291   DWORD rc;
1292   rc = GetExitCodeThread(th->th.th_info.ds.ds_thread, exit_val);
1293   if (rc == 0) {
1294     DWORD error = GetLastError();
1295     __kmp_fatal(KMP_MSG(FunctionError, "GetExitCodeThread()"), KMP_ERR(error),
1296                 __kmp_msg_null);
1297   }
1298   return (*exit_val == STILL_ACTIVE);
1299 }
1300 
1301 void __kmp_exit_thread(int exit_status) {
1302   ExitThread(exit_status);
1303 } // __kmp_exit_thread
1304 
1305 // This is a common part for both __kmp_reap_worker() and __kmp_reap_monitor().
1306 static void __kmp_reap_common(kmp_info_t *th) {
1307   DWORD exit_val;
1308 
1309   KMP_MB(); /* Flush all pending memory write invalidates.  */
1310 
1311   KA_TRACE(
1312       10, ("__kmp_reap_common: try to reap (%d)\n", th->th.th_info.ds.ds_gtid));
1313 
1314   /* 2006-10-19:
1315      There are two opposite situations:
1316      1. Windows* OS keep thread alive after it resets ds_alive flag and
1317      exits from thread function. (For example, see C70770/Q394281 "unloading of
1318      dll based on OMP is very slow".)
1319      2. Windows* OS may kill thread before it resets ds_alive flag.
1320 
1321      Right solution seems to be waiting for *either* thread termination *or*
1322      ds_alive resetting. */
1323   {
1324     // TODO: This code is very similar to KMP_WAIT. Need to generalize
1325     // KMP_WAIT to cover this usage also.
1326     void *obj = NULL;
1327     kmp_uint32 spins;
1328     kmp_uint64 time;
1329 #if USE_ITT_BUILD
1330     KMP_FSYNC_SPIN_INIT(obj, (void *)&th->th.th_info.ds.ds_alive);
1331 #endif /* USE_ITT_BUILD */
1332     KMP_INIT_YIELD(spins);
1333     KMP_INIT_BACKOFF(time);
1334     do {
1335 #if USE_ITT_BUILD
1336       KMP_FSYNC_SPIN_PREPARE(obj);
1337 #endif /* USE_ITT_BUILD */
1338       __kmp_is_thread_alive(th, &exit_val);
1339       KMP_YIELD_OVERSUB_ELSE_SPIN(spins, time);
1340     } while (exit_val == STILL_ACTIVE && TCR_4(th->th.th_info.ds.ds_alive));
1341 #if USE_ITT_BUILD
1342     if (exit_val == STILL_ACTIVE) {
1343       KMP_FSYNC_CANCEL(obj);
1344     } else {
1345       KMP_FSYNC_SPIN_ACQUIRED(obj);
1346     }
1347 #endif /* USE_ITT_BUILD */
1348   }
1349 
1350   __kmp_free_handle(th->th.th_info.ds.ds_thread);
1351 
1352   /* NOTE:  The ExitProcess(code) system call causes all threads to Terminate
1353      with a exit_val = code.  Because of this we can not rely on exit_val having
1354      any particular value. */
1355   kmp_intptr_t e = (kmp_intptr_t)exit_val;
1356   if (exit_val == STILL_ACTIVE) {
1357     KA_TRACE(1, ("__kmp_reap_common: thread still active.\n"));
1358   } else if ((void *)e != (void *)th) {
1359     KA_TRACE(1, ("__kmp_reap_common: ExitProcess / TerminateThread used?\n"));
1360   }
1361 
1362   KA_TRACE(10,
1363            ("__kmp_reap_common: done reaping (%d), handle = %" KMP_UINTPTR_SPEC
1364             "\n",
1365             th->th.th_info.ds.ds_gtid, th->th.th_info.ds.ds_thread));
1366 
1367   th->th.th_info.ds.ds_thread = 0;
1368   th->th.th_info.ds.ds_tid = KMP_GTID_DNE;
1369   th->th.th_info.ds.ds_gtid = KMP_GTID_DNE;
1370   th->th.th_info.ds.ds_thread_id = 0;
1371 
1372   KMP_MB(); /* Flush all pending memory write invalidates.  */
1373 }
1374 
1375 #if KMP_USE_MONITOR
1376 void __kmp_reap_monitor(kmp_info_t *th) {
1377   int status;
1378 
1379   KA_TRACE(10, ("__kmp_reap_monitor: try to reap %p\n",
1380                 (void *)th->th.th_info.ds.ds_thread));
1381 
1382   // If monitor has been created, its tid and gtid should be KMP_GTID_MONITOR.
1383   // If both tid and gtid are 0, it means the monitor did not ever start.
1384   // If both tid and gtid are KMP_GTID_DNE, the monitor has been shut down.
1385   KMP_DEBUG_ASSERT(th->th.th_info.ds.ds_tid == th->th.th_info.ds.ds_gtid);
1386   if (th->th.th_info.ds.ds_gtid != KMP_GTID_MONITOR) {
1387     KA_TRACE(10, ("__kmp_reap_monitor: monitor did not start, returning\n"));
1388     return;
1389   }
1390 
1391   KMP_MB(); /* Flush all pending memory write invalidates.  */
1392 
1393   status = SetEvent(__kmp_monitor_ev);
1394   if (status == FALSE) {
1395     DWORD error = GetLastError();
1396     __kmp_fatal(KMP_MSG(CantSetEvent), KMP_ERR(error), __kmp_msg_null);
1397   }
1398   KA_TRACE(10, ("__kmp_reap_monitor: reaping thread (%d)\n",
1399                 th->th.th_info.ds.ds_gtid));
1400   __kmp_reap_common(th);
1401 
1402   __kmp_free_handle(__kmp_monitor_ev);
1403 
1404   KMP_MB(); /* Flush all pending memory write invalidates.  */
1405 }
1406 #endif
1407 
1408 void __kmp_reap_worker(kmp_info_t *th) {
1409   KA_TRACE(10, ("__kmp_reap_worker: reaping thread (%d)\n",
1410                 th->th.th_info.ds.ds_gtid));
1411   __kmp_reap_common(th);
1412 }
1413 
1414 #if KMP_HANDLE_SIGNALS
1415 
1416 static void __kmp_team_handler(int signo) {
1417   if (__kmp_global.g.g_abort == 0) {
1418     // Stage 1 signal handler, let's shut down all of the threads.
1419     if (__kmp_debug_buf) {
1420       __kmp_dump_debug_buffer();
1421     }
1422     KMP_MB(); // Flush all pending memory write invalidates.
1423     TCW_4(__kmp_global.g.g_abort, signo);
1424     KMP_MB(); // Flush all pending memory write invalidates.
1425     TCW_4(__kmp_global.g.g_done, TRUE);
1426     KMP_MB(); // Flush all pending memory write invalidates.
1427   }
1428 } // __kmp_team_handler
1429 
1430 static sig_func_t __kmp_signal(int signum, sig_func_t handler) {
1431   sig_func_t old = signal(signum, handler);
1432   if (old == SIG_ERR) {
1433     int error = errno;
1434     __kmp_fatal(KMP_MSG(FunctionError, "signal"), KMP_ERR(error),
1435                 __kmp_msg_null);
1436   }
1437   return old;
1438 }
1439 
1440 static void __kmp_install_one_handler(int sig, sig_func_t handler,
1441                                       int parallel_init) {
1442   sig_func_t old;
1443   KMP_MB(); /* Flush all pending memory write invalidates.  */
1444   KB_TRACE(60, ("__kmp_install_one_handler: called: sig=%d\n", sig));
1445   if (parallel_init) {
1446     old = __kmp_signal(sig, handler);
1447     // SIG_DFL on Windows* OS in NULL or 0.
1448     if (old == __kmp_sighldrs[sig]) {
1449       __kmp_siginstalled[sig] = 1;
1450     } else { // Restore/keep user's handler if one previously installed.
1451       old = __kmp_signal(sig, old);
1452     }
1453   } else {
1454     // Save initial/system signal handlers to see if user handlers installed.
1455     // 2009-09-23: It is a dead code. On Windows* OS __kmp_install_signals
1456     // called once with parallel_init == TRUE.
1457     old = __kmp_signal(sig, SIG_DFL);
1458     __kmp_sighldrs[sig] = old;
1459     __kmp_signal(sig, old);
1460   }
1461   KMP_MB(); /* Flush all pending memory write invalidates.  */
1462 } // __kmp_install_one_handler
1463 
1464 static void __kmp_remove_one_handler(int sig) {
1465   if (__kmp_siginstalled[sig]) {
1466     sig_func_t old;
1467     KMP_MB(); // Flush all pending memory write invalidates.
1468     KB_TRACE(60, ("__kmp_remove_one_handler: called: sig=%d\n", sig));
1469     old = __kmp_signal(sig, __kmp_sighldrs[sig]);
1470     if (old != __kmp_team_handler) {
1471       KB_TRACE(10, ("__kmp_remove_one_handler: oops, not our handler, "
1472                     "restoring: sig=%d\n",
1473                     sig));
1474       old = __kmp_signal(sig, old);
1475     }
1476     __kmp_sighldrs[sig] = NULL;
1477     __kmp_siginstalled[sig] = 0;
1478     KMP_MB(); // Flush all pending memory write invalidates.
1479   }
1480 } // __kmp_remove_one_handler
1481 
1482 void __kmp_install_signals(int parallel_init) {
1483   KB_TRACE(10, ("__kmp_install_signals: called\n"));
1484   if (!__kmp_handle_signals) {
1485     KB_TRACE(10, ("__kmp_install_signals: KMP_HANDLE_SIGNALS is false - "
1486                   "handlers not installed\n"));
1487     return;
1488   }
1489   __kmp_install_one_handler(SIGINT, __kmp_team_handler, parallel_init);
1490   __kmp_install_one_handler(SIGILL, __kmp_team_handler, parallel_init);
1491   __kmp_install_one_handler(SIGABRT, __kmp_team_handler, parallel_init);
1492   __kmp_install_one_handler(SIGFPE, __kmp_team_handler, parallel_init);
1493   __kmp_install_one_handler(SIGSEGV, __kmp_team_handler, parallel_init);
1494   __kmp_install_one_handler(SIGTERM, __kmp_team_handler, parallel_init);
1495 } // __kmp_install_signals
1496 
1497 void __kmp_remove_signals(void) {
1498   int sig;
1499   KB_TRACE(10, ("__kmp_remove_signals: called\n"));
1500   for (sig = 1; sig < NSIG; ++sig) {
1501     __kmp_remove_one_handler(sig);
1502   }
1503 } // __kmp_remove_signals
1504 
1505 #endif // KMP_HANDLE_SIGNALS
1506 
1507 /* Put the thread to sleep for a time period */
1508 void __kmp_thread_sleep(int millis) {
1509   DWORD status;
1510 
1511   status = SleepEx((DWORD)millis, FALSE);
1512   if (status) {
1513     DWORD error = GetLastError();
1514     __kmp_fatal(KMP_MSG(FunctionError, "SleepEx()"), KMP_ERR(error),
1515                 __kmp_msg_null);
1516   }
1517 }
1518 
1519 // Determine whether the given address is mapped into the current address space.
1520 int __kmp_is_address_mapped(void *addr) {
1521   MEMORY_BASIC_INFORMATION lpBuffer;
1522   SIZE_T dwLength;
1523 
1524   dwLength = sizeof(MEMORY_BASIC_INFORMATION);
1525 
1526   VirtualQuery(addr, &lpBuffer, dwLength);
1527 
1528   return !(((lpBuffer.State == MEM_RESERVE) || (lpBuffer.State == MEM_FREE)) ||
1529            ((lpBuffer.Protect == PAGE_NOACCESS) ||
1530             (lpBuffer.Protect == PAGE_EXECUTE)));
1531 }
1532 
1533 kmp_uint64 __kmp_hardware_timestamp(void) {
1534   kmp_uint64 r = 0;
1535 
1536   QueryPerformanceCounter((LARGE_INTEGER *)&r);
1537   return r;
1538 }
1539 
1540 /* Free handle and check the error code */
1541 void __kmp_free_handle(kmp_thread_t tHandle) {
1542   /* called with parameter type HANDLE also, thus suppose kmp_thread_t defined
1543    * as HANDLE */
1544   BOOL rc;
1545   rc = CloseHandle(tHandle);
1546   if (!rc) {
1547     DWORD error = GetLastError();
1548     __kmp_fatal(KMP_MSG(CantCloseHandle), KMP_ERR(error), __kmp_msg_null);
1549   }
1550 }
1551 
1552 int __kmp_get_load_balance(int max) {
1553   static ULONG glb_buff_size = 100 * 1024;
1554 
1555   // Saved count of the running threads for the thread balance algorithm
1556   static int glb_running_threads = 0;
1557   static double glb_call_time = 0; /* Thread balance algorithm call time */
1558 
1559   int running_threads = 0; // Number of running threads in the system.
1560   NTSTATUS status = 0;
1561   ULONG buff_size = 0;
1562   ULONG info_size = 0;
1563   void *buffer = NULL;
1564   PSYSTEM_PROCESS_INFORMATION spi = NULL;
1565   int first_time = 1;
1566 
1567   double call_time = 0.0; // start, finish;
1568 
1569   __kmp_elapsed(&call_time);
1570 
1571   if (glb_call_time &&
1572       (call_time - glb_call_time < __kmp_load_balance_interval)) {
1573     running_threads = glb_running_threads;
1574     goto finish;
1575   }
1576   glb_call_time = call_time;
1577 
1578   // Do not spend time on running algorithm if we have a permanent error.
1579   if (NtQuerySystemInformation == NULL) {
1580     running_threads = -1;
1581     goto finish;
1582   }
1583 
1584   if (max <= 0) {
1585     max = INT_MAX;
1586   }
1587 
1588   do {
1589 
1590     if (first_time) {
1591       buff_size = glb_buff_size;
1592     } else {
1593       buff_size = 2 * buff_size;
1594     }
1595 
1596     buffer = KMP_INTERNAL_REALLOC(buffer, buff_size);
1597     if (buffer == NULL) {
1598       running_threads = -1;
1599       goto finish;
1600     }
1601     status = NtQuerySystemInformation(SystemProcessInformation, buffer,
1602                                       buff_size, &info_size);
1603     first_time = 0;
1604 
1605   } while (status == STATUS_INFO_LENGTH_MISMATCH);
1606   glb_buff_size = buff_size;
1607 
1608 #define CHECK(cond)                                                            \
1609   {                                                                            \
1610     KMP_DEBUG_ASSERT(cond);                                                    \
1611     if (!(cond)) {                                                             \
1612       running_threads = -1;                                                    \
1613       goto finish;                                                             \
1614     }                                                                          \
1615   }
1616 
1617   CHECK(buff_size >= info_size);
1618   spi = PSYSTEM_PROCESS_INFORMATION(buffer);
1619   for (;;) {
1620     ptrdiff_t offset = uintptr_t(spi) - uintptr_t(buffer);
1621     CHECK(0 <= offset &&
1622           offset + sizeof(SYSTEM_PROCESS_INFORMATION) < info_size);
1623     HANDLE pid = spi->ProcessId;
1624     ULONG num = spi->NumberOfThreads;
1625     CHECK(num >= 1);
1626     size_t spi_size =
1627         sizeof(SYSTEM_PROCESS_INFORMATION) + sizeof(SYSTEM_THREAD) * (num - 1);
1628     CHECK(offset + spi_size <
1629           info_size); // Make sure process info record fits the buffer.
1630     if (spi->NextEntryOffset != 0) {
1631       CHECK(spi_size <=
1632             spi->NextEntryOffset); // And do not overlap with the next record.
1633     }
1634     // pid == 0 corresponds to the System Idle Process. It always has running
1635     // threads on all cores. So, we don't consider the running threads of this
1636     // process.
1637     if (pid != 0) {
1638       for (int i = 0; i < num; ++i) {
1639         THREAD_STATE state = spi->Threads[i].State;
1640         // Count threads that have Ready or Running state.
1641         // !!! TODO: Why comment does not match the code???
1642         if (state == StateRunning) {
1643           ++running_threads;
1644           // Stop counting running threads if the number is already greater than
1645           // the number of available cores
1646           if (running_threads >= max) {
1647             goto finish;
1648           }
1649         }
1650       }
1651     }
1652     if (spi->NextEntryOffset == 0) {
1653       break;
1654     }
1655     spi = PSYSTEM_PROCESS_INFORMATION(uintptr_t(spi) + spi->NextEntryOffset);
1656   }
1657 
1658 #undef CHECK
1659 
1660 finish: // Clean up and exit.
1661 
1662   if (buffer != NULL) {
1663     KMP_INTERNAL_FREE(buffer);
1664   }
1665 
1666   glb_running_threads = running_threads;
1667 
1668   return running_threads;
1669 } //__kmp_get_load_balance()
1670 
1671 // Find symbol from the loaded modules
1672 void *__kmp_lookup_symbol(const char *name, bool next) {
1673   HANDLE process = GetCurrentProcess();
1674   DWORD needed;
1675   HMODULE *modules = nullptr;
1676   if (!EnumProcessModules(process, modules, 0, &needed))
1677     return nullptr;
1678   DWORD num_modules = needed / sizeof(HMODULE);
1679   modules = (HMODULE *)malloc(num_modules * sizeof(HMODULE));
1680   if (!EnumProcessModules(process, modules, needed, &needed)) {
1681     free(modules);
1682     return nullptr;
1683   }
1684   HMODULE curr_module = nullptr;
1685   if (next) {
1686     // Current module needs to be skipped if next flag is true
1687     if (!GetModuleHandleEx(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS,
1688                            (LPCTSTR)&__kmp_lookup_symbol, &curr_module)) {
1689       free(modules);
1690       return nullptr;
1691     }
1692   }
1693   void *proc = nullptr;
1694   for (uint32_t i = 0; i < num_modules; i++) {
1695     if (next && modules[i] == curr_module)
1696       continue;
1697     proc = (void *)GetProcAddress(modules[i], name);
1698     if (proc)
1699       break;
1700   }
1701   free(modules);
1702   return proc;
1703 }
1704 
1705 // Functions for hidden helper task
1706 void __kmp_hidden_helper_worker_thread_wait() {
1707   KMP_ASSERT(0 && "Hidden helper task is not supported on Windows");
1708 }
1709 
1710 void __kmp_do_initialize_hidden_helper_threads() {
1711   KMP_ASSERT(0 && "Hidden helper task is not supported on Windows");
1712 }
1713 
1714 void __kmp_hidden_helper_threads_initz_wait() {
1715   KMP_ASSERT(0 && "Hidden helper task is not supported on Windows");
1716 }
1717 
1718 void __kmp_hidden_helper_initz_release() {
1719   KMP_ASSERT(0 && "Hidden helper task is not supported on Windows");
1720 }
1721 
1722 void __kmp_hidden_helper_main_thread_wait() {
1723   KMP_ASSERT(0 && "Hidden helper task is not supported on Windows");
1724 }
1725 
1726 void __kmp_hidden_helper_main_thread_release() {
1727   KMP_ASSERT(0 && "Hidden helper task is not supported on Windows");
1728 }
1729 
1730 void __kmp_hidden_helper_worker_thread_signal() {
1731   KMP_ASSERT(0 && "Hidden helper task is not supported on Windows");
1732 }
1733 
1734 void __kmp_hidden_helper_threads_deinitz_wait() {
1735   KMP_ASSERT(0 && "Hidden helper task is not supported on Windows");
1736 }
1737 
1738 void __kmp_hidden_helper_threads_deinitz_release() {
1739   KMP_ASSERT(0 && "Hidden helper task is not supported on Windows");
1740 }
1741