1 /*
2 * z_Linux_util.cpp -- platform specific routines.
3 */
4
5 //===----------------------------------------------------------------------===//
6 //
7 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
8 // See https://llvm.org/LICENSE.txt for license information.
9 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "kmp.h"
14 #include "kmp_affinity.h"
15 #include "kmp_i18n.h"
16 #include "kmp_io.h"
17 #include "kmp_itt.h"
18 #include "kmp_lock.h"
19 #include "kmp_stats.h"
20 #include "kmp_str.h"
21 #include "kmp_wait_release.h"
22 #include "kmp_wrapper_getpid.h"
23
24 #if !KMP_OS_DRAGONFLY && !KMP_OS_FREEBSD && !KMP_OS_NETBSD && !KMP_OS_OPENBSD
25 #include <alloca.h>
26 #endif
27 #include <math.h> // HUGE_VAL.
28 #if KMP_OS_LINUX
29 #include <semaphore.h>
30 #endif // KMP_OS_LINUX
31 #include <sys/resource.h>
32 #if KMP_OS_AIX
33 #include <sys/ldr.h>
34 #include <libperfstat.h>
35 #else
36 #include <sys/syscall.h>
37 #endif
38 #include <sys/time.h>
39 #include <sys/times.h>
40 #include <unistd.h>
41
42 #if KMP_OS_LINUX
43 #include <sys/sysinfo.h>
44 #if KMP_USE_FUTEX
45 // We should really include <futex.h>, but that causes compatibility problems on
46 // different Linux* OS distributions that either require that you include (or
47 // break when you try to include) <pci/types.h>. Since all we need is the two
48 // macros below (which are part of the kernel ABI, so can't change) we just
49 // define the constants here and don't include <futex.h>
50 #ifndef FUTEX_WAIT
51 #define FUTEX_WAIT 0
52 #endif
53 #ifndef FUTEX_WAKE
54 #define FUTEX_WAKE 1
55 #endif
56 #endif
57 #elif KMP_OS_DARWIN
58 #include <mach/mach.h>
59 #include <sys/sysctl.h>
60 #elif KMP_OS_DRAGONFLY || KMP_OS_FREEBSD
61 #include <sys/types.h>
62 #include <sys/sysctl.h>
63 #include <sys/user.h>
64 #include <pthread_np.h>
65 #if KMP_OS_DRAGONFLY
66 #include <kvm.h>
67 #endif
68 #elif KMP_OS_NETBSD || KMP_OS_OPENBSD
69 #include <sys/types.h>
70 #include <sys/sysctl.h>
71 #if KMP_OS_NETBSD
72 #include <sched.h>
73 #endif
74 #elif KMP_OS_SOLARIS
75 #include <libproc.h>
76 #include <procfs.h>
77 #include <thread.h>
78 #include <sys/loadavg.h>
79 #endif
80
81 #include <ctype.h>
82 #include <dirent.h>
83 #include <fcntl.h>
84
85 struct kmp_sys_timer {
86 struct timespec start;
87 };
88
89 #ifndef TIMEVAL_TO_TIMESPEC
90 // Convert timeval to timespec.
91 #define TIMEVAL_TO_TIMESPEC(tv, ts) \
92 do { \
93 (ts)->tv_sec = (tv)->tv_sec; \
94 (ts)->tv_nsec = (tv)->tv_usec * 1000; \
95 } while (0)
96 #endif
97
98 // Convert timespec to nanoseconds.
99 #define TS2NS(timespec) \
100 (((timespec).tv_sec * (long int)1e9) + (timespec).tv_nsec)
101
102 static struct kmp_sys_timer __kmp_sys_timer_data;
103
104 #if KMP_HANDLE_SIGNALS
105 typedef void (*sig_func_t)(int);
106 STATIC_EFI2_WORKAROUND struct sigaction __kmp_sighldrs[NSIG];
107 static sigset_t __kmp_sigset;
108 #endif
109
110 static int __kmp_init_runtime = FALSE;
111
112 static int __kmp_fork_count = 0;
113
114 static pthread_condattr_t __kmp_suspend_cond_attr;
115 static pthread_mutexattr_t __kmp_suspend_mutex_attr;
116
117 static kmp_cond_align_t __kmp_wait_cv;
118 static kmp_mutex_align_t __kmp_wait_mx;
119
120 kmp_uint64 __kmp_ticks_per_msec = 1000000;
121 kmp_uint64 __kmp_ticks_per_usec = 1000;
122
123 #ifdef DEBUG_SUSPEND
__kmp_print_cond(char * buffer,kmp_cond_align_t * cond)124 static void __kmp_print_cond(char *buffer, kmp_cond_align_t *cond) {
125 KMP_SNPRINTF(buffer, 128, "(cond (lock (%ld, %d)), (descr (%p)))",
126 cond->c_cond.__c_lock.__status, cond->c_cond.__c_lock.__spinlock,
127 cond->c_cond.__c_waiting);
128 }
129 #endif
130
131 #if ((KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY || \
132 KMP_OS_AIX) && \
133 KMP_AFFINITY_SUPPORTED)
134
135 /* Affinity support */
136
__kmp_affinity_bind_thread(int which)137 void __kmp_affinity_bind_thread(int which) {
138 KMP_ASSERT2(KMP_AFFINITY_CAPABLE(),
139 "Illegal set affinity operation when not capable");
140
141 kmp_affin_mask_t *mask;
142 KMP_CPU_ALLOC_ON_STACK(mask);
143 KMP_CPU_ZERO(mask);
144 KMP_CPU_SET(which, mask);
145 __kmp_set_system_affinity(mask, TRUE);
146 KMP_CPU_FREE_FROM_STACK(mask);
147 }
148
149 #if KMP_OS_AIX
__kmp_affinity_determine_capable(const char * env_var)150 void __kmp_affinity_determine_capable(const char *env_var) {
151 // All versions of AIX support bindprocessor().
152
153 size_t mask_size = __kmp_xproc / CHAR_BIT;
154 // Round up to byte boundary.
155 if (__kmp_xproc % CHAR_BIT)
156 ++mask_size;
157
158 // Round up to the mask_size_type boundary.
159 if (mask_size % sizeof(__kmp_affin_mask_size))
160 mask_size += sizeof(__kmp_affin_mask_size) -
161 mask_size % sizeof(__kmp_affin_mask_size);
162 KMP_AFFINITY_ENABLE(mask_size);
163 KA_TRACE(10,
164 ("__kmp_affinity_determine_capable: "
165 "AIX OS affinity interface bindprocessor functional (mask size = "
166 "%" KMP_SIZE_T_SPEC ").\n",
167 __kmp_affin_mask_size));
168 }
169
170 #else // !KMP_OS_AIX
171
172 /* Determine if we can access affinity functionality on this version of
173 * Linux* OS by checking __NR_sched_{get,set}affinity system calls, and set
174 * __kmp_affin_mask_size to the appropriate value (0 means not capable). */
__kmp_affinity_determine_capable(const char * env_var)175 void __kmp_affinity_determine_capable(const char *env_var) {
176 // Check and see if the OS supports thread affinity.
177
178 #if KMP_OS_LINUX
179 #define KMP_CPU_SET_SIZE_LIMIT (1024 * 1024)
180 #define KMP_CPU_SET_TRY_SIZE CACHE_LINE
181 #elif KMP_OS_FREEBSD || KMP_OS_DRAGONFLY
182 #define KMP_CPU_SET_SIZE_LIMIT (sizeof(cpuset_t))
183 #elif KMP_OS_NETBSD
184 #define KMP_CPU_SET_SIZE_LIMIT (256)
185 #endif
186
187 int verbose = __kmp_affinity.flags.verbose;
188 int warnings = __kmp_affinity.flags.warnings;
189 enum affinity_type type = __kmp_affinity.type;
190
191 #if KMP_OS_LINUX
192 long gCode;
193 unsigned char *buf;
194 buf = (unsigned char *)KMP_INTERNAL_MALLOC(KMP_CPU_SET_SIZE_LIMIT);
195
196 // If the syscall returns a suggestion for the size,
197 // then we don't have to search for an appropriate size.
198 gCode = syscall(__NR_sched_getaffinity, 0, KMP_CPU_SET_TRY_SIZE, buf);
199 KA_TRACE(30, ("__kmp_affinity_determine_capable: "
200 "initial getaffinity call returned %ld errno = %d\n",
201 gCode, errno));
202
203 if (gCode < 0 && errno != EINVAL) {
204 // System call not supported
205 if (verbose ||
206 (warnings && (type != affinity_none) && (type != affinity_default) &&
207 (type != affinity_disabled))) {
208 int error = errno;
209 kmp_msg_t err_code = KMP_ERR(error);
210 __kmp_msg(kmp_ms_warning, KMP_MSG(GetAffSysCallNotSupported, env_var),
211 err_code, __kmp_msg_null);
212 if (__kmp_generate_warnings == kmp_warnings_off) {
213 __kmp_str_free(&err_code.str);
214 }
215 }
216 KMP_AFFINITY_DISABLE();
217 KMP_INTERNAL_FREE(buf);
218 return;
219 } else if (gCode > 0) {
220 // The optimal situation: the OS returns the size of the buffer it expects.
221 KMP_AFFINITY_ENABLE(gCode);
222 KA_TRACE(10, ("__kmp_affinity_determine_capable: "
223 "affinity supported (mask size %d)\n",
224 (int)__kmp_affin_mask_size));
225 KMP_INTERNAL_FREE(buf);
226 return;
227 }
228
229 // Call the getaffinity system call repeatedly with increasing set sizes
230 // until we succeed, or reach an upper bound on the search.
231 KA_TRACE(30, ("__kmp_affinity_determine_capable: "
232 "searching for proper set size\n"));
233 int size;
234 for (size = 1; size <= KMP_CPU_SET_SIZE_LIMIT; size *= 2) {
235 gCode = syscall(__NR_sched_getaffinity, 0, size, buf);
236 KA_TRACE(30, ("__kmp_affinity_determine_capable: "
237 "getaffinity for mask size %ld returned %ld errno = %d\n",
238 size, gCode, errno));
239
240 if (gCode < 0) {
241 if (errno == ENOSYS) {
242 // We shouldn't get here
243 KA_TRACE(30, ("__kmp_affinity_determine_capable: "
244 "inconsistent OS call behavior: errno == ENOSYS for mask "
245 "size %d\n",
246 size));
247 if (verbose ||
248 (warnings && (type != affinity_none) &&
249 (type != affinity_default) && (type != affinity_disabled))) {
250 int error = errno;
251 kmp_msg_t err_code = KMP_ERR(error);
252 __kmp_msg(kmp_ms_warning, KMP_MSG(GetAffSysCallNotSupported, env_var),
253 err_code, __kmp_msg_null);
254 if (__kmp_generate_warnings == kmp_warnings_off) {
255 __kmp_str_free(&err_code.str);
256 }
257 }
258 KMP_AFFINITY_DISABLE();
259 KMP_INTERNAL_FREE(buf);
260 return;
261 }
262 continue;
263 }
264
265 KMP_AFFINITY_ENABLE(gCode);
266 KA_TRACE(10, ("__kmp_affinity_determine_capable: "
267 "affinity supported (mask size %d)\n",
268 (int)__kmp_affin_mask_size));
269 KMP_INTERNAL_FREE(buf);
270 return;
271 }
272 #elif KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY
273 long gCode;
274 unsigned char *buf;
275 buf = (unsigned char *)KMP_INTERNAL_MALLOC(KMP_CPU_SET_SIZE_LIMIT);
276 gCode = pthread_getaffinity_np(pthread_self(), KMP_CPU_SET_SIZE_LIMIT,
277 reinterpret_cast<cpuset_t *>(buf));
278 KA_TRACE(30, ("__kmp_affinity_determine_capable: "
279 "initial getaffinity call returned %d errno = %d\n",
280 gCode, errno));
281 if (gCode == 0) {
282 KMP_AFFINITY_ENABLE(KMP_CPU_SET_SIZE_LIMIT);
283 KA_TRACE(10, ("__kmp_affinity_determine_capable: "
284 "affinity supported (mask size %d)\n",
285 (int)__kmp_affin_mask_size));
286 KMP_INTERNAL_FREE(buf);
287 return;
288 }
289 #endif
290 KMP_INTERNAL_FREE(buf);
291
292 // Affinity is not supported
293 KMP_AFFINITY_DISABLE();
294 KA_TRACE(10, ("__kmp_affinity_determine_capable: "
295 "cannot determine mask size - affinity not supported\n"));
296 if (verbose || (warnings && (type != affinity_none) &&
297 (type != affinity_default) && (type != affinity_disabled))) {
298 KMP_WARNING(AffCantGetMaskSize, env_var);
299 }
300 }
301 #endif // KMP_OS_AIX
302 #endif // (KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || \
303 KMP_OS_DRAGONFLY || KMP_OS_AIX) && KMP_AFFINITY_SUPPORTED
304
305 #if KMP_USE_FUTEX
306
__kmp_futex_determine_capable()307 int __kmp_futex_determine_capable() {
308 int loc = 0;
309 long rc = syscall(__NR_futex, &loc, FUTEX_WAKE, 1, NULL, NULL, 0);
310 int retval = (rc == 0) || (errno != ENOSYS);
311
312 KA_TRACE(10,
313 ("__kmp_futex_determine_capable: rc = %d errno = %d\n", rc, errno));
314 KA_TRACE(10, ("__kmp_futex_determine_capable: futex syscall%s supported\n",
315 retval ? "" : " not"));
316
317 return retval;
318 }
319
320 #endif // KMP_USE_FUTEX
321
322 #if (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_WASM) && (!KMP_ASM_INTRINS)
323 /* Only 32-bit "add-exchange" instruction on IA-32 architecture causes us to
324 use compare_and_store for these routines */
325
__kmp_test_then_or8(volatile kmp_int8 * p,kmp_int8 d)326 kmp_int8 __kmp_test_then_or8(volatile kmp_int8 *p, kmp_int8 d) {
327 kmp_int8 old_value, new_value;
328
329 old_value = TCR_1(*p);
330 new_value = old_value | d;
331
332 while (!KMP_COMPARE_AND_STORE_REL8(p, old_value, new_value)) {
333 KMP_CPU_PAUSE();
334 old_value = TCR_1(*p);
335 new_value = old_value | d;
336 }
337 return old_value;
338 }
339
__kmp_test_then_and8(volatile kmp_int8 * p,kmp_int8 d)340 kmp_int8 __kmp_test_then_and8(volatile kmp_int8 *p, kmp_int8 d) {
341 kmp_int8 old_value, new_value;
342
343 old_value = TCR_1(*p);
344 new_value = old_value & d;
345
346 while (!KMP_COMPARE_AND_STORE_REL8(p, old_value, new_value)) {
347 KMP_CPU_PAUSE();
348 old_value = TCR_1(*p);
349 new_value = old_value & d;
350 }
351 return old_value;
352 }
353
__kmp_test_then_or32(volatile kmp_uint32 * p,kmp_uint32 d)354 kmp_uint32 __kmp_test_then_or32(volatile kmp_uint32 *p, kmp_uint32 d) {
355 kmp_uint32 old_value, new_value;
356
357 old_value = TCR_4(*p);
358 new_value = old_value | d;
359
360 while (!KMP_COMPARE_AND_STORE_REL32(p, old_value, new_value)) {
361 KMP_CPU_PAUSE();
362 old_value = TCR_4(*p);
363 new_value = old_value | d;
364 }
365 return old_value;
366 }
367
__kmp_test_then_and32(volatile kmp_uint32 * p,kmp_uint32 d)368 kmp_uint32 __kmp_test_then_and32(volatile kmp_uint32 *p, kmp_uint32 d) {
369 kmp_uint32 old_value, new_value;
370
371 old_value = TCR_4(*p);
372 new_value = old_value & d;
373
374 while (!KMP_COMPARE_AND_STORE_REL32(p, old_value, new_value)) {
375 KMP_CPU_PAUSE();
376 old_value = TCR_4(*p);
377 new_value = old_value & d;
378 }
379 return old_value;
380 }
381
382 #if KMP_ARCH_X86 || KMP_ARCH_WASM
__kmp_test_then_add8(volatile kmp_int8 * p,kmp_int8 d)383 kmp_int8 __kmp_test_then_add8(volatile kmp_int8 *p, kmp_int8 d) {
384 kmp_int8 old_value, new_value;
385
386 old_value = TCR_1(*p);
387 new_value = old_value + d;
388
389 while (!KMP_COMPARE_AND_STORE_REL8(p, old_value, new_value)) {
390 KMP_CPU_PAUSE();
391 old_value = TCR_1(*p);
392 new_value = old_value + d;
393 }
394 return old_value;
395 }
396
__kmp_test_then_add64(volatile kmp_int64 * p,kmp_int64 d)397 kmp_int64 __kmp_test_then_add64(volatile kmp_int64 *p, kmp_int64 d) {
398 kmp_int64 old_value, new_value;
399
400 old_value = TCR_8(*p);
401 new_value = old_value + d;
402
403 while (!KMP_COMPARE_AND_STORE_REL64(p, old_value, new_value)) {
404 KMP_CPU_PAUSE();
405 old_value = TCR_8(*p);
406 new_value = old_value + d;
407 }
408 return old_value;
409 }
410 #endif /* KMP_ARCH_X86 */
411
__kmp_test_then_or64(volatile kmp_uint64 * p,kmp_uint64 d)412 kmp_uint64 __kmp_test_then_or64(volatile kmp_uint64 *p, kmp_uint64 d) {
413 kmp_uint64 old_value, new_value;
414
415 old_value = TCR_8(*p);
416 new_value = old_value | d;
417 while (!KMP_COMPARE_AND_STORE_REL64(p, old_value, new_value)) {
418 KMP_CPU_PAUSE();
419 old_value = TCR_8(*p);
420 new_value = old_value | d;
421 }
422 return old_value;
423 }
424
__kmp_test_then_and64(volatile kmp_uint64 * p,kmp_uint64 d)425 kmp_uint64 __kmp_test_then_and64(volatile kmp_uint64 *p, kmp_uint64 d) {
426 kmp_uint64 old_value, new_value;
427
428 old_value = TCR_8(*p);
429 new_value = old_value & d;
430 while (!KMP_COMPARE_AND_STORE_REL64(p, old_value, new_value)) {
431 KMP_CPU_PAUSE();
432 old_value = TCR_8(*p);
433 new_value = old_value & d;
434 }
435 return old_value;
436 }
437
438 #endif /* (KMP_ARCH_X86 || KMP_ARCH_X86_64) && (! KMP_ASM_INTRINS) */
439
__kmp_terminate_thread(int gtid)440 void __kmp_terminate_thread(int gtid) {
441 int status;
442 kmp_info_t *th = __kmp_threads[gtid];
443
444 if (!th)
445 return;
446
447 #ifdef KMP_CANCEL_THREADS
448 KA_TRACE(10, ("__kmp_terminate_thread: kill (%d)\n", gtid));
449 status = pthread_cancel(th->th.th_info.ds.ds_thread);
450 if (status != 0 && status != ESRCH) {
451 __kmp_fatal(KMP_MSG(CantTerminateWorkerThread), KMP_ERR(status),
452 __kmp_msg_null);
453 }
454 #endif
455 KMP_YIELD(TRUE);
456 } //
457
458 /* Set thread stack info.
459 If values are unreasonable, assume call failed and use incremental stack
460 refinement method instead. Returns TRUE if the stack parameters could be
461 determined exactly, FALSE if incremental refinement is necessary. */
__kmp_set_stack_info(int gtid,kmp_info_t * th)462 static kmp_int32 __kmp_set_stack_info(int gtid, kmp_info_t *th) {
463 int stack_data;
464 #if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \
465 KMP_OS_HURD || KMP_OS_SOLARIS || KMP_OS_AIX
466 int status;
467 size_t size = 0;
468 void *addr = 0;
469
470 /* Always do incremental stack refinement for ubermaster threads since the
471 initial thread stack range can be reduced by sibling thread creation so
472 pthread_attr_getstack may cause thread gtid aliasing */
473 if (!KMP_UBER_GTID(gtid)) {
474
475 #if KMP_OS_SOLARIS
476 stack_t s;
477 if ((status = thr_stksegment(&s)) < 0) {
478 KMP_CHECK_SYSFAIL("thr_stksegment", status);
479 }
480
481 addr = s.ss_sp;
482 size = s.ss_size;
483 KA_TRACE(60, ("__kmp_set_stack_info: T#%d thr_stksegment returned size:"
484 " %lu, low addr: %p\n",
485 gtid, size, addr));
486 #else
487 pthread_attr_t attr;
488 /* Fetch the real thread attributes */
489 status = pthread_attr_init(&attr);
490 KMP_CHECK_SYSFAIL("pthread_attr_init", status);
491 #if KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD
492 status = pthread_attr_get_np(pthread_self(), &attr);
493 KMP_CHECK_SYSFAIL("pthread_attr_get_np", status);
494 #else
495 status = pthread_getattr_np(pthread_self(), &attr);
496 KMP_CHECK_SYSFAIL("pthread_getattr_np", status);
497 #endif
498 status = pthread_attr_getstack(&attr, &addr, &size);
499 KMP_CHECK_SYSFAIL("pthread_attr_getstack", status);
500 KA_TRACE(60,
501 ("__kmp_set_stack_info: T#%d pthread_attr_getstack returned size:"
502 " %lu, low addr: %p\n",
503 gtid, size, addr));
504 status = pthread_attr_destroy(&attr);
505 KMP_CHECK_SYSFAIL("pthread_attr_destroy", status);
506 #endif
507 }
508
509 if (size != 0 && addr != 0) { // was stack parameter determination successful?
510 /* Store the correct base and size */
511 TCW_PTR(th->th.th_info.ds.ds_stackbase, (((char *)addr) + size));
512 TCW_PTR(th->th.th_info.ds.ds_stacksize, size);
513 TCW_4(th->th.th_info.ds.ds_stackgrow, FALSE);
514 return TRUE;
515 }
516 #endif /* KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD \
517 || KMP_OS_HURD || KMP_OS_SOLARIS */
518 /* Use incremental refinement starting from initial conservative estimate */
519 TCW_PTR(th->th.th_info.ds.ds_stacksize, 0);
520 TCW_PTR(th->th.th_info.ds.ds_stackbase, &stack_data);
521 TCW_4(th->th.th_info.ds.ds_stackgrow, TRUE);
522 return FALSE;
523 }
524
__kmp_launch_worker(void * thr)525 static void *__kmp_launch_worker(void *thr) {
526 int status, old_type, old_state;
527 #ifdef KMP_BLOCK_SIGNALS
528 sigset_t new_set, old_set;
529 #endif /* KMP_BLOCK_SIGNALS */
530 void *exit_val;
531 #if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \
532 KMP_OS_OPENBSD || KMP_OS_HURD || KMP_OS_SOLARIS || KMP_OS_AIX
533 void *volatile padding = 0;
534 #endif
535 int gtid;
536
537 gtid = ((kmp_info_t *)thr)->th.th_info.ds.ds_gtid;
538 __kmp_gtid_set_specific(gtid);
539 #ifdef KMP_TDATA_GTID
540 __kmp_gtid = gtid;
541 #endif
542 #if KMP_STATS_ENABLED
543 // set thread local index to point to thread-specific stats
544 __kmp_stats_thread_ptr = ((kmp_info_t *)thr)->th.th_stats;
545 __kmp_stats_thread_ptr->startLife();
546 KMP_SET_THREAD_STATE(IDLE);
547 KMP_INIT_PARTITIONED_TIMERS(OMP_idle);
548 #endif
549
550 #if USE_ITT_BUILD
551 __kmp_itt_thread_name(gtid);
552 #endif /* USE_ITT_BUILD */
553
554 #if KMP_AFFINITY_SUPPORTED
555 __kmp_affinity_bind_init_mask(gtid);
556 #endif
557
558 #ifdef KMP_CANCEL_THREADS
559 status = pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, &old_type);
560 KMP_CHECK_SYSFAIL("pthread_setcanceltype", status);
561 // josh todo: isn't PTHREAD_CANCEL_ENABLE default for newly-created threads?
562 status = pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &old_state);
563 KMP_CHECK_SYSFAIL("pthread_setcancelstate", status);
564 #endif
565
566 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
567 // Set FP control regs to be a copy of the parallel initialization thread's.
568 __kmp_clear_x87_fpu_status_word();
569 __kmp_load_x87_fpu_control_word(&__kmp_init_x87_fpu_control_word);
570 __kmp_load_mxcsr(&__kmp_init_mxcsr);
571 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
572
573 #ifdef KMP_BLOCK_SIGNALS
574 status = sigfillset(&new_set);
575 KMP_CHECK_SYSFAIL_ERRNO("sigfillset", status);
576 status = pthread_sigmask(SIG_BLOCK, &new_set, &old_set);
577 KMP_CHECK_SYSFAIL("pthread_sigmask", status);
578 #endif /* KMP_BLOCK_SIGNALS */
579
580 #if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \
581 KMP_OS_OPENBSD || KMP_OS_HURD || KMP_OS_SOLARIS || KMP_OS_AIX
582 if (__kmp_stkoffset > 0 && gtid > 0) {
583 padding = KMP_ALLOCA(gtid * __kmp_stkoffset);
584 (void)padding;
585 }
586 #endif
587
588 KMP_MB();
589 __kmp_set_stack_info(gtid, (kmp_info_t *)thr);
590
591 __kmp_check_stack_overlap((kmp_info_t *)thr);
592
593 exit_val = __kmp_launch_thread((kmp_info_t *)thr);
594
595 #ifdef KMP_BLOCK_SIGNALS
596 status = pthread_sigmask(SIG_SETMASK, &old_set, NULL);
597 KMP_CHECK_SYSFAIL("pthread_sigmask", status);
598 #endif /* KMP_BLOCK_SIGNALS */
599
600 return exit_val;
601 }
602
603 #if KMP_USE_MONITOR
604 /* The monitor thread controls all of the threads in the complex */
605
__kmp_launch_monitor(void * thr)606 static void *__kmp_launch_monitor(void *thr) {
607 int status, old_type, old_state;
608 #ifdef KMP_BLOCK_SIGNALS
609 sigset_t new_set;
610 #endif /* KMP_BLOCK_SIGNALS */
611 struct timespec interval;
612
613 KMP_MB(); /* Flush all pending memory write invalidates. */
614
615 KA_TRACE(10, ("__kmp_launch_monitor: #1 launched\n"));
616
617 /* register us as the monitor thread */
618 __kmp_gtid_set_specific(KMP_GTID_MONITOR);
619 #ifdef KMP_TDATA_GTID
620 __kmp_gtid = KMP_GTID_MONITOR;
621 #endif
622
623 KMP_MB();
624
625 #if USE_ITT_BUILD
626 // Instruct Intel(R) Threading Tools to ignore monitor thread.
627 __kmp_itt_thread_ignore();
628 #endif /* USE_ITT_BUILD */
629
630 __kmp_set_stack_info(((kmp_info_t *)thr)->th.th_info.ds.ds_gtid,
631 (kmp_info_t *)thr);
632
633 __kmp_check_stack_overlap((kmp_info_t *)thr);
634
635 #ifdef KMP_CANCEL_THREADS
636 status = pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, &old_type);
637 KMP_CHECK_SYSFAIL("pthread_setcanceltype", status);
638 // josh todo: isn't PTHREAD_CANCEL_ENABLE default for newly-created threads?
639 status = pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &old_state);
640 KMP_CHECK_SYSFAIL("pthread_setcancelstate", status);
641 #endif
642
643 #if KMP_REAL_TIME_FIX
644 // This is a potential fix which allows application with real-time scheduling
645 // policy work. However, decision about the fix is not made yet, so it is
646 // disabled by default.
647 { // Are program started with real-time scheduling policy?
648 int sched = sched_getscheduler(0);
649 if (sched == SCHED_FIFO || sched == SCHED_RR) {
650 // Yes, we are a part of real-time application. Try to increase the
651 // priority of the monitor.
652 struct sched_param param;
653 int max_priority = sched_get_priority_max(sched);
654 int rc;
655 KMP_WARNING(RealTimeSchedNotSupported);
656 sched_getparam(0, ¶m);
657 if (param.sched_priority < max_priority) {
658 param.sched_priority += 1;
659 rc = sched_setscheduler(0, sched, ¶m);
660 if (rc != 0) {
661 int error = errno;
662 kmp_msg_t err_code = KMP_ERR(error);
663 __kmp_msg(kmp_ms_warning, KMP_MSG(CantChangeMonitorPriority),
664 err_code, KMP_MSG(MonitorWillStarve), __kmp_msg_null);
665 if (__kmp_generate_warnings == kmp_warnings_off) {
666 __kmp_str_free(&err_code.str);
667 }
668 }
669 } else {
670 // We cannot abort here, because number of CPUs may be enough for all
671 // the threads, including the monitor thread, so application could
672 // potentially work...
673 __kmp_msg(kmp_ms_warning, KMP_MSG(RunningAtMaxPriority),
674 KMP_MSG(MonitorWillStarve), KMP_HNT(RunningAtMaxPriority),
675 __kmp_msg_null);
676 }
677 }
678 // AC: free thread that waits for monitor started
679 TCW_4(__kmp_global.g.g_time.dt.t_value, 0);
680 }
681 #endif // KMP_REAL_TIME_FIX
682
683 KMP_MB(); /* Flush all pending memory write invalidates. */
684
685 if (__kmp_monitor_wakeups == 1) {
686 interval.tv_sec = 1;
687 interval.tv_nsec = 0;
688 } else {
689 interval.tv_sec = 0;
690 interval.tv_nsec = (KMP_NSEC_PER_SEC / __kmp_monitor_wakeups);
691 }
692
693 KA_TRACE(10, ("__kmp_launch_monitor: #2 monitor\n"));
694
695 while (!TCR_4(__kmp_global.g.g_done)) {
696 struct timespec now;
697 struct timeval tval;
698
699 /* This thread monitors the state of the system */
700
701 KA_TRACE(15, ("__kmp_launch_monitor: update\n"));
702
703 status = gettimeofday(&tval, NULL);
704 KMP_CHECK_SYSFAIL_ERRNO("gettimeofday", status);
705 TIMEVAL_TO_TIMESPEC(&tval, &now);
706
707 now.tv_sec += interval.tv_sec;
708 now.tv_nsec += interval.tv_nsec;
709
710 if (now.tv_nsec >= KMP_NSEC_PER_SEC) {
711 now.tv_sec += 1;
712 now.tv_nsec -= KMP_NSEC_PER_SEC;
713 }
714
715 status = pthread_mutex_lock(&__kmp_wait_mx.m_mutex);
716 KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
717 // AC: the monitor should not fall asleep if g_done has been set
718 if (!TCR_4(__kmp_global.g.g_done)) { // check once more under mutex
719 status = pthread_cond_timedwait(&__kmp_wait_cv.c_cond,
720 &__kmp_wait_mx.m_mutex, &now);
721 if (status != 0) {
722 if (status != ETIMEDOUT && status != EINTR) {
723 KMP_SYSFAIL("pthread_cond_timedwait", status);
724 }
725 }
726 }
727 status = pthread_mutex_unlock(&__kmp_wait_mx.m_mutex);
728 KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
729
730 TCW_4(__kmp_global.g.g_time.dt.t_value,
731 TCR_4(__kmp_global.g.g_time.dt.t_value) + 1);
732
733 KMP_MB(); /* Flush all pending memory write invalidates. */
734 }
735
736 KA_TRACE(10, ("__kmp_launch_monitor: #3 cleanup\n"));
737
738 #ifdef KMP_BLOCK_SIGNALS
739 status = sigfillset(&new_set);
740 KMP_CHECK_SYSFAIL_ERRNO("sigfillset", status);
741 status = pthread_sigmask(SIG_UNBLOCK, &new_set, NULL);
742 KMP_CHECK_SYSFAIL("pthread_sigmask", status);
743 #endif /* KMP_BLOCK_SIGNALS */
744
745 KA_TRACE(10, ("__kmp_launch_monitor: #4 finished\n"));
746
747 if (__kmp_global.g.g_abort != 0) {
748 /* now we need to terminate the worker threads */
749 /* the value of t_abort is the signal we caught */
750
751 int gtid;
752
753 KA_TRACE(10, ("__kmp_launch_monitor: #5 terminate sig=%d\n",
754 __kmp_global.g.g_abort));
755
756 /* terminate the OpenMP worker threads */
757 /* TODO this is not valid for sibling threads!!
758 * the uber master might not be 0 anymore.. */
759 for (gtid = 1; gtid < __kmp_threads_capacity; ++gtid)
760 __kmp_terminate_thread(gtid);
761
762 __kmp_cleanup();
763
764 KA_TRACE(10, ("__kmp_launch_monitor: #6 raise sig=%d\n",
765 __kmp_global.g.g_abort));
766
767 if (__kmp_global.g.g_abort > 0)
768 raise(__kmp_global.g.g_abort);
769 }
770
771 KA_TRACE(10, ("__kmp_launch_monitor: #7 exit\n"));
772
773 return thr;
774 }
775 #endif // KMP_USE_MONITOR
776
__kmp_create_worker(int gtid,kmp_info_t * th,size_t stack_size)777 void __kmp_create_worker(int gtid, kmp_info_t *th, size_t stack_size) {
778 pthread_t handle;
779 pthread_attr_t thread_attr;
780 int status;
781
782 th->th.th_info.ds.ds_gtid = gtid;
783
784 #if KMP_STATS_ENABLED
785 // sets up worker thread stats
786 __kmp_acquire_tas_lock(&__kmp_stats_lock, gtid);
787
788 // th->th.th_stats is used to transfer thread-specific stats-pointer to
789 // __kmp_launch_worker. So when thread is created (goes into
790 // __kmp_launch_worker) it will set its thread local pointer to
791 // th->th.th_stats
792 if (!KMP_UBER_GTID(gtid)) {
793 th->th.th_stats = __kmp_stats_list->push_back(gtid);
794 } else {
795 // For root threads, __kmp_stats_thread_ptr is set in __kmp_register_root(),
796 // so set the th->th.th_stats field to it.
797 th->th.th_stats = __kmp_stats_thread_ptr;
798 }
799 __kmp_release_tas_lock(&__kmp_stats_lock, gtid);
800
801 #endif // KMP_STATS_ENABLED
802
803 if (KMP_UBER_GTID(gtid)) {
804 KA_TRACE(10, ("__kmp_create_worker: uber thread (%d)\n", gtid));
805 th->th.th_info.ds.ds_thread = pthread_self();
806 __kmp_set_stack_info(gtid, th);
807 __kmp_check_stack_overlap(th);
808 return;
809 }
810
811 KA_TRACE(10, ("__kmp_create_worker: try to create thread (%d)\n", gtid));
812
813 KMP_MB(); /* Flush all pending memory write invalidates. */
814
815 #ifdef KMP_THREAD_ATTR
816 status = pthread_attr_init(&thread_attr);
817 if (status != 0) {
818 __kmp_fatal(KMP_MSG(CantInitThreadAttrs), KMP_ERR(status), __kmp_msg_null);
819 }
820 status = pthread_attr_setdetachstate(&thread_attr, PTHREAD_CREATE_JOINABLE);
821 if (status != 0) {
822 __kmp_fatal(KMP_MSG(CantSetWorkerState), KMP_ERR(status), __kmp_msg_null);
823 }
824
825 /* Set stack size for this thread now.
826 The multiple of 2 is there because on some machines, requesting an unusual
827 stacksize causes the thread to have an offset before the dummy alloca()
828 takes place to create the offset. Since we want the user to have a
829 sufficient stacksize AND support a stack offset, we alloca() twice the
830 offset so that the upcoming alloca() does not eliminate any premade offset,
831 and also gives the user the stack space they requested for all threads */
832 stack_size += gtid * __kmp_stkoffset * 2;
833
834 KA_TRACE(10, ("__kmp_create_worker: T#%d, default stacksize = %lu bytes, "
835 "__kmp_stksize = %lu bytes, final stacksize = %lu bytes\n",
836 gtid, KMP_DEFAULT_STKSIZE, __kmp_stksize, stack_size));
837
838 #ifdef _POSIX_THREAD_ATTR_STACKSIZE
839 status = pthread_attr_setstacksize(&thread_attr, stack_size);
840 #ifdef KMP_BACKUP_STKSIZE
841 if (status != 0) {
842 if (!__kmp_env_stksize) {
843 stack_size = KMP_BACKUP_STKSIZE + gtid * __kmp_stkoffset;
844 __kmp_stksize = KMP_BACKUP_STKSIZE;
845 KA_TRACE(10, ("__kmp_create_worker: T#%d, default stacksize = %lu bytes, "
846 "__kmp_stksize = %lu bytes, (backup) final stacksize = %lu "
847 "bytes\n",
848 gtid, KMP_DEFAULT_STKSIZE, __kmp_stksize, stack_size));
849 status = pthread_attr_setstacksize(&thread_attr, stack_size);
850 }
851 }
852 #endif /* KMP_BACKUP_STKSIZE */
853 if (status != 0) {
854 __kmp_fatal(KMP_MSG(CantSetWorkerStackSize, stack_size), KMP_ERR(status),
855 KMP_HNT(ChangeWorkerStackSize), __kmp_msg_null);
856 }
857 #endif /* _POSIX_THREAD_ATTR_STACKSIZE */
858
859 #endif /* KMP_THREAD_ATTR */
860
861 status =
862 pthread_create(&handle, &thread_attr, __kmp_launch_worker, (void *)th);
863 if (status != 0 || !handle) { // ??? Why do we check handle??
864 #ifdef _POSIX_THREAD_ATTR_STACKSIZE
865 if (status == EINVAL) {
866 __kmp_fatal(KMP_MSG(CantSetWorkerStackSize, stack_size), KMP_ERR(status),
867 KMP_HNT(IncreaseWorkerStackSize), __kmp_msg_null);
868 }
869 if (status == ENOMEM) {
870 __kmp_fatal(KMP_MSG(CantSetWorkerStackSize, stack_size), KMP_ERR(status),
871 KMP_HNT(DecreaseWorkerStackSize), __kmp_msg_null);
872 }
873 #endif /* _POSIX_THREAD_ATTR_STACKSIZE */
874 if (status == EAGAIN) {
875 __kmp_fatal(KMP_MSG(NoResourcesForWorkerThread), KMP_ERR(status),
876 KMP_HNT(Decrease_NUM_THREADS), __kmp_msg_null);
877 }
878 KMP_SYSFAIL("pthread_create", status);
879 }
880
881 th->th.th_info.ds.ds_thread = handle;
882
883 #ifdef KMP_THREAD_ATTR
884 status = pthread_attr_destroy(&thread_attr);
885 if (status) {
886 kmp_msg_t err_code = KMP_ERR(status);
887 __kmp_msg(kmp_ms_warning, KMP_MSG(CantDestroyThreadAttrs), err_code,
888 __kmp_msg_null);
889 if (__kmp_generate_warnings == kmp_warnings_off) {
890 __kmp_str_free(&err_code.str);
891 }
892 }
893 #endif /* KMP_THREAD_ATTR */
894
895 KMP_MB(); /* Flush all pending memory write invalidates. */
896
897 KA_TRACE(10, ("__kmp_create_worker: done creating thread (%d)\n", gtid));
898
899 } // __kmp_create_worker
900
901 #if KMP_USE_MONITOR
__kmp_create_monitor(kmp_info_t * th)902 void __kmp_create_monitor(kmp_info_t *th) {
903 pthread_t handle;
904 pthread_attr_t thread_attr;
905 size_t size;
906 int status;
907 int auto_adj_size = FALSE;
908
909 if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME) {
910 // We don't need monitor thread in case of MAX_BLOCKTIME
911 KA_TRACE(10, ("__kmp_create_monitor: skipping monitor thread because of "
912 "MAX blocktime\n"));
913 th->th.th_info.ds.ds_tid = 0; // this makes reap_monitor no-op
914 th->th.th_info.ds.ds_gtid = 0;
915 return;
916 }
917 KA_TRACE(10, ("__kmp_create_monitor: try to create monitor\n"));
918
919 KMP_MB(); /* Flush all pending memory write invalidates. */
920
921 th->th.th_info.ds.ds_tid = KMP_GTID_MONITOR;
922 th->th.th_info.ds.ds_gtid = KMP_GTID_MONITOR;
923 #if KMP_REAL_TIME_FIX
924 TCW_4(__kmp_global.g.g_time.dt.t_value,
925 -1); // Will use it for synchronization a bit later.
926 #else
927 TCW_4(__kmp_global.g.g_time.dt.t_value, 0);
928 #endif // KMP_REAL_TIME_FIX
929
930 #ifdef KMP_THREAD_ATTR
931 if (__kmp_monitor_stksize == 0) {
932 __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE;
933 auto_adj_size = TRUE;
934 }
935 status = pthread_attr_init(&thread_attr);
936 if (status != 0) {
937 __kmp_fatal(KMP_MSG(CantInitThreadAttrs), KMP_ERR(status), __kmp_msg_null);
938 }
939 status = pthread_attr_setdetachstate(&thread_attr, PTHREAD_CREATE_JOINABLE);
940 if (status != 0) {
941 __kmp_fatal(KMP_MSG(CantSetMonitorState), KMP_ERR(status), __kmp_msg_null);
942 }
943
944 #ifdef _POSIX_THREAD_ATTR_STACKSIZE
945 status = pthread_attr_getstacksize(&thread_attr, &size);
946 KMP_CHECK_SYSFAIL("pthread_attr_getstacksize", status);
947 #else
948 size = __kmp_sys_min_stksize;
949 #endif /* _POSIX_THREAD_ATTR_STACKSIZE */
950 #endif /* KMP_THREAD_ATTR */
951
952 if (__kmp_monitor_stksize == 0) {
953 __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE;
954 }
955 if (__kmp_monitor_stksize < __kmp_sys_min_stksize) {
956 __kmp_monitor_stksize = __kmp_sys_min_stksize;
957 }
958
959 KA_TRACE(10, ("__kmp_create_monitor: default stacksize = %lu bytes,"
960 "requested stacksize = %lu bytes\n",
961 size, __kmp_monitor_stksize));
962
963 retry:
964
965 /* Set stack size for this thread now. */
966 #ifdef _POSIX_THREAD_ATTR_STACKSIZE
967 KA_TRACE(10, ("__kmp_create_monitor: setting stacksize = %lu bytes,",
968 __kmp_monitor_stksize));
969 status = pthread_attr_setstacksize(&thread_attr, __kmp_monitor_stksize);
970 if (status != 0) {
971 if (auto_adj_size) {
972 __kmp_monitor_stksize *= 2;
973 goto retry;
974 }
975 kmp_msg_t err_code = KMP_ERR(status);
976 __kmp_msg(kmp_ms_warning, // should this be fatal? BB
977 KMP_MSG(CantSetMonitorStackSize, (long int)__kmp_monitor_stksize),
978 err_code, KMP_HNT(ChangeMonitorStackSize), __kmp_msg_null);
979 if (__kmp_generate_warnings == kmp_warnings_off) {
980 __kmp_str_free(&err_code.str);
981 }
982 }
983 #endif /* _POSIX_THREAD_ATTR_STACKSIZE */
984
985 status =
986 pthread_create(&handle, &thread_attr, __kmp_launch_monitor, (void *)th);
987
988 if (status != 0) {
989 #ifdef _POSIX_THREAD_ATTR_STACKSIZE
990 if (status == EINVAL) {
991 if (auto_adj_size && (__kmp_monitor_stksize < (size_t)0x40000000)) {
992 __kmp_monitor_stksize *= 2;
993 goto retry;
994 }
995 __kmp_fatal(KMP_MSG(CantSetMonitorStackSize, __kmp_monitor_stksize),
996 KMP_ERR(status), KMP_HNT(IncreaseMonitorStackSize),
997 __kmp_msg_null);
998 }
999 if (status == ENOMEM) {
1000 __kmp_fatal(KMP_MSG(CantSetMonitorStackSize, __kmp_monitor_stksize),
1001 KMP_ERR(status), KMP_HNT(DecreaseMonitorStackSize),
1002 __kmp_msg_null);
1003 }
1004 #endif /* _POSIX_THREAD_ATTR_STACKSIZE */
1005 if (status == EAGAIN) {
1006 __kmp_fatal(KMP_MSG(NoResourcesForMonitorThread), KMP_ERR(status),
1007 KMP_HNT(DecreaseNumberOfThreadsInUse), __kmp_msg_null);
1008 }
1009 KMP_SYSFAIL("pthread_create", status);
1010 }
1011
1012 th->th.th_info.ds.ds_thread = handle;
1013
1014 #if KMP_REAL_TIME_FIX
1015 // Wait for the monitor thread is really started and set its *priority*.
1016 KMP_DEBUG_ASSERT(sizeof(kmp_uint32) ==
1017 sizeof(__kmp_global.g.g_time.dt.t_value));
1018 __kmp_wait_4((kmp_uint32 volatile *)&__kmp_global.g.g_time.dt.t_value, -1,
1019 &__kmp_neq_4, NULL);
1020 #endif // KMP_REAL_TIME_FIX
1021
1022 #ifdef KMP_THREAD_ATTR
1023 status = pthread_attr_destroy(&thread_attr);
1024 if (status != 0) {
1025 kmp_msg_t err_code = KMP_ERR(status);
1026 __kmp_msg(kmp_ms_warning, KMP_MSG(CantDestroyThreadAttrs), err_code,
1027 __kmp_msg_null);
1028 if (__kmp_generate_warnings == kmp_warnings_off) {
1029 __kmp_str_free(&err_code.str);
1030 }
1031 }
1032 #endif
1033
1034 KMP_MB(); /* Flush all pending memory write invalidates. */
1035
1036 KA_TRACE(10, ("__kmp_create_monitor: monitor created %#.8lx\n",
1037 th->th.th_info.ds.ds_thread));
1038
1039 } // __kmp_create_monitor
1040 #endif // KMP_USE_MONITOR
1041
__kmp_exit_thread(int exit_status)1042 void __kmp_exit_thread(int exit_status) {
1043 #if KMP_OS_WASI
1044 // TODO: the wasm32-wasi-threads target does not yet support pthread_exit.
1045 #else
1046 pthread_exit((void *)(intptr_t)exit_status);
1047 #endif
1048 } // __kmp_exit_thread
1049
1050 #if KMP_USE_MONITOR
1051 void __kmp_resume_monitor();
1052
__kmp_reap_monitor(kmp_info_t * th)1053 extern "C" void __kmp_reap_monitor(kmp_info_t *th) {
1054 int status;
1055 void *exit_val;
1056
1057 KA_TRACE(10, ("__kmp_reap_monitor: try to reap monitor thread with handle"
1058 " %#.8lx\n",
1059 th->th.th_info.ds.ds_thread));
1060
1061 // If monitor has been created, its tid and gtid should be KMP_GTID_MONITOR.
1062 // If both tid and gtid are 0, it means the monitor did not ever start.
1063 // If both tid and gtid are KMP_GTID_DNE, the monitor has been shut down.
1064 KMP_DEBUG_ASSERT(th->th.th_info.ds.ds_tid == th->th.th_info.ds.ds_gtid);
1065 if (th->th.th_info.ds.ds_gtid != KMP_GTID_MONITOR) {
1066 KA_TRACE(10, ("__kmp_reap_monitor: monitor did not start, returning\n"));
1067 return;
1068 }
1069
1070 KMP_MB(); /* Flush all pending memory write invalidates. */
1071
1072 /* First, check to see whether the monitor thread exists to wake it up. This
1073 is to avoid performance problem when the monitor sleeps during
1074 blocktime-size interval */
1075
1076 status = pthread_kill(th->th.th_info.ds.ds_thread, 0);
1077 if (status != ESRCH) {
1078 __kmp_resume_monitor(); // Wake up the monitor thread
1079 }
1080 KA_TRACE(10, ("__kmp_reap_monitor: try to join with monitor\n"));
1081 status = pthread_join(th->th.th_info.ds.ds_thread, &exit_val);
1082 if (exit_val != th) {
1083 __kmp_fatal(KMP_MSG(ReapMonitorError), KMP_ERR(status), __kmp_msg_null);
1084 }
1085
1086 th->th.th_info.ds.ds_tid = KMP_GTID_DNE;
1087 th->th.th_info.ds.ds_gtid = KMP_GTID_DNE;
1088
1089 KA_TRACE(10, ("__kmp_reap_monitor: done reaping monitor thread with handle"
1090 " %#.8lx\n",
1091 th->th.th_info.ds.ds_thread));
1092
1093 KMP_MB(); /* Flush all pending memory write invalidates. */
1094 }
1095 #else
1096 // Empty symbol to export (see exports_so.txt) when
1097 // monitor thread feature is disabled
__kmp_reap_monitor(kmp_info_t * th)1098 extern "C" void __kmp_reap_monitor(kmp_info_t *th) { (void)th; }
1099 #endif // KMP_USE_MONITOR
1100
__kmp_reap_worker(kmp_info_t * th)1101 void __kmp_reap_worker(kmp_info_t *th) {
1102 int status;
1103 void *exit_val;
1104
1105 KMP_MB(); /* Flush all pending memory write invalidates. */
1106
1107 KA_TRACE(
1108 10, ("__kmp_reap_worker: try to reap T#%d\n", th->th.th_info.ds.ds_gtid));
1109
1110 status = pthread_join(th->th.th_info.ds.ds_thread, &exit_val);
1111 #ifdef KMP_DEBUG
1112 /* Don't expose these to the user until we understand when they trigger */
1113 if (status != 0) {
1114 __kmp_fatal(KMP_MSG(ReapWorkerError), KMP_ERR(status), __kmp_msg_null);
1115 }
1116 if (exit_val != th) {
1117 KA_TRACE(10, ("__kmp_reap_worker: worker T#%d did not reap properly, "
1118 "exit_val = %p\n",
1119 th->th.th_info.ds.ds_gtid, exit_val));
1120 }
1121 #else
1122 (void)status; // unused variable
1123 #endif /* KMP_DEBUG */
1124
1125 KA_TRACE(10, ("__kmp_reap_worker: done reaping T#%d\n",
1126 th->th.th_info.ds.ds_gtid));
1127
1128 KMP_MB(); /* Flush all pending memory write invalidates. */
1129 }
1130
1131 #if KMP_HANDLE_SIGNALS
1132
__kmp_null_handler(int signo)1133 static void __kmp_null_handler(int signo) {
1134 // Do nothing, for doing SIG_IGN-type actions.
1135 } // __kmp_null_handler
1136
__kmp_team_handler(int signo)1137 static void __kmp_team_handler(int signo) {
1138 if (__kmp_global.g.g_abort == 0) {
1139 /* Stage 1 signal handler, let's shut down all of the threads */
1140 #ifdef KMP_DEBUG
1141 __kmp_debug_printf("__kmp_team_handler: caught signal = %d\n", signo);
1142 #endif
1143 switch (signo) {
1144 case SIGHUP:
1145 case SIGINT:
1146 case SIGQUIT:
1147 case SIGILL:
1148 case SIGABRT:
1149 case SIGFPE:
1150 case SIGBUS:
1151 case SIGSEGV:
1152 #ifdef SIGSYS
1153 case SIGSYS:
1154 #endif
1155 case SIGTERM:
1156 if (__kmp_debug_buf) {
1157 __kmp_dump_debug_buffer();
1158 }
1159 __kmp_unregister_library(); // cleanup shared memory
1160 KMP_MB(); // Flush all pending memory write invalidates.
1161 TCW_4(__kmp_global.g.g_abort, signo);
1162 KMP_MB(); // Flush all pending memory write invalidates.
1163 TCW_4(__kmp_global.g.g_done, TRUE);
1164 KMP_MB(); // Flush all pending memory write invalidates.
1165 break;
1166 default:
1167 #ifdef KMP_DEBUG
1168 __kmp_debug_printf("__kmp_team_handler: unknown signal type");
1169 #endif
1170 break;
1171 }
1172 }
1173 } // __kmp_team_handler
1174
__kmp_sigaction(int signum,const struct sigaction * act,struct sigaction * oldact)1175 static void __kmp_sigaction(int signum, const struct sigaction *act,
1176 struct sigaction *oldact) {
1177 int rc = sigaction(signum, act, oldact);
1178 KMP_CHECK_SYSFAIL_ERRNO("sigaction", rc);
1179 }
1180
__kmp_install_one_handler(int sig,sig_func_t handler_func,int parallel_init)1181 static void __kmp_install_one_handler(int sig, sig_func_t handler_func,
1182 int parallel_init) {
1183 KMP_MB(); // Flush all pending memory write invalidates.
1184 KB_TRACE(60,
1185 ("__kmp_install_one_handler( %d, ..., %d )\n", sig, parallel_init));
1186 if (parallel_init) {
1187 struct sigaction new_action;
1188 struct sigaction old_action;
1189 new_action.sa_handler = handler_func;
1190 new_action.sa_flags = 0;
1191 sigfillset(&new_action.sa_mask);
1192 __kmp_sigaction(sig, &new_action, &old_action);
1193 if (old_action.sa_handler == __kmp_sighldrs[sig].sa_handler) {
1194 sigaddset(&__kmp_sigset, sig);
1195 } else {
1196 // Restore/keep user's handler if one previously installed.
1197 __kmp_sigaction(sig, &old_action, NULL);
1198 }
1199 } else {
1200 // Save initial/system signal handlers to see if user handlers installed.
1201 __kmp_sigaction(sig, NULL, &__kmp_sighldrs[sig]);
1202 }
1203 KMP_MB(); // Flush all pending memory write invalidates.
1204 } // __kmp_install_one_handler
1205
__kmp_remove_one_handler(int sig)1206 static void __kmp_remove_one_handler(int sig) {
1207 KB_TRACE(60, ("__kmp_remove_one_handler( %d )\n", sig));
1208 if (sigismember(&__kmp_sigset, sig)) {
1209 struct sigaction old;
1210 KMP_MB(); // Flush all pending memory write invalidates.
1211 __kmp_sigaction(sig, &__kmp_sighldrs[sig], &old);
1212 if ((old.sa_handler != __kmp_team_handler) &&
1213 (old.sa_handler != __kmp_null_handler)) {
1214 // Restore the users signal handler.
1215 KB_TRACE(10, ("__kmp_remove_one_handler: oops, not our handler, "
1216 "restoring: sig=%d\n",
1217 sig));
1218 __kmp_sigaction(sig, &old, NULL);
1219 }
1220 sigdelset(&__kmp_sigset, sig);
1221 KMP_MB(); // Flush all pending memory write invalidates.
1222 }
1223 } // __kmp_remove_one_handler
1224
__kmp_install_signals(int parallel_init)1225 void __kmp_install_signals(int parallel_init) {
1226 KB_TRACE(10, ("__kmp_install_signals( %d )\n", parallel_init));
1227 if (__kmp_handle_signals || !parallel_init) {
1228 // If ! parallel_init, we do not install handlers, just save original
1229 // handlers. Let us do it even __handle_signals is 0.
1230 sigemptyset(&__kmp_sigset);
1231 __kmp_install_one_handler(SIGHUP, __kmp_team_handler, parallel_init);
1232 __kmp_install_one_handler(SIGINT, __kmp_team_handler, parallel_init);
1233 __kmp_install_one_handler(SIGQUIT, __kmp_team_handler, parallel_init);
1234 __kmp_install_one_handler(SIGILL, __kmp_team_handler, parallel_init);
1235 __kmp_install_one_handler(SIGABRT, __kmp_team_handler, parallel_init);
1236 __kmp_install_one_handler(SIGFPE, __kmp_team_handler, parallel_init);
1237 __kmp_install_one_handler(SIGBUS, __kmp_team_handler, parallel_init);
1238 __kmp_install_one_handler(SIGSEGV, __kmp_team_handler, parallel_init);
1239 #ifdef SIGSYS
1240 __kmp_install_one_handler(SIGSYS, __kmp_team_handler, parallel_init);
1241 #endif // SIGSYS
1242 __kmp_install_one_handler(SIGTERM, __kmp_team_handler, parallel_init);
1243 #ifdef SIGPIPE
1244 __kmp_install_one_handler(SIGPIPE, __kmp_team_handler, parallel_init);
1245 #endif // SIGPIPE
1246 }
1247 } // __kmp_install_signals
1248
__kmp_remove_signals(void)1249 void __kmp_remove_signals(void) {
1250 int sig;
1251 KB_TRACE(10, ("__kmp_remove_signals()\n"));
1252 for (sig = 1; sig < NSIG; ++sig) {
1253 __kmp_remove_one_handler(sig);
1254 }
1255 } // __kmp_remove_signals
1256
1257 #endif // KMP_HANDLE_SIGNALS
1258
__kmp_enable(int new_state)1259 void __kmp_enable(int new_state) {
1260 #ifdef KMP_CANCEL_THREADS
1261 int status, old_state;
1262 status = pthread_setcancelstate(new_state, &old_state);
1263 KMP_CHECK_SYSFAIL("pthread_setcancelstate", status);
1264 KMP_DEBUG_ASSERT(old_state == PTHREAD_CANCEL_DISABLE);
1265 #endif
1266 }
1267
__kmp_disable(int * old_state)1268 void __kmp_disable(int *old_state) {
1269 #ifdef KMP_CANCEL_THREADS
1270 int status;
1271 status = pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, old_state);
1272 KMP_CHECK_SYSFAIL("pthread_setcancelstate", status);
1273 #endif
1274 }
1275
__kmp_atfork_prepare(void)1276 static void __kmp_atfork_prepare(void) {
1277 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
1278 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
1279 }
1280
__kmp_atfork_parent(void)1281 static void __kmp_atfork_parent(void) {
1282 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
1283 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
1284 }
1285
1286 /* Reset the library so execution in the child starts "all over again" with
1287 clean data structures in initial states. Don't worry about freeing memory
1288 allocated by parent, just abandon it to be safe. */
__kmp_atfork_child(void)1289 static void __kmp_atfork_child(void) {
1290 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
1291 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
1292 /* TODO make sure this is done right for nested/sibling */
1293 // ATT: Memory leaks are here? TODO: Check it and fix.
1294 /* KMP_ASSERT( 0 ); */
1295
1296 ++__kmp_fork_count;
1297
1298 #if KMP_AFFINITY_SUPPORTED
1299 #if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY || \
1300 KMP_OS_AIX
1301 // reset the affinity in the child to the initial thread
1302 // affinity in the parent
1303 kmp_set_thread_affinity_mask_initial();
1304 #endif
1305 // Set default not to bind threads tightly in the child (we're expecting
1306 // over-subscription after the fork and this can improve things for
1307 // scripting languages that use OpenMP inside process-parallel code).
1308 if (__kmp_nested_proc_bind.bind_types != NULL) {
1309 __kmp_nested_proc_bind.bind_types[0] = proc_bind_false;
1310 }
1311 for (kmp_affinity_t *affinity : __kmp_affinities)
1312 *affinity = KMP_AFFINITY_INIT(affinity->env_var);
1313 __kmp_affin_fullMask = nullptr;
1314 __kmp_affin_origMask = nullptr;
1315 __kmp_topology = nullptr;
1316 #endif // KMP_AFFINITY_SUPPORTED
1317
1318 #if KMP_USE_MONITOR
1319 __kmp_init_monitor = 0;
1320 #endif
1321 __kmp_init_parallel = FALSE;
1322 __kmp_init_middle = FALSE;
1323 __kmp_init_serial = FALSE;
1324 TCW_4(__kmp_init_gtid, FALSE);
1325 __kmp_init_common = FALSE;
1326
1327 TCW_4(__kmp_init_user_locks, FALSE);
1328 #if !KMP_USE_DYNAMIC_LOCK
1329 __kmp_user_lock_table.used = 1;
1330 __kmp_user_lock_table.allocated = 0;
1331 __kmp_user_lock_table.table = NULL;
1332 __kmp_lock_blocks = NULL;
1333 #endif
1334
1335 __kmp_all_nth = 0;
1336 TCW_4(__kmp_nth, 0);
1337
1338 __kmp_thread_pool = NULL;
1339 __kmp_thread_pool_insert_pt = NULL;
1340 __kmp_team_pool = NULL;
1341
1342 /* Must actually zero all the *cache arguments passed to __kmpc_threadprivate
1343 here so threadprivate doesn't use stale data */
1344 KA_TRACE(10, ("__kmp_atfork_child: checking cache address list %p\n",
1345 __kmp_threadpriv_cache_list));
1346
1347 while (__kmp_threadpriv_cache_list != NULL) {
1348
1349 if (*__kmp_threadpriv_cache_list->addr != NULL) {
1350 KC_TRACE(50, ("__kmp_atfork_child: zeroing cache at address %p\n",
1351 &(*__kmp_threadpriv_cache_list->addr)));
1352
1353 *__kmp_threadpriv_cache_list->addr = NULL;
1354 }
1355 __kmp_threadpriv_cache_list = __kmp_threadpriv_cache_list->next;
1356 }
1357
1358 __kmp_init_runtime = FALSE;
1359
1360 /* reset statically initialized locks */
1361 __kmp_init_bootstrap_lock(&__kmp_initz_lock);
1362 __kmp_init_bootstrap_lock(&__kmp_stdio_lock);
1363 __kmp_init_bootstrap_lock(&__kmp_console_lock);
1364 __kmp_init_bootstrap_lock(&__kmp_task_team_lock);
1365
1366 #if USE_ITT_BUILD
1367 __kmp_itt_reset(); // reset ITT's global state
1368 #endif /* USE_ITT_BUILD */
1369
1370 {
1371 // Child process often get terminated without any use of OpenMP. That might
1372 // cause mapped shared memory file to be left unattended. Thus we postpone
1373 // library registration till middle initialization in the child process.
1374 __kmp_need_register_serial = FALSE;
1375 __kmp_serial_initialize();
1376 }
1377
1378 /* This is necessary to make sure no stale data is left around */
1379 /* AC: customers complain that we use unsafe routines in the atfork
1380 handler. Mathworks: dlsym() is unsafe. We call dlsym and dlopen
1381 in dynamic_link when check the presence of shared tbbmalloc library.
1382 Suggestion is to make the library initialization lazier, similar
1383 to what done for __kmpc_begin(). */
1384 // TODO: synchronize all static initializations with regular library
1385 // startup; look at kmp_global.cpp and etc.
1386 //__kmp_internal_begin ();
1387 }
1388
__kmp_register_atfork(void)1389 void __kmp_register_atfork(void) {
1390 if (__kmp_need_register_atfork) {
1391 #if !KMP_OS_WASI
1392 int status = pthread_atfork(__kmp_atfork_prepare, __kmp_atfork_parent,
1393 __kmp_atfork_child);
1394 KMP_CHECK_SYSFAIL("pthread_atfork", status);
1395 #endif
1396 __kmp_need_register_atfork = FALSE;
1397 }
1398 }
1399
__kmp_suspend_initialize(void)1400 void __kmp_suspend_initialize(void) {
1401 int status;
1402 status = pthread_mutexattr_init(&__kmp_suspend_mutex_attr);
1403 KMP_CHECK_SYSFAIL("pthread_mutexattr_init", status);
1404 status = pthread_condattr_init(&__kmp_suspend_cond_attr);
1405 KMP_CHECK_SYSFAIL("pthread_condattr_init", status);
1406 }
1407
__kmp_suspend_initialize_thread(kmp_info_t * th)1408 void __kmp_suspend_initialize_thread(kmp_info_t *th) {
1409 int old_value = KMP_ATOMIC_LD_RLX(&th->th.th_suspend_init_count);
1410 int new_value = __kmp_fork_count + 1;
1411 // Return if already initialized
1412 if (old_value == new_value)
1413 return;
1414 // Wait, then return if being initialized
1415 if (old_value == -1 || !__kmp_atomic_compare_store(
1416 &th->th.th_suspend_init_count, old_value, -1)) {
1417 while (KMP_ATOMIC_LD_ACQ(&th->th.th_suspend_init_count) != new_value) {
1418 KMP_CPU_PAUSE();
1419 }
1420 } else {
1421 // Claim to be the initializer and do initializations
1422 int status;
1423 status = pthread_cond_init(&th->th.th_suspend_cv.c_cond,
1424 &__kmp_suspend_cond_attr);
1425 KMP_CHECK_SYSFAIL("pthread_cond_init", status);
1426 status = pthread_mutex_init(&th->th.th_suspend_mx.m_mutex,
1427 &__kmp_suspend_mutex_attr);
1428 KMP_CHECK_SYSFAIL("pthread_mutex_init", status);
1429 KMP_ATOMIC_ST_REL(&th->th.th_suspend_init_count, new_value);
1430 }
1431 }
1432
__kmp_suspend_uninitialize_thread(kmp_info_t * th)1433 void __kmp_suspend_uninitialize_thread(kmp_info_t *th) {
1434 if (KMP_ATOMIC_LD_ACQ(&th->th.th_suspend_init_count) > __kmp_fork_count) {
1435 /* this means we have initialize the suspension pthread objects for this
1436 thread in this instance of the process */
1437 int status;
1438
1439 status = pthread_cond_destroy(&th->th.th_suspend_cv.c_cond);
1440 if (status != 0 && status != EBUSY) {
1441 KMP_SYSFAIL("pthread_cond_destroy", status);
1442 }
1443 status = pthread_mutex_destroy(&th->th.th_suspend_mx.m_mutex);
1444 if (status != 0 && status != EBUSY) {
1445 KMP_SYSFAIL("pthread_mutex_destroy", status);
1446 }
1447 --th->th.th_suspend_init_count;
1448 KMP_DEBUG_ASSERT(KMP_ATOMIC_LD_RLX(&th->th.th_suspend_init_count) ==
1449 __kmp_fork_count);
1450 }
1451 }
1452
1453 // return true if lock obtained, false otherwise
__kmp_try_suspend_mx(kmp_info_t * th)1454 int __kmp_try_suspend_mx(kmp_info_t *th) {
1455 return (pthread_mutex_trylock(&th->th.th_suspend_mx.m_mutex) == 0);
1456 }
1457
__kmp_lock_suspend_mx(kmp_info_t * th)1458 void __kmp_lock_suspend_mx(kmp_info_t *th) {
1459 int status = pthread_mutex_lock(&th->th.th_suspend_mx.m_mutex);
1460 KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
1461 }
1462
__kmp_unlock_suspend_mx(kmp_info_t * th)1463 void __kmp_unlock_suspend_mx(kmp_info_t *th) {
1464 int status = pthread_mutex_unlock(&th->th.th_suspend_mx.m_mutex);
1465 KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
1466 }
1467
1468 /* This routine puts the calling thread to sleep after setting the
1469 sleep bit for the indicated flag variable to true. */
1470 template <class C>
__kmp_suspend_template(int th_gtid,C * flag)1471 static inline void __kmp_suspend_template(int th_gtid, C *flag) {
1472 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_suspend);
1473 kmp_info_t *th = __kmp_threads[th_gtid];
1474 int status;
1475 typename C::flag_t old_spin;
1476
1477 KF_TRACE(30, ("__kmp_suspend_template: T#%d enter for flag = %p\n", th_gtid,
1478 flag->get()));
1479
1480 __kmp_suspend_initialize_thread(th);
1481
1482 __kmp_lock_suspend_mx(th);
1483
1484 KF_TRACE(10, ("__kmp_suspend_template: T#%d setting sleep bit for spin(%p)\n",
1485 th_gtid, flag->get()));
1486
1487 /* TODO: shouldn't this use release semantics to ensure that
1488 __kmp_suspend_initialize_thread gets called first? */
1489 old_spin = flag->set_sleeping();
1490 TCW_PTR(th->th.th_sleep_loc, (void *)flag);
1491 th->th.th_sleep_loc_type = flag->get_type();
1492 if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME &&
1493 __kmp_pause_status != kmp_soft_paused) {
1494 flag->unset_sleeping();
1495 TCW_PTR(th->th.th_sleep_loc, NULL);
1496 th->th.th_sleep_loc_type = flag_unset;
1497 __kmp_unlock_suspend_mx(th);
1498 return;
1499 }
1500 KF_TRACE(5, ("__kmp_suspend_template: T#%d set sleep bit for spin(%p)==%x,"
1501 " was %x\n",
1502 th_gtid, flag->get(), flag->load(), old_spin));
1503
1504 if (flag->done_check_val(old_spin) || flag->done_check()) {
1505 flag->unset_sleeping();
1506 TCW_PTR(th->th.th_sleep_loc, NULL);
1507 th->th.th_sleep_loc_type = flag_unset;
1508 KF_TRACE(5, ("__kmp_suspend_template: T#%d false alarm, reset sleep bit "
1509 "for spin(%p)\n",
1510 th_gtid, flag->get()));
1511 } else {
1512 /* Encapsulate in a loop as the documentation states that this may
1513 "with low probability" return when the condition variable has
1514 not been signaled or broadcast */
1515 int deactivated = FALSE;
1516
1517 while (flag->is_sleeping()) {
1518 #ifdef DEBUG_SUSPEND
1519 char buffer[128];
1520 __kmp_suspend_count++;
1521 __kmp_print_cond(buffer, &th->th.th_suspend_cv);
1522 __kmp_printf("__kmp_suspend_template: suspending T#%d: %s\n", th_gtid,
1523 buffer);
1524 #endif
1525 // Mark the thread as no longer active (only in the first iteration of the
1526 // loop).
1527 if (!deactivated) {
1528 th->th.th_active = FALSE;
1529 if (th->th.th_active_in_pool) {
1530 th->th.th_active_in_pool = FALSE;
1531 KMP_ATOMIC_DEC(&__kmp_thread_pool_active_nth);
1532 KMP_DEBUG_ASSERT(TCR_4(__kmp_thread_pool_active_nth) >= 0);
1533 }
1534 deactivated = TRUE;
1535 }
1536
1537 KMP_DEBUG_ASSERT(th->th.th_sleep_loc);
1538 KMP_DEBUG_ASSERT(flag->get_type() == th->th.th_sleep_loc_type);
1539
1540 #if USE_SUSPEND_TIMEOUT
1541 struct timespec now;
1542 struct timeval tval;
1543 int msecs;
1544
1545 status = gettimeofday(&tval, NULL);
1546 KMP_CHECK_SYSFAIL_ERRNO("gettimeofday", status);
1547 TIMEVAL_TO_TIMESPEC(&tval, &now);
1548
1549 msecs = (4 * __kmp_dflt_blocktime) + 200;
1550 now.tv_sec += msecs / 1000;
1551 now.tv_nsec += (msecs % 1000) * 1000;
1552
1553 KF_TRACE(15, ("__kmp_suspend_template: T#%d about to perform "
1554 "pthread_cond_timedwait\n",
1555 th_gtid));
1556 status = pthread_cond_timedwait(&th->th.th_suspend_cv.c_cond,
1557 &th->th.th_suspend_mx.m_mutex, &now);
1558 #else
1559 KF_TRACE(15, ("__kmp_suspend_template: T#%d about to perform"
1560 " pthread_cond_wait\n",
1561 th_gtid));
1562 status = pthread_cond_wait(&th->th.th_suspend_cv.c_cond,
1563 &th->th.th_suspend_mx.m_mutex);
1564 #endif // USE_SUSPEND_TIMEOUT
1565
1566 if ((status != 0) && (status != EINTR) && (status != ETIMEDOUT)) {
1567 KMP_SYSFAIL("pthread_cond_wait", status);
1568 }
1569
1570 KMP_DEBUG_ASSERT(flag->get_type() == flag->get_ptr_type());
1571
1572 if (!flag->is_sleeping() &&
1573 ((status == EINTR) || (status == ETIMEDOUT))) {
1574 // if interrupt or timeout, and thread is no longer sleeping, we need to
1575 // make sure sleep_loc gets reset; however, this shouldn't be needed if
1576 // we woke up with resume
1577 flag->unset_sleeping();
1578 TCW_PTR(th->th.th_sleep_loc, NULL);
1579 th->th.th_sleep_loc_type = flag_unset;
1580 }
1581 #ifdef KMP_DEBUG
1582 if (status == ETIMEDOUT) {
1583 if (flag->is_sleeping()) {
1584 KF_TRACE(100,
1585 ("__kmp_suspend_template: T#%d timeout wakeup\n", th_gtid));
1586 } else {
1587 KF_TRACE(2, ("__kmp_suspend_template: T#%d timeout wakeup, sleep bit "
1588 "not set!\n",
1589 th_gtid));
1590 TCW_PTR(th->th.th_sleep_loc, NULL);
1591 th->th.th_sleep_loc_type = flag_unset;
1592 }
1593 } else if (flag->is_sleeping()) {
1594 KF_TRACE(100,
1595 ("__kmp_suspend_template: T#%d spurious wakeup\n", th_gtid));
1596 }
1597 #endif
1598 } // while
1599
1600 // Mark the thread as active again (if it was previous marked as inactive)
1601 if (deactivated) {
1602 th->th.th_active = TRUE;
1603 if (TCR_4(th->th.th_in_pool)) {
1604 KMP_ATOMIC_INC(&__kmp_thread_pool_active_nth);
1605 th->th.th_active_in_pool = TRUE;
1606 }
1607 }
1608 }
1609 // We may have had the loop variable set before entering the loop body;
1610 // so we need to reset sleep_loc.
1611 TCW_PTR(th->th.th_sleep_loc, NULL);
1612 th->th.th_sleep_loc_type = flag_unset;
1613
1614 KMP_DEBUG_ASSERT(!flag->is_sleeping());
1615 KMP_DEBUG_ASSERT(!th->th.th_sleep_loc);
1616 #ifdef DEBUG_SUSPEND
1617 {
1618 char buffer[128];
1619 __kmp_print_cond(buffer, &th->th.th_suspend_cv);
1620 __kmp_printf("__kmp_suspend_template: T#%d has awakened: %s\n", th_gtid,
1621 buffer);
1622 }
1623 #endif
1624
1625 __kmp_unlock_suspend_mx(th);
1626 KF_TRACE(30, ("__kmp_suspend_template: T#%d exit\n", th_gtid));
1627 }
1628
1629 template <bool C, bool S>
__kmp_suspend_32(int th_gtid,kmp_flag_32<C,S> * flag)1630 void __kmp_suspend_32(int th_gtid, kmp_flag_32<C, S> *flag) {
1631 __kmp_suspend_template(th_gtid, flag);
1632 }
1633 template <bool C, bool S>
__kmp_suspend_64(int th_gtid,kmp_flag_64<C,S> * flag)1634 void __kmp_suspend_64(int th_gtid, kmp_flag_64<C, S> *flag) {
1635 __kmp_suspend_template(th_gtid, flag);
1636 }
1637 template <bool C, bool S>
__kmp_atomic_suspend_64(int th_gtid,kmp_atomic_flag_64<C,S> * flag)1638 void __kmp_atomic_suspend_64(int th_gtid, kmp_atomic_flag_64<C, S> *flag) {
1639 __kmp_suspend_template(th_gtid, flag);
1640 }
__kmp_suspend_oncore(int th_gtid,kmp_flag_oncore * flag)1641 void __kmp_suspend_oncore(int th_gtid, kmp_flag_oncore *flag) {
1642 __kmp_suspend_template(th_gtid, flag);
1643 }
1644
1645 template void __kmp_suspend_32<false, false>(int, kmp_flag_32<false, false> *);
1646 template void __kmp_suspend_64<false, true>(int, kmp_flag_64<false, true> *);
1647 template void __kmp_suspend_64<true, false>(int, kmp_flag_64<true, false> *);
1648 template void
1649 __kmp_atomic_suspend_64<false, true>(int, kmp_atomic_flag_64<false, true> *);
1650 template void
1651 __kmp_atomic_suspend_64<true, false>(int, kmp_atomic_flag_64<true, false> *);
1652
1653 /* This routine signals the thread specified by target_gtid to wake up
1654 after setting the sleep bit indicated by the flag argument to FALSE.
1655 The target thread must already have called __kmp_suspend_template() */
1656 template <class C>
__kmp_resume_template(int target_gtid,C * flag)1657 static inline void __kmp_resume_template(int target_gtid, C *flag) {
1658 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_resume);
1659 kmp_info_t *th = __kmp_threads[target_gtid];
1660 int status;
1661
1662 #ifdef KMP_DEBUG
1663 int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
1664 #endif
1665
1666 KF_TRACE(30, ("__kmp_resume_template: T#%d wants to wakeup T#%d enter\n",
1667 gtid, target_gtid));
1668 KMP_DEBUG_ASSERT(gtid != target_gtid);
1669
1670 __kmp_suspend_initialize_thread(th);
1671
1672 __kmp_lock_suspend_mx(th);
1673
1674 if (!flag || flag != th->th.th_sleep_loc) {
1675 // coming from __kmp_null_resume_wrapper, or thread is now sleeping on a
1676 // different location; wake up at new location
1677 flag = (C *)CCAST(void *, th->th.th_sleep_loc);
1678 }
1679
1680 // First, check if the flag is null or its type has changed. If so, someone
1681 // else woke it up.
1682 if (!flag) { // Thread doesn't appear to be sleeping on anything
1683 KF_TRACE(5, ("__kmp_resume_template: T#%d exiting, thread T#%d already "
1684 "awake: flag(%p)\n",
1685 gtid, target_gtid, (void *)NULL));
1686 __kmp_unlock_suspend_mx(th);
1687 return;
1688 } else if (flag->get_type() != th->th.th_sleep_loc_type) {
1689 // Flag type does not appear to match this function template; possibly the
1690 // thread is sleeping on something else. Try null resume again.
1691 KF_TRACE(
1692 5,
1693 ("__kmp_resume_template: T#%d retrying, thread T#%d Mismatch flag(%p), "
1694 "spin(%p) type=%d ptr_type=%d\n",
1695 gtid, target_gtid, flag, flag->get(), flag->get_type(),
1696 th->th.th_sleep_loc_type));
1697 __kmp_unlock_suspend_mx(th);
1698 __kmp_null_resume_wrapper(th);
1699 return;
1700 } else { // if multiple threads are sleeping, flag should be internally
1701 // referring to a specific thread here
1702 if (!flag->is_sleeping()) {
1703 KF_TRACE(5, ("__kmp_resume_template: T#%d exiting, thread T#%d already "
1704 "awake: flag(%p): %u\n",
1705 gtid, target_gtid, flag->get(), (unsigned int)flag->load()));
1706 __kmp_unlock_suspend_mx(th);
1707 return;
1708 }
1709 }
1710 KMP_DEBUG_ASSERT(flag);
1711 flag->unset_sleeping();
1712 TCW_PTR(th->th.th_sleep_loc, NULL);
1713 th->th.th_sleep_loc_type = flag_unset;
1714
1715 KF_TRACE(5, ("__kmp_resume_template: T#%d about to wakeup T#%d, reset "
1716 "sleep bit for flag's loc(%p): %u\n",
1717 gtid, target_gtid, flag->get(), (unsigned int)flag->load()));
1718
1719 #ifdef DEBUG_SUSPEND
1720 {
1721 char buffer[128];
1722 __kmp_print_cond(buffer, &th->th.th_suspend_cv);
1723 __kmp_printf("__kmp_resume_template: T#%d resuming T#%d: %s\n", gtid,
1724 target_gtid, buffer);
1725 }
1726 #endif
1727 status = pthread_cond_signal(&th->th.th_suspend_cv.c_cond);
1728 KMP_CHECK_SYSFAIL("pthread_cond_signal", status);
1729 __kmp_unlock_suspend_mx(th);
1730 KF_TRACE(30, ("__kmp_resume_template: T#%d exiting after signaling wake up"
1731 " for T#%d\n",
1732 gtid, target_gtid));
1733 }
1734
1735 template <bool C, bool S>
__kmp_resume_32(int target_gtid,kmp_flag_32<C,S> * flag)1736 void __kmp_resume_32(int target_gtid, kmp_flag_32<C, S> *flag) {
1737 __kmp_resume_template(target_gtid, flag);
1738 }
1739 template <bool C, bool S>
__kmp_resume_64(int target_gtid,kmp_flag_64<C,S> * flag)1740 void __kmp_resume_64(int target_gtid, kmp_flag_64<C, S> *flag) {
1741 __kmp_resume_template(target_gtid, flag);
1742 }
1743 template <bool C, bool S>
__kmp_atomic_resume_64(int target_gtid,kmp_atomic_flag_64<C,S> * flag)1744 void __kmp_atomic_resume_64(int target_gtid, kmp_atomic_flag_64<C, S> *flag) {
1745 __kmp_resume_template(target_gtid, flag);
1746 }
__kmp_resume_oncore(int target_gtid,kmp_flag_oncore * flag)1747 void __kmp_resume_oncore(int target_gtid, kmp_flag_oncore *flag) {
1748 __kmp_resume_template(target_gtid, flag);
1749 }
1750
1751 template void __kmp_resume_32<false, true>(int, kmp_flag_32<false, true> *);
1752 template void __kmp_resume_32<false, false>(int, kmp_flag_32<false, false> *);
1753 template void __kmp_resume_64<false, true>(int, kmp_flag_64<false, true> *);
1754 template void
1755 __kmp_atomic_resume_64<false, true>(int, kmp_atomic_flag_64<false, true> *);
1756
1757 #if KMP_USE_MONITOR
__kmp_resume_monitor()1758 void __kmp_resume_monitor() {
1759 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_resume);
1760 int status;
1761 #ifdef KMP_DEBUG
1762 int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
1763 KF_TRACE(30, ("__kmp_resume_monitor: T#%d wants to wakeup T#%d enter\n", gtid,
1764 KMP_GTID_MONITOR));
1765 KMP_DEBUG_ASSERT(gtid != KMP_GTID_MONITOR);
1766 #endif
1767 status = pthread_mutex_lock(&__kmp_wait_mx.m_mutex);
1768 KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
1769 #ifdef DEBUG_SUSPEND
1770 {
1771 char buffer[128];
1772 __kmp_print_cond(buffer, &__kmp_wait_cv.c_cond);
1773 __kmp_printf("__kmp_resume_monitor: T#%d resuming T#%d: %s\n", gtid,
1774 KMP_GTID_MONITOR, buffer);
1775 }
1776 #endif
1777 status = pthread_cond_signal(&__kmp_wait_cv.c_cond);
1778 KMP_CHECK_SYSFAIL("pthread_cond_signal", status);
1779 status = pthread_mutex_unlock(&__kmp_wait_mx.m_mutex);
1780 KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
1781 KF_TRACE(30, ("__kmp_resume_monitor: T#%d exiting after signaling wake up"
1782 " for T#%d\n",
1783 gtid, KMP_GTID_MONITOR));
1784 }
1785 #endif // KMP_USE_MONITOR
1786
__kmp_yield()1787 void __kmp_yield() { sched_yield(); }
1788
__kmp_gtid_set_specific(int gtid)1789 void __kmp_gtid_set_specific(int gtid) {
1790 if (__kmp_init_gtid) {
1791 int status;
1792 status = pthread_setspecific(__kmp_gtid_threadprivate_key,
1793 (void *)(intptr_t)(gtid + 1));
1794 KMP_CHECK_SYSFAIL("pthread_setspecific", status);
1795 } else {
1796 KA_TRACE(50, ("__kmp_gtid_set_specific: runtime shutdown, returning\n"));
1797 }
1798 }
1799
__kmp_gtid_get_specific()1800 int __kmp_gtid_get_specific() {
1801 int gtid;
1802 if (!__kmp_init_gtid) {
1803 KA_TRACE(50, ("__kmp_gtid_get_specific: runtime shutdown, returning "
1804 "KMP_GTID_SHUTDOWN\n"));
1805 return KMP_GTID_SHUTDOWN;
1806 }
1807 gtid = (int)(size_t)pthread_getspecific(__kmp_gtid_threadprivate_key);
1808 if (gtid == 0) {
1809 gtid = KMP_GTID_DNE;
1810 } else {
1811 gtid--;
1812 }
1813 KA_TRACE(50, ("__kmp_gtid_get_specific: key:%d gtid:%d\n",
1814 __kmp_gtid_threadprivate_key, gtid));
1815 return gtid;
1816 }
1817
__kmp_read_cpu_time(void)1818 double __kmp_read_cpu_time(void) {
1819 /*clock_t t;*/
1820 struct tms buffer;
1821
1822 /*t =*/times(&buffer);
1823
1824 return (double)(buffer.tms_utime + buffer.tms_cutime) /
1825 (double)CLOCKS_PER_SEC;
1826 }
1827
__kmp_read_system_info(struct kmp_sys_info * info)1828 int __kmp_read_system_info(struct kmp_sys_info *info) {
1829 int status;
1830 struct rusage r_usage;
1831
1832 memset(info, 0, sizeof(*info));
1833
1834 status = getrusage(RUSAGE_SELF, &r_usage);
1835 KMP_CHECK_SYSFAIL_ERRNO("getrusage", status);
1836
1837 #if !KMP_OS_WASI
1838 // The maximum resident set size utilized (in kilobytes)
1839 info->maxrss = r_usage.ru_maxrss;
1840 // The number of page faults serviced without any I/O
1841 info->minflt = r_usage.ru_minflt;
1842 // The number of page faults serviced that required I/O
1843 info->majflt = r_usage.ru_majflt;
1844 // The number of times a process was "swapped" out of memory
1845 info->nswap = r_usage.ru_nswap;
1846 // The number of times the file system had to perform input
1847 info->inblock = r_usage.ru_inblock;
1848 // The number of times the file system had to perform output
1849 info->oublock = r_usage.ru_oublock;
1850 // The number of times a context switch was voluntarily
1851 info->nvcsw = r_usage.ru_nvcsw;
1852 // The number of times a context switch was forced
1853 info->nivcsw = r_usage.ru_nivcsw;
1854 #endif
1855
1856 return (status != 0);
1857 }
1858
__kmp_read_system_time(double * delta)1859 void __kmp_read_system_time(double *delta) {
1860 double t_ns;
1861 struct timeval tval;
1862 struct timespec stop;
1863 int status;
1864
1865 status = gettimeofday(&tval, NULL);
1866 KMP_CHECK_SYSFAIL_ERRNO("gettimeofday", status);
1867 TIMEVAL_TO_TIMESPEC(&tval, &stop);
1868 t_ns = (double)(TS2NS(stop) - TS2NS(__kmp_sys_timer_data.start));
1869 *delta = (t_ns * 1e-9);
1870 }
1871
__kmp_clear_system_time(void)1872 void __kmp_clear_system_time(void) {
1873 struct timeval tval;
1874 int status;
1875 status = gettimeofday(&tval, NULL);
1876 KMP_CHECK_SYSFAIL_ERRNO("gettimeofday", status);
1877 TIMEVAL_TO_TIMESPEC(&tval, &__kmp_sys_timer_data.start);
1878 }
1879
__kmp_get_xproc(void)1880 static int __kmp_get_xproc(void) {
1881
1882 int r = 0;
1883
1884 #if KMP_OS_LINUX
1885
1886 __kmp_type_convert(sysconf(_SC_NPROCESSORS_CONF), &(r));
1887
1888 #elif KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_OPENBSD || \
1889 KMP_OS_HURD || KMP_OS_SOLARIS || KMP_OS_WASI || KMP_OS_AIX
1890
1891 __kmp_type_convert(sysconf(_SC_NPROCESSORS_ONLN), &(r));
1892
1893 #elif KMP_OS_DARWIN
1894
1895 size_t len = sizeof(r);
1896 sysctlbyname("hw.logicalcpu", &r, &len, NULL, 0);
1897
1898 #else
1899
1900 #error "Unknown or unsupported OS."
1901
1902 #endif
1903
1904 return r > 0 ? r : 2; /* guess value of 2 if OS told us 0 */
1905
1906 } // __kmp_get_xproc
1907
__kmp_read_from_file(char const * path,char const * format,...)1908 int __kmp_read_from_file(char const *path, char const *format, ...) {
1909 int result;
1910 va_list args;
1911
1912 va_start(args, format);
1913 FILE *f = fopen(path, "rb");
1914 if (f == NULL) {
1915 va_end(args);
1916 return 0;
1917 }
1918 result = vfscanf(f, format, args);
1919 fclose(f);
1920 va_end(args);
1921
1922 return result;
1923 }
1924
__kmp_runtime_initialize(void)1925 void __kmp_runtime_initialize(void) {
1926 int status;
1927 pthread_mutexattr_t mutex_attr;
1928 pthread_condattr_t cond_attr;
1929
1930 if (__kmp_init_runtime) {
1931 return;
1932 }
1933
1934 #if (KMP_ARCH_X86 || KMP_ARCH_X86_64)
1935 if (!__kmp_cpuinfo.initialized) {
1936 __kmp_query_cpuid(&__kmp_cpuinfo);
1937 }
1938 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
1939
1940 __kmp_xproc = __kmp_get_xproc();
1941
1942 #if !KMP_32_BIT_ARCH
1943 struct rlimit rlim;
1944 // read stack size of calling thread, save it as default for worker threads;
1945 // this should be done before reading environment variables
1946 status = getrlimit(RLIMIT_STACK, &rlim);
1947 if (status == 0) { // success?
1948 __kmp_stksize = rlim.rlim_cur;
1949 __kmp_check_stksize(&__kmp_stksize); // check value and adjust if needed
1950 }
1951 #endif /* KMP_32_BIT_ARCH */
1952
1953 if (sysconf(_SC_THREADS)) {
1954
1955 /* Query the maximum number of threads */
1956 __kmp_type_convert(sysconf(_SC_THREAD_THREADS_MAX), &(__kmp_sys_max_nth));
1957 #ifdef __ve__
1958 if (__kmp_sys_max_nth == -1) {
1959 // VE's pthread supports only up to 64 threads per a VE process.
1960 // So we use that KMP_MAX_NTH (predefined as 64) here.
1961 __kmp_sys_max_nth = KMP_MAX_NTH;
1962 }
1963 #else
1964 if (__kmp_sys_max_nth == -1) {
1965 /* Unlimited threads for NPTL */
1966 __kmp_sys_max_nth = INT_MAX;
1967 } else if (__kmp_sys_max_nth <= 1) {
1968 /* Can't tell, just use PTHREAD_THREADS_MAX */
1969 __kmp_sys_max_nth = KMP_MAX_NTH;
1970 }
1971 #endif
1972
1973 /* Query the minimum stack size */
1974 __kmp_sys_min_stksize = sysconf(_SC_THREAD_STACK_MIN);
1975 if (__kmp_sys_min_stksize <= 1) {
1976 __kmp_sys_min_stksize = KMP_MIN_STKSIZE;
1977 }
1978 }
1979
1980 /* Set up minimum number of threads to switch to TLS gtid */
1981 __kmp_tls_gtid_min = KMP_TLS_GTID_MIN;
1982
1983 status = pthread_key_create(&__kmp_gtid_threadprivate_key,
1984 __kmp_internal_end_dest);
1985 KMP_CHECK_SYSFAIL("pthread_key_create", status);
1986 status = pthread_mutexattr_init(&mutex_attr);
1987 KMP_CHECK_SYSFAIL("pthread_mutexattr_init", status);
1988 status = pthread_mutex_init(&__kmp_wait_mx.m_mutex, &mutex_attr);
1989 KMP_CHECK_SYSFAIL("pthread_mutex_init", status);
1990 status = pthread_mutexattr_destroy(&mutex_attr);
1991 KMP_CHECK_SYSFAIL("pthread_mutexattr_destroy", status);
1992 status = pthread_condattr_init(&cond_attr);
1993 KMP_CHECK_SYSFAIL("pthread_condattr_init", status);
1994 status = pthread_cond_init(&__kmp_wait_cv.c_cond, &cond_attr);
1995 KMP_CHECK_SYSFAIL("pthread_cond_init", status);
1996 status = pthread_condattr_destroy(&cond_attr);
1997 KMP_CHECK_SYSFAIL("pthread_condattr_destroy", status);
1998 #if USE_ITT_BUILD
1999 __kmp_itt_initialize();
2000 #endif /* USE_ITT_BUILD */
2001
2002 __kmp_init_runtime = TRUE;
2003 }
2004
__kmp_runtime_destroy(void)2005 void __kmp_runtime_destroy(void) {
2006 int status;
2007
2008 if (!__kmp_init_runtime) {
2009 return; // Nothing to do.
2010 }
2011
2012 #if USE_ITT_BUILD
2013 __kmp_itt_destroy();
2014 #endif /* USE_ITT_BUILD */
2015
2016 status = pthread_key_delete(__kmp_gtid_threadprivate_key);
2017 KMP_CHECK_SYSFAIL("pthread_key_delete", status);
2018
2019 status = pthread_mutex_destroy(&__kmp_wait_mx.m_mutex);
2020 if (status != 0 && status != EBUSY) {
2021 KMP_SYSFAIL("pthread_mutex_destroy", status);
2022 }
2023 status = pthread_cond_destroy(&__kmp_wait_cv.c_cond);
2024 if (status != 0 && status != EBUSY) {
2025 KMP_SYSFAIL("pthread_cond_destroy", status);
2026 }
2027 #if KMP_AFFINITY_SUPPORTED
2028 __kmp_affinity_uninitialize();
2029 #endif
2030
2031 __kmp_init_runtime = FALSE;
2032 }
2033
2034 /* Put the thread to sleep for a time period */
2035 /* NOTE: not currently used anywhere */
__kmp_thread_sleep(int millis)2036 void __kmp_thread_sleep(int millis) { sleep((millis + 500) / 1000); }
2037
2038 /* Calculate the elapsed wall clock time for the user */
__kmp_elapsed(double * t)2039 void __kmp_elapsed(double *t) {
2040 int status;
2041 #ifdef FIX_SGI_CLOCK
2042 struct timespec ts;
2043
2044 status = clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &ts);
2045 KMP_CHECK_SYSFAIL_ERRNO("clock_gettime", status);
2046 *t =
2047 (double)ts.tv_nsec * (1.0 / (double)KMP_NSEC_PER_SEC) + (double)ts.tv_sec;
2048 #else
2049 struct timeval tv;
2050
2051 status = gettimeofday(&tv, NULL);
2052 KMP_CHECK_SYSFAIL_ERRNO("gettimeofday", status);
2053 *t =
2054 (double)tv.tv_usec * (1.0 / (double)KMP_USEC_PER_SEC) + (double)tv.tv_sec;
2055 #endif
2056 }
2057
2058 /* Calculate the elapsed wall clock tick for the user */
__kmp_elapsed_tick(double * t)2059 void __kmp_elapsed_tick(double *t) { *t = 1 / (double)CLOCKS_PER_SEC; }
2060
2061 /* Return the current time stamp in nsec */
__kmp_now_nsec()2062 kmp_uint64 __kmp_now_nsec() {
2063 struct timeval t;
2064 gettimeofday(&t, NULL);
2065 kmp_uint64 nsec = (kmp_uint64)KMP_NSEC_PER_SEC * (kmp_uint64)t.tv_sec +
2066 (kmp_uint64)1000 * (kmp_uint64)t.tv_usec;
2067 return nsec;
2068 }
2069
2070 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
2071 /* Measure clock ticks per millisecond */
__kmp_initialize_system_tick()2072 void __kmp_initialize_system_tick() {
2073 kmp_uint64 now, nsec2, diff;
2074 kmp_uint64 delay = 1000000; // ~450 usec on most machines.
2075 kmp_uint64 nsec = __kmp_now_nsec();
2076 kmp_uint64 goal = __kmp_hardware_timestamp() + delay;
2077 while ((now = __kmp_hardware_timestamp()) < goal)
2078 ;
2079 nsec2 = __kmp_now_nsec();
2080 diff = nsec2 - nsec;
2081 if (diff > 0) {
2082 double tpus = 1000.0 * (double)(delay + (now - goal)) / (double)diff;
2083 if (tpus > 0.0) {
2084 __kmp_ticks_per_msec = (kmp_uint64)(tpus * 1000.0);
2085 __kmp_ticks_per_usec = (kmp_uint64)tpus;
2086 }
2087 }
2088 }
2089 #endif
2090
2091 /* Determine whether the given address is mapped into the current address
2092 space. */
2093
__kmp_is_address_mapped(void * addr)2094 int __kmp_is_address_mapped(void *addr) {
2095
2096 int found = 0;
2097 int rc;
2098
2099 #if KMP_OS_LINUX || KMP_OS_HURD
2100
2101 /* On GNUish OSes, read the /proc/<pid>/maps pseudo-file to get all the
2102 address ranges mapped into the address space. */
2103
2104 char *name = __kmp_str_format("/proc/%d/maps", getpid());
2105 FILE *file = NULL;
2106
2107 file = fopen(name, "r");
2108 KMP_ASSERT(file != NULL);
2109
2110 for (;;) {
2111
2112 void *beginning = NULL;
2113 void *ending = NULL;
2114 char perms[5];
2115
2116 rc = fscanf(file, "%p-%p %4s %*[^\n]\n", &beginning, &ending, perms);
2117 if (rc == EOF) {
2118 break;
2119 }
2120 KMP_ASSERT(rc == 3 &&
2121 KMP_STRLEN(perms) == 4); // Make sure all fields are read.
2122
2123 // Ending address is not included in the region, but beginning is.
2124 if ((addr >= beginning) && (addr < ending)) {
2125 perms[2] = 0; // 3th and 4th character does not matter.
2126 if (strcmp(perms, "rw") == 0) {
2127 // Memory we are looking for should be readable and writable.
2128 found = 1;
2129 }
2130 break;
2131 }
2132 }
2133
2134 // Free resources.
2135 fclose(file);
2136 KMP_INTERNAL_FREE(name);
2137 #elif KMP_OS_FREEBSD
2138 char *buf;
2139 size_t lstsz;
2140 int mib[] = {CTL_KERN, KERN_PROC, KERN_PROC_VMMAP, getpid()};
2141 rc = sysctl(mib, 4, NULL, &lstsz, NULL, 0);
2142 if (rc < 0)
2143 return 0;
2144 // We pass from number of vm entry's semantic
2145 // to size of whole entry map list.
2146 lstsz = lstsz * 4 / 3;
2147 buf = reinterpret_cast<char *>(KMP_INTERNAL_MALLOC(lstsz));
2148 rc = sysctl(mib, 4, buf, &lstsz, NULL, 0);
2149 if (rc < 0) {
2150 KMP_INTERNAL_FREE(buf);
2151 return 0;
2152 }
2153
2154 char *lw = buf;
2155 char *up = buf + lstsz;
2156
2157 while (lw < up) {
2158 struct kinfo_vmentry *cur = reinterpret_cast<struct kinfo_vmentry *>(lw);
2159 size_t cursz = cur->kve_structsize;
2160 if (cursz == 0)
2161 break;
2162 void *start = reinterpret_cast<void *>(cur->kve_start);
2163 void *end = reinterpret_cast<void *>(cur->kve_end);
2164 // Readable/Writable addresses within current map entry
2165 if ((addr >= start) && (addr < end)) {
2166 if ((cur->kve_protection & KVME_PROT_READ) != 0 &&
2167 (cur->kve_protection & KVME_PROT_WRITE) != 0) {
2168 found = 1;
2169 break;
2170 }
2171 }
2172 lw += cursz;
2173 }
2174 KMP_INTERNAL_FREE(buf);
2175 #elif KMP_OS_DRAGONFLY
2176 char err[_POSIX2_LINE_MAX];
2177 kinfo_proc *proc;
2178 vmspace sp;
2179 vm_map *cur;
2180 vm_map_entry entry, *c;
2181 struct proc p;
2182 kvm_t *fd;
2183 uintptr_t uaddr;
2184 int num;
2185
2186 fd = kvm_openfiles(nullptr, nullptr, nullptr, O_RDONLY, err);
2187 if (!fd) {
2188 return 0;
2189 }
2190
2191 proc = kvm_getprocs(fd, KERN_PROC_PID, getpid(), &num);
2192
2193 if (kvm_read(fd, static_cast<uintptr_t>(proc->kp_paddr), &p, sizeof(p)) !=
2194 sizeof(p) ||
2195 kvm_read(fd, reinterpret_cast<uintptr_t>(p.p_vmspace), &sp, sizeof(sp)) !=
2196 sizeof(sp)) {
2197 kvm_close(fd);
2198 return 0;
2199 }
2200
2201 (void)rc;
2202 cur = &sp.vm_map;
2203 uaddr = reinterpret_cast<uintptr_t>(addr);
2204 for (c = kvm_vm_map_entry_first(fd, cur, &entry); c;
2205 c = kvm_vm_map_entry_next(fd, c, &entry)) {
2206 if ((uaddr >= entry.ba.start) && (uaddr <= entry.ba.end)) {
2207 if ((entry.protection & VM_PROT_READ) != 0 &&
2208 (entry.protection & VM_PROT_WRITE) != 0) {
2209 found = 1;
2210 break;
2211 }
2212 }
2213 }
2214
2215 kvm_close(fd);
2216 #elif KMP_OS_SOLARIS
2217 prmap_t *cur, *map;
2218 void *buf;
2219 uintptr_t uaddr;
2220 ssize_t rd;
2221 int err;
2222 int file;
2223
2224 pid_t pid = getpid();
2225 struct ps_prochandle *fd = Pgrab(pid, PGRAB_RDONLY, &err);
2226 ;
2227
2228 if (!fd) {
2229 return 0;
2230 }
2231
2232 char *name = __kmp_str_format("/proc/%d/map", pid);
2233 size_t sz = (1 << 20);
2234 file = open(name, O_RDONLY);
2235 if (file == -1) {
2236 KMP_INTERNAL_FREE(name);
2237 return 0;
2238 }
2239
2240 buf = KMP_INTERNAL_MALLOC(sz);
2241
2242 while (sz > 0 && (rd = pread(file, buf, sz, 0)) == sz) {
2243 void *newbuf;
2244 sz <<= 1;
2245 newbuf = KMP_INTERNAL_REALLOC(buf, sz);
2246 buf = newbuf;
2247 }
2248
2249 map = reinterpret_cast<prmap_t *>(buf);
2250 uaddr = reinterpret_cast<uintptr_t>(addr);
2251
2252 for (cur = map; rd > 0; cur++, rd = -sizeof(*map)) {
2253 if ((uaddr >= cur->pr_vaddr) && (uaddr < cur->pr_vaddr)) {
2254 if ((cur->pr_mflags & MA_READ) != 0 && (cur->pr_mflags & MA_WRITE) != 0) {
2255 found = 1;
2256 break;
2257 }
2258 }
2259 }
2260
2261 KMP_INTERNAL_FREE(map);
2262 close(file);
2263 KMP_INTERNAL_FREE(name);
2264 #elif KMP_OS_DARWIN
2265
2266 /* On OS X*, /proc pseudo filesystem is not available. Try to read memory
2267 using vm interface. */
2268
2269 int buffer;
2270 vm_size_t count;
2271 rc = vm_read_overwrite(
2272 mach_task_self(), // Task to read memory of.
2273 (vm_address_t)(addr), // Address to read from.
2274 1, // Number of bytes to be read.
2275 (vm_address_t)(&buffer), // Address of buffer to save read bytes in.
2276 &count // Address of var to save number of read bytes in.
2277 );
2278 if (rc == 0) {
2279 // Memory successfully read.
2280 found = 1;
2281 }
2282
2283 #elif KMP_OS_NETBSD
2284
2285 int mib[5];
2286 mib[0] = CTL_VM;
2287 mib[1] = VM_PROC;
2288 mib[2] = VM_PROC_MAP;
2289 mib[3] = getpid();
2290 mib[4] = sizeof(struct kinfo_vmentry);
2291
2292 size_t size;
2293 rc = sysctl(mib, __arraycount(mib), NULL, &size, NULL, 0);
2294 KMP_ASSERT(!rc);
2295 KMP_ASSERT(size);
2296
2297 size = size * 4 / 3;
2298 struct kinfo_vmentry *kiv = (struct kinfo_vmentry *)KMP_INTERNAL_MALLOC(size);
2299 KMP_ASSERT(kiv);
2300
2301 rc = sysctl(mib, __arraycount(mib), kiv, &size, NULL, 0);
2302 KMP_ASSERT(!rc);
2303 KMP_ASSERT(size);
2304
2305 for (size_t i = 0; i < size; i++) {
2306 if (kiv[i].kve_start >= (uint64_t)addr &&
2307 kiv[i].kve_end <= (uint64_t)addr) {
2308 found = 1;
2309 break;
2310 }
2311 }
2312 KMP_INTERNAL_FREE(kiv);
2313 #elif KMP_OS_OPENBSD
2314
2315 int mib[3];
2316 mib[0] = CTL_KERN;
2317 mib[1] = KERN_PROC_VMMAP;
2318 mib[2] = getpid();
2319
2320 size_t size;
2321 uint64_t end;
2322 rc = sysctl(mib, 3, NULL, &size, NULL, 0);
2323 KMP_ASSERT(!rc);
2324 KMP_ASSERT(size);
2325 end = size;
2326
2327 struct kinfo_vmentry kiv = {.kve_start = 0};
2328
2329 while ((rc = sysctl(mib, 3, &kiv, &size, NULL, 0)) == 0) {
2330 KMP_ASSERT(size);
2331 if (kiv.kve_end == end)
2332 break;
2333
2334 if (kiv.kve_start >= (uint64_t)addr && kiv.kve_end <= (uint64_t)addr) {
2335 found = 1;
2336 break;
2337 }
2338 kiv.kve_start += 1;
2339 }
2340 #elif KMP_OS_WASI
2341 found = (int)addr < (__builtin_wasm_memory_size(0) * PAGESIZE);
2342 #elif KMP_OS_AIX
2343
2344 uint32_t loadQueryBufSize = 4096u; // Default loadquery buffer size.
2345 char *loadQueryBuf;
2346
2347 for (;;) {
2348 loadQueryBuf = (char *)KMP_INTERNAL_MALLOC(loadQueryBufSize);
2349 if (loadQueryBuf == NULL) {
2350 return 0;
2351 }
2352
2353 rc = loadquery(L_GETXINFO | L_IGNOREUNLOAD, loadQueryBuf, loadQueryBufSize);
2354 if (rc < 0) {
2355 KMP_INTERNAL_FREE(loadQueryBuf);
2356 if (errno != ENOMEM) {
2357 return 0;
2358 }
2359 // errno == ENOMEM; double the size.
2360 loadQueryBufSize <<= 1;
2361 continue;
2362 }
2363 // Obtained the load info successfully.
2364 break;
2365 }
2366
2367 struct ld_xinfo *curLdInfo = (struct ld_xinfo *)loadQueryBuf;
2368
2369 // Loop through the load info to find if there is a match.
2370 for (;;) {
2371 uintptr_t curDataStart = (uintptr_t)curLdInfo->ldinfo_dataorg;
2372 uintptr_t curDataEnd = curDataStart + curLdInfo->ldinfo_datasize;
2373
2374 // The data segment is readable and writable.
2375 if (curDataStart <= (uintptr_t)addr && (uintptr_t)addr < curDataEnd) {
2376 found = 1;
2377 break;
2378 }
2379 if (curLdInfo->ldinfo_next == 0u) {
2380 // Reached the end of load info.
2381 break;
2382 }
2383 curLdInfo = (struct ld_xinfo *)((char *)curLdInfo + curLdInfo->ldinfo_next);
2384 }
2385 KMP_INTERNAL_FREE(loadQueryBuf);
2386
2387 #else
2388
2389 #error "Unknown or unsupported OS"
2390
2391 #endif
2392
2393 return found;
2394
2395 } // __kmp_is_address_mapped
2396
2397 #ifdef USE_LOAD_BALANCE
2398
2399 #if KMP_OS_DARWIN || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \
2400 KMP_OS_OPENBSD || KMP_OS_SOLARIS
2401
2402 // The function returns the rounded value of the system load average
2403 // during given time interval which depends on the value of
2404 // __kmp_load_balance_interval variable (default is 60 sec, other values
2405 // may be 300 sec or 900 sec).
2406 // It returns -1 in case of error.
__kmp_get_load_balance(int max)2407 int __kmp_get_load_balance(int max) {
2408 double averages[3];
2409 int ret_avg = 0;
2410
2411 int res = getloadavg(averages, 3);
2412
2413 // Check __kmp_load_balance_interval to determine which of averages to use.
2414 // getloadavg() may return the number of samples less than requested that is
2415 // less than 3.
2416 if (__kmp_load_balance_interval < 180 && (res >= 1)) {
2417 ret_avg = (int)averages[0]; // 1 min
2418 } else if ((__kmp_load_balance_interval >= 180 &&
2419 __kmp_load_balance_interval < 600) &&
2420 (res >= 2)) {
2421 ret_avg = (int)averages[1]; // 5 min
2422 } else if ((__kmp_load_balance_interval >= 600) && (res == 3)) {
2423 ret_avg = (int)averages[2]; // 15 min
2424 } else { // Error occurred
2425 return -1;
2426 }
2427
2428 return ret_avg;
2429 }
2430
2431 #elif KMP_OS_AIX
2432
2433 // The function returns number of running (not sleeping) threads, or -1 in case
2434 // of error.
__kmp_get_load_balance(int max)2435 int __kmp_get_load_balance(int max) {
2436
2437 static int glb_running_threads = 0; // Saved count of the running threads for
2438 // the thread balance algorithm.
2439 static double glb_call_time = 0; // Thread balance algorithm call time.
2440 int running_threads = 0; // Number of running threads in the system.
2441
2442 double call_time = 0.0;
2443
2444 __kmp_elapsed(&call_time);
2445
2446 if (glb_call_time &&
2447 (call_time - glb_call_time < __kmp_load_balance_interval))
2448 return glb_running_threads;
2449
2450 glb_call_time = call_time;
2451
2452 if (max <= 0) {
2453 max = INT_MAX;
2454 }
2455
2456 // Check how many perfstat_cpu_t structures are available.
2457 int logical_cpus = perfstat_cpu(NULL, NULL, sizeof(perfstat_cpu_t), 0);
2458 if (logical_cpus <= 0) {
2459 glb_call_time = -1;
2460 return -1;
2461 }
2462
2463 perfstat_cpu_t *cpu_stat = (perfstat_cpu_t *)KMP_INTERNAL_MALLOC(
2464 logical_cpus * sizeof(perfstat_cpu_t));
2465 if (cpu_stat == NULL) {
2466 glb_call_time = -1;
2467 return -1;
2468 }
2469
2470 // Set first CPU as the name of the first logical CPU for which the info is
2471 // desired.
2472 perfstat_id_t first_cpu_name;
2473 strcpy(first_cpu_name.name, FIRST_CPU);
2474
2475 // Get the stat info of logical CPUs.
2476 int rc = perfstat_cpu(&first_cpu_name, cpu_stat, sizeof(perfstat_cpu_t),
2477 logical_cpus);
2478 KMP_DEBUG_ASSERT(rc == logical_cpus);
2479 if (rc <= 0) {
2480 KMP_INTERNAL_FREE(cpu_stat);
2481 glb_call_time = -1;
2482 return -1;
2483 }
2484 for (int i = 0; i < logical_cpus; ++i) {
2485 running_threads += cpu_stat[i].runque;
2486 if (running_threads >= max)
2487 break;
2488 }
2489
2490 // There _might_ be a timing hole where the thread executing this
2491 // code gets skipped in the load balance, and running_threads is 0.
2492 // Assert in the debug builds only!!!
2493 KMP_DEBUG_ASSERT(running_threads > 0);
2494 if (running_threads <= 0)
2495 running_threads = 1;
2496
2497 KMP_INTERNAL_FREE(cpu_stat);
2498
2499 glb_running_threads = running_threads;
2500
2501 return running_threads;
2502 }
2503
2504 #else // Linux* OS
2505
2506 // The function returns number of running (not sleeping) threads, or -1 in case
2507 // of error. Error could be reported if Linux* OS kernel too old (without
2508 // "/proc" support). Counting running threads stops if max running threads
2509 // encountered.
__kmp_get_load_balance(int max)2510 int __kmp_get_load_balance(int max) {
2511 static int permanent_error = 0;
2512 static int glb_running_threads = 0; // Saved count of the running threads for
2513 // the thread balance algorithm
2514 static double glb_call_time = 0; /* Thread balance algorithm call time */
2515
2516 int running_threads = 0; // Number of running threads in the system.
2517
2518 DIR *proc_dir = NULL; // Handle of "/proc/" directory.
2519 struct dirent *proc_entry = NULL;
2520
2521 kmp_str_buf_t task_path; // "/proc/<pid>/task/<tid>/" path.
2522 DIR *task_dir = NULL; // Handle of "/proc/<pid>/task/<tid>/" directory.
2523 struct dirent *task_entry = NULL;
2524 int task_path_fixed_len;
2525
2526 kmp_str_buf_t stat_path; // "/proc/<pid>/task/<tid>/stat" path.
2527 int stat_file = -1;
2528 int stat_path_fixed_len;
2529
2530 #ifdef KMP_DEBUG
2531 int total_processes = 0; // Total number of processes in system.
2532 #endif
2533
2534 double call_time = 0.0;
2535
2536 __kmp_str_buf_init(&task_path);
2537 __kmp_str_buf_init(&stat_path);
2538
2539 __kmp_elapsed(&call_time);
2540
2541 if (glb_call_time &&
2542 (call_time - glb_call_time < __kmp_load_balance_interval)) {
2543 running_threads = glb_running_threads;
2544 goto finish;
2545 }
2546
2547 glb_call_time = call_time;
2548
2549 // Do not spend time on scanning "/proc/" if we have a permanent error.
2550 if (permanent_error) {
2551 running_threads = -1;
2552 goto finish;
2553 }
2554
2555 if (max <= 0) {
2556 max = INT_MAX;
2557 }
2558
2559 // Open "/proc/" directory.
2560 proc_dir = opendir("/proc");
2561 if (proc_dir == NULL) {
2562 // Cannot open "/proc/". Probably the kernel does not support it. Return an
2563 // error now and in subsequent calls.
2564 running_threads = -1;
2565 permanent_error = 1;
2566 goto finish;
2567 }
2568
2569 // Initialize fixed part of task_path. This part will not change.
2570 __kmp_str_buf_cat(&task_path, "/proc/", 6);
2571 task_path_fixed_len = task_path.used; // Remember number of used characters.
2572
2573 proc_entry = readdir(proc_dir);
2574 while (proc_entry != NULL) {
2575 // Proc entry is a directory and name starts with a digit. Assume it is a
2576 // process' directory.
2577 if (proc_entry->d_type == DT_DIR && isdigit(proc_entry->d_name[0])) {
2578
2579 #ifdef KMP_DEBUG
2580 ++total_processes;
2581 #endif
2582 // Make sure init process is the very first in "/proc", so we can replace
2583 // strcmp( proc_entry->d_name, "1" ) == 0 with simpler total_processes ==
2584 // 1. We are going to check that total_processes == 1 => d_name == "1" is
2585 // true (where "=>" is implication). Since C++ does not have => operator,
2586 // let us replace it with its equivalent: a => b == ! a || b.
2587 KMP_DEBUG_ASSERT(total_processes != 1 ||
2588 strcmp(proc_entry->d_name, "1") == 0);
2589
2590 // Construct task_path.
2591 task_path.used = task_path_fixed_len; // Reset task_path to "/proc/".
2592 __kmp_str_buf_cat(&task_path, proc_entry->d_name,
2593 KMP_STRLEN(proc_entry->d_name));
2594 __kmp_str_buf_cat(&task_path, "/task", 5);
2595
2596 task_dir = opendir(task_path.str);
2597 if (task_dir == NULL) {
2598 // Process can finish between reading "/proc/" directory entry and
2599 // opening process' "task/" directory. So, in general case we should not
2600 // complain, but have to skip this process and read the next one. But on
2601 // systems with no "task/" support we will spend lot of time to scan
2602 // "/proc/" tree again and again without any benefit. "init" process
2603 // (its pid is 1) should exist always, so, if we cannot open
2604 // "/proc/1/task/" directory, it means "task/" is not supported by
2605 // kernel. Report an error now and in the future.
2606 if (strcmp(proc_entry->d_name, "1") == 0) {
2607 running_threads = -1;
2608 permanent_error = 1;
2609 goto finish;
2610 }
2611 } else {
2612 // Construct fixed part of stat file path.
2613 __kmp_str_buf_clear(&stat_path);
2614 __kmp_str_buf_cat(&stat_path, task_path.str, task_path.used);
2615 __kmp_str_buf_cat(&stat_path, "/", 1);
2616 stat_path_fixed_len = stat_path.used;
2617
2618 task_entry = readdir(task_dir);
2619 while (task_entry != NULL) {
2620 // It is a directory and name starts with a digit.
2621 if (proc_entry->d_type == DT_DIR && isdigit(task_entry->d_name[0])) {
2622
2623 // Construct complete stat file path. Easiest way would be:
2624 // __kmp_str_buf_print( & stat_path, "%s/%s/stat", task_path.str,
2625 // task_entry->d_name );
2626 // but seriae of __kmp_str_buf_cat works a bit faster.
2627 stat_path.used =
2628 stat_path_fixed_len; // Reset stat path to its fixed part.
2629 __kmp_str_buf_cat(&stat_path, task_entry->d_name,
2630 KMP_STRLEN(task_entry->d_name));
2631 __kmp_str_buf_cat(&stat_path, "/stat", 5);
2632
2633 // Note: Low-level API (open/read/close) is used. High-level API
2634 // (fopen/fclose) works ~ 30 % slower.
2635 stat_file = open(stat_path.str, O_RDONLY);
2636 if (stat_file == -1) {
2637 // We cannot report an error because task (thread) can terminate
2638 // just before reading this file.
2639 } else {
2640 /* Content of "stat" file looks like:
2641 24285 (program) S ...
2642
2643 It is a single line (if program name does not include funny
2644 symbols). First number is a thread id, then name of executable
2645 file name in paretheses, then state of the thread. We need just
2646 thread state.
2647
2648 Good news: Length of program name is 15 characters max. Longer
2649 names are truncated.
2650
2651 Thus, we need rather short buffer: 15 chars for program name +
2652 2 parenthesis, + 3 spaces + ~7 digits of pid = 37.
2653
2654 Bad news: Program name may contain special symbols like space,
2655 closing parenthesis, or even new line. This makes parsing
2656 "stat" file not 100 % reliable. In case of fanny program names
2657 parsing may fail (report incorrect thread state).
2658
2659 Parsing "status" file looks more promissing (due to different
2660 file structure and escaping special symbols) but reading and
2661 parsing of "status" file works slower.
2662 -- ln
2663 */
2664 char buffer[65];
2665 ssize_t len;
2666 len = read(stat_file, buffer, sizeof(buffer) - 1);
2667 if (len >= 0) {
2668 buffer[len] = 0;
2669 // Using scanf:
2670 // sscanf( buffer, "%*d (%*s) %c ", & state );
2671 // looks very nice, but searching for a closing parenthesis
2672 // works a bit faster.
2673 char *close_parent = strstr(buffer, ") ");
2674 if (close_parent != NULL) {
2675 char state = *(close_parent + 2);
2676 if (state == 'R') {
2677 ++running_threads;
2678 if (running_threads >= max) {
2679 goto finish;
2680 }
2681 }
2682 }
2683 }
2684 close(stat_file);
2685 stat_file = -1;
2686 }
2687 }
2688 task_entry = readdir(task_dir);
2689 }
2690 closedir(task_dir);
2691 task_dir = NULL;
2692 }
2693 }
2694 proc_entry = readdir(proc_dir);
2695 }
2696
2697 // There _might_ be a timing hole where the thread executing this
2698 // code get skipped in the load balance, and running_threads is 0.
2699 // Assert in the debug builds only!!!
2700 KMP_DEBUG_ASSERT(running_threads > 0);
2701 if (running_threads <= 0) {
2702 running_threads = 1;
2703 }
2704
2705 finish: // Clean up and exit.
2706 if (proc_dir != NULL) {
2707 closedir(proc_dir);
2708 }
2709 __kmp_str_buf_free(&task_path);
2710 if (task_dir != NULL) {
2711 closedir(task_dir);
2712 }
2713 __kmp_str_buf_free(&stat_path);
2714 if (stat_file != -1) {
2715 close(stat_file);
2716 }
2717
2718 glb_running_threads = running_threads;
2719
2720 return running_threads;
2721
2722 } // __kmp_get_load_balance
2723
2724 #endif // KMP_OS_DARWIN
2725
2726 #endif // USE_LOAD_BALANCE
2727
2728 #if !(KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_MIC || \
2729 ((KMP_OS_LINUX || KMP_OS_DARWIN) && KMP_ARCH_AARCH64) || \
2730 KMP_ARCH_PPC64 || KMP_ARCH_RISCV64 || KMP_ARCH_LOONGARCH64 || \
2731 KMP_ARCH_ARM || KMP_ARCH_VE || KMP_ARCH_S390X || KMP_ARCH_PPC_XCOFF || \
2732 KMP_ARCH_AARCH64_32)
2733
2734 // Because WebAssembly will use `call_indirect` to invoke the microtask and
2735 // WebAssembly indirect calls check that the called signature is a precise
2736 // match, we need to cast each microtask function pointer back from `void *` to
2737 // its original type.
2738 typedef void (*microtask_t0)(int *, int *);
2739 typedef void (*microtask_t1)(int *, int *, void *);
2740 typedef void (*microtask_t2)(int *, int *, void *, void *);
2741 typedef void (*microtask_t3)(int *, int *, void *, void *, void *);
2742 typedef void (*microtask_t4)(int *, int *, void *, void *, void *, void *);
2743 typedef void (*microtask_t5)(int *, int *, void *, void *, void *, void *,
2744 void *);
2745 typedef void (*microtask_t6)(int *, int *, void *, void *, void *, void *,
2746 void *, void *);
2747 typedef void (*microtask_t7)(int *, int *, void *, void *, void *, void *,
2748 void *, void *, void *);
2749 typedef void (*microtask_t8)(int *, int *, void *, void *, void *, void *,
2750 void *, void *, void *, void *);
2751 typedef void (*microtask_t9)(int *, int *, void *, void *, void *, void *,
2752 void *, void *, void *, void *, void *);
2753 typedef void (*microtask_t10)(int *, int *, void *, void *, void *, void *,
2754 void *, void *, void *, void *, void *, void *);
2755 typedef void (*microtask_t11)(int *, int *, void *, void *, void *, void *,
2756 void *, void *, void *, void *, void *, void *,
2757 void *);
2758 typedef void (*microtask_t12)(int *, int *, void *, void *, void *, void *,
2759 void *, void *, void *, void *, void *, void *,
2760 void *, void *);
2761 typedef void (*microtask_t13)(int *, int *, void *, void *, void *, void *,
2762 void *, void *, void *, void *, void *, void *,
2763 void *, void *, void *);
2764 typedef void (*microtask_t14)(int *, int *, void *, void *, void *, void *,
2765 void *, void *, void *, void *, void *, void *,
2766 void *, void *, void *, void *);
2767 typedef void (*microtask_t15)(int *, int *, void *, void *, void *, void *,
2768 void *, void *, void *, void *, void *, void *,
2769 void *, void *, void *, void *, void *);
2770
2771 // we really only need the case with 1 argument, because CLANG always build
2772 // a struct of pointers to shared variables referenced in the outlined function
__kmp_invoke_microtask(microtask_t pkfn,int gtid,int tid,int argc,void * p_argv[],void ** exit_frame_ptr)2773 int __kmp_invoke_microtask(microtask_t pkfn, int gtid, int tid, int argc,
2774 void *p_argv[]
2775 #if OMPT_SUPPORT
2776 ,
2777 void **exit_frame_ptr
2778 #endif
2779 ) {
2780 #if OMPT_SUPPORT
2781 *exit_frame_ptr = OMPT_GET_FRAME_ADDRESS(0);
2782 #endif
2783
2784 switch (argc) {
2785 default:
2786 fprintf(stderr, "Too many args to microtask: %d!\n", argc);
2787 fflush(stderr);
2788 exit(-1);
2789 case 0:
2790 (*(microtask_t0)pkfn)(>id, &tid);
2791 break;
2792 case 1:
2793 (*(microtask_t1)pkfn)(>id, &tid, p_argv[0]);
2794 break;
2795 case 2:
2796 (*(microtask_t2)pkfn)(>id, &tid, p_argv[0], p_argv[1]);
2797 break;
2798 case 3:
2799 (*(microtask_t3)pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2]);
2800 break;
2801 case 4:
2802 (*(microtask_t4)pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2],
2803 p_argv[3]);
2804 break;
2805 case 5:
2806 (*(microtask_t5)pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2],
2807 p_argv[3], p_argv[4]);
2808 break;
2809 case 6:
2810 (*(microtask_t6)pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2],
2811 p_argv[3], p_argv[4], p_argv[5]);
2812 break;
2813 case 7:
2814 (*(microtask_t7)pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2],
2815 p_argv[3], p_argv[4], p_argv[5], p_argv[6]);
2816 break;
2817 case 8:
2818 (*(microtask_t8)pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2],
2819 p_argv[3], p_argv[4], p_argv[5], p_argv[6],
2820 p_argv[7]);
2821 break;
2822 case 9:
2823 (*(microtask_t9)pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2],
2824 p_argv[3], p_argv[4], p_argv[5], p_argv[6], p_argv[7],
2825 p_argv[8]);
2826 break;
2827 case 10:
2828 (*(microtask_t10)pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2],
2829 p_argv[3], p_argv[4], p_argv[5], p_argv[6],
2830 p_argv[7], p_argv[8], p_argv[9]);
2831 break;
2832 case 11:
2833 (*(microtask_t11)pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2],
2834 p_argv[3], p_argv[4], p_argv[5], p_argv[6],
2835 p_argv[7], p_argv[8], p_argv[9], p_argv[10]);
2836 break;
2837 case 12:
2838 (*(microtask_t12)pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2],
2839 p_argv[3], p_argv[4], p_argv[5], p_argv[6],
2840 p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2841 p_argv[11]);
2842 break;
2843 case 13:
2844 (*(microtask_t13)pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2],
2845 p_argv[3], p_argv[4], p_argv[5], p_argv[6],
2846 p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2847 p_argv[11], p_argv[12]);
2848 break;
2849 case 14:
2850 (*(microtask_t14)pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2],
2851 p_argv[3], p_argv[4], p_argv[5], p_argv[6],
2852 p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2853 p_argv[11], p_argv[12], p_argv[13]);
2854 break;
2855 case 15:
2856 (*(microtask_t15)pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2],
2857 p_argv[3], p_argv[4], p_argv[5], p_argv[6],
2858 p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2859 p_argv[11], p_argv[12], p_argv[13], p_argv[14]);
2860 break;
2861 }
2862
2863 return 1;
2864 }
2865
2866 #endif
2867
2868 #if KMP_OS_LINUX
2869 // Functions for hidden helper task
2870 namespace {
2871 // Condition variable for initializing hidden helper team
2872 pthread_cond_t hidden_helper_threads_initz_cond_var;
2873 pthread_mutex_t hidden_helper_threads_initz_lock;
2874 volatile int hidden_helper_initz_signaled = FALSE;
2875
2876 // Condition variable for deinitializing hidden helper team
2877 pthread_cond_t hidden_helper_threads_deinitz_cond_var;
2878 pthread_mutex_t hidden_helper_threads_deinitz_lock;
2879 volatile int hidden_helper_deinitz_signaled = FALSE;
2880
2881 // Condition variable for the wrapper function of main thread
2882 pthread_cond_t hidden_helper_main_thread_cond_var;
2883 pthread_mutex_t hidden_helper_main_thread_lock;
2884 volatile int hidden_helper_main_thread_signaled = FALSE;
2885
2886 // Semaphore for worker threads. We don't use condition variable here in case
2887 // that when multiple signals are sent at the same time, only one thread might
2888 // be waken.
2889 sem_t hidden_helper_task_sem;
2890 } // namespace
2891
__kmp_hidden_helper_worker_thread_wait()2892 void __kmp_hidden_helper_worker_thread_wait() {
2893 int status = sem_wait(&hidden_helper_task_sem);
2894 KMP_CHECK_SYSFAIL("sem_wait", status);
2895 }
2896
__kmp_do_initialize_hidden_helper_threads()2897 void __kmp_do_initialize_hidden_helper_threads() {
2898 // Initialize condition variable
2899 int status =
2900 pthread_cond_init(&hidden_helper_threads_initz_cond_var, nullptr);
2901 KMP_CHECK_SYSFAIL("pthread_cond_init", status);
2902
2903 status = pthread_cond_init(&hidden_helper_threads_deinitz_cond_var, nullptr);
2904 KMP_CHECK_SYSFAIL("pthread_cond_init", status);
2905
2906 status = pthread_cond_init(&hidden_helper_main_thread_cond_var, nullptr);
2907 KMP_CHECK_SYSFAIL("pthread_cond_init", status);
2908
2909 status = pthread_mutex_init(&hidden_helper_threads_initz_lock, nullptr);
2910 KMP_CHECK_SYSFAIL("pthread_mutex_init", status);
2911
2912 status = pthread_mutex_init(&hidden_helper_threads_deinitz_lock, nullptr);
2913 KMP_CHECK_SYSFAIL("pthread_mutex_init", status);
2914
2915 status = pthread_mutex_init(&hidden_helper_main_thread_lock, nullptr);
2916 KMP_CHECK_SYSFAIL("pthread_mutex_init", status);
2917
2918 // Initialize the semaphore
2919 status = sem_init(&hidden_helper_task_sem, 0, 0);
2920 KMP_CHECK_SYSFAIL("sem_init", status);
2921
2922 // Create a new thread to finish initialization
2923 pthread_t handle;
2924 status = pthread_create(
2925 &handle, nullptr,
2926 [](void *) -> void * {
2927 __kmp_hidden_helper_threads_initz_routine();
2928 return nullptr;
2929 },
2930 nullptr);
2931 KMP_CHECK_SYSFAIL("pthread_create", status);
2932 }
2933
__kmp_hidden_helper_threads_initz_wait()2934 void __kmp_hidden_helper_threads_initz_wait() {
2935 // Initial thread waits here for the completion of the initialization. The
2936 // condition variable will be notified by main thread of hidden helper teams.
2937 int status = pthread_mutex_lock(&hidden_helper_threads_initz_lock);
2938 KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
2939
2940 if (!TCR_4(hidden_helper_initz_signaled)) {
2941 status = pthread_cond_wait(&hidden_helper_threads_initz_cond_var,
2942 &hidden_helper_threads_initz_lock);
2943 KMP_CHECK_SYSFAIL("pthread_cond_wait", status);
2944 }
2945
2946 status = pthread_mutex_unlock(&hidden_helper_threads_initz_lock);
2947 KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
2948 }
2949
__kmp_hidden_helper_initz_release()2950 void __kmp_hidden_helper_initz_release() {
2951 // After all initialization, reset __kmp_init_hidden_helper_threads to false.
2952 int status = pthread_mutex_lock(&hidden_helper_threads_initz_lock);
2953 KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
2954
2955 status = pthread_cond_signal(&hidden_helper_threads_initz_cond_var);
2956 KMP_CHECK_SYSFAIL("pthread_cond_wait", status);
2957
2958 TCW_SYNC_4(hidden_helper_initz_signaled, TRUE);
2959
2960 status = pthread_mutex_unlock(&hidden_helper_threads_initz_lock);
2961 KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
2962 }
2963
__kmp_hidden_helper_main_thread_wait()2964 void __kmp_hidden_helper_main_thread_wait() {
2965 // The main thread of hidden helper team will be blocked here. The
2966 // condition variable can only be signal in the destructor of RTL.
2967 int status = pthread_mutex_lock(&hidden_helper_main_thread_lock);
2968 KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
2969
2970 if (!TCR_4(hidden_helper_main_thread_signaled)) {
2971 status = pthread_cond_wait(&hidden_helper_main_thread_cond_var,
2972 &hidden_helper_main_thread_lock);
2973 KMP_CHECK_SYSFAIL("pthread_cond_wait", status);
2974 }
2975
2976 status = pthread_mutex_unlock(&hidden_helper_main_thread_lock);
2977 KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
2978 }
2979
__kmp_hidden_helper_main_thread_release()2980 void __kmp_hidden_helper_main_thread_release() {
2981 // The initial thread of OpenMP RTL should call this function to wake up the
2982 // main thread of hidden helper team.
2983 int status = pthread_mutex_lock(&hidden_helper_main_thread_lock);
2984 KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
2985
2986 status = pthread_cond_signal(&hidden_helper_main_thread_cond_var);
2987 KMP_CHECK_SYSFAIL("pthread_cond_signal", status);
2988
2989 // The hidden helper team is done here
2990 TCW_SYNC_4(hidden_helper_main_thread_signaled, TRUE);
2991
2992 status = pthread_mutex_unlock(&hidden_helper_main_thread_lock);
2993 KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
2994 }
2995
__kmp_hidden_helper_worker_thread_signal()2996 void __kmp_hidden_helper_worker_thread_signal() {
2997 int status = sem_post(&hidden_helper_task_sem);
2998 KMP_CHECK_SYSFAIL("sem_post", status);
2999 }
3000
__kmp_hidden_helper_threads_deinitz_wait()3001 void __kmp_hidden_helper_threads_deinitz_wait() {
3002 // Initial thread waits here for the completion of the deinitialization. The
3003 // condition variable will be notified by main thread of hidden helper teams.
3004 int status = pthread_mutex_lock(&hidden_helper_threads_deinitz_lock);
3005 KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
3006
3007 if (!TCR_4(hidden_helper_deinitz_signaled)) {
3008 status = pthread_cond_wait(&hidden_helper_threads_deinitz_cond_var,
3009 &hidden_helper_threads_deinitz_lock);
3010 KMP_CHECK_SYSFAIL("pthread_cond_wait", status);
3011 }
3012
3013 status = pthread_mutex_unlock(&hidden_helper_threads_deinitz_lock);
3014 KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
3015 }
3016
__kmp_hidden_helper_threads_deinitz_release()3017 void __kmp_hidden_helper_threads_deinitz_release() {
3018 int status = pthread_mutex_lock(&hidden_helper_threads_deinitz_lock);
3019 KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
3020
3021 status = pthread_cond_signal(&hidden_helper_threads_deinitz_cond_var);
3022 KMP_CHECK_SYSFAIL("pthread_cond_wait", status);
3023
3024 TCW_SYNC_4(hidden_helper_deinitz_signaled, TRUE);
3025
3026 status = pthread_mutex_unlock(&hidden_helper_threads_deinitz_lock);
3027 KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
3028 }
3029 #else // KMP_OS_LINUX
__kmp_hidden_helper_worker_thread_wait()3030 void __kmp_hidden_helper_worker_thread_wait() {
3031 KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
3032 }
3033
__kmp_do_initialize_hidden_helper_threads()3034 void __kmp_do_initialize_hidden_helper_threads() {
3035 KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
3036 }
3037
__kmp_hidden_helper_threads_initz_wait()3038 void __kmp_hidden_helper_threads_initz_wait() {
3039 KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
3040 }
3041
__kmp_hidden_helper_initz_release()3042 void __kmp_hidden_helper_initz_release() {
3043 KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
3044 }
3045
__kmp_hidden_helper_main_thread_wait()3046 void __kmp_hidden_helper_main_thread_wait() {
3047 KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
3048 }
3049
__kmp_hidden_helper_main_thread_release()3050 void __kmp_hidden_helper_main_thread_release() {
3051 KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
3052 }
3053
__kmp_hidden_helper_worker_thread_signal()3054 void __kmp_hidden_helper_worker_thread_signal() {
3055 KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
3056 }
3057
__kmp_hidden_helper_threads_deinitz_wait()3058 void __kmp_hidden_helper_threads_deinitz_wait() {
3059 KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
3060 }
3061
__kmp_hidden_helper_threads_deinitz_release()3062 void __kmp_hidden_helper_threads_deinitz_release() {
3063 KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
3064 }
3065 #endif // KMP_OS_LINUX
3066
__kmp_detect_shm()3067 bool __kmp_detect_shm() {
3068 DIR *dir = opendir("/dev/shm");
3069 if (dir) { // /dev/shm exists
3070 closedir(dir);
3071 return true;
3072 } else if (ENOENT == errno) { // /dev/shm does not exist
3073 return false;
3074 } else { // opendir() failed
3075 return false;
3076 }
3077 }
3078
__kmp_detect_tmp()3079 bool __kmp_detect_tmp() {
3080 DIR *dir = opendir("/tmp");
3081 if (dir) { // /tmp exists
3082 closedir(dir);
3083 return true;
3084 } else if (ENOENT == errno) { // /tmp does not exist
3085 return false;
3086 } else { // opendir() failed
3087 return false;
3088 }
3089 }
3090
3091 // end of file //
3092