3  * kmp.h -- KPTS runtime header file.
6 //===----------------------------------------------------------------------===//
10 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
12 //===----------------------------------------------------------------------===//
22    the Altix.  Requires user code to be linked with -lrt. */
43 #define TASK_STACK_INDEX_MASK (TASK_STACK_BLOCK_SIZE - 1)
66 // Some WASI targets (e.g., wasm32-wasi-threads) do not support thread
106 // OMPD_SKIP_HWLOC used in libompd/omp-icv.cpp to avoid OMPD depending on hwloc
151 #include "ompt-internal.h"
155 #include "ompd-specific.h"
165 // 0 - no fast memory allocation, alignment: 8-byte on x86, 16-byte on x64.
166 // 3 - fast allocation using sync, non-sync free lists of any size, non-self
195 #define KMP_NSEC_PER_USEC 1000L
208   /*! Use c-style ident structure */
211   /*! Entry point generated by auto-parallelization */
253 /*  but currently used for storing region-specific ITT */
258                        The string is composed of semi-colon separated fields
287 /* ------------------------------------------------------------------------ */
289 /* Pack two 32-bit signed integers into a 64-bit signed integer */
290 /* ToDo: Fix word ordering for big-endian machines. */
317 /* ------------------------------------------------------------------------ */
322 /* ------------------------------------------------------------------------ */
418       (kmp_sch_static_chunked - kmp_sch_lower + kmp_nm_lower),
454      These modifiers can be or-ed into non-static schedules by the compiler to
535 // OpenMP 3.1 - Nested num threads array
544 /* -- fast reduction stuff ------------------------------------------------ */
564 // that are packed together into 0-th byte and 1-st byte:
571 // so that the bits of 0-th byte are empty, so no need to execute a shift
607 /* -- end of fast reduction stuff ----------------------------------------- */
629   KMP_HW_UNKNOWN = -1,
712   for (i = (mask)->begin(); (int)i != (mask)->end(); i = (mask)->next(i))
713 #define KMP_CPU_SET(i, mask) (mask)->set(i)
714 #define KMP_CPU_ISSET(i, mask) (mask)->is_set(i)
715 #define KMP_CPU_CLR(i, mask) (mask)->clear(i)
716 #define KMP_CPU_ZERO(mask) (mask)->zero()
717 #define KMP_CPU_ISEMPTY(mask) (mask)->empty()
718 #define KMP_CPU_COPY(dest, src) (dest)->copy(src)
719 #define KMP_CPU_AND(dest, src) (dest)->bitwise_and(src)
720 #define KMP_CPU_COMPLEMENT(max_bit_number, mask) (mask)->bitwise_not()
721 #define KMP_CPU_UNION(dest, src) (dest)->bitwise_or(src)
722 #define KMP_CPU_EQUAL(dest, src) (dest)->is_equal(src)
723 #define KMP_CPU_ALLOC(ptr) (ptr = __kmp_affinity_dispatch->allocate_mask())
724 #define KMP_CPU_FREE(ptr) __kmp_affinity_dispatch->deallocate_mask(ptr)
729 #define KMP_CPU_INDEX(arr, i) __kmp_affinity_dispatch->index_mask_array(arr, i)
731   (arr = __kmp_affinity_dispatch->allocate_mask_array(n))
733   __kmp_affinity_dispatch->deallocate_mask_array(arr)
737   (mask)->get_system_affinity(abort_bool)
739   (mask)->set_system_affinity(abort_bool)
740 #define __kmp_get_proc_group(mask) (mask)->get_proc_group()
772     // for (int i = mask->begin(); i != mask->end(); i = mask->next(i))
777     virtual int set_process_affinity(bool abort_on_error) const { return -1; }  in set_process_affinity()
780     virtual int set_system_affinity(bool abort_on_error) const { return -1; }  in set_system_affinity()
782     virtual int get_system_affinity(bool abort_on_error) { return -1; }  in get_system_affinity()
784     // Return the appropriate index, or -1 for an invalid mask.
785     virtual int get_proc_group() const { return -1; }  in get_proc_group()
788       int max_cpu = -1;  in get_max_cpu()
942     nullptr, affinity_default, KMP_HW_UNKNOWN, -1, KMP_AFFINITY_ATTRS_UNKNOWN, \
995 #define KMP_PLACE_ALL (-1)
996 #define KMP_PLACE_UNDEFINED (-2)
1027 extern int __kmp_hws_abs_flag; // absolute or per-item number requested
1029 /* ------------------------------------------------------------------------ */
1032   (sizeof(type) + (sz - ((sizeof(type) - 1) % (sz)) - 1))
1034 // We need to avoid using -1 as a GTID as +1 is added to the gtid
1036 #define KMP_GTID_DNE (-2) /* Does not exist */
1037 #define KMP_GTID_SHUTDOWN (-3) /* Library is shutting down */
1038 #define KMP_GTID_MONITOR (-4) /* Monitor thread ID */
1039 #define KMP_GTID_UNKNOWN (-5) /* Is not known */
1040 #define KMP_GTID_MIN (-6) /* Minimal gtid for low bound check in DEBUG */
1080 #define omp_atv_default ((omp_uintptr_t)-1)
1162 /* ------------------------------------------------------------------------ */
1168 /* ------------------------------------------------------------------------ */
1171   (~((kmp_uint64)1 << ((sizeof(kmp_uint64) * (1 << 3)) - 1)))
1197 #define KMP_MAX_STKSIZE 0x10000000 /* 256Mb max size on 32-bit AIX */
1199 #define KMP_MAX_STKSIZE (~((size_t)1 << ((sizeof(size_t) * (1 << 3)) - 1)))
1221   (~((size_t)1 << ((sizeof(size_t) * (1 << 3)) - 1)))
1246 #define KMP_MAX_MONITOR_WAKEUPS (1000) // max times monitor can wake up per sec
1260   (((blocktime) + (KMP_BLOCKTIME_MULTIPLIER / (monitor_wakeups)) - 1) /        \
1283 #define KMP_BLOCKING(goal, count) ((count) % 1000 != 0 || (goal) > KMP_NOW())
1292 #define KMP_MAX_INTERVAL (INT_MAX - 1)
1296 #define KMP_MAX_CHUNK (INT_MAX - 1)
1339 #define KMP_INIT_WAIT 64U /* initial number of spin-tests   */
1340 #define KMP_NEXT_WAIT 32U /* susequent number of spin-tests */
1342 #define KMP_INIT_WAIT 1024U /* initial number of spin-tests   */
1343 #define KMP_NEXT_WAIT 512U /* susequent number of spin-tests */
1346 #define KMP_INIT_WAIT 1024U /* initial number of spin-tests   */
1347 #define KMP_NEXT_WAIT 512U /* susequent number of spin-tests */
1350 #define KMP_INIT_WAIT 1024U /* initial number of spin-tests   */
1351 #define KMP_NEXT_WAIT 512U /* susequent number of spin-tests */
1354 #define KMP_INIT_WAIT 1024U /* initial number of spin-tests   */
1355 #define KMP_NEXT_WAIT 512U /* susequent number of spin-tests */
1358 #define KMP_INIT_WAIT 1024U /* initial number of spin-tests   */
1359 #define KMP_NEXT_WAIT 512U /* susequent number of spin-tests */
1362 #define KMP_INIT_WAIT 1024U /* initial number of spin-tests   */
1363 #define KMP_NEXT_WAIT 512U /* susequent number of spin-tests */
1366 #define KMP_INIT_WAIT 1024U /* initial number of spin-tests   */
1367 #define KMP_NEXT_WAIT 512U /* susequent number of spin-tests */
1370 #define KMP_INIT_WAIT 1024U /* initial number of spin-tests   */
1371 #define KMP_NEXT_WAIT 512U /* susequent number of spin-tests */
1374 #define KMP_INIT_WAIT 1024U /* initial number of spin-tests   */
1375 #define KMP_NEXT_WAIT 512U /* susequent number of spin-tests */
1378 #define KMP_INIT_WAIT 1024U /* initial number of spin-tests   */
1379 #define KMP_NEXT_WAIT 512U /* susequent number of spin-tests */
1386   kmp_uint32 ecx;  member
1417                        : "=a"(p->eax), "=b"(p->ebx), "=c"(p->ecx), "=d"(p->edx)  in __kmp_x86_cpuid()
1430   // 32-bit protected mode x87 FPU state  in __kmp_clear_x87_fpu_status_word()
1442                        "andw $0x7f00, %1\n\t" // clear 0-7,15 bits of FP SW  in __kmp_clear_x87_fpu_status_word()
1468 // User-level Monitor/Mwait
1488                    //   in 32-bit mode: a, b, c, and d;  in __kmp_tpause()
1489                    //   in 64-bit mode: any integer register  in __kmp_tpause()
1518                    //   in 32-bit mode: a, b, c, and d;  in __kmp_umwait()
1519                    //   in 64-bit mode: any integer register  in __kmp_umwait()
1553 // Performance testing on KNC (C0QS-7120 P/A/X/D, 61-core, 16 GB Memory) showed
1611       (count) -= 2;                                                            \
1640         (count) -= 2;                                                          \
1655       (count) -= 2;                                                            \
1664 /* ------------------------------------------------------------------------ */
1666 /* ------------------------------------------------------------------------ */
1708 /* ---------------------------------------------------------------------- */
1709 /* ---------------------------------------------------------------------- */
1760 /* ---------------------------------------------------------------------------
1764 /*  for non-array objects:  __kmpc_threadprivate_register()  */
1862   ((((kmp_uintptr_t)x) >> KMP_HASH_SHIFT) & (KMP_HASH_TABLE_SIZE - 1))
1872 /* ------------------------------------------------------------------------ */
1910   //    b) all parm1-4 are on the same cache line.
1911   // Because of parm1-4 are used together, performance seems to be better
1936   kmp_int64 count; // current chunk number for static & static-steal scheduling
1937   kmp_int64 ub; /* upper-bound */
1939   kmp_int64 lb; /* lower-bound */
1946   /* parm[1-4] are used in different ways by different scheduling algorithms */
1950   //    b) all parm1-4 are in the same cache line.
1951   // Because of parm1-4 are used together, performance seems to be better
1997   kmp_int64 lb; /* lower-bound */
1998   kmp_int64 ub; /* upper-bound */
2002   /* parm[1-4] are used in different ways by different scheduling algorithms */
2038   /* chunk index under dynamic, number of idle threads under static-steal;
2044   kmp_int32 ordered_dummy[KMP_MAX_ORDERED - 1];
2048   /* chunk index under dynamic, number of idle threads under static-steal;
2054   kmp_int64 ordered_dummy[KMP_MAX_ORDERED - 3];
2096 /* ------------------------------------------------------------------------ */
2125   4 // Special state; worker resets appropriate flag on wake-up
2136   flag_oncore, /**< special 64-bit flag for on-core barrier (hierarchical) */
2141   bs_plain_barrier = 0, /* 0, All non-fork/join barriers (except reduction
2160                            bp_hyper_bar = 2, /* Hypercube-embedded tree with min
2183   int thread_limit; /* internal control for thread-limit-var */
2184   int task_thread_limit; /* internal control for thread-limit-var of a task*/
2202   // it to be padded -- instead, we fit b_go into the same cache line with
2240 /* Team barrier needs only non-volatile arrived counter */
2262    to write outside of the structure if compiled on pre-NPTL threads.  */
2283   /* A manual-reset event that's used to block and release waiting threads. */
2325    them to system-wide ids. GetThreadId() function is available only in
2326    Longhorn and Server 2003. :-( In contrast, GetCurrentThreadId() is available
2370   ((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.blocktime)
2372   ((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.bt_set)
2375   ((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.bt_intervals)
2379   ((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.dynamic)
2381   ((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.nproc)
2383   ((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.sched)
2386   (((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.blocktime) =     \
2391   (((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.bt_intervals) =  \
2396   (((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.bt_set) = (xval))
2399   (((xthread)->th.th_current_task->td_icvs.dynamic) = (xval))
2401   (((xthread)->th.th_current_task->td_icvs.dynamic) ? (FTN_TRUE) : (FTN_FALSE))
2404   (((xthread)->th.th_current_task->td_icvs.nproc) = (xval))
2407   (((xthread)->th.th_current_task->td_icvs.thread_limit) = (xval))
2410   (((xthread)->th.th_current_task->td_icvs.max_active_levels) = (xval))
2413   ((xthread)->th.th_current_task->td_icvs.max_active_levels)
2416   (((xthread)->th.th_current_task->td_icvs.sched) = (xval))
2419   (((xthread)->th.th_current_task->td_icvs.proc_bind) = (xval))
2421   ((xthread)->th.th_current_task->td_icvs.proc_bind)
2445 #define KMP_TASK_TO_TASKDATA(task) (((kmp_taskdata_t *)task) - 1)
2451   (TRUE == TCR_SYNC_4((task_team)->tt.tt_found_tasks))
2550   kmp_task_t *task; /* non-NULL if depnode is active, used under lock */
2740   unsigned native : 1; /* 1==gcc-compiled task, 0==intel */
2819   // GEH: shouldn't this be volatile since used in while-spin?
2831 #define TASK_DEQUE_MASK(td) ((td).td_deque_size - 1)
2848       tt_threads_lock; /* Lock used to allocate per-thread part of task team */
2857       *tt_threads_data; /* Array of per-thread structures for task team */
2891 // Free lists keep same-size free memory slots for fast memory allocation
2894   void *th_free_list_self; // Self-allocated tasks free list
2895   void *th_free_list_sync; // Self-allocated tasks stolen/returned by other
2897   void *th_free_list_other; // Non-self free list (to be returned to owner's
3019   volatile kmp_uint32 th_spin_here; /* thread-local location for spinning */
3029   /* Tasking-related data for the thread */
3033   kmp_uint32 th_reap_state; // Non-zero indicates thread is not
3120   (4 * CACHE_LINE -                                                            \
3126   (2 * CACHE_LINE - ((3 * KMP_PTR_SKIP + 2 * sizeof(int)) % CACHE_LINE))
3132   // ---------------------------------------------------------------------------
3138   // [0] - parallel / [1] - worksharing task reduction data shared by taskgroups
3143   // ---------------------------------------------------------------------------
3160   // --------------------------------------------------------------------------
3192   kmp_r_sched_t t_sched; // run-time schedule for the team
3198   int t_size_changed; // team size was changed?: 0: no, 1: yes, -1: changed via
3246   /* cache-aligned */
3249   /* non cache-aligned */
3264   // TODO: GEH - combine r_active with r_in_parallel then r_active ==
3266   // TODO: GEH - then replace r_active with t_active_levels if we can to reduce
3302 /* ------------------------------------------------------------------------ */
3408     __kmp_exit_lock; /* exit() is not always thread-safe */
3415                              __kmp_threads expansion to co-exist */
3456 /* ------------------------------------------------------------------------- */
3463 extern size_t __kmp_sys_min_stksize; /* system-defined minimum stack size */
3464 extern int __kmp_sys_max_nth; /* system-imposed maximum number of threads */
3465 // maximum total number of concurrently-existing threads on device
3467 // maximum total number of concurrently-existing threads in a contention group
3489     if (*bt > INT_MAX / 1000) {  in __kmp_aux_convert_blocktime()
3490       *bt = INT_MAX / 1000;  in __kmp_aux_convert_blocktime()
3493     *bt = *bt * 1000;  in __kmp_aux_convert_blocktime()
3516                                0 - not set, will be set at runtime
3517                                1 - using stack search
3518                                2 - dynamic TLS (pthread_getspecific(Linux* OS/OS
3520                                3 - static TLS (__declspec(thread) __kmp_gtid),
3530 extern int __kmp_inherit_fp_control; // copy fp creg(s) parent->workers at fork
3586 /* ------------------------------------------------------------------------- */
3607 /* ------------------------------------------------------------------------- */
3612 #define __kmp_get_team() (__kmp_threads[(__kmp_get_gtid())]->th.th_team)
3616 // AT: 1. nproc = __kmp_threads[ ( gtid ) ] -> th.th_team -> t.t_nproc;
3617 // AT: 2. nproc = __kmp_threads[ ( gtid ) ] -> th.th_team_nproc;
3619   (__kmp_threads[(gtid)]->th.th_team->t.t_nproc)
3625           __kmp_threads[gtid] == __kmp_root[gtid]->r.r_uber_thread);  in KMP_UBER_GTID()
3630   return __kmp_threads[gtid]->th.th_info.ds.ds_tid;  in __kmp_tid_from_gtid()
3635   return team->t.t_threads[tid]->th.th_info.ds.ds_gtid;  in __kmp_gtid_from_tid()
3640   return thr->th.th_info.ds.ds_gtid;  in __kmp_gtid_from_thread()
3650   return __kmp_threads[gtid]->th.th_team;  in __kmp_team_from_gtid()
3660 extern int __kmp_umwait_enabled; // Runtime check if user-level mwait enabled
3669 extern int __kmp_tpause_enabled; // 0 (default), 1 (KMP_TPAUSE is non-zero)
3672 /* ------------------------------------------------------------------------- */
3692 /* ------------------------------------------------------------------------ */
3945   kmp_root_t *r = __kmp_threads[gtid]->th.th_root;  in __kmp_assign_root_init_mask()
3946   if (r->r.r_uber_thread == __kmp_threads[gtid] && !r->r.r_affinity_assigned) {  in __kmp_assign_root_init_mask()
3949     r->r.r_affinity_assigned = TRUE;  in __kmp_assign_root_init_mask()
3956   kmp_root_t *r = th->th.th_root;  in __kmp_reset_root_init_mask()
3957   if (r->r.r_uber_thread == th && r->r.r_affinity_assigned) {  in __kmp_reset_root_init_mask()
3959     KMP_CPU_COPY(th->th.th_affin_mask, __kmp_affin_origMask);  in __kmp_reset_root_init_mask()
3960     r->r.r_affinity_assigned = FALSE;  in __kmp_reset_root_init_mask()
4047 /* ------------------------------------------------------------------------ */
4158       __kmp_tasking_mode != tskm_task_teams || team->t.t_nproc == 1 ||         \
4159       thr->th.th_task_team == team->t.t_task_team[thr->th.th_task_state])
4171 /* ------------------------------------------------------------------------ */
4184 /* ------------------------------------------------------------------------ */
4292 /* ------------------------------------------------------------------------ */
4545 #define KMP_DEVICE_DEFAULT -1 // This is libomptarget's default device.
4546 #define KMP_DEVICE_ALL -11 // This is libomptarget's "all devices".
4551 // as the internal equivalent of the externally-visible omp_pause_resource_t.
4554   kmp_soft_paused = 1, // status is soft-paused, or, requesting soft pause
4555   kmp_hard_paused = 2 // status is hard-paused, or, requesting hard pause
4613   (team->t.t_threads[0] == __kmp_hidden_helper_main_thread)
4618   ((gtid) % (__kmp_hidden_helper_threads_num - 1) + 2)
4626       gtid - __kmp_hidden_helper_threads_num >= 0) {  in __kmp_adjust_gtid_for_hidden_helpers()
4627     adjusted_gtid -= __kmp_hidden_helper_threads_num;  in __kmp_adjust_gtid_for_hidden_helpers()
4711 /// This class safely opens and closes a C-style FILE* object using RAII
4754   /// Instead of erroring out, return non-zero when