2  * kmp_runtime.cpp -- KPTS runtime support library
5 //===----------------------------------------------------------------------===//
9 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
11 //===----------------------------------------------------------------------===//
33 #include "ompt-specific.h"
36 #include "ompd-specific.h"
76 /* ------------------------------------------------------------------------ */
121   int new_size = level + thr->th.th_set_nested_nth_sz;  in __kmp_override_nested_nth()
122   new_nested_nth->nth = (int *)KMP_INTERNAL_MALLOC(new_size * sizeof(int));  in __kmp_override_nested_nth()
124     new_nested_nth->nth[i] = 0;  in __kmp_override_nested_nth()
126     new_nested_nth->nth[i] = thr->th.th_set_nested_nth[j];  in __kmp_override_nested_nth()
127   new_nested_nth->size = new_nested_nth->used = new_size;  in __kmp_override_nested_nth()
143       1000,  in __kmp_get_global_thread_id()
147   /* JPH - to handle the case where __kmpc_end(0) is called immediately prior to  in __kmp_get_global_thread_id()
149      by caller. Had to handle KMP_GTID_DNE at all call-sites, or else guarantee  in __kmp_get_global_thread_id()
157     KA_TRACE(1000, ("*** __kmp_get_global_thread_id: using TDATA\n"));  in __kmp_get_global_thread_id()
162     KA_TRACE(1000, ("*** __kmp_get_global_thread_id: using keyed TLS\n"));  in __kmp_get_global_thread_id()
165   KA_TRACE(1000, ("*** __kmp_get_global_thread_id: using internal alg.\n"));  in __kmp_get_global_thread_id()
173         non-NULL.  in __kmp_get_global_thread_id()
179      TODO: Fix it.  --ln  */  in __kmp_get_global_thread_id()
187     stack_size = (size_t)TCR_PTR(thr->th.th_info.ds.ds_stacksize);  in __kmp_get_global_thread_id()
188     stack_base = (char *)TCR_PTR(thr->th.th_info.ds.ds_stackbase);  in __kmp_get_global_thread_id()
190     /* stack grows down -- search through all of the active threads */  in __kmp_get_global_thread_id()
193       size_t stack_diff = stack_base - stack_addr;  in __kmp_get_global_thread_id()
201         // thread-specific data will be reset to NULL.  in __kmp_get_global_thread_id()
210   KA_TRACE(1000,  in __kmp_get_global_thread_id()
229   if (!TCR_4(other_threads[i]->th.th_info.ds.ds_stackgrow)) {  in __kmp_get_global_thread_id()
233   stack_base = (char *)other_threads[i]->th.th_info.ds.ds_stackbase;  in __kmp_get_global_thread_id()
235     TCW_PTR(other_threads[i]->th.th_info.ds.ds_stackbase, stack_addr);  in __kmp_get_global_thread_id()
236     TCW_PTR(other_threads[i]->th.th_info.ds.ds_stacksize,  in __kmp_get_global_thread_id()
237             other_threads[i]->th.th_info.ds.ds_stacksize + stack_addr -  in __kmp_get_global_thread_id()
240     TCW_PTR(other_threads[i]->th.th_info.ds.ds_stacksize,  in __kmp_get_global_thread_id()
241             stack_base - stack_addr);  in __kmp_get_global_thread_id()
246     char *stack_end = (char *)other_threads[i]->th.th_info.ds.ds_stackbase;  in __kmp_get_global_thread_id()
247     char *stack_beg = stack_end - other_threads[i]->th.th_info.ds.ds_stacksize;  in __kmp_get_global_thread_id()
249                                  other_threads[i]->th.th_info.ds.ds_stacksize,  in __kmp_get_global_thread_id()
263     KA_TRACE(1000, ("*** __kmp_get_global_thread_id_reg: using TDATA\n"));  in __kmp_get_global_thread_id_reg()
268     KA_TRACE(1000, ("*** __kmp_get_global_thread_id_reg: using keyed TLS\n"));  in __kmp_get_global_thread_id_reg()
271     KA_TRACE(1000,  in __kmp_get_global_thread_id_reg()
306     stack_end = (char *)th->th.th_info.ds.ds_stackbase;  in __kmp_check_stack_overlap()
307     stack_beg = stack_end - th->th.th_info.ds.ds_stacksize;  in __kmp_check_stack_overlap()
313           gtid, stack_beg, stack_end, th->th.th_info.ds.ds_stacksize,  in __kmp_check_stack_overlap()
315           (th->th.th_info.ds.ds_stackgrow) ? "initial" : "actual");  in __kmp_check_stack_overlap()
318           gtid, stack_beg, stack_end, th->th.th_info.ds.ds_stacksize,  in __kmp_check_stack_overlap()
320           (th->th.th_info.ds.ds_stackgrow) ? "initial" : "actual");  in __kmp_check_stack_overlap()
331       stack_end = (char *)th->th.th_info.ds.ds_stackbase;  in __kmp_check_stack_overlap()
332       stack_beg = stack_end - th->th.th_info.ds.ds_stacksize;  in __kmp_check_stack_overlap()
340             (char *)TCR_PTR(f_th->th.th_info.ds.ds_stackbase);  in __kmp_check_stack_overlap()
342             other_stack_end - (size_t)TCR_PTR(f_th->th.th_info.ds.ds_stacksize);  in __kmp_check_stack_overlap()
349                 -1, other_stack_beg, other_stack_end,  in __kmp_check_stack_overlap()
350                 (size_t)TCR_PTR(f_th->th.th_info.ds.ds_stacksize),  in __kmp_check_stack_overlap()
362 /* ------------------------------------------------------------------------ */
387     if (p1 <= p2 && (char *)p2 - (char *)p1 == size) {  in __kmp_print_storage_map_gtid()
399           p1 = (void *)((size_t)p1 & ~((size_t)page_size - 1));  in __kmp_print_storage_map_gtid()
400           p2 = (void *)(((size_t)p2 - 1) & ~((size_t)page_size - 1));  in __kmp_print_storage_map_gtid()
416             __kmp_printf_no_lock("    %p-%p memNode %d\n", last, (char *)p1 - 1,  in __kmp_print_storage_map_gtid()
420           __kmp_printf_no_lock("    %p-%p memNode %d\n", p1,  in __kmp_print_storage_map_gtid()
421                                (char *)p1 + (page_size - 1),  in __kmp_print_storage_map_gtid()
424             __kmp_printf_no_lock("    %p-%p memNode %d\n", p2,  in __kmp_print_storage_map_gtid()
425                                  (char *)p2 + (page_size - 1),  in __kmp_print_storage_map_gtid()
471   /* On Windows* OS by default abort() causes pop-up error box, which stalls  in __kmp_abort_process()
472      nightly testing. Unfortunately, we cannot reliably suppress pop-up error  in __kmp_abort_process()
479      avoid pop-up error box. */  in __kmp_abort_process()
505   __kmp_print_storage_map_gtid(gtid, &thr->th.th_info, &thr->th.th_team,  in __kmp_print_thread_storage_map()
508   __kmp_print_storage_map_gtid(gtid, &thr->th.th_local, &thr->th.th_pri_head,  in __kmp_print_thread_storage_map()
512       gtid, &thr->th.th_bar[0], &thr->th.th_bar[bs_last_barrier],  in __kmp_print_thread_storage_map()
515   __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_plain_barrier],  in __kmp_print_thread_storage_map()
516                                &thr->th.th_bar[bs_plain_barrier + 1],  in __kmp_print_thread_storage_map()
520   __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_forkjoin_barrier],  in __kmp_print_thread_storage_map()
521                                &thr->th.th_bar[bs_forkjoin_barrier + 1],  in __kmp_print_thread_storage_map()
526   __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_reduction_barrier],  in __kmp_print_thread_storage_map()
527                                &thr->th.th_bar[bs_reduction_barrier + 1],  in __kmp_print_thread_storage_map()
538   int num_disp_buff = team->t.t_max_nproc > 1 ? __kmp_dispatch_num_buffers : 2;  in __kmp_print_team_storage_map()
539   __kmp_print_storage_map_gtid(-1, team, team + 1, sizeof(kmp_team_t), "%s_%d",  in __kmp_print_team_storage_map()
542   __kmp_print_storage_map_gtid(-1, &team->t.t_bar[0],  in __kmp_print_team_storage_map()
543                                &team->t.t_bar[bs_last_barrier],  in __kmp_print_team_storage_map()
547   __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_plain_barrier],  in __kmp_print_team_storage_map()
548                                &team->t.t_bar[bs_plain_barrier + 1],  in __kmp_print_team_storage_map()
552   __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_forkjoin_barrier],  in __kmp_print_team_storage_map()
553                                &team->t.t_bar[bs_forkjoin_barrier + 1],  in __kmp_print_team_storage_map()
558   __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_reduction_barrier],  in __kmp_print_team_storage_map()
559                                &team->t.t_bar[bs_reduction_barrier + 1],  in __kmp_print_team_storage_map()
565       -1, &team->t.t_dispatch[0], &team->t.t_dispatch[num_thr],  in __kmp_print_team_storage_map()
569       -1, &team->t.t_threads[0], &team->t.t_threads[num_thr],  in __kmp_print_team_storage_map()
572   __kmp_print_storage_map_gtid(-1, &team->t.t_disp_buffer[0],  in __kmp_print_team_storage_map()
573                                &team->t.t_disp_buffer[num_disp_buff],  in __kmp_print_team_storage_map()
584 /* ------------------------------------------------------------------------ */
592 /* ------------------------------------------------------------------------ */
646 /* __kmp_parallel_deo -- Wait until it's our turn. */
654     if (__kmp_threads[gtid]->th.th_root->r.r_active)  in __kmp_parallel_deo()
662   if (!team->t.t_serialized) {  in __kmp_parallel_deo()
664     KMP_WAIT(&team->t.t_ordered.dt.t_value, __kmp_tid_from_gtid(gtid), KMP_EQ,  in __kmp_parallel_deo()
671 /* __kmp_parallel_dxo -- Signal the next task. */
680     if (__kmp_threads[gtid]->th.th_root->r.r_active)  in __kmp_parallel_dxo()
684   if (!team->t.t_serialized) {  in __kmp_parallel_dxo()
689     team->t.t_ordered.dt.t_value = ((tid + 1) % team->t.t_nproc);  in __kmp_parallel_dxo()
696 /* ------------------------------------------------------------------------ */
709   team = th->th.th_team;  in __kmp_enter_single()
712   th->th.th_ident = id_ref;  in __kmp_enter_single()
714   if (team->t.t_serialized) {  in __kmp_enter_single()
717     kmp_int32 old_this = th->th.th_local.this_construct;  in __kmp_enter_single()
719     ++th->th.th_local.this_construct;  in __kmp_enter_single()
720     /* try to set team count to thread count--success means thread got the  in __kmp_enter_single()
723     if (team->t.t_construct == old_this) {  in __kmp_enter_single()
724       status = __kmp_atomic_compare_store_acq(&team->t.t_construct, old_this,  in __kmp_enter_single()
725                                               th->th.th_local.this_construct);  in __kmp_enter_single()
729         KMP_MASTER_GTID(gtid) && th->th.th_teams_microtask == NULL &&  in __kmp_enter_single()
730         team->t.t_active_level == 1) {  in __kmp_enter_single()
773   kmp_info_t *this_thr = parent_team->t.t_threads[master_tid];  in __kmp_reserve_threads()
775   // If dyn-var is set, dynamically adjust the number of desired threads,  in __kmp_reserve_threads()
798     new_nthreads = __kmp_avail_proc - __kmp_nth +  in __kmp_reserve_threads()
799                    (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc);  in __kmp_reserve_threads()
815       new_nthreads = __kmp_get_random(parent_team->t.t_threads[master_tid]);  in __kmp_reserve_threads()
834   if (__kmp_nth + new_nthreads -  in __kmp_reserve_threads()
835           (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) >  in __kmp_reserve_threads()
837     int tl_nthreads = __kmp_max_nth - __kmp_nth +  in __kmp_reserve_threads()
838                       (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc);  in __kmp_reserve_threads()
843     // If dyn-var is false, emit a 1-time warning.  in __kmp_reserve_threads()
863   int cg_nthreads = this_thr->th.th_cg_roots->cg_nthreads;  in __kmp_reserve_threads()
864   int max_cg_threads = this_thr->th.th_cg_roots->cg_thread_limit;  in __kmp_reserve_threads()
865   if (cg_nthreads + new_nthreads -  in __kmp_reserve_threads()
866           (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) >  in __kmp_reserve_threads()
868     int tl_nthreads = max_cg_threads - cg_nthreads +  in __kmp_reserve_threads()
869                       (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc);  in __kmp_reserve_threads()
874     // If dyn-var is false, emit a 1-time warning.  in __kmp_reserve_threads()
898     --capacity;  in __kmp_reserve_threads()
904     capacity -= __kmp_hidden_helper_threads_num;  in __kmp_reserve_threads()
906   if (__kmp_nth + new_nthreads -  in __kmp_reserve_threads()
907           (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) >  in __kmp_reserve_threads()
910     int slotsRequired = __kmp_nth + new_nthreads -  in __kmp_reserve_threads()
911                         (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) -  in __kmp_reserve_threads()
916       new_nthreads -= (slotsRequired - slotsAdded);  in __kmp_reserve_threads()
919       // If dyn-var is false, emit a 1-time warning.  in __kmp_reserve_threads()
949   if (this_thr->th.th_nt_strict && new_nthreads < set_nthreads) {  in __kmp_reserve_threads()
950     __kmpc_error(this_thr->th.th_nt_loc, this_thr->th.th_nt_sev,  in __kmp_reserve_threads()
951                  this_thr->th.th_nt_msg);  in __kmp_reserve_threads()
965   KA_TRACE(10, ("__kmp_fork_team_threads: new_nprocs = %d\n", team->t.t_nproc));  in __kmp_fork_team_threads()
970   master_th->th.th_info.ds.ds_tid = 0;  in __kmp_fork_team_threads()
971   master_th->th.th_team = team;  in __kmp_fork_team_threads()
972   master_th->th.th_team_nproc = team->t.t_nproc;  in __kmp_fork_team_threads()
973   master_th->th.th_team_master = master_th;  in __kmp_fork_team_threads()
974   master_th->th.th_team_serialized = FALSE;  in __kmp_fork_team_threads()
975   master_th->th.th_dispatch = &team->t.t_dispatch[0];  in __kmp_fork_team_threads()
980   kmp_hot_team_ptr_t *hot_teams = master_th->th.th_hot_teams;  in __kmp_fork_team_threads()
983     int level = team->t.t_active_level - 1; // index in array of hot teams  in __kmp_fork_team_threads()
984     if (master_th->th.th_teams_microtask) { // are we inside the teams?  in __kmp_fork_team_threads()
985       if (master_th->th.th_teams_size.nteams > 1) {  in __kmp_fork_team_threads()
989       if (team->t.t_pkfn != (microtask_t)__kmp_teams_master &&  in __kmp_fork_team_threads()
990           master_th->th.th_teams_level == team->t.t_level) {  in __kmp_fork_team_threads()
993       } // team->t.t_level will be increased inside parallel  in __kmp_fork_team_threads()
1003         hot_teams[level].hot_team_nth = team->t.t_nproc;  in __kmp_fork_team_threads()
1010   use_hot_team = team == root->r.r_hot_team;  in __kmp_fork_team_threads()
1015     team->t.t_threads[0] = master_th;  in __kmp_fork_team_threads()
1019     for (i = 1; i < team->t.t_nproc; i++) {  in __kmp_fork_team_threads()
1023       team->t.t_threads[i] = thr;  in __kmp_fork_team_threads()
1025       KMP_DEBUG_ASSERT(thr->th.th_team == team);  in __kmp_fork_team_threads()
1029                     __kmp_gtid_from_tid(0, team), team->t.t_id, 0,  in __kmp_fork_team_threads()
1030                     __kmp_gtid_from_tid(i, team), team->t.t_id, i,  in __kmp_fork_team_threads()
1031                     team->t.t_bar[bs_forkjoin_barrier].b_arrived,  in __kmp_fork_team_threads()
1032                     team->t.t_bar[bs_plain_barrier].b_arrived));  in __kmp_fork_team_threads()
1033       thr->th.th_teams_microtask = master_th->th.th_teams_microtask;  in __kmp_fork_team_threads()
1034       thr->th.th_teams_level = master_th->th.th_teams_level;  in __kmp_fork_team_threads()
1035       thr->th.th_teams_size = master_th->th.th_teams_size;  in __kmp_fork_team_threads()
1038         kmp_balign_t *balign = team->t.t_threads[i]->th.th_bar;  in __kmp_fork_team_threads()
1040           balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;  in __kmp_fork_team_threads()
1043           balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;  in __kmp_fork_team_threads()
1058     if (team->t.t_nproc > 1 &&  in __kmp_fork_team_threads()
1060       team->t.b->update_num_threads(team->t.t_nproc);  in __kmp_fork_team_threads()
1061       __kmp_add_threads_to_team(team, team->t.t_nproc);  in __kmp_fork_team_threads()
1068       KMP_DEBUG_ASSERT_TASKTEAM_INVARIANT(team->t.t_parent, master_th);  in __kmp_fork_team_threads()
1073            __kmp_gtid_from_thread(master_th), master_th->th.th_task_team,  in __kmp_fork_team_threads()
1074            team->t.t_parent, team->t.t_task_team[master_th->th.th_task_state],  in __kmp_fork_team_threads()
1078       KMP_CHECK_UPDATE(team->t.t_primary_task_state,  in __kmp_fork_team_threads()
1079                        master_th->th.th_task_state);  in __kmp_fork_team_threads()
1083       if (team->t.t_nproc > 1) {  in __kmp_fork_team_threads()
1084         KMP_DEBUG_ASSERT(team->t.t_threads[1]->th.th_task_state == 0 ||  in __kmp_fork_team_threads()
1085                          team->t.t_threads[1]->th.th_task_state == 1);  in __kmp_fork_team_threads()
1086         KMP_CHECK_UPDATE(master_th->th.th_task_state,  in __kmp_fork_team_threads()
1087                          team->t.t_threads[1]->th.th_task_state);  in __kmp_fork_team_threads()
1089         master_th->th.th_task_state = 0;  in __kmp_fork_team_threads()
1093       KMP_CHECK_UPDATE(team->t.t_primary_task_state,  in __kmp_fork_team_threads()
1094                        master_th->th.th_task_state);  in __kmp_fork_team_threads()
1096       master_th->th.th_task_state = 0;  in __kmp_fork_team_threads()
1100   if (__kmp_display_affinity && team->t.t_display_affinity != 1) {  in __kmp_fork_team_threads()
1101     for (i = 0; i < team->t.t_nproc; i++) {  in __kmp_fork_team_threads()
1102       kmp_info_t *thr = team->t.t_threads[i];  in __kmp_fork_team_threads()
1103       if (thr->th.th_prev_num_threads != team->t.t_nproc ||  in __kmp_fork_team_threads()
1104           thr->th.th_prev_level != team->t.t_level) {  in __kmp_fork_team_threads()
1105         team->t.t_display_affinity = 1;  in __kmp_fork_team_threads()
1135     // writes that would put the cache-line into a written state, causing all  in propagateFPControl()
1137     KMP_CHECK_UPDATE(team->t.t_x87_fpu_control_word, x87_fpu_control_word);  in propagateFPControl()
1138     KMP_CHECK_UPDATE(team->t.t_mxcsr, mxcsr);  in propagateFPControl()
1141     KMP_CHECK_UPDATE(team->t.t_fp_control_saved, TRUE);  in propagateFPControl()
1143     // Similarly here. Don't write to this cache-line in the team structure  in propagateFPControl()
1145     KMP_CHECK_UPDATE(team->t.t_fp_control_saved, FALSE);  in propagateFPControl()
1152   if (__kmp_inherit_fp_control && team->t.t_fp_control_saved) {  in updateHWFPControl()
1161     if (team->t.t_x87_fpu_control_word != x87_fpu_control_word) {  in updateHWFPControl()
1163       __kmp_load_x87_fpu_control_word(&team->t.t_x87_fpu_control_word);  in updateHWFPControl()
1166     if (team->t.t_mxcsr != mxcsr) {  in updateHWFPControl()
1167       __kmp_load_mxcsr(&team->t.t_mxcsr);  in updateHWFPControl()
1189   if (loc != NULL && (loc->flags & KMP_IDENT_AUTOPAR))  in __kmp_serialized_parallel()
1197   serial_team = this_thr->th.th_serial_team;  in __kmp_serialized_parallel()
1203   kmp_proc_bind_t proc_bind = this_thr->th.th_set_proc_bind;  in __kmp_serialized_parallel()
1204   if (this_thr->th.th_current_task->td_icvs.proc_bind == proc_bind_false) {  in __kmp_serialized_parallel()
1208     // of proc-bind-var for this parallel region.  in __kmp_serialized_parallel()
1209     proc_bind = this_thr->th.th_current_task->td_icvs.proc_bind;  in __kmp_serialized_parallel()
1212   this_thr->th.th_set_proc_bind = proc_bind_default;  in __kmp_serialized_parallel()
1215   this_thr->th.th_set_nproc = 0;  in __kmp_serialized_parallel()
1221       this_thr->th.ompt_thread_info.state != ompt_state_overhead) {  in __kmp_serialized_parallel()
1226     parent_task_info->frame.enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);  in __kmp_serialized_parallel()
1231           &(parent_task_info->task_data), &(parent_task_info->frame),  in __kmp_serialized_parallel()
1238   if (this_thr->th.th_team != serial_team) {  in __kmp_serialized_parallel()
1240     int level = this_thr->th.th_team->t.t_level;  in __kmp_serialized_parallel()
1242     if (serial_team->t.t_serialized) {  in __kmp_serialized_parallel()
1250           __kmp_allocate_team(this_thr->th.th_root, 1, 1,  in __kmp_serialized_parallel()
1254                               proc_bind, &this_thr->th.th_current_task->td_icvs,  in __kmp_serialized_parallel()
1260       new_team->t.t_threads[0] = this_thr;  in __kmp_serialized_parallel()
1261       new_team->t.t_parent = this_thr->th.th_team;  in __kmp_serialized_parallel()
1263       this_thr->th.th_serial_team = serial_team;  in __kmp_serialized_parallel()
1281     KMP_DEBUG_ASSERT(serial_team->t.t_threads);  in __kmp_serialized_parallel()
1282     KMP_DEBUG_ASSERT(serial_team->t.t_threads[0] == this_thr);  in __kmp_serialized_parallel()
1283     KMP_DEBUG_ASSERT(this_thr->th.th_team != serial_team);  in __kmp_serialized_parallel()
1284     serial_team->t.t_ident = loc;  in __kmp_serialized_parallel()
1285     serial_team->t.t_serialized = 1;  in __kmp_serialized_parallel()
1286     serial_team->t.t_nproc = 1;  in __kmp_serialized_parallel()
1287     serial_team->t.t_parent = this_thr->th.th_team;  in __kmp_serialized_parallel()
1288     if (this_thr->th.th_team->t.t_nested_nth)  in __kmp_serialized_parallel()
1289       serial_team->t.t_nested_nth = this_thr->th.th_team->t.t_nested_nth;  in __kmp_serialized_parallel()
1291       serial_team->t.t_nested_nth = &__kmp_nested_nth;  in __kmp_serialized_parallel()
1293     serial_team->t.t_primary_task_state = this_thr->th.th_task_state;  in __kmp_serialized_parallel()
1294     serial_team->t.t_sched.sched = this_thr->th.th_team->t.t_sched.sched;  in __kmp_serialized_parallel()
1295     this_thr->th.th_team = serial_team;  in __kmp_serialized_parallel()
1296     serial_team->t.t_master_tid = this_thr->th.th_info.ds.ds_tid;  in __kmp_serialized_parallel()
1299                   this_thr->th.th_current_task));  in __kmp_serialized_parallel()
1300     KMP_ASSERT(this_thr->th.th_current_task->td_flags.executing == 1);  in __kmp_serialized_parallel()
1301     this_thr->th.th_current_task->td_flags.executing = 0;  in __kmp_serialized_parallel()
1307        team->t.t_serialized? */  in __kmp_serialized_parallel()
1308     copy_icvs(&this_thr->th.th_current_task->td_icvs,  in __kmp_serialized_parallel()
1309               &this_thr->th.th_current_task->td_parent->td_icvs);  in __kmp_serialized_parallel()
1314     if (this_thr->th.th_team->t.t_nested_nth)  in __kmp_serialized_parallel()
1315       nested_nth = this_thr->th.th_team->t.t_nested_nth;  in __kmp_serialized_parallel()
1316     if (nested_nth->used && (level + 1 < nested_nth->used)) {  in __kmp_serialized_parallel()
1317       this_thr->th.th_current_task->td_icvs.nproc = nested_nth->nth[level + 1];  in __kmp_serialized_parallel()
1322       this_thr->th.th_current_task->td_icvs.proc_bind =  in __kmp_serialized_parallel()
1327     serial_team->t.t_pkfn = (microtask_t)(~0); // For the debugger.  in __kmp_serialized_parallel()
1329     this_thr->th.th_info.ds.ds_tid = 0;  in __kmp_serialized_parallel()
1332     this_thr->th.th_team_nproc = 1;  in __kmp_serialized_parallel()
1333     this_thr->th.th_team_master = this_thr;  in __kmp_serialized_parallel()
1334     this_thr->th.th_team_serialized = 1;  in __kmp_serialized_parallel()
1335     this_thr->th.th_task_team = NULL;  in __kmp_serialized_parallel()
1336     this_thr->th.th_task_state = 0;  in __kmp_serialized_parallel()
1338     serial_team->t.t_level = serial_team->t.t_parent->t.t_level + 1;  in __kmp_serialized_parallel()
1339     serial_team->t.t_active_level = serial_team->t.t_parent->t.t_active_level;  in __kmp_serialized_parallel()
1340     serial_team->t.t_def_allocator = this_thr->th.th_def_allocator; // save  in __kmp_serialized_parallel()
1345     KMP_DEBUG_ASSERT(serial_team->t.t_dispatch);  in __kmp_serialized_parallel()
1346     if (!serial_team->t.t_dispatch->th_disp_buffer) {  in __kmp_serialized_parallel()
1347       serial_team->t.t_dispatch->th_disp_buffer =  in __kmp_serialized_parallel()
1351     this_thr->th.th_dispatch = serial_team->t.t_dispatch;  in __kmp_serialized_parallel()
1358     KMP_DEBUG_ASSERT(this_thr->th.th_team == serial_team);  in __kmp_serialized_parallel()
1359     KMP_DEBUG_ASSERT(serial_team->t.t_threads);  in __kmp_serialized_parallel()
1360     KMP_DEBUG_ASSERT(serial_team->t.t_threads[0] == this_thr);  in __kmp_serialized_parallel()
1361     ++serial_team->t.t_serialized;  in __kmp_serialized_parallel()
1362     this_thr->th.th_team_serialized = serial_team->t.t_serialized;  in __kmp_serialized_parallel()
1365     int level = this_thr->th.th_team->t.t_level;  in __kmp_serialized_parallel()
1370     if (serial_team->t.t_nested_nth)  in __kmp_serialized_parallel()
1371       nested_nth = serial_team->t.t_nested_nth;  in __kmp_serialized_parallel()
1372     if (nested_nth->used && (level + 1 < nested_nth->used)) {  in __kmp_serialized_parallel()
1373       this_thr->th.th_current_task->td_icvs.nproc = nested_nth->nth[level + 1];  in __kmp_serialized_parallel()
1376     serial_team->t.t_level++;  in __kmp_serialized_parallel()
1379                   global_tid, serial_team, serial_team->t.t_level));  in __kmp_serialized_parallel()
1382     KMP_DEBUG_ASSERT(serial_team->t.t_dispatch);  in __kmp_serialized_parallel()
1387       disp_buffer->next = serial_team->t.t_dispatch->th_disp_buffer;  in __kmp_serialized_parallel()
1388       serial_team->t.t_dispatch->th_disp_buffer = disp_buffer;  in __kmp_serialized_parallel()
1390     this_thr->th.th_dispatch = serial_team->t.t_dispatch;  in __kmp_serialized_parallel()
1397   KMP_CHECK_UPDATE(serial_team->t.t_cancel_request, cancel_noreq);  in __kmp_serialized_parallel()
1402     if (this_thr->th.th_prev_level != serial_team->t.t_level ||  in __kmp_serialized_parallel()
1403         this_thr->th.th_prev_num_threads != 1) {  in __kmp_serialized_parallel()
1404       // NULL means use the affinity-format-var ICV  in __kmp_serialized_parallel()
1406       this_thr->th.th_prev_level = serial_team->t.t_level;  in __kmp_serialized_parallel()
1407       this_thr->th.th_prev_num_threads = 1;  in __kmp_serialized_parallel()
1414   serial_team->t.ompt_team_info.master_return_address = codeptr;  in __kmp_serialized_parallel()
1416       this_thr->th.ompt_thread_info.state != ompt_state_overhead) {  in __kmp_serialized_parallel()
1417     OMPT_CUR_TASK_INFO(this_thr)->frame.exit_frame.ptr =  in __kmp_serialized_parallel()
1433       OMPT_CUR_TASK_INFO(this_thr)->thread_num =  in __kmp_serialized_parallel()
1438     this_thr->th.ompt_thread_info.state = ompt_state_work_parallel;  in __kmp_serialized_parallel()
1439     OMPT_CUR_TASK_INFO(this_thr)->frame.exit_frame.ptr =  in __kmp_serialized_parallel()
1449   return (master_th->th.th_teams_microtask && ap &&  in __kmp_is_fork_in_teams()
1476   parent_team->t.t_ident = loc;  in __kmp_fork_in_teams()
1478   parent_team->t.t_argc = argc;  in __kmp_fork_in_teams()
1479   argv = (void **)parent_team->t.t_argv;  in __kmp_fork_in_teams()
1480   for (i = argc - 1; i >= 0; --i) {  in __kmp_fork_in_teams()
1484   if (parent_team == master_th->th.th_serial_team) {  in __kmp_fork_in_teams()
1487     KMP_DEBUG_ASSERT(parent_team->t.t_serialized > 1);  in __kmp_fork_in_teams()
1492       parent_team->t.t_serialized--;  in __kmp_fork_in_teams()
1497     parent_team->t.t_pkfn = microtask;  in __kmp_fork_in_teams()
1517         OMPT_CUR_TASK_INFO(master_th)->thread_num = __kmp_tid_from_gtid(gtid);  in __kmp_fork_in_teams()
1520             1, OMPT_CUR_TASK_INFO(master_th)->thread_num, ompt_task_implicit);  in __kmp_fork_in_teams()
1524       master_th->th.ompt_thread_info.state = ompt_state_work_parallel;  in __kmp_fork_in_teams()
1532     parent_team->t.t_serialized--;  in __kmp_fork_in_teams()
1537       __kmp_invoke_microtask(microtask, gtid, 0, argc, parent_team->t.t_argv  in __kmp_fork_in_teams()
1548       OMPT_CUR_TASK_INFO(master_th)->frame.exit_frame = ompt_data_none;  in __kmp_fork_in_teams()
1552             OMPT_CUR_TASK_INFO(master_th)->thread_num, ompt_task_implicit);  in __kmp_fork_in_teams()
1561       master_th->th.ompt_thread_info.state = ompt_state_overhead;  in __kmp_fork_in_teams()
1567   parent_team->t.t_pkfn = microtask;  in __kmp_fork_in_teams()
1568   parent_team->t.t_invoke = invoker;  in __kmp_fork_in_teams()
1569   KMP_ATOMIC_INC(&root->r.r_in_parallel);  in __kmp_fork_in_teams()
1570   parent_team->t.t_active_level++;  in __kmp_fork_in_teams()
1571   parent_team->t.t_level++;  in __kmp_fork_in_teams()
1572   parent_team->t.t_def_allocator = master_th->th.th_def_allocator; // save  in __kmp_fork_in_teams()
1579   master_th->th.th_teams_size.nth = parent_team->t.t_nproc;  in __kmp_fork_in_teams()
1592     if (master_set_numthreads <= master_th->th.th_teams_size.nth) {  in __kmp_fork_in_teams()
1594       kmp_info_t **other_threads = parent_team->t.t_threads;  in __kmp_fork_in_teams()
1597       int old_proc = master_th->th.th_teams_size.nth;  in __kmp_fork_in_teams()
1602       parent_team->t.t_nproc = master_set_numthreads;  in __kmp_fork_in_teams()
1604         other_threads[i]->th.th_team_nproc = master_set_numthreads;  in __kmp_fork_in_teams()
1608     master_th->th.th_set_nproc = 0;  in __kmp_fork_in_teams()
1621   kmp_proc_bind_t proc_bind = master_th->th.th_set_proc_bind;  in __kmp_fork_in_teams()
1624   if (master_th->th.th_current_task->td_icvs.proc_bind == proc_bind_false) {  in __kmp_fork_in_teams()
1627     // No proc_bind clause specified; use current proc-bind-var  in __kmp_fork_in_teams()
1629       proc_bind = master_th->th.th_current_task->td_icvs.proc_bind;  in __kmp_fork_in_teams()
1632        This overrides proc-bind-var for this parallel region, but does not  in __kmp_fork_in_teams()
1633        change proc-bind-var. */  in __kmp_fork_in_teams()
1634     // Figure the value of proc-bind-var for the child threads.  in __kmp_fork_in_teams()
1637          master_th->th.th_current_task->td_icvs.proc_bind)) {  in __kmp_fork_in_teams()
1641   KMP_CHECK_UPDATE(parent_team->t.t_proc_bind, proc_bind);  in __kmp_fork_in_teams()
1642   // Need to change the bind-var ICV to correct value for each implicit task  in __kmp_fork_in_teams()
1644       master_th->th.th_current_task->td_icvs.proc_bind != proc_bind_icv) {  in __kmp_fork_in_teams()
1645     kmp_info_t **other_threads = parent_team->t.t_threads;  in __kmp_fork_in_teams()
1646     for (i = 0; i < master_th->th.th_team_nproc; ++i) {  in __kmp_fork_in_teams()
1647       other_threads[i]->th.th_current_task->td_icvs.proc_bind = proc_bind_icv;  in __kmp_fork_in_teams()
1651   master_th->th.th_set_proc_bind = proc_bind_default;  in __kmp_fork_in_teams()
1657       parent_team->t.t_active_level == 1 // only report frames at level 1  in __kmp_fork_in_teams()
1658       && master_th->th.th_teams_size.nteams == 1) {  in __kmp_fork_in_teams()
1660     master_th->th.th_frame_time = tmp_time;  in __kmp_fork_in_teams()
1661     parent_team->t.t_region_time = tmp_time;  in __kmp_fork_in_teams()
1664     KMP_DEBUG_ASSERT(parent_team->t.t_stack_id == NULL);  in __kmp_fork_in_teams()
1666     parent_team->t.t_stack_id = __kmp_itt_stack_caller_create();  in __kmp_fork_in_teams()
1686                 parent_team->t.t_id, parent_team->t.t_pkfn));  in __kmp_fork_in_teams()
1688   if (!parent_team->t.t_invoke(gtid)) {  in __kmp_fork_in_teams()
1692                 parent_team->t.t_id, parent_team->t.t_pkfn));  in __kmp_fork_in_teams()
1729   master_th->th.th_serial_team->t.t_pkfn = microtask;  in __kmp_serial_fork_call()
1734     master_th->th.th_serial_team->t.t_ident = loc;  in __kmp_serial_fork_call()
1737       master_th->th.th_serial_team->t.t_level--;  in __kmp_serial_fork_call()
1753         exit_frame_p = &(task_info->frame.exit_frame.ptr);  in __kmp_serial_fork_call()
1755           OMPT_CUR_TASK_INFO(master_th)->thread_num = __kmp_tid_from_gtid(gtid);  in __kmp_serial_fork_call()
1758               &(task_info->task_data), 1,  in __kmp_serial_fork_call()
1759               OMPT_CUR_TASK_INFO(master_th)->thread_num, ompt_task_implicit);  in __kmp_serial_fork_call()
1763         master_th->th.ompt_thread_info.state = ompt_state_work_parallel;  in __kmp_serial_fork_call()
1772         __kmp_invoke_microtask(microtask, gtid, 0, argc, parent_team->t.t_argv  in __kmp_serial_fork_call()
1785               ompt_scope_end, NULL, &(task_info->task_data), 1,  in __kmp_serial_fork_call()
1786               OMPT_CUR_TASK_INFO(master_th)->thread_num, ompt_task_implicit);  in __kmp_serial_fork_call()
1795         master_th->th.ompt_thread_info.state = ompt_state_overhead;  in __kmp_serial_fork_call()
1799       KMP_DEBUG_ASSERT(master_th->th.th_team == master_th->th.th_serial_team);  in __kmp_serial_fork_call()
1800       team = master_th->th.th_team;  in __kmp_serial_fork_call()
1801       // team->t.t_pkfn = microtask;  in __kmp_serial_fork_call()
1802       team->t.t_invoke = invoker;  in __kmp_serial_fork_call()
1804       team->t.t_argc = argc;  in __kmp_serial_fork_call()
1805       argv = (void **)team->t.t_argv;  in __kmp_serial_fork_call()
1806       for (i = argc - 1; i >= 0; --i)  in __kmp_serial_fork_call()
1810       team->t.t_level--;  in __kmp_serial_fork_call()
1818               ompt_scope_end, NULL, &(task_info->task_data), 0,  in __kmp_serial_fork_call()
1819               OMPT_CUR_TASK_INFO(master_th)->thread_num, ompt_task_initial);  in __kmp_serial_fork_call()
1827         master_th->th.ompt_thread_info.state = ompt_state_overhead;  in __kmp_serial_fork_call()
1832       for (i = argc - 1; i >= 0; --i)  in __kmp_serial_fork_call()
1849         exit_frame_p = &(task_info->frame.exit_frame.ptr);  in __kmp_serial_fork_call()
1858           OMPT_CUR_TASK_INFO(master_th)->thread_num = __kmp_tid_from_gtid(gtid);  in __kmp_serial_fork_call()
1862         master_th->th.ompt_thread_info.state = ompt_state_work_parallel;  in __kmp_serial_fork_call()
1884               ompt_scope_end, NULL, &(task_info->task_data), 1,  in __kmp_serial_fork_call()
1885               OMPT_CUR_TASK_INFO(master_th)->thread_num, ompt_task_implicit);  in __kmp_serial_fork_call()
1895         master_th->th.ompt_thread_info.state = ompt_state_overhead;  in __kmp_serial_fork_call()
1975     parent_team = master_th->th.th_team;  in __kmp_fork_call()
1976     master_tid = master_th->th.th_info.ds.ds_tid;  in __kmp_fork_call()
1977     master_this_cons = master_th->th.th_local.this_construct;  in __kmp_fork_call()
1978     root = master_th->th.th_root;  in __kmp_fork_call()
1979     master_active = root->r.r_active;  in __kmp_fork_call()
1980     master_set_numthreads = master_th->th.th_set_nproc;  in __kmp_fork_call()
1982         master_th->th.th_current_task->td_icvs.task_thread_limit;  in __kmp_fork_call()
2001     level = parent_team->t.t_level;  in __kmp_fork_call()
2002     // used to launch non-serial teams even if nested is not allowed  in __kmp_fork_call()
2003     active_level = parent_team->t.t_active_level;  in __kmp_fork_call()
2005     teams_level = master_th->th.th_teams_level;  in __kmp_fork_call()
2007     p_hot_teams = &master_th->th.th_hot_teams;  in __kmp_fork_call()
2011       (*p_hot_teams)[0].hot_team = root->r.r_hot_team;  in __kmp_fork_call()
2031       master_th->th.ompt_thread_info.state = ompt_state_overhead;  in __kmp_fork_call()
2035     master_th->th.th_ident = loc;  in __kmp_fork_call()
2058          (parent_team->t.t_active_level >=  in __kmp_fork_call()
2059           master_th->th.th_current_task->td_icvs.max_active_levels)) ||  in __kmp_fork_call()
2085           // Free lock for single thread execution here; for multi-thread  in __kmp_fork_call()
2095     master_th->th.th_set_nproc = 0;  in __kmp_fork_call()
2111                   parent_team->t.t_active_level, master_th,  in __kmp_fork_call()
2112                   master_th->th.th_current_task,  in __kmp_fork_call()
2113                   master_th->th.th_current_task->td_icvs.max_active_levels));  in __kmp_fork_call()
2114     // TODO: GEH - cannot do this assertion because root thread not set up as  in __kmp_fork_call()
2116     // KMP_ASSERT( master_th->th.th_current_task->td_flags.executing == 1 );  in __kmp_fork_call()
2117     master_th->th.th_current_task->td_flags.executing = 0;  in __kmp_fork_call()
2119     if (!master_th->th.th_teams_microtask || level > teams_level) {  in __kmp_fork_call()
2121       KMP_ATOMIC_INC(&root->r.r_in_parallel);  in __kmp_fork_call()
2125     int nthreads_icv = master_th->th.th_current_task->td_icvs.nproc;  in __kmp_fork_call()
2127     if (!master_th->th.th_set_nested_nth &&  in __kmp_fork_call()
2128         (level + 1 < parent_team->t.t_nested_nth->used) &&  in __kmp_fork_call()
2129         (parent_team->t.t_nested_nth->nth[level + 1] != nthreads_icv)) {  in __kmp_fork_call()
2130       nthreads_icv = parent_team->t.t_nested_nth->nth[level + 1];  in __kmp_fork_call()
2131     } else if (master_th->th.th_set_nested_nth) {  in __kmp_fork_call()
2133       if ((level + 1 < nested_nth->used) &&  in __kmp_fork_call()
2134           (nested_nth->nth[level + 1] != nthreads_icv))  in __kmp_fork_call()
2135         nthreads_icv = nested_nth->nth[level + 1];  in __kmp_fork_call()
2143     kmp_proc_bind_t proc_bind = master_th->th.th_set_proc_bind;  in __kmp_fork_call()
2146     if (master_th->th.th_current_task->td_icvs.proc_bind == proc_bind_false) {  in __kmp_fork_call()
2149       // No proc_bind clause specified; use current proc-bind-var for this  in __kmp_fork_call()
2152         proc_bind = master_th->th.th_current_task->td_icvs.proc_bind;  in __kmp_fork_call()
2155       if (master_th->th.th_teams_microtask &&  in __kmp_fork_call()
2160          This overrides proc-bind-var for this parallel region, but does not  in __kmp_fork_call()
2161          change proc-bind-var. */  in __kmp_fork_call()
2162       // Figure the value of proc-bind-var for the child threads.  in __kmp_fork_call()
2165            master_th->th.th_current_task->td_icvs.proc_bind)) {  in __kmp_fork_call()
2168         if (!master_th->th.th_teams_microtask ||  in __kmp_fork_call()
2175     master_th->th.th_set_proc_bind = proc_bind_default;  in __kmp_fork_call()
2179       copy_icvs(&new_icvs, &master_th->th.th_current_task->td_icvs);  in __kmp_fork_call()
2197         copy_icvs((kmp_internal_control_t *)team->t.b->team_icvs, &new_icvs);  in __kmp_fork_call()
2206                                  &master_th->th.th_current_task->td_icvs,  in __kmp_fork_call()
2209         copy_icvs((kmp_internal_control_t *)team->t.b->team_icvs,  in __kmp_fork_call()
2210                   &master_th->th.th_current_task->td_icvs);  in __kmp_fork_call()
2213         10, ("__kmp_fork_call: after __kmp_allocate_team - team = %p\n", team));  in __kmp_fork_call()
2216     KMP_CHECK_UPDATE(team->t.t_master_tid, master_tid);  in __kmp_fork_call()
2217     KMP_CHECK_UPDATE(team->t.t_master_this_cons, master_this_cons);  in __kmp_fork_call()
2218     KMP_CHECK_UPDATE(team->t.t_ident, loc);  in __kmp_fork_call()
2219     KMP_CHECK_UPDATE(team->t.t_parent, parent_team);  in __kmp_fork_call()
2220     KMP_CHECK_UPDATE_SYNC(team->t.t_pkfn, microtask);  in __kmp_fork_call()
2222     KMP_CHECK_UPDATE_SYNC(team->t.ompt_team_info.master_return_address,  in __kmp_fork_call()
2225     KMP_CHECK_UPDATE(team->t.t_invoke, invoker); // TODO move to root, maybe  in __kmp_fork_call()
2226     // TODO: parent_team->t.t_level == INT_MAX ???  in __kmp_fork_call()
2227     if (!master_th->th.th_teams_microtask || level > teams_level) {  in __kmp_fork_call()
2228       int new_level = parent_team->t.t_level + 1;  in __kmp_fork_call()
2229       KMP_CHECK_UPDATE(team->t.t_level, new_level);  in __kmp_fork_call()
2230       new_level = parent_team->t.t_active_level + 1;  in __kmp_fork_call()
2231       KMP_CHECK_UPDATE(team->t.t_active_level, new_level);  in __kmp_fork_call()
2234       int new_level = parent_team->t.t_level;  in __kmp_fork_call()
2235       KMP_CHECK_UPDATE(team->t.t_level, new_level);  in __kmp_fork_call()
2236       new_level = parent_team->t.t_active_level;  in __kmp_fork_call()
2237       KMP_CHECK_UPDATE(team->t.t_active_level, new_level);  in __kmp_fork_call()
2240     // set primary thread's schedule as new run-time schedule  in __kmp_fork_call()
2241     KMP_CHECK_UPDATE(team->t.t_sched.sched, new_sched.sched);  in __kmp_fork_call()
2243     KMP_CHECK_UPDATE(team->t.t_cancel_request, cancel_noreq);  in __kmp_fork_call()
2244     KMP_CHECK_UPDATE(team->t.t_def_allocator, master_th->th.th_def_allocator);  in __kmp_fork_call()
2247     if (team->t.t_nested_nth &&  in __kmp_fork_call()
2248         team->t.t_nested_nth != parent_team->t.t_nested_nth) {  in __kmp_fork_call()
2249       KMP_INTERNAL_FREE(team->t.t_nested_nth->nth);  in __kmp_fork_call()
2250       KMP_INTERNAL_FREE(team->t.t_nested_nth);  in __kmp_fork_call()
2251       team->t.t_nested_nth = NULL;  in __kmp_fork_call()
2253     team->t.t_nested_nth = parent_team->t.t_nested_nth;  in __kmp_fork_call()
2254     if (master_th->th.th_set_nested_nth) {  in __kmp_fork_call()
2257       team->t.t_nested_nth = nested_nth;  in __kmp_fork_call()
2258       KMP_INTERNAL_FREE(master_th->th.th_set_nested_nth);  in __kmp_fork_call()
2259       master_th->th.th_set_nested_nth = NULL;  in __kmp_fork_call()
2260       master_th->th.th_set_nested_nth_sz = 0;  in __kmp_fork_call()
2261       master_th->th.th_nt_strict = false;  in __kmp_fork_call()
2273         ("__kmp_fork_call: T#%d(%d:%d)->(%d:0) created a team of %d threads\n",  in __kmp_fork_call()
2274          gtid, parent_team->t.t_id, team->t.t_master_tid, team->t.t_id,  in __kmp_fork_call()
2275          team->t.t_nproc));  in __kmp_fork_call()
2276     KMP_DEBUG_ASSERT(team != root->r.r_hot_team ||  in __kmp_fork_call()
2277                      (team->t.t_master_tid == 0 &&  in __kmp_fork_call()
2278                       (team->t.t_parent == root->r.r_root_team ||  in __kmp_fork_call()
2279                        team->t.t_parent->t.t_serialized)));  in __kmp_fork_call()
2283     argv = (void **)team->t.t_argv;  in __kmp_fork_call()
2285       for (i = argc - 1; i >= 0; --i) {  in __kmp_fork_call()
2293         KMP_CHECK_UPDATE(argv[i], team->t.t_parent->t.t_argv[i]);  in __kmp_fork_call()
2298     KMP_CHECK_UPDATE(team->t.t_master_active, master_active);  in __kmp_fork_call()
2299     if (!root->r.r_active) // Only do assignment if it prevents cache ping-pong  in __kmp_fork_call()
2300       root->r.r_active = TRUE;  in __kmp_fork_call()
2304                          &master_th->th.th_current_task->td_icvs, loc);  in __kmp_fork_call()
2307     master_th->th.ompt_thread_info.state = ompt_state_work_parallel;  in __kmp_fork_call()
2313     if (team->t.t_active_level == 1 // only report frames at level 1  in __kmp_fork_call()
2314         && !master_th->th.th_teams_microtask) { // not in teams construct  in __kmp_fork_call()
2322         // Internal fork - report frame begin  in __kmp_fork_call()
2323         master_th->th.th_frame_time = tmp_time;  in __kmp_fork_call()
2325           team->t.t_region_time = tmp_time;  in __kmp_fork_call()
2332           __kmp_itt_region_forking(gtid, team->t.t_nproc, 0);  in __kmp_fork_call()
2338     KMP_DEBUG_ASSERT(team == __kmp_threads[gtid]->th.th_team);  in __kmp_fork_call()
2348         KMP_DEBUG_ASSERT(team->t.t_stack_id == NULL);  in __kmp_fork_call()
2349         team->t.t_stack_id = __kmp_itt_stack_caller_create();  in __kmp_fork_call()
2350       } else if (parent_team->t.t_serialized) {  in __kmp_fork_call()
2355         KMP_DEBUG_ASSERT(parent_team->t.t_stack_id == NULL);  in __kmp_fork_call()
2356         parent_team->t.t_stack_id = __kmp_itt_stack_caller_create();  in __kmp_fork_call()
2377                   team->t.t_id, team->t.t_pkfn));  in __kmp_fork_call()
2388   if (!team->t.t_invoke(gtid)) {  in __kmp_fork_call()
2400                 team->t.t_id, team->t.t_pkfn));  in __kmp_fork_call()
2406     master_th->th.ompt_thread_info.state = ompt_state_overhead;  in __kmp_fork_call()
2417   thread->th.ompt_thread_info.state =  in __kmp_join_restore_state()
2418       ((team->t.t_serialized) ? ompt_state_work_serial  in __kmp_join_restore_state()
2428         parallel_data, &(task_info->task_data), flags, codeptr);  in __kmp_join_ompt()
2431   task_info->frame.enter_frame = ompt_data_none;  in __kmp_join_ompt()
2454   root = master_th->th.th_root;  in __kmp_join_call()
2455   team = master_th->th.th_team;  in __kmp_join_call()
2456   parent_team = team->t.t_parent;  in __kmp_join_call()
2458   master_th->th.th_ident = loc;  in __kmp_join_call()
2461   void *team_microtask = (void *)team->t.t_pkfn;  in __kmp_join_call()
2463   // __kmpc_end_serialized_parallel to call hooks for OMPT end-implicit-task  in __kmp_join_call()
2464   // and end-parallel events.  in __kmp_join_call()
2466       !(team->t.t_serialized && fork_context == fork_context_gnu)) {  in __kmp_join_call()
2467     master_th->th.ompt_thread_info.state = ompt_state_overhead;  in __kmp_join_call()
2476                   team->t.t_task_team[master_th->th.th_task_state],  in __kmp_join_call()
2477                   master_th->th.th_task_team));  in __kmp_join_call()
2482   if (team->t.t_serialized) {  in __kmp_join_call()
2483     if (master_th->th.th_teams_microtask) {  in __kmp_join_call()
2485       int level = team->t.t_level;  in __kmp_join_call()
2486       int tlevel = master_th->th.th_teams_level;  in __kmp_join_call()
2489         //     so do it here - at the end of teams construct  in __kmp_join_call()
2490         team->t.t_level++;  in __kmp_join_call()
2495         team->t.t_serialized++;  in __kmp_join_call()
2512   master_active = team->t.t_master_active;  in __kmp_join_call()
2520       KMP_DEBUG_ASSERT(team->t.t_stack_id != NULL);  in __kmp_join_call()
2522       __kmp_itt_stack_caller_destroy((__itt_caller)team->t.t_stack_id);  in __kmp_join_call()
2523       team->t.t_stack_id = NULL;  in __kmp_join_call()
2527     master_th->th.th_task_state =  in __kmp_join_call()
2530     if (__itt_stack_caller_create_ptr && parent_team->t.t_serialized) {  in __kmp_join_call()
2531       KMP_DEBUG_ASSERT(parent_team->t.t_stack_id != NULL);  in __kmp_join_call()
2535       __kmp_itt_stack_caller_destroy((__itt_caller)parent_team->t.t_stack_id);  in __kmp_join_call()
2536       parent_team->t.t_stack_id = NULL;  in __kmp_join_call()
2544   ompt_data_t *parallel_data = &(team->t.ompt_team_info.parallel_data);  in __kmp_join_call()
2545   void *codeptr = team->t.ompt_team_info.master_return_address;  in __kmp_join_call()
2550   if (team->t.t_active_level == 1 &&  in __kmp_join_call()
2551       (!master_th->th.th_teams_microtask || /* not in teams construct */  in __kmp_join_call()
2552        master_th->th.th_teams_size.nteams == 1)) {  in __kmp_join_call()
2553     master_th->th.th_ident = loc;  in __kmp_join_call()
2558       __kmp_itt_frame_submit(gtid, team->t.t_region_time,  in __kmp_join_call()
2559                              master_th->th.th_frame_time, 0, loc,  in __kmp_join_call()
2560                              master_th->th.th_team_nproc, 1);  in __kmp_join_call()
2570     master_th->th.th_first_place = team->t.t_first_place;  in __kmp_join_call()
2571     master_th->th.th_last_place = team->t.t_last_place;  in __kmp_join_call()
2575   if (master_th->th.th_teams_microtask && !exit_teams &&  in __kmp_join_call()
2576       team->t.t_pkfn != (microtask_t)__kmp_teams_master &&  in __kmp_join_call()
2577       team->t.t_level == master_th->th.th_teams_level + 1) {  in __kmp_join_call()
2586         int ompt_team_size = team->t.t_nproc;  in __kmp_join_call()
2588             ompt_scope_end, NULL, &(task_info->task_data), ompt_team_size,  in __kmp_join_call()
2589             OMPT_CUR_TASK_INFO(master_th)->thread_num, ompt_task_implicit);  in __kmp_join_call()
2591       task_info->frame.exit_frame = ompt_data_none;  in __kmp_join_call()
2592       task_info->task_data = ompt_data_none;  in __kmp_join_call()
2598     team->t.t_level--;  in __kmp_join_call()
2599     team->t.t_active_level--;  in __kmp_join_call()
2600     KMP_ATOMIC_DEC(&root->r.r_in_parallel);  in __kmp_join_call()
2606     if (master_th->th.th_team_nproc < master_th->th.th_teams_size.nth) {  in __kmp_join_call()
2607       int old_num = master_th->th.th_team_nproc;  in __kmp_join_call()
2608       int new_num = master_th->th.th_teams_size.nth;  in __kmp_join_call()
2609       kmp_info_t **other_threads = team->t.t_threads;  in __kmp_join_call()
2610       team->t.t_nproc = new_num;  in __kmp_join_call()
2612         other_threads[i]->th.th_team_nproc = new_num;  in __kmp_join_call()
2614       // Adjust states of non-used threads of the team  in __kmp_join_call()
2616         // Re-initialize thread's barrier data.  in __kmp_join_call()
2618         kmp_balign_t *balign = other_threads[i]->th.th_bar;  in __kmp_join_call()
2620           balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;  in __kmp_join_call()
2623           balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;  in __kmp_join_call()
2628           other_threads[i]->th.th_task_state = master_th->th.th_task_state;  in __kmp_join_call()
2644   master_th->th.th_info.ds.ds_tid = team->t.t_master_tid;  in __kmp_join_call()
2645   master_th->th.th_local.this_construct = team->t.t_master_this_cons;  in __kmp_join_call()
2647   master_th->th.th_dispatch = &parent_team->t.t_dispatch[team->t.t_master_tid];  in __kmp_join_call()
2654   if (!master_th->th.th_teams_microtask ||  in __kmp_join_call()
2655       team->t.t_level > master_th->th.th_teams_level) {  in __kmp_join_call()
2657     KMP_ATOMIC_DEC(&root->r.r_in_parallel);  in __kmp_join_call()
2659   KMP_DEBUG_ASSERT(root->r.r_in_parallel >= 0);  in __kmp_join_call()
2668       int ompt_team_size = (flags == ompt_task_initial) ? 0 : team->t.t_nproc;  in __kmp_join_call()
2670           ompt_scope_end, NULL, &(task_info->task_data), ompt_team_size,  in __kmp_join_call()
2671           OMPT_CUR_TASK_INFO(master_th)->thread_num, flags);  in __kmp_join_call()
2673     task_info->frame.exit_frame = ompt_data_none;  in __kmp_join_call()
2674     task_info->task_data = ompt_data_none;  in __kmp_join_call()
2682   master_th->th.th_def_allocator = team->t.t_def_allocator;  in __kmp_join_call()
2690   if (root->r.r_active != master_active)  in __kmp_join_call()
2691     root->r.r_active = master_active;  in __kmp_join_call()
2701   master_th->th.th_team = parent_team;  in __kmp_join_call()
2702   master_th->th.th_team_nproc = parent_team->t.t_nproc;  in __kmp_join_call()
2703   master_th->th.th_team_master = parent_team->t.t_threads[0];  in __kmp_join_call()
2704   master_th->th.th_team_serialized = parent_team->t.t_serialized;  in __kmp_join_call()
2707   if (parent_team->t.t_serialized &&  in __kmp_join_call()
2708       parent_team != master_th->th.th_serial_team &&  in __kmp_join_call()
2709       parent_team != root->r.r_root_team) {  in __kmp_join_call()
2711                     master_th->th.th_serial_team USE_NESTED_HOT_ARG(NULL));  in __kmp_join_call()
2712     master_th->th.th_serial_team = parent_team;  in __kmp_join_call()
2717     KMP_DEBUG_ASSERT(team->t.t_primary_task_state == 0 ||  in __kmp_join_call()
2718                      team->t.t_primary_task_state == 1);  in __kmp_join_call()
2719     master_th->th.th_task_state = (kmp_uint8)team->t.t_primary_task_state;  in __kmp_join_call()
2722     master_th->th.th_task_team =  in __kmp_join_call()
2723         parent_team->t.t_task_team[master_th->th.th_task_state];  in __kmp_join_call()
2726               __kmp_gtid_from_thread(master_th), master_th->th.th_task_team,  in __kmp_join_call()
2730   // TODO: GEH - cannot do this assertion because root thread not set up as  in __kmp_join_call()
2732   // KMP_ASSERT( master_th->th.th_current_task->td_flags.executing == 0 );  in __kmp_join_call()
2733   master_th->th.th_current_task->td_flags.executing = 1;  in __kmp_join_call()
2738   if (master_th->th.th_team->t.t_level == 0 && __kmp_affinity.flags.reset) {  in __kmp_join_call()
2761   if (thread->th.th_team != thread->th.th_serial_team) {  in __kmp_save_internal_controls()
2764   if (thread->th.th_team->t.t_serialized > 1) {  in __kmp_save_internal_controls()
2767     if (thread->th.th_team->t.t_control_stack_top == NULL) {  in __kmp_save_internal_controls()
2770       if (thread->th.th_team->t.t_control_stack_top->serial_nesting_level !=  in __kmp_save_internal_controls()
2771           thread->th.th_team->t.t_serialized) {  in __kmp_save_internal_controls()
2780       copy_icvs(control, &thread->th.th_current_task->td_icvs);  in __kmp_save_internal_controls()
2782       control->serial_nesting_level = thread->th.th_team->t.t_serialized;  in __kmp_save_internal_controls()
2784       control->next = thread->th.th_team->t.t_control_stack_top;  in __kmp_save_internal_controls()
2785       thread->th.th_team->t.t_control_stack_top = control;  in __kmp_save_internal_controls()
2805   if (thread->th.th_current_task->td_icvs.nproc == new_nth)  in __kmp_set_num_threads()
2815   root = thread->th.th_root;  in __kmp_set_num_threads()
2816   if (__kmp_init_parallel && (!root->r.r_active) &&  in __kmp_set_num_threads()
2817       (root->r.r_hot_team->t.t_nproc > new_nth)  in __kmp_set_num_threads()
2822     kmp_team_t *hot_team = root->r.r_hot_team;  in __kmp_set_num_threads()
2828       __kmp_resize_dist_barrier(hot_team, hot_team->t.t_nproc, new_nth);  in __kmp_set_num_threads()
2831     for (f = new_nth; f < hot_team->t.t_nproc; f++) {  in __kmp_set_num_threads()
2832       KMP_DEBUG_ASSERT(hot_team->t.t_threads[f] != NULL);  in __kmp_set_num_threads()
2836         hot_team->t.t_threads[f]->th.th_task_team = NULL;  in __kmp_set_num_threads()
2838       __kmp_free_thread(hot_team->t.t_threads[f]);  in __kmp_set_num_threads()
2839       hot_team->t.t_threads[f] = NULL;  in __kmp_set_num_threads()
2841     hot_team->t.t_nproc = new_nth;  in __kmp_set_num_threads()
2843     if (thread->th.th_hot_teams) {  in __kmp_set_num_threads()
2844       KMP_DEBUG_ASSERT(hot_team == thread->th.th_hot_teams[0].hot_team);  in __kmp_set_num_threads()
2845       thread->th.th_hot_teams[0].hot_team_nth = new_nth;  in __kmp_set_num_threads()
2850       hot_team->t.b->update_num_threads(new_nth);  in __kmp_set_num_threads()
2858       KMP_DEBUG_ASSERT(hot_team->t.t_threads[f] != NULL);  in __kmp_set_num_threads()
2859       hot_team->t.t_threads[f]->th.th_team_nproc = new_nth;  in __kmp_set_num_threads()
2862     hot_team->t.t_size_changed = -1;  in __kmp_set_num_threads()
2919   KMP_DEBUG_ASSERT(thread->th.th_current_task);  in __kmp_get_max_active_levels()
2922                 gtid, thread->th.th_current_task,  in __kmp_get_max_active_levels()
2923                 thread->th.th_current_task->td_icvs.max_active_levels));  in __kmp_get_max_active_levels()
2924   return thread->th.th_current_task->td_icvs.max_active_levels;  in __kmp_get_max_active_levels()
2927 // nteams-var per-device ICV
2933 // teams-thread-limit-var per-device ICV
2943 /* Changes def_sched_var ICV values (run-time schedule kind and chunk) */
2954   // Valid parameters should fit in one of two intervals - standard or extended:  in __kmp_set_schedule()
2956   // 2008-01-25: 0,  1 - 4,       5,         100,     101 - 102, 103  in __kmp_set_schedule()
2978       thread->th.th_current_task->td_icvs.sched.r_sched_type = kmp_sch_static;  in __kmp_set_schedule()
2980       thread->th.th_current_task->td_icvs.sched.r_sched_type =  in __kmp_set_schedule()
2981           __kmp_sch_map[kind - kmp_sched_lower - 1];  in __kmp_set_schedule()
2984     //    __kmp_sch_map[ kind - kmp_sched_lower_ext + kmp_sched_upper_std -  in __kmp_set_schedule()
2985     //    kmp_sched_lower - 2 ];  in __kmp_set_schedule()
2986     thread->th.th_current_task->td_icvs.sched.r_sched_type =  in __kmp_set_schedule()
2987         __kmp_sch_map[kind - kmp_sched_lower_ext + kmp_sched_upper_std -  in __kmp_set_schedule()
2988                       kmp_sched_lower - 2];  in __kmp_set_schedule()
2991       orig_kind, &(thread->th.th_current_task->td_icvs.sched.r_sched_type));  in __kmp_set_schedule()
2994     thread->th.th_current_task->td_icvs.sched.chunk = KMP_DEFAULT_CHUNK;  in __kmp_set_schedule()
2996     thread->th.th_current_task->td_icvs.sched.chunk = chunk;  in __kmp_set_schedule()
3010   th_type = thread->th.th_current_task->td_icvs.sched.r_sched_type;  in __kmp_get_schedule()
3046   *chunk = thread->th.th_current_task->td_icvs.sched.chunk;  in __kmp_get_schedule()
3062     return -1;  in __kmp_get_ancestor_thread_num()
3064   team = thr->th.th_team;  in __kmp_get_ancestor_thread_num()
3065   ii = team->t.t_level;  in __kmp_get_ancestor_thread_num()
3067     return -1;  in __kmp_get_ancestor_thread_num()
3069   if (thr->th.th_teams_microtask) {  in __kmp_get_ancestor_thread_num()
3071     int tlevel = thr->th.th_teams_level; // the level of the teams construct  in __kmp_get_ancestor_thread_num()
3088   dd = team->t.t_serialized;  in __kmp_get_ancestor_thread_num()
3091     for (dd = team->t.t_serialized; (dd > 0) && (ii > level); dd--, ii--) {  in __kmp_get_ancestor_thread_num()
3093     if ((team->t.t_serialized) && (!dd)) {  in __kmp_get_ancestor_thread_num()
3094       team = team->t.t_parent;  in __kmp_get_ancestor_thread_num()
3098       team = team->t.t_parent;  in __kmp_get_ancestor_thread_num()
3099       dd = team->t.t_serialized;  in __kmp_get_ancestor_thread_num()
3100       ii--;  in __kmp_get_ancestor_thread_num()
3104   return (dd > 1) ? (0) : (team->t.t_master_tid);  in __kmp_get_ancestor_thread_num()
3120     return -1;  in __kmp_get_team_size()
3122   team = thr->th.th_team;  in __kmp_get_team_size()
3123   ii = team->t.t_level;  in __kmp_get_team_size()
3125     return -1;  in __kmp_get_team_size()
3127   if (thr->th.th_teams_microtask) {  in __kmp_get_team_size()
3129     int tlevel = thr->th.th_teams_level; // the level of the teams construct  in __kmp_get_team_size()
3144     for (dd = team->t.t_serialized; (dd > 0) && (ii > level); dd--, ii--) {  in __kmp_get_team_size()
3146     if (team->t.t_serialized && (!dd)) {  in __kmp_get_team_size()
3147       team = team->t.t_parent;  in __kmp_get_team_size()
3151       team = team->t.t_parent;  in __kmp_get_team_size()
3152       ii--;  in __kmp_get_team_size()
3156   return team->t.t_nproc;  in __kmp_get_team_size()
3168   // KMP_SCHEDULE multiple times, and thus have different run-time schedules in  in __kmp_get_schedule_global()
3198   if (!realloc || argc > team->t.t_max_argc) {  in __kmp_alloc_argv_entries()
3202                    team->t.t_id, argc, (realloc) ? team->t.t_max_argc : 0));  in __kmp_alloc_argv_entries()
3204     if (realloc && team->t.t_argv != &team->t.t_inline_argv[0])  in __kmp_alloc_argv_entries()
3205       __kmp_free((void *)team->t.t_argv);  in __kmp_alloc_argv_entries()
3209       team->t.t_max_argc = KMP_INLINE_ARGV_ENTRIES;  in __kmp_alloc_argv_entries()
3212                      team->t.t_id, team->t.t_max_argc));  in __kmp_alloc_argv_entries()
3213       team->t.t_argv = &team->t.t_inline_argv[0];  in __kmp_alloc_argv_entries()
3216             -1, &team->t.t_inline_argv[0],  in __kmp_alloc_argv_entries()
3217             &team->t.t_inline_argv[KMP_INLINE_ARGV_ENTRIES],  in __kmp_alloc_argv_entries()
3219             team->t.t_id);  in __kmp_alloc_argv_entries()
3223       team->t.t_max_argc = (argc <= (KMP_MIN_MALLOC_ARGV_ENTRIES >> 1))  in __kmp_alloc_argv_entries()
3228                      team->t.t_id, team->t.t_max_argc));  in __kmp_alloc_argv_entries()
3229       team->t.t_argv =  in __kmp_alloc_argv_entries()
3230           (void **)__kmp_page_allocate(sizeof(void *) * team->t.t_max_argc);  in __kmp_alloc_argv_entries()
3232         __kmp_print_storage_map_gtid(-1, &team->t.t_argv[0],  in __kmp_alloc_argv_entries()
3233                                      &team->t.t_argv[team->t.t_max_argc],  in __kmp_alloc_argv_entries()
3234                                      sizeof(void *) * team->t.t_max_argc,  in __kmp_alloc_argv_entries()
3235                                      "team_%d.t_argv", team->t.t_id);  in __kmp_alloc_argv_entries()
3244   team->t.t_threads =  in __kmp_allocate_team_arrays()
3246   team->t.t_disp_buffer = (dispatch_shared_info_t *)__kmp_allocate(  in __kmp_allocate_team_arrays()
3248   team->t.t_dispatch =  in __kmp_allocate_team_arrays()
3250   team->t.t_implicit_task_taskdata =  in __kmp_allocate_team_arrays()
3252   team->t.t_max_nproc = max_nth;  in __kmp_allocate_team_arrays()
3256     team->t.t_disp_buffer[i].buffer_index = i;  in __kmp_allocate_team_arrays()
3257     team->t.t_disp_buffer[i].doacross_buf_idx = i;  in __kmp_allocate_team_arrays()
3264   for (i = 0; i < team->t.t_max_nproc; ++i) {  in __kmp_free_team_arrays()
3265     if (team->t.t_dispatch[i].th_disp_buffer != NULL) {  in __kmp_free_team_arrays()
3266       __kmp_free(team->t.t_dispatch[i].th_disp_buffer);  in __kmp_free_team_arrays()
3267       team->t.t_dispatch[i].th_disp_buffer = NULL;  in __kmp_free_team_arrays()
3273   __kmp_free(team->t.t_threads);  in __kmp_free_team_arrays()
3274   __kmp_free(team->t.t_disp_buffer);  in __kmp_free_team_arrays()
3275   __kmp_free(team->t.t_dispatch);  in __kmp_free_team_arrays()
3276   __kmp_free(team->t.t_implicit_task_taskdata);  in __kmp_free_team_arrays()
3277   team->t.t_threads = NULL;  in __kmp_free_team_arrays()
3278   team->t.t_disp_buffer = NULL;  in __kmp_free_team_arrays()
3279   team->t.t_dispatch = NULL;  in __kmp_free_team_arrays()
3280   team->t.t_implicit_task_taskdata = 0;  in __kmp_free_team_arrays()
3284   kmp_info_t **oldThreads = team->t.t_threads;  in __kmp_reallocate_team_arrays()
3286   __kmp_free(team->t.t_disp_buffer);  in __kmp_reallocate_team_arrays()
3287   __kmp_free(team->t.t_dispatch);  in __kmp_reallocate_team_arrays()
3288   __kmp_free(team->t.t_implicit_task_taskdata);  in __kmp_reallocate_team_arrays()
3291   KMP_MEMCPY(team->t.t_threads, oldThreads,  in __kmp_reallocate_team_arrays()
3292              team->t.t_nproc * sizeof(kmp_info_t *));  in __kmp_reallocate_team_arrays()
3337       0; // probably =team->t.t_serial like in save_inter_controls  in __kmp_get_x_global_icvs()
3338   copy_icvs(&gx_icvs, &team->t.t_threads[0]->th.th_current_task->td_icvs);  in __kmp_get_x_global_icvs()
3353   KMP_ASSERT(!root->r.r_begin);  in __kmp_initialize_root()
3356   __kmp_init_lock(&root->r.r_begin_lock);  in __kmp_initialize_root()
3357   root->r.r_begin = FALSE;  in __kmp_initialize_root()
3358   root->r.r_active = FALSE;  in __kmp_initialize_root()
3359   root->r.r_in_parallel = 0;  in __kmp_initialize_root()
3360   root->r.r_blocktime = __kmp_dflt_blocktime;  in __kmp_initialize_root()
3362   root->r.r_affinity_assigned = FALSE;  in __kmp_initialize_root()
3381   // Non-NULL value should be assigned to make the debugger display the root  in __kmp_initialize_root()
3383   TCW_SYNC_PTR(root_team->t.t_pkfn, (microtask_t)(~0));  in __kmp_initialize_root()
3388   root->r.r_root_team = root_team;  in __kmp_initialize_root()
3389   root_team->t.t_control_stack_top = NULL;  in __kmp_initialize_root()
3392   root_team->t.t_threads[0] = NULL;  in __kmp_initialize_root()
3393   root_team->t.t_nproc = 1;  in __kmp_initialize_root()
3394   root_team->t.t_serialized = 1;  in __kmp_initialize_root()
3395   // TODO???: root_team->t.t_max_active_levels = __kmp_dflt_max_active_levels;  in __kmp_initialize_root()
3396   root_team->t.t_sched.sched = r_sched.sched;  in __kmp_initialize_root()
3397   root_team->t.t_nested_nth = &__kmp_nested_nth;  in __kmp_initialize_root()
3401        root_team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE));  in __kmp_initialize_root()
3420   root->r.r_hot_team = hot_team;  in __kmp_initialize_root()
3421   root_team->t.t_control_stack_top = NULL;  in __kmp_initialize_root()
3423   /* first-time initialization */  in __kmp_initialize_root()
3424   hot_team->t.t_parent = root_team;  in __kmp_initialize_root()
3427   hot_team_max_nth = hot_team->t.t_max_nproc;  in __kmp_initialize_root()
3429     hot_team->t.t_threads[f] = NULL;  in __kmp_initialize_root()
3431   hot_team->t.t_nproc = 1;  in __kmp_initialize_root()
3432   // TODO???: hot_team->t.t_max_active_levels = __kmp_dflt_max_active_levels;  in __kmp_initialize_root()
3433   hot_team->t.t_sched.sched = r_sched.sched;  in __kmp_initialize_root()
3434   hot_team->t.t_size_changed = 0;  in __kmp_initialize_root()
3435   hot_team->t.t_nested_nth = &__kmp_nested_nth;  in __kmp_initialize_root()
3463   __kmp_print_structure_team_accum(list, team->t.t_parent);  in __kmp_print_structure_team_accum()
3464   __kmp_print_structure_team_accum(list, team->t.t_next_pool);  in __kmp_print_structure_team_accum()
3468   while (l->next != NULL && l->entry != team) {  in __kmp_print_structure_team_accum()
3469     l = l->next;  in __kmp_print_structure_team_accum()
3471   if (l->next != NULL) {  in __kmp_print_structure_team_accum()
3477   while (l->next != NULL && l->entry->t.t_id <= team->t.t_id) {  in __kmp_print_structure_team_accum()
3478     l = l->next;  in __kmp_print_structure_team_accum()
3486     l->entry = team;  in __kmp_print_structure_team_accum()
3487     l->next = item;  in __kmp_print_structure_team_accum()
3496     __kmp_printf("%2x %p\n", team->t.t_id, team);  in __kmp_print_structure_team()
3498     __kmp_printf(" - (nil)\n");  in __kmp_print_structure_team()
3506     __kmp_printf("%2d %p\n", thread->th.th_info.ds.ds_gtid, thread);  in __kmp_print_structure_thread()
3508     __kmp_printf(" - (nil)\n");  in __kmp_print_structure_thread()
3519   list->entry = NULL;  in __kmp_print_structure()
3520   list->next = NULL;  in __kmp_print_structure()
3522   __kmp_printf("\n------------------------------\nGlobal Thread "  in __kmp_print_structure()
3523                "Table\n------------------------------\n");  in __kmp_print_structure()
3539   __kmp_printf("\n------------------------------\nThreads\n--------------------"  in __kmp_print_structure()
3540                "----------\n");  in __kmp_print_structure()
3547         __kmp_printf("    Our Root:        %p\n", thread->th.th_root);  in __kmp_print_structure()
3548         __kmp_print_structure_team("    Our Team:     ", thread->th.th_team);  in __kmp_print_structure()
3550                                    thread->th.th_serial_team);  in __kmp_print_structure()
3551         __kmp_printf("    Threads:      %2d\n", thread->th.th_team_nproc);  in __kmp_print_structure()
3553                                      thread->th.th_team_master);  in __kmp_print_structure()
3554         __kmp_printf("    Serialized?:  %2d\n", thread->th.th_team_serialized);  in __kmp_print_structure()
3555         __kmp_printf("    Set NProc:    %2d\n", thread->th.th_set_nproc);  in __kmp_print_structure()
3556         __kmp_printf("    Set Proc Bind: %2d\n", thread->th.th_set_proc_bind);  in __kmp_print_structure()
3558                                      thread->th.th_next_pool);  in __kmp_print_structure()
3560         __kmp_print_structure_team_accum(list, thread->th.th_team);  in __kmp_print_structure()
3561         __kmp_print_structure_team_accum(list, thread->th.th_serial_team);  in __kmp_print_structure()
3569   __kmp_printf("\n------------------------------\nUbers\n----------------------"  in __kmp_print_structure()
3570                "--------\n");  in __kmp_print_structure()
3577         __kmp_print_structure_team("    Root Team:    ", root->r.r_root_team);  in __kmp_print_structure()
3578         __kmp_print_structure_team("    Hot Team:     ", root->r.r_hot_team);  in __kmp_print_structure()
3580                                      root->r.r_uber_thread);  in __kmp_print_structure()
3581         __kmp_printf("    Active?:      %2d\n", root->r.r_active);  in __kmp_print_structure()
3583                      KMP_ATOMIC_LD_RLX(&root->r.r_in_parallel));  in __kmp_print_structure()
3585         __kmp_print_structure_team_accum(list, root->r.r_root_team);  in __kmp_print_structure()
3586         __kmp_print_structure_team_accum(list, root->r.r_hot_team);  in __kmp_print_structure()
3593   __kmp_printf("\n------------------------------\nTeams\n----------------------"  in __kmp_print_structure()
3594                "--------\n");  in __kmp_print_structure()
3595   while (list->next != NULL) {  in __kmp_print_structure()
3596     kmp_team_p const *team = list->entry;  in __kmp_print_structure()
3598     __kmp_printf("Team %2x %p:\n", team->t.t_id, team);  in __kmp_print_structure()
3599     __kmp_print_structure_team("    Parent Team:      ", team->t.t_parent);  in __kmp_print_structure()
3600     __kmp_printf("    Primary TID:      %2d\n", team->t.t_master_tid);  in __kmp_print_structure()
3601     __kmp_printf("    Max threads:      %2d\n", team->t.t_max_nproc);  in __kmp_print_structure()
3602     __kmp_printf("    Levels of serial: %2d\n", team->t.t_serialized);  in __kmp_print_structure()
3603     __kmp_printf("    Number threads:   %2d\n", team->t.t_nproc);  in __kmp_print_structure()
3604     for (i = 0; i < team->t.t_nproc; ++i) {  in __kmp_print_structure()
3606       __kmp_print_structure_thread("", team->t.t_threads[i]);  in __kmp_print_structure()
3608     __kmp_print_structure_team("    Next in pool:     ", team->t.t_next_pool);  in __kmp_print_structure()
3610     list = list->next;  in __kmp_print_structure()
3614   __kmp_printf("\n------------------------------\nPools\n----------------------"  in __kmp_print_structure()
3615                "--------\n");  in __kmp_print_structure()
3625     list = list->next;  in __kmp_print_structure()
3632 //---------------------------------------------------------------------------
3633 //  Stuff for per-thread fast random number generator
3648 //---------------------------------------------------------------------------
3651   unsigned x = thread->th.th_x;  in __kmp_get_random()
3654   thread->th.th_x = x * thread->th.th_a + 1;  in __kmp_get_random()
3657                 thread->th.th_info.ds.ds_tid, r));  in __kmp_get_random()
3661 //--------------------------------------------------------
3664   unsigned seed = thread->th.th_info.ds.ds_tid;  in __kmp_init_random()
3666   thread->th.th_a =  in __kmp_init_random()
3668   thread->th.th_x = (seed + 1) * thread->th.th_a + 1;  in __kmp_init_random()
3670            ("__kmp_init_random: THREAD: %u; A: %u\n", seed, thread->th.th_a));  in __kmp_init_random()
3683              ->r.r_active) { // AC: reclaim only roots died in non-active state  in __kmp_reclaim_dead_roots()
3726     nNeed -= added;  in __kmp_expand_threads()
3750   // instead of having it performed here. -BB  in __kmp_expand_threads()
3755   if (__kmp_sys_max_nth - __kmp_threads_capacity < nNeed) {  in __kmp_expand_threads()
3756     /* possible expansion too small -- give up */  in __kmp_expand_threads()
3778   node->threads = __kmp_threads;  in __kmp_expand_threads()
3779   node->next = __kmp_old_threads_list;  in __kmp_expand_threads()
3784   added += newCapacity - __kmp_threads_capacity;  in __kmp_expand_threads()
3812   /* 2007-03-02:  in __kmp_register_root()
3815      work as expected -- it may return false (that means there is at least one  in __kmp_register_root()
3828     --capacity;  in __kmp_register_root()
3835     capacity -= __kmp_hidden_helper_threads_num;  in __kmp_register_root()
3913     KMP_DEBUG_ASSERT(!root->r.r_root_team);  in __kmp_register_root()
3918   __kmp_stats_thread_ptr = __kmp_stats_list->push_back(gtid);  in __kmp_register_root()
3919   __kmp_stats_thread_ptr->startLife();  in __kmp_register_root()
3926   if (root->r.r_uber_thread) {  in __kmp_register_root()
3927     root_thread = root->r.r_uber_thread;  in __kmp_register_root()
3933     root_thread->th.th_info.ds.ds_gtid = gtid;  in __kmp_register_root()
3935     root_thread->th.ompt_thread_info.thread_data = ompt_data_none;  in __kmp_register_root()
3937     root_thread->th.th_root = root;  in __kmp_register_root()
3939       root_thread->th.th_cons = __kmp_allocate_cons_stack(gtid);  in __kmp_register_root()
3946     KMP_DEBUG_ASSERT(root_thread->th.th_local.bget_data == NULL);  in __kmp_register_root()
3953   if (!root_thread->th.th_serial_team) {  in __kmp_register_root()
3956     root_thread->th.th_serial_team = __kmp_allocate_team(  in __kmp_register_root()
3963   KMP_ASSERT(root_thread->th.th_serial_team);  in __kmp_register_root()
3965                 root_thread->th.th_serial_team));  in __kmp_register_root()
3970   root->r.r_root_team->t.t_threads[0] = root_thread;  in __kmp_register_root()
3971   root->r.r_hot_team->t.t_threads[0] = root_thread;  in __kmp_register_root()
3972   root_thread->th.th_serial_team->t.t_threads[0] = root_thread;  in __kmp_register_root()
3974   root_thread->th.th_serial_team->t.t_serialized = 0;  in __kmp_register_root()
3975   root->r.r_uber_thread = root_thread;  in __kmp_register_root()
3978   __kmp_initialize_info(root_thread, root->r.r_root_team, 0, gtid);  in __kmp_register_root()
3996                 gtid, __kmp_gtid_from_tid(0, root->r.r_hot_team),  in __kmp_register_root()
3997                 root->r.r_hot_team->t.t_id, 0, KMP_INIT_BARRIER_STATE,  in __kmp_register_root()
4002       root_thread->th.th_bar[b].bb.b_arrived = KMP_INIT_BARRIER_STATE;  in __kmp_register_root()
4004       root_thread->th.th_bar[b].bb.b_worker_arrived = 0;  in __kmp_register_root()
4008   KMP_DEBUG_ASSERT(root->r.r_hot_team->t.t_bar[bs_forkjoin_barrier].b_arrived ==  in __kmp_register_root()
4012   root_thread->th.th_current_place = KMP_PLACE_UNDEFINED;  in __kmp_register_root()
4013   root_thread->th.th_new_place = KMP_PLACE_UNDEFINED;  in __kmp_register_root()
4014   root_thread->th.th_first_place = KMP_PLACE_UNDEFINED;  in __kmp_register_root()
4015   root_thread->th.th_last_place = KMP_PLACE_UNDEFINED;  in __kmp_register_root()
4017   root_thread->th.th_def_allocator = __kmp_def_allocator;  in __kmp_register_root()
4018   root_thread->th.th_prev_level = 0;  in __kmp_register_root()
4019   root_thread->th.th_prev_num_threads = 1;  in __kmp_register_root()
4022   tmp->cg_root = root_thread;  in __kmp_register_root()
4023   tmp->cg_thread_limit = __kmp_cg_max_nth;  in __kmp_register_root()
4024   tmp->cg_nthreads = 1;  in __kmp_register_root()
4028   tmp->up = NULL;  in __kmp_register_root()
4029   root_thread->th.th_cg_roots = tmp;  in __kmp_register_root()
4071   kmp_hot_team_ptr_t *hot_teams = thr->th.th_hot_teams;  in __kmp_free_hot_teams()
4078   n = nth - 1; // primary thread is not freed  in __kmp_free_hot_teams()
4079   if (level < max_level - 1) {  in __kmp_free_hot_teams()
4081       kmp_info_t *th = team->t.t_threads[i];  in __kmp_free_hot_teams()
4083       if (i > 0 && th->th.th_hot_teams) {  in __kmp_free_hot_teams()
4084         __kmp_free(th->th.th_hot_teams);  in __kmp_free_hot_teams()
4085         th->th.th_hot_teams = NULL;  in __kmp_free_hot_teams()
4097   kmp_team_t *root_team = root->r.r_root_team;  in __kmp_reset_root()
4098   kmp_team_t *hot_team = root->r.r_hot_team;  in __kmp_reset_root()
4099   int n = hot_team->t.t_nproc;  in __kmp_reset_root()
4102   KMP_DEBUG_ASSERT(!root->r.r_active);  in __kmp_reset_root()
4104   root->r.r_root_team = NULL;  in __kmp_reset_root()
4105   root->r.r_hot_team = NULL;  in __kmp_reset_root()
4112     for (i = 0; i < hot_team->t.t_nproc; ++i) {  in __kmp_reset_root()
4113       kmp_info_t *th = hot_team->t.t_threads[i];  in __kmp_reset_root()
4117       if (th->th.th_hot_teams) {  in __kmp_reset_root()
4118         __kmp_free(th->th.th_hot_teams);  in __kmp_reset_root()
4119         th->th.th_hot_teams = NULL;  in __kmp_reset_root()
4138            (LPVOID) & (root->r.r_uber_thread->th),  in __kmp_reset_root()
4139            root->r.r_uber_thread->th.th_info.ds.ds_thread));  in __kmp_reset_root()
4140   __kmp_free_handle(root->r.r_uber_thread->th.th_info.ds.ds_thread);  in __kmp_reset_root()
4159         &(root->r.r_uber_thread->th.ompt_thread_info.thread_data));  in __kmp_reset_root()
4164         __kmp_nth - 1); // __kmp_reap_thread will decrement __kmp_all_nth.  in __kmp_reset_root()
4165   i = root->r.r_uber_thread->th.th_cg_roots->cg_nthreads--;  in __kmp_reset_root()
4168                  root->r.r_uber_thread, root->r.r_uber_thread->th.th_cg_roots,  in __kmp_reset_root()
4169                  root->r.r_uber_thread->th.th_cg_roots->cg_nthreads));  in __kmp_reset_root()
4172     KMP_DEBUG_ASSERT(root->r.r_uber_thread ==  in __kmp_reset_root()
4173                      root->r.r_uber_thread->th.th_cg_roots->cg_root);  in __kmp_reset_root()
4174     KMP_DEBUG_ASSERT(root->r.r_uber_thread->th.th_cg_roots->up == NULL);  in __kmp_reset_root()
4175     __kmp_free(root->r.r_uber_thread->th.th_cg_roots);  in __kmp_reset_root()
4176     root->r.r_uber_thread->th.th_cg_roots = NULL;  in __kmp_reset_root()
4178   __kmp_reap_thread(root->r.r_uber_thread, 1);  in __kmp_reset_root()
4182   root->r.r_uber_thread = NULL;  in __kmp_reset_root()
4184   root->r.r_begin = FALSE;  in __kmp_reset_root()
4206   KMP_ASSERT(root == __kmp_threads[gtid]->th.th_root);  in __kmp_unregister_root_current_thread()
4207   KMP_ASSERT(root->r.r_active == FALSE);  in __kmp_unregister_root_current_thread()
4212   kmp_team_t *team = thread->th.th_team;  in __kmp_unregister_root_current_thread()
4213   kmp_task_team_t *task_team = thread->th.th_task_team;  in __kmp_unregister_root_current_thread()
4216   if (task_team != NULL && (task_team->tt.tt_found_proxy_tasks ||  in __kmp_unregister_root_current_thread()
4217                             task_team->tt.tt_hidden_helper_task_encountered)) {  in __kmp_unregister_root_current_thread()
4220     thread->th.ompt_thread_info.state = ompt_state_undefined;  in __kmp_unregister_root_current_thread()
4245   KMP_ASSERT(root == __kmp_threads[gtid]->th.th_root);  in __kmp_unregister_root_other_thread()
4246   KMP_ASSERT(root->r.r_active == FALSE);  in __kmp_unregister_root_other_thread()
4261   kmp_team_t *steam = this_thr->th.th_serial_team;  in __kmp_task_info()
4262   kmp_team_t *team = this_thr->th.th_team;  in __kmp_task_info()
4267       gtid, tid, this_thr, team, steam, this_thr->th.th_current_task,  in __kmp_task_info()
4268       team->t.t_implicit_task_taskdata[tid].td_parent);  in __kmp_task_info()
4277   /* this_thr->th.th_info.ds.ds_gtid is setup in  in __kmp_initialize_info()
4279      this_thr->th.th_serial_team is setup in __kmp_allocate_thread */  in __kmp_initialize_info()
4281   KMP_DEBUG_ASSERT(this_thr->th.th_serial_team);  in __kmp_initialize_info()
4283   KMP_DEBUG_ASSERT(team->t.t_threads);  in __kmp_initialize_info()
4284   KMP_DEBUG_ASSERT(team->t.t_dispatch);  in __kmp_initialize_info()
4285   kmp_info_t *master = team->t.t_threads[0];  in __kmp_initialize_info()
4287   KMP_DEBUG_ASSERT(master->th.th_root);  in __kmp_initialize_info()
4291   TCW_SYNC_PTR(this_thr->th.th_team, team);  in __kmp_initialize_info()
4293   this_thr->th.th_info.ds.ds_tid = tid;  in __kmp_initialize_info()
4294   this_thr->th.th_set_nproc = 0;  in __kmp_initialize_info()
4298     this_thr->th.th_reap_state = KMP_NOT_SAFE_TO_REAP;  in __kmp_initialize_info()
4299   else // no tasking --> always safe to reap  in __kmp_initialize_info()
4300     this_thr->th.th_reap_state = KMP_SAFE_TO_REAP;  in __kmp_initialize_info()
4301   this_thr->th.th_set_proc_bind = proc_bind_default;  in __kmp_initialize_info()
4304   this_thr->th.th_new_place = this_thr->th.th_current_place;  in __kmp_initialize_info()
4306   this_thr->th.th_root = master->th.th_root;  in __kmp_initialize_info()
4309   this_thr->th.th_team_nproc = team->t.t_nproc;  in __kmp_initialize_info()
4310   this_thr->th.th_team_master = master;  in __kmp_initialize_info()
4311   this_thr->th.th_team_serialized = team->t.t_serialized;  in __kmp_initialize_info()
4313   KMP_DEBUG_ASSERT(team->t.t_implicit_task_taskdata);  in __kmp_initialize_info()
4316                 tid, gtid, this_thr, this_thr->th.th_current_task));  in __kmp_initialize_info()
4318   __kmp_init_implicit_task(this_thr->th.th_team_master->th.th_ident, this_thr,  in __kmp_initialize_info()
4322                 tid, gtid, this_thr, this_thr->th.th_current_task));  in __kmp_initialize_info()
4323   // TODO: Initialize ICVs from parent; GEH - isn't that already done in  in __kmp_initialize_info()
4327   this_thr->th.th_dispatch = &team->t.t_dispatch[tid];  in __kmp_initialize_info()
4329   this_thr->th.th_local.this_construct = 0;  in __kmp_initialize_info()
4331   if (!this_thr->th.th_pri_common) {  in __kmp_initialize_info()
4332     this_thr->th.th_pri_common =  in __kmp_initialize_info()
4336           gtid, this_thr->th.th_pri_common, this_thr->th.th_pri_common + 1,  in __kmp_initialize_info()
4339     this_thr->th.th_pri_head = NULL;  in __kmp_initialize_info()
4343       this_thr->th.th_cg_roots != master->th.th_cg_roots) { // CG root not set  in __kmp_initialize_info()
4345     KMP_DEBUG_ASSERT(master->th.th_cg_roots);  in __kmp_initialize_info()
4346     kmp_cg_root_t *tmp = this_thr->th.th_cg_roots;  in __kmp_initialize_info()
4349       int i = tmp->cg_nthreads--;  in __kmp_initialize_info()
4352                      this_thr, tmp, tmp->cg_root, tmp->cg_nthreads));  in __kmp_initialize_info()
4354         __kmp_free(tmp); // last thread left CG --> free it  in __kmp_initialize_info()
4357     this_thr->th.th_cg_roots = master->th.th_cg_roots;  in __kmp_initialize_info()
4359     this_thr->th.th_cg_roots->cg_nthreads++;  in __kmp_initialize_info()
4362                    this_thr, this_thr->th.th_cg_roots,  in __kmp_initialize_info()
4363                    this_thr->th.th_cg_roots->cg_root,  in __kmp_initialize_info()
4364                    this_thr->th.th_cg_roots->cg_nthreads));  in __kmp_initialize_info()
4365     this_thr->th.th_current_task->td_icvs.thread_limit =  in __kmp_initialize_info()
4366         this_thr->th.th_cg_roots->cg_thread_limit;  in __kmp_initialize_info()
4371     volatile kmp_disp_t *dispatch = this_thr->th.th_dispatch;  in __kmp_initialize_info()
4375         (team->t.t_max_nproc == 1 ? 1 : __kmp_dispatch_num_buffers);  in __kmp_initialize_info()
4377                   team->t.t_max_nproc));  in __kmp_initialize_info()
4379     KMP_DEBUG_ASSERT(team->t.t_dispatch);  in __kmp_initialize_info()
4380     KMP_DEBUG_ASSERT(dispatch == &team->t.t_dispatch[tid]);  in __kmp_initialize_info()
4382     dispatch->th_disp_index = 0;  in __kmp_initialize_info()
4383     dispatch->th_doacross_buf_idx = 0;  in __kmp_initialize_info()
4384     if (!dispatch->th_disp_buffer) {  in __kmp_initialize_info()
4385       dispatch->th_disp_buffer =  in __kmp_initialize_info()
4390             gtid, &dispatch->th_disp_buffer[0],  in __kmp_initialize_info()
4391             &dispatch->th_disp_buffer[team->t.t_max_nproc == 1  in __kmp_initialize_info()
4397             gtid, team->t.t_id, gtid);  in __kmp_initialize_info()
4400       memset(&dispatch->th_disp_buffer[0], '\0', disp_size);  in __kmp_initialize_info()
4403     dispatch->th_dispatch_pr_current = 0;  in __kmp_initialize_info()
4404     dispatch->th_dispatch_sh_current = 0;  in __kmp_initialize_info()
4406     dispatch->th_deo_fcn = 0; /* ORDERED     */  in __kmp_initialize_info()
4407     dispatch->th_dxo_fcn = 0; /* END ORDERED */  in __kmp_initialize_info()
4410   this_thr->th.th_next_pool = NULL;  in __kmp_initialize_info()
4412   KMP_DEBUG_ASSERT(!this_thr->th.th_spin_here);  in __kmp_initialize_info()
4413   KMP_DEBUG_ASSERT(this_thr->th.th_next_waiting == 0);  in __kmp_initialize_info()
4441     __kmp_thread_pool = (volatile kmp_info_t *)new_thr->th.th_next_pool;  in __kmp_allocate_thread()
4445     TCW_4(new_thr->th.th_in_pool, FALSE);  in __kmp_allocate_thread()
4448     if (new_thr->th.th_active_in_pool == TRUE) {  in __kmp_allocate_thread()
4449       KMP_DEBUG_ASSERT(new_thr->th.th_active == TRUE);  in __kmp_allocate_thread()
4451       new_thr->th.th_active_in_pool = FALSE;  in __kmp_allocate_thread()
4456                   __kmp_get_gtid(), new_thr->th.th_info.ds.ds_gtid));  in __kmp_allocate_thread()
4457     KMP_ASSERT(!new_thr->th.th_team);  in __kmp_allocate_thread()
4462                           new_thr->th.th_info.ds.ds_gtid);  in __kmp_allocate_thread()
4463     KMP_DEBUG_ASSERT(new_thr->th.th_serial_team);  in __kmp_allocate_thread()
4467     new_thr->th.th_task_state = 0;  in __kmp_allocate_thread()
4471       KMP_DEBUG_ASSERT(new_thr->th.th_used_in_team.load() == 0);  in __kmp_allocate_thread()
4489     kmp_balign_t *balign = new_thr->th.th_bar;  in __kmp_allocate_thread()
4495                   __kmp_get_gtid(), new_thr, new_thr->th.th_info.ds.ds_gtid));  in __kmp_allocate_thread()
4555   new_thr->th.th_nt_strict = false;  in __kmp_allocate_thread()
4556   new_thr->th.th_nt_loc = NULL;  in __kmp_allocate_thread()
4557   new_thr->th.th_nt_sev = severity_fatal;  in __kmp_allocate_thread()
4558   new_thr->th.th_nt_msg = NULL;  in __kmp_allocate_thread()
4567       &new_thr->th.th_sleep_loc, sizeof(new_thr->th.th_sleep_loc));  in __kmp_allocate_thread()
4570       &new_thr->th.th_reap_state, sizeof(new_thr->th.th_reap_state));  in __kmp_allocate_thread()
4574       &new_thr->th.th_suspend_init, sizeof(new_thr->th.th_suspend_init));  in __kmp_allocate_thread()
4578                             &new_thr->th.th_suspend_init_count,  in __kmp_allocate_thread()
4579                             sizeof(new_thr->th.th_suspend_init_count));  in __kmp_allocate_thread()
4584                             CCAST(kmp_uint64 *, &new_thr->th.th_bar[0].bb.b_go),  in __kmp_allocate_thread()
4585                             sizeof(new_thr->th.th_bar[0].bb.b_go));  in __kmp_allocate_thread()
4588                             CCAST(kmp_uint64 *, &new_thr->th.th_bar[1].bb.b_go),  in __kmp_allocate_thread()
4589                             sizeof(new_thr->th.th_bar[1].bb.b_go));  in __kmp_allocate_thread()
4592                             CCAST(kmp_uint64 *, &new_thr->th.th_bar[2].bb.b_go),  in __kmp_allocate_thread()
4593                             sizeof(new_thr->th.th_bar[2].bb.b_go));  in __kmp_allocate_thread()
4603     new_thr->th.th_serial_team = serial_team =  in __kmp_allocate_thread()
4612   serial_team->t.t_serialized = 0; // AC: the team created in reserve, not for  in __kmp_allocate_thread()
4614   serial_team->t.t_threads[0] = new_thr;  in __kmp_allocate_thread()
4627   KMP_DEBUG_ASSERT(new_thr->th.th_local.bget_data == NULL);  in __kmp_allocate_thread()
4639   kmp_balign_t *balign = new_thr->th.th_bar;  in __kmp_allocate_thread()
4647   TCW_PTR(new_thr->th.th_sleep_loc, NULL);  in __kmp_allocate_thread()
4648   new_thr->th.th_sleep_loc_type = flag_unset;  in __kmp_allocate_thread()
4650   new_thr->th.th_spin_here = FALSE;  in __kmp_allocate_thread()
4651   new_thr->th.th_next_waiting = 0;  in __kmp_allocate_thread()
4653   new_thr->th.th_blocking = false;  in __kmp_allocate_thread()
4657   new_thr->th.th_current_place = KMP_PLACE_UNDEFINED;  in __kmp_allocate_thread()
4658   new_thr->th.th_new_place = KMP_PLACE_UNDEFINED;  in __kmp_allocate_thread()
4659   new_thr->th.th_first_place = KMP_PLACE_UNDEFINED;  in __kmp_allocate_thread()
4660   new_thr->th.th_last_place = KMP_PLACE_UNDEFINED;  in __kmp_allocate_thread()
4662   new_thr->th.th_def_allocator = __kmp_def_allocator;  in __kmp_allocate_thread()
4663   new_thr->th.th_prev_level = 0;  in __kmp_allocate_thread()
4664   new_thr->th.th_prev_num_threads = 1;  in __kmp_allocate_thread()
4666   TCW_4(new_thr->th.th_in_pool, FALSE);  in __kmp_allocate_thread()
4667   new_thr->th.th_active_in_pool = FALSE;  in __kmp_allocate_thread()
4668   TCW_4(new_thr->th.th_active, TRUE);  in __kmp_allocate_thread()
4670   new_thr->th.th_set_nested_nth = NULL;  in __kmp_allocate_thread()
4671   new_thr->th.th_set_nested_nth_sz = 0;  in __kmp_allocate_thread()
4723    IF YOU TOUCH THIS ROUTINE, RUN EPCC C SYNCBENCH ON A BIG-IRON MACHINE!!! */
4728                 team->t.t_threads[0], team));  in __kmp_reinitialize_team()
4730   KMP_DEBUG_ASSERT((!TCR_4(__kmp_init_parallel)) || new_icvs->nproc);  in __kmp_reinitialize_team()
4731   KMP_CHECK_UPDATE(team->t.t_ident, loc);  in __kmp_reinitialize_team()
4733   KMP_CHECK_UPDATE(team->t.t_id, KMP_GEN_TEAM_ID());  in __kmp_reinitialize_team()
4735   __kmp_init_implicit_task(loc, team->t.t_threads[0], team, 0, FALSE);  in __kmp_reinitialize_team()
4736   copy_icvs(&team->t.t_implicit_task_taskdata[0].td_icvs, new_icvs);  in __kmp_reinitialize_team()
4739                 team->t.t_threads[0], team));  in __kmp_reinitialize_team()
4752   KMP_DEBUG_ASSERT(new_nproc <= team->t.t_max_nproc);  in __kmp_initialize_team()
4753   KMP_DEBUG_ASSERT(team->t.t_threads);  in __kmp_initialize_team()
4756   team->t.t_master_tid = 0; /* not needed */  in __kmp_initialize_team()
4757   /* team->t.t_master_bar;        not needed */  in __kmp_initialize_team()
4758   team->t.t_serialized = new_nproc > 1 ? 0 : 1;  in __kmp_initialize_team()
4759   team->t.t_nproc = new_nproc;  in __kmp_initialize_team()
4761   /* team->t.t_parent     = NULL; TODO not needed & would mess up hot team */  in __kmp_initialize_team()
4762   team->t.t_next_pool = NULL;  in __kmp_initialize_team()
4763   /* memset( team->t.t_threads, 0, sizeof(kmp_info_t*)*new_nproc ); would mess  in __kmp_initialize_team()
4766   TCW_SYNC_PTR(team->t.t_pkfn, NULL); /* not needed */  in __kmp_initialize_team()
4767   team->t.t_invoke = NULL; /* not needed */  in __kmp_initialize_team()
4769   // TODO???: team->t.t_max_active_levels       = new_max_active_levels;  in __kmp_initialize_team()
4770   team->t.t_sched.sched = new_icvs->sched.sched;  in __kmp_initialize_team()
4773   team->t.t_fp_control_saved = FALSE; /* not needed */  in __kmp_initialize_team()
4774   team->t.t_x87_fpu_control_word = 0; /* not needed */  in __kmp_initialize_team()
4775   team->t.t_mxcsr = 0; /* not needed */  in __kmp_initialize_team()
4778   team->t.t_construct = 0;  in __kmp_initialize_team()
4780   team->t.t_ordered.dt.t_value = 0;  in __kmp_initialize_team()
4781   team->t.t_master_active = FALSE;  in __kmp_initialize_team()
4784   team->t.t_copypriv_data = NULL; /* not necessary, but nice for debugging */  in __kmp_initialize_team()
4787   team->t.t_copyin_counter = 0; /* for barrier-free copyin implementation */  in __kmp_initialize_team()
4790   team->t.t_control_stack_top = NULL;  in __kmp_initialize_team()
4801   th->th.th_first_place = first;  in __kmp_set_thread_place()
4802   th->th.th_last_place = last;  in __kmp_set_thread_place()
4803   th->th.th_new_place = newp;  in __kmp_set_thread_place()
4804   if (newp != th->th.th_current_place) {  in __kmp_set_thread_place()
4805     if (__kmp_display_affinity && team->t.t_display_affinity != 1)  in __kmp_set_thread_place()
4806       team->t.t_display_affinity = 1;  in __kmp_set_thread_place()
4808     th->th.th_topology_ids = __kmp_affinity.ids[th->th.th_new_place];  in __kmp_set_thread_place()
4809     th->th.th_topology_attrs = __kmp_affinity.attrs[th->th.th_new_place];  in __kmp_set_thread_place()
4822   kmp_info_t *master_th = team->t.t_threads[0];  in __kmp_partition_places()
4824   kmp_proc_bind_t proc_bind = team->t.t_proc_bind;  in __kmp_partition_places()
4825   int first_place = master_th->th.th_first_place;  in __kmp_partition_places()
4826   int last_place = master_th->th.th_last_place;  in __kmp_partition_places()
4827   int masters_place = master_th->th.th_current_place;  in __kmp_partition_places()
4829   team->t.t_first_place = first_place;  in __kmp_partition_places()
4830   team->t.t_last_place = last_place;  in __kmp_partition_places()
4834                 proc_bind, __kmp_gtid_from_thread(team->t.t_threads[0]),  in __kmp_partition_places()
4835                 team->t.t_id, masters_place, first_place, last_place));  in __kmp_partition_places()
4841     // Not an issue -- we don't rebind primary thread for any proc_bind policy.  in __kmp_partition_places()
4842     KMP_DEBUG_ASSERT(team->t.t_nproc == 1);  in __kmp_partition_places()
4847     int n_th = team->t.t_nproc;  in __kmp_partition_places()
4849       kmp_info_t *th = team->t.t_threads[f];  in __kmp_partition_places()
4855                      __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id,  in __kmp_partition_places()
4862     int n_th = team->t.t_nproc;  in __kmp_partition_places()
4865       n_places = last_place - first_place + 1;  in __kmp_partition_places()
4867       n_places = num_masks - first_place + last_place + 1;  in __kmp_partition_places()
4872         kmp_info_t *th = team->t.t_threads[f];  in __kmp_partition_places()
4877         } else if (place == (num_masks - 1)) {  in __kmp_partition_places()
4886                        __kmp_gtid_from_thread(team->t.t_threads[f]),  in __kmp_partition_places()
4887                        team->t.t_id, f, place, first_place, last_place));  in __kmp_partition_places()
4893       rem = n_th - (S * n_places);  in __kmp_partition_places()
4898         kmp_info_t *th = team->t.t_threads[f];  in __kmp_partition_places()
4910           } else if (place == (num_masks - 1)) {  in __kmp_partition_places()
4917           rem--;  in __kmp_partition_places()
4921           } else if (place == (num_masks - 1)) {  in __kmp_partition_places()
4933                   __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id, f,  in __kmp_partition_places()
4934                   th->th.th_new_place, first_place, last_place));  in __kmp_partition_places()
4942     int n_th = team->t.t_nproc;  in __kmp_partition_places()
4946       n_places = last_place - first_place + 1;  in __kmp_partition_places()
4948       n_places = num_masks - first_place + last_place + 1;  in __kmp_partition_places()
4951       int place = -1;  in __kmp_partition_places()
4958         rem = n_places - n_th * S;  in __kmp_partition_places()
4965           kmp_info_t *th = team->t.t_threads[f];  in __kmp_partition_places()
4973             } else if (place == (num_masks - 1)) {  in __kmp_partition_places()
4983             } else if (place == (num_masks - 1)) {  in __kmp_partition_places()
4988             rem--;  in __kmp_partition_places()
4996           } else if (place == (num_masks - 1)) {  in __kmp_partition_places()
5005                     __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id,  in __kmp_partition_places()
5006                     f, th->th.th_new_place, th->th.th_first_place,  in __kmp_partition_places()
5007                     th->th.th_last_place, num_masks));  in __kmp_partition_places()
5024           last = static_cast<int>(current + spacing) - 1;  in __kmp_partition_places()
5028               first -= n_places;  in __kmp_partition_places()
5029               last -= n_places;  in __kmp_partition_places()
5032                 first--;  in __kmp_partition_places()
5035                 KMP_DEBUG_ASSERT(f == (n_th - 1));  in __kmp_partition_places()
5036                 last--;  in __kmp_partition_places()
5045             last = (n_places - 1);  in __kmp_partition_places()
5055             th = team->t.t_threads[f];  in __kmp_partition_places()
5061                       __kmp_gtid_from_thread(team->t.t_threads[f]),  in __kmp_partition_places()
5062                       team->t.t_id, f, th->th.th_new_place,  in __kmp_partition_places()
5063                       th->th.th_first_place, th->th.th_last_place, spacing));  in __kmp_partition_places()
5072       rem = n_th - (S * n_places);  in __kmp_partition_places()
5080         kmp_info_t *th = team->t.t_threads[f];  in __kmp_partition_places()
5092           } else if (place == (num_masks - 1)) {  in __kmp_partition_places()
5099           rem--;  in __kmp_partition_places()
5103           } else if (place == (num_masks - 1)) {  in __kmp_partition_places()
5114                        __kmp_gtid_from_thread(team->t.t_threads[f]),  in __kmp_partition_places()
5115                        team->t.t_id, f, th->th.th_new_place,  in __kmp_partition_places()
5116                        th->th.th_first_place, th->th.th_last_place));  in __kmp_partition_places()
5126   KA_TRACE(20, ("__kmp_partition_places: exit T#%d\n", team->t.t_id));  in __kmp_partition_places()
5144   int use_hot_team = !root->r.r_active;  in __kmp_allocate_team()
5156     team = master->th.th_team;  in __kmp_allocate_team()
5157     level = team->t.t_active_level;  in __kmp_allocate_team()
5158     if (master->th.th_teams_microtask) { // in teams construct?  in __kmp_allocate_team()
5159       if (master->th.th_teams_size.nteams > 1 &&  in __kmp_allocate_team()
5161               team->t.t_pkfn ==  in __kmp_allocate_team()
5163               master->th.th_teams_level <  in __kmp_allocate_team()
5164                   team->t.t_level)) { // or nested parallel inside the teams  in __kmp_allocate_team()
5170       if ((master->th.th_teams_size.nteams == 1 &&  in __kmp_allocate_team()
5171            master->th.th_teams_level >= team->t.t_level) ||  in __kmp_allocate_team()
5172           (team->t.t_pkfn == (microtask_t)__kmp_teams_master))  in __kmp_allocate_team()
5175     hot_teams = master->th.th_hot_teams;  in __kmp_allocate_team()
5194     team = root->r.r_hot_team;  in __kmp_allocate_team()
5200                     team->t.t_task_team[0], team->t.t_task_team[1]));  in __kmp_allocate_team()
5204     if (team->t.t_nproc != new_nproc &&  in __kmp_allocate_team()
5207       int old_nthr = team->t.t_nproc;  in __kmp_allocate_team()
5214       team->t.t_proc_bind = proc_bind_default;  in __kmp_allocate_team()
5218     if (team->t.t_nproc == new_nproc) { // Check changes in number of threads  in __kmp_allocate_team()
5222       if (team->t.t_size_changed == -1) {  in __kmp_allocate_team()
5223         team->t.t_size_changed = 1;  in __kmp_allocate_team()
5225         KMP_CHECK_UPDATE(team->t.t_size_changed, 0);  in __kmp_allocate_team()
5228       // TODO???: team->t.t_max_active_levels = new_max_active_levels;  in __kmp_allocate_team()
5229       kmp_r_sched_t new_sched = new_icvs->sched;  in __kmp_allocate_team()
5230       // set primary thread's schedule as new run-time schedule  in __kmp_allocate_team()
5231       KMP_CHECK_UPDATE(team->t.t_sched.sched, new_sched.sched);  in __kmp_allocate_team()
5234                               root->r.r_uber_thread->th.th_ident);  in __kmp_allocate_team()
5237                     team->t.t_threads[0], team));  in __kmp_allocate_team()
5238       __kmp_push_current_task_to_thread(team->t.t_threads[0], team, 0);  in __kmp_allocate_team()
5241       if ((team->t.t_size_changed == 0) &&  in __kmp_allocate_team()
5242           (team->t.t_proc_bind == new_proc_bind)) {  in __kmp_allocate_team()
5251                        team->t.t_id, new_proc_bind, team->t.t_first_place,  in __kmp_allocate_team()
5252                        team->t.t_last_place));  in __kmp_allocate_team()
5255           KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind);  in __kmp_allocate_team()
5260       KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind);  in __kmp_allocate_team()
5262     } else if (team->t.t_nproc > new_nproc) {  in __kmp_allocate_team()
5267       team->t.t_size_changed = 1;  in __kmp_allocate_team()
5276         for (f = new_nproc; f < team->t.t_nproc; f++) {  in __kmp_allocate_team()
5277           kmp_info_t *th = team->t.t_threads[f];  in __kmp_allocate_team()
5279           th->th.th_task_team = NULL;  in __kmp_allocate_team()
5286         KMP_DEBUG_ASSERT(hot_teams[level].hot_team_nth == team->t.t_nproc);  in __kmp_allocate_team()
5290         for (f = new_nproc; f < team->t.t_nproc; f++) {  in __kmp_allocate_team()
5291           KMP_DEBUG_ASSERT(team->t.t_threads[f]);  in __kmp_allocate_team()
5292           __kmp_free_thread(team->t.t_threads[f]);  in __kmp_allocate_team()
5293           team->t.t_threads[f] = NULL;  in __kmp_allocate_team()
5300         for (f = new_nproc; f < team->t.t_nproc; ++f) {  in __kmp_allocate_team()
5301           KMP_DEBUG_ASSERT(team->t.t_threads[f]);  in __kmp_allocate_team()
5302           kmp_balign_t *balign = team->t.t_threads[f]->th.th_bar;  in __kmp_allocate_team()
5312       team->t.t_nproc = new_nproc;  in __kmp_allocate_team()
5313       // TODO???: team->t.t_max_active_levels = new_max_active_levels;  in __kmp_allocate_team()
5314       KMP_CHECK_UPDATE(team->t.t_sched.sched, new_icvs->sched.sched);  in __kmp_allocate_team()
5316                               root->r.r_uber_thread->th.th_ident);  in __kmp_allocate_team()
5320         team->t.t_threads[f]->th.th_team_nproc = new_nproc;  in __kmp_allocate_team()
5326                     team->t.t_threads[0], team));  in __kmp_allocate_team()
5328       __kmp_push_current_task_to_thread(team->t.t_threads[0], team, 0);  in __kmp_allocate_team()
5331       for (f = 0; f < team->t.t_nproc; f++) {  in __kmp_allocate_team()
5332         KMP_DEBUG_ASSERT(team->t.t_threads[f] &&  in __kmp_allocate_team()
5333                          team->t.t_threads[f]->th.th_team_nproc ==  in __kmp_allocate_team()
5334                              team->t.t_nproc);  in __kmp_allocate_team()
5339         KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind);  in __kmp_allocate_team()
5344     } else { // team->t.t_nproc < new_nproc  in __kmp_allocate_team()
5349       int old_nproc = team->t.t_nproc; // save old value and use to update only  in __kmp_allocate_team()
5350       team->t.t_size_changed = 1;  in __kmp_allocate_team()
5356       kmp_info_t **other_threads = team->t.t_threads;  in __kmp_allocate_team()
5357       for (f = team->t.t_nproc; f < avail_threads; ++f) {  in __kmp_allocate_team()
5361         kmp_balign_t *balign = other_threads[f]->th.th_bar;  in __kmp_allocate_team()
5363           balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;  in __kmp_allocate_team()
5366           balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;  in __kmp_allocate_team()
5374         team->t.t_nproc = new_nproc; // just get reserved threads involved  in __kmp_allocate_team()
5378         team->t.t_nproc = hot_teams[level].hot_team_nth;  in __kmp_allocate_team()
5381         if (team->t.t_max_nproc < new_nproc) {  in __kmp_allocate_team()
5398         for (f = team->t.t_nproc; f < new_nproc; f++) {  in __kmp_allocate_team()
5401           team->t.t_threads[f] = new_worker;  in __kmp_allocate_team()
5406                     team->t.t_id, __kmp_gtid_from_tid(f, team), team->t.t_id, f,  in __kmp_allocate_team()
5407                     team->t.t_bar[bs_forkjoin_barrier].b_arrived,  in __kmp_allocate_team()
5408                     team->t.t_bar[bs_plain_barrier].b_arrived));  in __kmp_allocate_team()
5412             kmp_balign_t *balign = new_worker->th.th_bar;  in __kmp_allocate_team()
5414               balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;  in __kmp_allocate_team()
5418               balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;  in __kmp_allocate_team()
5440                             root->r.r_uber_thread->th.th_ident);  in __kmp_allocate_team()
5443       KMP_DEBUG_ASSERT(team->t.t_nproc == new_nproc);  in __kmp_allocate_team()
5444       for (f = 0; f < team->t.t_nproc; ++f)  in __kmp_allocate_team()
5445         __kmp_initialize_info(team->t.t_threads[f], team, f,  in __kmp_allocate_team()
5449       kmp_uint8 old_state = team->t.t_threads[old_nproc - 1]->th.th_task_state;  in __kmp_allocate_team()
5450       for (f = old_nproc; f < team->t.t_nproc; ++f)  in __kmp_allocate_team()
5451         team->t.t_threads[f]->th.th_task_state = old_state;  in __kmp_allocate_team()
5454       for (f = 0; f < team->t.t_nproc; ++f) {  in __kmp_allocate_team()
5455         KMP_DEBUG_ASSERT(team->t.t_threads[f] &&  in __kmp_allocate_team()
5456                          team->t.t_threads[f]->th.th_team_nproc ==  in __kmp_allocate_team()
5457                              team->t.t_nproc);  in __kmp_allocate_team()
5462         KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind);  in __kmp_allocate_team()
5469     if (master->th.th_teams_microtask) {  in __kmp_allocate_team()
5472         kmp_info_t *thr = team->t.t_threads[f];  in __kmp_allocate_team()
5473         thr->th.th_teams_microtask = master->th.th_teams_microtask;  in __kmp_allocate_team()
5474         thr->th.th_teams_level = master->th.th_teams_level;  in __kmp_allocate_team()
5475         thr->th.th_teams_size = master->th.th_teams_size;  in __kmp_allocate_team()
5483         kmp_info_t *thr = team->t.t_threads[f];  in __kmp_allocate_team()
5485         kmp_balign_t *balign = thr->th.th_bar;  in __kmp_allocate_team()
5487           balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;  in __kmp_allocate_team()
5490           balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;  in __kmp_allocate_team()
5499     KMP_CHECK_UPDATE(team->t.t_argc, argc);  in __kmp_allocate_team()
5500     // The hot team re-uses the previous task team,  in __kmp_allocate_team()
5501     // if untouched during the previous release->gather phase.  in __kmp_allocate_team()
5509                     team->t.t_task_team[0], team->t.t_task_team[1]));  in __kmp_allocate_team()
5527     if (team->t.t_max_nproc >= max_nproc) {  in __kmp_allocate_team()
5529       __kmp_team_pool = team->t.t_next_pool;  in __kmp_allocate_team()
5533         if (!team->t.b) { // Allocate barrier structure  in __kmp_allocate_team()
5534           team->t.b = distributedBarrier::allocate(__kmp_dflt_team_nth_ub);  in __kmp_allocate_team()
5543                     &team->t.t_task_team[0], &team->t.t_task_team[1]));  in __kmp_allocate_team()
5544       team->t.t_task_team[0] = NULL;  in __kmp_allocate_team()
5545       team->t.t_task_team[1] = NULL;  in __kmp_allocate_team()
5549       KMP_CHECK_UPDATE(team->t.t_argc, argc);  in __kmp_allocate_team()
5553                team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE));  in __kmp_allocate_team()
5557           team->t.t_bar[b].b_arrived = KMP_INIT_BARRIER_STATE;  in __kmp_allocate_team()
5559           team->t.t_bar[b].b_master_arrived = 0;  in __kmp_allocate_team()
5560           team->t.t_bar[b].b_team_arrived = 0;  in __kmp_allocate_team()
5565       team->t.t_proc_bind = new_proc_bind;  in __kmp_allocate_team()
5568                     team->t.t_id));  in __kmp_allocate_team()
5574       team->t.t_nested_nth = NULL;  in __kmp_allocate_team()
5582     // not sure if this is wise, but, will be redone during the hot-teams  in __kmp_allocate_team()
5584     /* TODO: Use technique to find the right size hot-team, don't reap them */  in __kmp_allocate_team()
5594   team->t.t_max_nproc = max_nproc;  in __kmp_allocate_team()
5598     team->t.b = distributedBarrier::allocate(__kmp_dflt_team_nth_ub);  in __kmp_allocate_team()
5610                 &team->t.t_task_team[0], &team->t.t_task_team[1]));  in __kmp_allocate_team()
5611   team->t.t_task_team[0] = NULL; // to be removed, as __kmp_allocate zeroes  in __kmp_allocate_team()
5613   team->t.t_task_team[1] = NULL; // to be removed, as __kmp_allocate zeroes  in __kmp_allocate_team()
5617     __kmp_print_team_storage_map("team", team, team->t.t_id, new_nproc);  in __kmp_allocate_team()
5622   team->t.t_argc = argc;  in __kmp_allocate_team()
5626             team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE));  in __kmp_allocate_team()
5630       team->t.t_bar[b].b_arrived = KMP_INIT_BARRIER_STATE;  in __kmp_allocate_team()
5632       team->t.t_bar[b].b_master_arrived = 0;  in __kmp_allocate_team()
5633       team->t.t_bar[b].b_team_arrived = 0;  in __kmp_allocate_team()
5638   team->t.t_proc_bind = new_proc_bind;  in __kmp_allocate_team()
5642   team->t.ompt_serialized_team_info = NULL;  in __kmp_allocate_team()
5647   team->t.t_nested_nth = NULL;  in __kmp_allocate_team()
5650                 team->t.t_id));  in __kmp_allocate_team()
5655 /* TODO implement hot-teams at all levels */
5664                 team->t.t_id));  in __kmp_free_team()
5669   KMP_DEBUG_ASSERT(team->t.t_nproc <= team->t.t_max_nproc);  in __kmp_free_team()
5670   KMP_DEBUG_ASSERT(team->t.t_threads);  in __kmp_free_team()
5672   int use_hot_team = team == root->r.r_hot_team;  in __kmp_free_team()
5676     level = team->t.t_active_level - 1;  in __kmp_free_team()
5677     if (master->th.th_teams_microtask) { // in teams construct?  in __kmp_free_team()
5678       if (master->th.th_teams_size.nteams > 1) {  in __kmp_free_team()
5682       if (team->t.t_pkfn != (microtask_t)__kmp_teams_master &&  in __kmp_free_team()
5683           master->th.th_teams_level == team->t.t_level) {  in __kmp_free_team()
5686       } // team->t.t_level will be increased inside parallel  in __kmp_free_team()
5689     kmp_hot_team_ptr_t *hot_teams = master->th.th_hot_teams;  in __kmp_free_team()
5699   TCW_SYNC_PTR(team->t.t_pkfn,  in __kmp_free_team()
5702   team->t.t_copyin_counter = 0; // init counter for possible reuse  in __kmp_free_team()
5706   /* if we are non-hot team, release our threads */  in __kmp_free_team()
5710       for (f = 1; f < team->t.t_nproc; ++f) {  in __kmp_free_team()
5711         KMP_DEBUG_ASSERT(team->t.t_threads[f]);  in __kmp_free_team()
5712         kmp_info_t *th = team->t.t_threads[f];  in __kmp_free_team()
5713         volatile kmp_uint32 *state = &th->th.th_reap_state;  in __kmp_free_team()
5724           if (th->th.th_sleep_loc)  in __kmp_free_team()
5733         kmp_task_team_t *task_team = team->t.t_task_team[tt_idx];  in __kmp_free_team()
5735           for (f = 0; f < team->t.t_nproc; ++f) { // threads unref task teams  in __kmp_free_team()
5736             KMP_DEBUG_ASSERT(team->t.t_threads[f]);  in __kmp_free_team()
5737             team->t.t_threads[f]->th.th_task_team = NULL;  in __kmp_free_team()
5742                __kmp_get_gtid(), task_team, team->t.t_id));  in __kmp_free_team()
5746           team->t.t_task_team[tt_idx] = NULL;  in __kmp_free_team()
5752     if (team->t.t_nested_nth && team->t.t_nested_nth != &__kmp_nested_nth &&  in __kmp_free_team()
5753         team->t.t_nested_nth != team->t.t_parent->t.t_nested_nth) {  in __kmp_free_team()
5754       KMP_INTERNAL_FREE(team->t.t_nested_nth->nth);  in __kmp_free_team()
5755       KMP_INTERNAL_FREE(team->t.t_nested_nth);  in __kmp_free_team()
5757     team->t.t_nested_nth = NULL;  in __kmp_free_team()
5759     // Reset pointer to parent team only for non-hot teams.  in __kmp_free_team()
5760     team->t.t_parent = NULL;  in __kmp_free_team()
5761     team->t.t_level = 0;  in __kmp_free_team()
5762     team->t.t_active_level = 0;  in __kmp_free_team()
5765     for (f = 1; f < team->t.t_nproc; ++f) {  in __kmp_free_team()
5766       KMP_DEBUG_ASSERT(team->t.t_threads[f]);  in __kmp_free_team()
5768         KMP_COMPARE_AND_STORE_ACQ32(&(team->t.t_threads[f]->th.th_used_in_team),  in __kmp_free_team()
5771       __kmp_free_thread(team->t.t_threads[f]);  in __kmp_free_team()
5775       if (team->t.b) {  in __kmp_free_team()
5777         team->t.b->go_release();  in __kmp_free_team()
5779           for (f = 1; f < team->t.t_nproc; ++f) {  in __kmp_free_team()
5780             if (team->t.b->sleep[f].sleep) {  in __kmp_free_team()
5782                   team->t.t_threads[f]->th.th_info.ds.ds_gtid,  in __kmp_free_team()
5788         for (int f = 1; f < team->t.t_nproc; ++f) {  in __kmp_free_team()
5789           while (team->t.t_threads[f]->th.th_used_in_team.load() != 0)  in __kmp_free_team()
5795     for (f = 1; f < team->t.t_nproc; ++f) {  in __kmp_free_team()
5796       team->t.t_threads[f] = NULL;  in __kmp_free_team()
5799     if (team->t.t_max_nproc > 1 &&  in __kmp_free_team()
5801       distributedBarrier::deallocate(team->t.b);  in __kmp_free_team()
5802       team->t.b = NULL;  in __kmp_free_team()
5806     team->t.t_next_pool = CCAST(kmp_team_t *, __kmp_team_pool);  in __kmp_free_team()
5810     KMP_DEBUG_ASSERT(team->t.t_threads[1] &&  in __kmp_free_team()
5811                      team->t.t_threads[1]->th.th_cg_roots);  in __kmp_free_team()
5812     if (team->t.t_threads[1]->th.th_cg_roots->cg_root == team->t.t_threads[1]) {  in __kmp_free_team()
5813       // Clean up the CG root nodes on workers so that this team can be re-used  in __kmp_free_team()
5814       for (f = 1; f < team->t.t_nproc; ++f) {  in __kmp_free_team()
5815         kmp_info_t *thr = team->t.t_threads[f];  in __kmp_free_team()
5816         KMP_DEBUG_ASSERT(thr && thr->th.th_cg_roots &&  in __kmp_free_team()
5817                          thr->th.th_cg_roots->cg_root == thr);  in __kmp_free_team()
5819         kmp_cg_root_t *tmp = thr->th.th_cg_roots;  in __kmp_free_team()
5820         thr->th.th_cg_roots = tmp->up;  in __kmp_free_team()
5823                        thr, tmp, thr->th.th_cg_roots, tmp->cg_nthreads));  in __kmp_free_team()
5824         int i = tmp->cg_nthreads--;  in __kmp_free_team()
5829         if (thr->th.th_cg_roots)  in __kmp_free_team()
5830           thr->th.th_current_task->td_icvs.thread_limit =  in __kmp_free_team()
5831               thr->th.th_cg_roots->cg_thread_limit;  in __kmp_free_team()
5841   kmp_team_t *next_pool = team->t.t_next_pool;  in __kmp_reap_team()
5844   KMP_DEBUG_ASSERT(team->t.t_dispatch);  in __kmp_reap_team()
5845   KMP_DEBUG_ASSERT(team->t.t_disp_buffer);  in __kmp_reap_team()
5846   KMP_DEBUG_ASSERT(team->t.t_threads);  in __kmp_reap_team()
5847   KMP_DEBUG_ASSERT(team->t.t_argv);  in __kmp_reap_team()
5853   if (team->t.t_argv != &team->t.t_inline_argv[0])  in __kmp_reap_team()
5854     __kmp_free((void *)team->t.t_argv);  in __kmp_reap_team()
5864 // Changes for Quad issue 527845: We need a predictable OMP tid <-> gtid
5872 // With single-level parallelism, threads will always be added to the tail
5885 // Now, for single-level parallelism, the OMP tid is always == gtid.
5891                 __kmp_get_gtid(), this_th->th.th_info.ds.ds_gtid));  in __kmp_free_thread()
5898   kmp_balign_t *balign = this_th->th.th_bar;  in __kmp_free_thread()
5905   this_th->th.th_task_state = 0;  in __kmp_free_thread()
5906   this_th->th.th_reap_state = KMP_SAFE_TO_REAP;  in __kmp_free_thread()
5909   TCW_PTR(this_th->th.th_team, NULL);  in __kmp_free_thread()
5910   TCW_PTR(this_th->th.th_root, NULL);  in __kmp_free_thread()
5911   TCW_PTR(this_th->th.th_dispatch, NULL); /* NOT NEEDED */  in __kmp_free_thread()
5913   while (this_th->th.th_cg_roots) {  in __kmp_free_thread()
5914     this_th->th.th_cg_roots->cg_nthreads--;  in __kmp_free_thread()
5917                    this_th, this_th->th.th_cg_roots,  in __kmp_free_thread()
5918                    this_th->th.th_cg_roots->cg_root,  in __kmp_free_thread()
5919                    this_th->th.th_cg_roots->cg_nthreads));  in __kmp_free_thread()
5920     kmp_cg_root_t *tmp = this_th->th.th_cg_roots;  in __kmp_free_thread()
5921     if (tmp->cg_root == this_th) { // Thread is a cg_root  in __kmp_free_thread()
5922       KMP_DEBUG_ASSERT(tmp->cg_nthreads == 0);  in __kmp_free_thread()
5925       this_th->th.th_cg_roots = tmp->up;  in __kmp_free_thread()
5928       if (tmp->cg_nthreads == 0) { // last thread leaves contention group  in __kmp_free_thread()
5931       this_th->th.th_cg_roots = NULL;  in __kmp_free_thread()
5937    * -> multiple threads can share the data and try to free the task at  in __kmp_free_thread()
5942   this_th->th.th_current_task = NULL;  in __kmp_free_thread()
5945   // point, then we need to re-scan the entire list.  in __kmp_free_thread()
5946   gtid = this_th->th.th_info.ds.ds_gtid;  in __kmp_free_thread()
5949     if (__kmp_thread_pool_insert_pt->th.th_info.ds.ds_gtid > gtid) {  in __kmp_free_thread()
5960     scan = &(__kmp_thread_pool_insert_pt->th.th_next_pool);  in __kmp_free_thread()
5964   for (; (*scan != NULL) && ((*scan)->th.th_info.ds.ds_gtid < gtid);  in __kmp_free_thread()
5965        scan = &((*scan)->th.th_next_pool))  in __kmp_free_thread()
5970   TCW_PTR(this_th->th.th_next_pool, *scan);  in __kmp_free_thread()
5972   KMP_DEBUG_ASSERT((this_th->th.th_next_pool == NULL) ||  in __kmp_free_thread()
5973                    (this_th->th.th_info.ds.ds_gtid <  in __kmp_free_thread()
5974                     this_th->th.th_next_pool->th.th_info.ds.ds_gtid));  in __kmp_free_thread()
5975   TCW_4(this_th->th.th_in_pool, TRUE);  in __kmp_free_thread()
5978   if (this_th->th.th_active == TRUE) {  in __kmp_free_thread()
5980     this_th->th.th_active_in_pool = TRUE;  in __kmp_free_thread()
5984     KMP_DEBUG_ASSERT(this_th->th.th_active_in_pool == FALSE);  in __kmp_free_thread()
5989   TCW_4(__kmp_nth, __kmp_nth - 1);  in __kmp_free_thread()
6005 /* ------------------------------------------------------------------------ */
6015   int gtid = this_thr->th.th_info.ds.ds_gtid;  in __kmp_launch_thread()
6023     this_thr->th.th_cons = __kmp_allocate_cons_stack(gtid); // ATT: Memory leak?  in __kmp_launch_thread()
6034     thread_data = &(this_thr->th.ompt_thread_info.thread_data);  in __kmp_launch_thread()
6037     this_thr->th.ompt_thread_info.state = ompt_state_overhead;  in __kmp_launch_thread()
6038     this_thr->th.ompt_thread_info.wait_id = 0;  in __kmp_launch_thread()
6039     this_thr->th.ompt_thread_info.idle_frame = OMPT_GET_FRAME_ADDRESS(0);  in __kmp_launch_thread()
6040     this_thr->th.ompt_thread_info.parallel_flags = 0;  in __kmp_launch_thread()
6045     this_thr->th.ompt_thread_info.state = ompt_state_idle;  in __kmp_launch_thread()
6062       this_thr->th.ompt_thread_info.state = ompt_state_overhead;  in __kmp_launch_thread()
6066     pteam = &this_thr->th.th_team;  in __kmp_launch_thread()
6071       if (TCR_SYNC_PTR((*pteam)->t.t_pkfn) != NULL) {  in __kmp_launch_thread()
6075                   gtid, (*pteam)->t.t_id, __kmp_tid_from_gtid(gtid),  in __kmp_launch_thread()
6076                   (*pteam)->t.t_pkfn));  in __kmp_launch_thread()
6082           this_thr->th.ompt_thread_info.state = ompt_state_work_parallel;  in __kmp_launch_thread()
6086         rc = (*pteam)->t.t_invoke(gtid);  in __kmp_launch_thread()
6091                       gtid, (*pteam)->t.t_id, __kmp_tid_from_gtid(gtid),  in __kmp_launch_thread()
6092                       (*pteam)->t.t_pkfn));  in __kmp_launch_thread()
6097         __ompt_get_task_info_object(0)->frame.exit_frame = ompt_data_none;  in __kmp_launch_thread()
6099         this_thr->th.ompt_thread_info.state = ompt_state_overhead;  in __kmp_launch_thread()
6118   this_thr->th.th_task_team = NULL;  in __kmp_launch_thread()
6131 /* ------------------------------------------------------------------------ */
6136   __kmp_type_convert((kmp_intptr_t)specific_gtid - 1, &gtid);  in __kmp_internal_end_dest()
6139   /* NOTE: the gtid is stored as gitd+1 in the thread-local-storage  in __kmp_internal_end_dest()
6140    * this is because 0 is reserved for the nothing-stored case */  in __kmp_internal_end_dest()
6180   __kmp_internal_end_library(-1);  in __kmp_internal_end_atexit()
6193   gtid = thread->th.th_info.ds.ds_gtid;  in __kmp_reap_thread()
6203             !KMP_COMPARE_AND_STORE_ACQ32(&(thread->th.th_used_in_team), 0, 3))  in __kmp_reap_thread()
6209         kmp_flag_64<> flag(&thread->th.th_bar[bs_forkjoin_barrier].bb.b_go,  in __kmp_reap_thread()
6221     // There is a small timing hole here - if the worker thread was just waking  in __kmp_reap_thread()
6228     if (thread->th.th_active_in_pool) {  in __kmp_reap_thread()
6229       thread->th.th_active_in_pool = FALSE;  in __kmp_reap_thread()
6247   --__kmp_all_nth;  in __kmp_reap_thread()
6263     if (thread->th.th_cons) {  in __kmp_reap_thread()
6264       __kmp_free_cons_stack(thread->th.th_cons);  in __kmp_reap_thread()
6265       thread->th.th_cons = NULL;  in __kmp_reap_thread()
6269   if (thread->th.th_pri_common != NULL) {  in __kmp_reap_thread()
6270     __kmp_free(thread->th.th_pri_common);  in __kmp_reap_thread()
6271     thread->th.th_pri_common = NULL;  in __kmp_reap_thread()
6275   if (thread->th.th_local.bget_data != NULL) {  in __kmp_reap_thread()
6281   if (thread->th.th_affin_mask != NULL) {  in __kmp_reap_thread()
6282     KMP_CPU_FREE(thread->th.th_affin_mask);  in __kmp_reap_thread()
6283     thread->th.th_affin_mask = NULL;  in __kmp_reap_thread()
6288   if (thread->th.th_hier_bar_data != NULL) {  in __kmp_reap_thread()
6289     __kmp_free(thread->th.th_hier_bar_data);  in __kmp_reap_thread()
6290     thread->th.th_hier_bar_data = NULL;  in __kmp_reap_thread()
6294   __kmp_reap_team(thread->th.th_serial_team);  in __kmp_reap_thread()
6295   thread->th.th_serial_team = NULL;  in __kmp_reap_thread()
6308         kmp_itthash_entry_t *next = bucket->next_in_bucket;  in __kmp_itthash_clean()
6318         kmp_itthash_entry_t *next = bucket->next_in_bucket;  in __kmp_itthash_clean()
6344       if (__kmp_root[i]->r.r_active)  in __kmp_internal_end()
6351     // 2009-09-08 (lev): Other alive roots found. Why do we kill the monitor??  in __kmp_internal_end()
6377         KMP_ASSERT(!__kmp_root[i]->r.r_active); // TODO: can they be active?  in __kmp_internal_end()
6389       __kmp_thread_pool = thread->th.th_next_pool;  in __kmp_internal_end()
6391       KMP_DEBUG_ASSERT(thread->th.th_reap_state == KMP_SAFE_TO_REAP);  in __kmp_internal_end()
6392       thread->th.th_next_pool = NULL;  in __kmp_internal_end()
6393       thread->th.th_in_pool = FALSE;  in __kmp_internal_end()
6402       __kmp_team_pool = team->t.t_next_pool;  in __kmp_internal_end()
6404       team->t.t_next_pool = NULL;  in __kmp_internal_end()
6413     // until all threads either exit the final spin-waiting loop or begin  in __kmp_internal_end()
6417       while (thr && KMP_ATOMIC_LD_ACQ(&thr->th.th_blocking))  in __kmp_internal_end()
6462   // 2009-09-06: We do not set g_abort without setting g_done. This check looks  in __kmp_internal_end_library()
6504       if (__kmp_root[gtid]->r.r_active) {  in __kmp_internal_end_library()
6505         __kmp_global.g.g_abort = -1;  in __kmp_internal_end_library()
6585   // 2009-09-06: We do not set g_abort without setting g_done. This check looks  in __kmp_internal_end_thread()
6629       if (__kmp_root[gtid]->r.r_active) {  in __kmp_internal_end_thread()
6630         __kmp_global.g.g_abort = -1;  in __kmp_internal_end_thread()
6646         __kmp_threads[gtid]->th.th_task_team = NULL;  in __kmp_internal_end_thread()
6685   /* should we finish the run-time?  are all siblings done? */  in __kmp_internal_end_thread()
6714 // -----------------------------------------------------------------------------
6757       __kmp_str_format("%p-%lx-%s", &__kmp_registration_flag,  in __kmp_register_library_startup()
6772       int fd1 = -1;  in __kmp_register_library_startup()
6776       if ((fd1 == -1) && (errno == EEXIST)) {  in __kmp_register_library_startup()
6780         if (fd1 == -1) { // file didn't open  in __kmp_register_library_startup()
6788         if (ftruncate(fd1, SHM_SIZE) == -1) { // error occured setting size;  in __kmp_register_library_startup()
6809       if (fd1 != -1)  in __kmp_register_library_startup()
6820       int fd1 = -1;  in __kmp_register_library_startup()
6824       if ((fd1 == -1) && (errno == EEXIST)) {  in __kmp_register_library_startup()
6828         if (fd1 == -1) { // file didn't open if (fd1 == -1) {  in __kmp_register_library_startup()
6837         if (ftruncate(fd1, SHM_SIZE) == -1) { // error occured setting size;  in __kmp_register_library_startup()
6858       if (fd1 != -1)  in __kmp_register_library_startup()
6862       // no /dev/shm and no /tmp -- fall back to environment variable  in __kmp_register_library_startup()
6880       int neighbor = 0; // 0 -- unknown status, 1 -- alive, 2 -- dead.  in __kmp_register_library_startup()
6885       __kmp_str_split(tail, '-', &flag_addr_str, &tail);  in __kmp_register_library_startup()
6886       __kmp_str_split(tail, '-', &flag_val_str, &tail);  in __kmp_register_library_startup()
6894           // First, check whether environment-encoded address is mapped into  in __kmp_register_library_startup()
6907       case 0: // Cannot parse environment variable -- neighbor status unknown.  in __kmp_register_library_startup()
6968     if (fd1 != -1) { // File opened successfully  in __kmp_unregister_library()
6978     if (fd1 != -1) { // File opened successfully  in __kmp_unregister_library()
7027 // -----------------------------------------------------------------------------
7036   if ((cs_p->eax & 0xff0) == 0xB10) {  in __kmp_check_mic_type()
7038   } else if ((cs_p->eax & 0xf0ff0) == 0x50670) {  in __kmp_check_mic_type()
7051   __kmp_waitpkg_enabled = ((buf.ecx >> 5) & 1);  in __kmp_user_level_mwait_init()
7059 // Spurious, non-existent value that should always fail to return anything.
7071   // use them to find if the user-level mwait is enabled. Otherwise, forcibly  in __kmp_user_level_mwait_init()
7211 // AC: do not use analytical here, because it is non-monotonous  in __kmp_do_serial_initialize()
7301   // case, set them to NULL -- some memory may leak, but subsequent code will  in __kmp_do_serial_initialize()
7313   /* Since allocation is cache-aligned, just add extra padding at the end */  in __kmp_do_serial_initialize()
7483     // already explicitly set its nthreads-var with a call to  in __kmp_do_middle_initialize()
7489       if (thread->th.th_current_task->td_icvs.nproc != 0)  in __kmp_do_middle_initialize()
7645 /* ------------------------------------------------------------------------ */
7654   this_thr->th.th_local.this_construct = 0;  in __kmp_run_before_invoked_task()
7656   KMP_CACHE_PREFETCH(&this_thr->th.th_bar[bs_forkjoin_barrier].bb.b_arrived);  in __kmp_run_before_invoked_task()
7658   dispatch = (kmp_disp_t *)TCR_PTR(this_thr->th.th_dispatch);  in __kmp_run_before_invoked_task()
7660   KMP_DEBUG_ASSERT(team->t.t_dispatch);  in __kmp_run_before_invoked_task()
7661   // KMP_DEBUG_ASSERT( this_thr->th.th_dispatch == &team->t.t_dispatch[  in __kmp_run_before_invoked_task()
7662   // this_thr->th.th_info.ds.ds_tid ] );  in __kmp_run_before_invoked_task()
7664   dispatch->th_disp_index = 0; /* reset the dispatch buffer counter */  in __kmp_run_before_invoked_task()
7665   dispatch->th_doacross_buf_idx = 0; // reset doacross dispatch buffer counter  in __kmp_run_before_invoked_task()
7667     __kmp_push_parallel(gtid, team->t.t_ident);  in __kmp_run_before_invoked_task()
7675     __kmp_pop_parallel(gtid, team->t.t_ident);  in __kmp_run_after_invoked_task()
7684   kmp_team_t *team = this_thr->th.th_team;  in __kmp_invoke_task_func()
7690     if (team->t.t_stack_id != NULL) {  in __kmp_invoke_task_func()
7691       __kmp_itt_stack_callee_enter((__itt_caller)team->t.t_stack_id);  in __kmp_invoke_task_func()
7693       KMP_DEBUG_ASSERT(team->t.t_parent->t.t_stack_id != NULL);  in __kmp_invoke_task_func()
7695           (__itt_caller)team->t.t_parent->t.t_stack_id);  in __kmp_invoke_task_func()
7711     exit_frame_p = &(team->t.t_implicit_task_taskdata[tid]  in __kmp_invoke_task_func()
7718       &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data);  in __kmp_invoke_task_func()
7719   my_parallel_data = &(team->t.ompt_team_info.parallel_data);  in __kmp_invoke_task_func()
7721     ompt_team_size = team->t.t_nproc;  in __kmp_invoke_task_func()
7725     OMPT_CUR_TASK_INFO(this_thr)->thread_num = __kmp_tid_from_gtid(gtid);  in __kmp_invoke_task_func()
7739   rc = __kmp_invoke_microtask((microtask_t)TCR_SYNC_PTR(team->t.t_pkfn), gtid,  in __kmp_invoke_task_func()
7740                               tid, (int)team->t.t_argc, (void **)team->t.t_argv  in __kmp_invoke_task_func()
7748   this_thr->th.ompt_thread_info.parallel_flags = ompt_parallel_team;  in __kmp_invoke_task_func()
7761     if (team->t.t_stack_id != NULL) {  in __kmp_invoke_task_func()
7762       __kmp_itt_stack_callee_leave((__itt_caller)team->t.t_stack_id);  in __kmp_invoke_task_func()
7764       KMP_DEBUG_ASSERT(team->t.t_parent->t.t_stack_id != NULL);  in __kmp_invoke_task_func()
7766           (__itt_caller)team->t.t_parent->t.t_stack_id);  in __kmp_invoke_task_func()
7778   kmp_team_t *team = thr->th.th_team;  in __kmp_teams_master()
7779   ident_t *loc = team->t.t_ident;  in __kmp_teams_master()
7780   thr->th.th_set_nproc = thr->th.th_teams_size.nth;  in __kmp_teams_master()
7781   KMP_DEBUG_ASSERT(thr->th.th_teams_microtask);  in __kmp_teams_master()
7782   KMP_DEBUG_ASSERT(thr->th.th_set_nproc);  in __kmp_teams_master()
7784                 __kmp_tid_from_gtid(gtid), thr->th.th_teams_microtask));  in __kmp_teams_master()
7788   tmp->cg_root = thr; // Make thr the CG root  in __kmp_teams_master()
7790   tmp->cg_thread_limit = thr->th.th_current_task->td_icvs.thread_limit;  in __kmp_teams_master()
7791   tmp->cg_nthreads = 1; // Init counter to one active thread, this one  in __kmp_teams_master()
7795   tmp->up = thr->th.th_cg_roots;  in __kmp_teams_master()
7796   thr->th.th_cg_roots = tmp;  in __kmp_teams_master()
7803   __kmp_fork_call(loc, gtid, fork_context_intel, team->t.t_argc,  in __kmp_teams_master()
7804                   (microtask_t)thr->th.th_teams_microtask, // "wrapped" task  in __kmp_teams_master()
7810   if (thr->th.th_team_nproc < thr->th.th_teams_size.nth)  in __kmp_teams_master()
7811     thr->th.th_teams_size.nth = thr->th.th_team_nproc;  in __kmp_teams_master()
7825   kmp_team_t *team = this_thr->th.th_team;  in __kmp_invoke_teams_master()
7827   if (!__kmp_threads[gtid]->th.th_team->t.t_serialized)  in __kmp_invoke_teams_master()
7828     KMP_DEBUG_ASSERT((void *)__kmp_threads[gtid]->th.th_team->t.t_pkfn ==  in __kmp_invoke_teams_master()
7835       &team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data;  in __kmp_invoke_teams_master()
7836   ompt_data_t *parallel_data = &team->t.ompt_team_info.parallel_data;  in __kmp_invoke_teams_master()
7839         ompt_scope_begin, parallel_data, task_data, team->t.t_nproc, tid,  in __kmp_invoke_teams_master()
7841     OMPT_CUR_TASK_INFO(this_thr)->thread_num = tid;  in __kmp_invoke_teams_master()
7846   this_thr->th.ompt_thread_info.parallel_flags = ompt_parallel_league;  in __kmp_invoke_teams_master()
7860     thr->th.th_set_nproc = num_threads;  in __kmp_push_num_threads()
7870     thr->th.th_set_nproc = num_threads_list[0];  in __kmp_push_num_threads_list()
7871   thr->th.th_set_nested_nth =  in __kmp_push_num_threads_list()
7874     thr->th.th_set_nested_nth[i] = num_threads_list[i];  in __kmp_push_num_threads_list()
7875   thr->th.th_set_nested_nth_sz = list_length;  in __kmp_push_num_threads_list()
7881   thr->th.th_nt_strict = true;  in __kmp_set_strict_num_threads()
7882   thr->th.th_nt_loc = loc;  in __kmp_set_strict_num_threads()
7885     thr->th.th_nt_sev = sev;  in __kmp_set_strict_num_threads()
7887     thr->th.th_nt_sev = severity_fatal;  in __kmp_set_strict_num_threads()
7890     thr->th.th_nt_msg = msg;  in __kmp_set_strict_num_threads()
7892     thr->th.th_nt_msg = "Cannot form team with number of threads specified by "  in __kmp_set_strict_num_threads()
7913     // num_threads = min(num_threads, nthreads-var, thread-limit-var)  in __kmp_push_thread_limit()
7914     // no thread_limit clause specified -  do not change thread-limit-var ICV  in __kmp_push_thread_limit()
7916       num_threads = __kmp_dflt_team_nth; // honor nthreads-var ICV  in __kmp_push_thread_limit()
7918     if (num_threads > thr->th.th_current_task->td_icvs.thread_limit) {  in __kmp_push_thread_limit()
7919       num_threads = thr->th.th_current_task->td_icvs.thread_limit;  in __kmp_push_thread_limit()
7920     } // prevent team size to exceed thread-limit-var  in __kmp_push_thread_limit()
7935     thr->th.th_current_task->td_icvs.thread_limit = num_threads;  in __kmp_push_thread_limit()
7936     // num_threads = min(num_threads, nthreads-var)  in __kmp_push_thread_limit()
7938       num_threads = __kmp_dflt_team_nth; // honor nthreads-var ICV  in __kmp_push_thread_limit()
7956   thr->th.th_teams_size.nth = num_threads;  in __kmp_push_thread_limit()
7989   thr->th.th_set_nproc = thr->th.th_teams_size.nteams = num_teams;  in __kmp_push_num_teams()
8046   thr->th.th_set_nproc = thr->th.th_teams_size.nteams = num_teams;  in __kmp_push_num_teams_51()
8054   thr->th.th_set_proc_bind = proc_bind;  in __kmp_push_proc_bind()
8067   KMP_DEBUG_ASSERT(this_thr->th.th_team == team);  in __kmp_internal_fork()
8071   team->t.t_construct = 0; /* no single directives seen yet */  in __kmp_internal_fork()
8072   team->t.t_ordered.dt.t_value =  in __kmp_internal_fork()
8076   KMP_DEBUG_ASSERT(team->t.t_disp_buffer);  in __kmp_internal_fork()
8077   if (team->t.t_max_nproc > 1) {  in __kmp_internal_fork()
8080       team->t.t_disp_buffer[i].buffer_index = i;  in __kmp_internal_fork()
8081       team->t.t_disp_buffer[i].doacross_buf_idx = i;  in __kmp_internal_fork()
8084     team->t.t_disp_buffer[0].buffer_index = 0;  in __kmp_internal_fork()
8085     team->t.t_disp_buffer[0].doacross_buf_idx = 0;  in __kmp_internal_fork()
8089   KMP_ASSERT(this_thr->th.th_team == team);  in __kmp_internal_fork()
8092   for (f = 0; f < team->t.t_nproc; f++) {  in __kmp_internal_fork()
8093     KMP_DEBUG_ASSERT(team->t.t_threads[f] &&  in __kmp_internal_fork()
8094                      team->t.t_threads[f]->th.th_team_nproc == team->t.t_nproc);  in __kmp_internal_fork()
8106   KMP_DEBUG_ASSERT(this_thr->th.th_team == team);  in __kmp_internal_join()
8114       __kmp_threads[gtid]->th.th_team_nproc != team->t.t_nproc) {  in __kmp_internal_join()
8117     __kmp_printf("__kmp_threads[%d]->th.th_team_nproc=%d, TEAM: %p, "  in __kmp_internal_join()
8118                  "team->t.t_nproc=%d\n",  in __kmp_internal_join()
8119                  gtid, __kmp_threads[gtid]->th.th_team_nproc, team,  in __kmp_internal_join()
8120                  team->t.t_nproc);  in __kmp_internal_join()
8124                    __kmp_threads[gtid]->th.th_team_nproc == team->t.t_nproc);  in __kmp_internal_join()
8129   ompt_state_t ompt_state = this_thr->th.ompt_thread_info.state;  in __kmp_internal_join()
8133     int ds_tid = this_thr->th.th_info.ds.ds_tid;  in __kmp_internal_join()
8135     this_thr->th.ompt_thread_info.state = ompt_state_overhead;  in __kmp_internal_join()
8141       codeptr = OMPT_CUR_TEAM_INFO(this_thr)->master_return_address;  in __kmp_internal_join()
8144     if (this_thr->th.ompt_thread_info.parallel_flags & ompt_parallel_league)  in __kmp_internal_join()
8164   KMP_ASSERT(this_thr->th.th_team == team);  in __kmp_internal_join()
8167 /* ------------------------------------------------------------------------ */
8178   if (root->r.r_active) {  in __kmp_active_hot_team_nproc()
8181   hot_team = root->r.r_hot_team;  in __kmp_active_hot_team_nproc()
8183     return hot_team->t.t_nproc - 1; // Don't count primary thread  in __kmp_active_hot_team_nproc()
8186   // Skip the primary thread - it is accounted for elsewhere.  in __kmp_active_hot_team_nproc()
8188   for (i = 1; i < hot_team->t.t_nproc; i++) {  in __kmp_active_hot_team_nproc()
8189     if (hot_team->t.t_threads[i]->th.th_active) {  in __kmp_active_hot_team_nproc()
8208   KMP_DEBUG_ASSERT(root->r.r_root_team->t.t_threads[0]  in __kmp_load_balance_nproc()
8209                        ->th.th_current_task->td_icvs.dynamic == TRUE);  in __kmp_load_balance_nproc()
8240     retval = __kmp_avail_proc - __kmp_nth +  in __kmp_load_balance_nproc()
8241              (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc);  in __kmp_load_balance_nproc()
8260   retval = __kmp_avail_proc - system_active + team_curr_active;  in __kmp_load_balance_nproc()
8274 /* ------------------------------------------------------------------------ */
8322     kmp_old_threads_list_t *next = ptr->next;  in __kmp_cleanup()
8323     __kmp_free(ptr->threads);  in __kmp_cleanup()
8378 /* ------------------------------------------------------------------------ */
8387   // By default __kmpc_begin() is no-op.  in __kmp_ignore_mppbeg()
8398   // By default __kmpc_end() is no-op.  in __kmp_ignore_mppend()
8409   root = __kmp_threads[gtid]->th.th_root;  in __kmp_internal_begin()
8412   if (root->r.r_begin)  in __kmp_internal_begin()
8414   __kmp_acquire_lock(&root->r.r_begin_lock, gtid);  in __kmp_internal_begin()
8415   if (root->r.r_begin) {  in __kmp_internal_begin()
8416     __kmp_release_lock(&root->r.r_begin_lock, gtid);  in __kmp_internal_begin()
8420   root->r.r_begin = TRUE;  in __kmp_internal_begin()
8422   __kmp_release_lock(&root->r.r_begin_lock, gtid);  in __kmp_internal_begin()
8425 /* ------------------------------------------------------------------------ */
8437   root = thread->th.th_root;  in __kmp_user_set_library()
8441   if (root->r.r_in_parallel) { /* Must be called in serial section of top-level  in __kmp_user_set_library()
8449     thread->th.th_set_nproc = 0;  in __kmp_user_set_library()
8453     thread->th.th_set_nproc = 0;  in __kmp_user_set_library()
8458     thread->th.th_set_nproc = 0;  in __kmp_user_set_library()
8474   if (arg & (0x1000 - 1)) {  in __kmp_aux_set_stacksize()
8475     arg &= ~(0x1000 - 1);  in __kmp_aux_set_stacksize()
8526   if (thr->th.th_teams_microtask) {  in __kmp_aux_get_team_info()
8527     kmp_team_t *team = thr->th.th_team;  in __kmp_aux_get_team_info()
8528     int tlevel = thr->th.th_teams_level; // the level of the teams construct  in __kmp_aux_get_team_info()
8529     int ii = team->t.t_level;  in __kmp_aux_get_team_info()
8530     teams_serialized = team->t.t_serialized;  in __kmp_aux_get_team_info()
8534       for (teams_serialized = team->t.t_serialized;  in __kmp_aux_get_team_info()
8535            (teams_serialized > 0) && (ii > level); teams_serialized--, ii--) {  in __kmp_aux_get_team_info()
8537       if (team->t.t_serialized && (!teams_serialized)) {  in __kmp_aux_get_team_info()
8538         team = team->t.t_parent;  in __kmp_aux_get_team_info()
8542         team = team->t.t_parent;  in __kmp_aux_get_team_info()
8543         ii--;  in __kmp_aux_get_team_info()
8558       return team->t.t_master_tid;  in __kmp_aux_get_team_num()
8571       return team->t.t_parent->t.t_nproc;  in __kmp_aux_get_num_teams()
8577 /* ------------------------------------------------------------------------ */
8593  * L {thread_level}      - omp_get_level()
8594  * n {thread_num}        - omp_get_thread_num()
8595  * h {host}              - name of host machine
8596  * P {process_id}        - process id (integer)
8597  * T {thread_identifier} - native thread identifier (integer)
8598  * N {num_threads}       - omp_get_num_threads()
8599  * A {ancestor_tnum}     - omp_get_ancestor_thread_num(omp_get_level()-1)
8600  * a {thread_affinity}   - comma separated list of integers or integer ranges
8603  * Implementation-specific field types can be added
8611   char short_name; // from spec e.g., L -> thread level
8612   const char *long_name; // from spec thread_level -> thread level
8682     format[format_index++] = '-';  in __kmp_aux_capture_affinity_field()
8743     rc = __kmp_str_buf_print(field_buffer, format, th->th.th_team->t.t_level);  in __kmp_aux_capture_affinity_field()
8761     rc = __kmp_str_buf_print(field_buffer, format, th->th.th_team->t.t_nproc);  in __kmp_aux_capture_affinity_field()
8765         __kmp_get_ancestor_thread_num(gtid, th->th.th_team->t.t_level - 1);  in __kmp_aux_capture_affinity_field()
8772     __kmp_affinity_str_buf_mask(&buf, th->th.th_affin_mask);  in __kmp_aux_capture_affinity_field()
8817   // If format is NULL or zero-length string, then we use  in __kmp_aux_capture_affinity()
8818   // affinity-format-var ICV  in __kmp_aux_capture_affinity()
8852 /* ------------------------------------------------------------------------ */
8868   set__blocktime_team(thread->th.th_team, tid, blocktime);  in __kmp_aux_set_blocktime()
8869   set__blocktime_team(thread->th.th_serial_team, 0, blocktime);  in __kmp_aux_set_blocktime()
8875   set__bt_intervals_team(thread->th.th_team, tid, bt_intervals);  in __kmp_aux_set_blocktime()
8876   set__bt_intervals_team(thread->th.th_serial_team, 0, bt_intervals);  in __kmp_aux_set_blocktime()
8882   set__bt_set_team(thread->th.th_team, tid, bt_set);  in __kmp_aux_set_blocktime()
8883   set__bt_set_team(thread->th.th_serial_team, 0, bt_set);  in __kmp_aux_set_blocktime()
8887                 __kmp_gtid_from_tid(tid, thread->th.th_team),  in __kmp_aux_set_blocktime()
8888                 thread->th.th_team->t.t_id, tid, blocktime, bt_intervals,  in __kmp_aux_set_blocktime()
8892                 __kmp_gtid_from_tid(tid, thread->th.th_team),  in __kmp_aux_set_blocktime()
8893                 thread->th.th_team->t.t_id, tid, blocktime));  in __kmp_aux_set_blocktime()
8908 /* ------------------------------------------------------------------------ */
8919   // If ( reduce_data!=NULL && reduce_func!=NULL ): the tree-reduction method  in __kmp_determine_reduction_method()
8921   // If loc->flags contains KMP_IDENT_ATOMIC_REDUCE, the atomic reduce method  in __kmp_determine_reduction_method()
8934    ((loc->flags & (KMP_IDENT_ATOMIC_REDUCE)) == (KMP_IDENT_ATOMIC_REDUCE)))  in __kmp_determine_reduction_method()
8992       if (num_vars <= 2) { // && ( team_size <= 8 ) due to false-sharing ???  in __kmp_determine_reduction_method()
9070   return ((__kmp_entry_thread()->th.th_local.packed_reduction_method) >> 8);  in __kmp_get_reduce_method()
9074 // Spin-wait code checks __kmp_pause_status and reacts accordingly.
9081   __kmp_internal_end_thread(-1);  in __kmp_hard_pause()
9092         kmp_flag_64<> fl(&thread->th.th_bar[bs_forkjoin_barrier].bb.b_go,  in __kmp_resume_if_soft_paused()
9162   kmp_info_t **other_threads = team->t.t_threads;  in __kmp_resize_dist_barrier()
9169     if (team->t.t_threads[f]->th.th_used_in_team.load() == 0) {  in __kmp_resize_dist_barrier()
9175     if (team->t.t_threads[f]->th.th_used_in_team.load() == 3) {  in __kmp_resize_dist_barrier()
9176       while (team->t.t_threads[f]->th.th_used_in_team.load() == 3)  in __kmp_resize_dist_barrier()
9180     KMP_DEBUG_ASSERT(team->t.t_threads[f]->th.th_used_in_team.load() == 1);  in __kmp_resize_dist_barrier()
9182     team->t.t_threads[f]->th.th_used_in_team.store(2);  in __kmp_resize_dist_barrier()
9183     KMP_DEBUG_ASSERT(team->t.t_threads[f]->th.th_used_in_team.load() == 2);  in __kmp_resize_dist_barrier()
9186   team->t.b->go_release();  in __kmp_resize_dist_barrier()
9192   int count = old_nthreads - 1;  in __kmp_resize_dist_barrier()
9194     count = old_nthreads - 1;  in __kmp_resize_dist_barrier()
9196       if (other_threads[f]->th.th_used_in_team.load() != 0) {  in __kmp_resize_dist_barrier()
9199               void *, other_threads[f]->th.th_sleep_loc);  in __kmp_resize_dist_barrier()
9200           __kmp_atomic_resume_64(other_threads[f]->th.th_info.ds.ds_gtid, flag);  in __kmp_resize_dist_barrier()
9203         KMP_DEBUG_ASSERT(team->t.t_threads[f]->th.th_used_in_team.load() == 0);  in __kmp_resize_dist_barrier()
9204         count--;  in __kmp_resize_dist_barrier()
9209   team->t.b->update_num_threads(new_nthreads);  in __kmp_resize_dist_barrier()
9210   team->t.b->go_reset();  in __kmp_resize_dist_barrier()
9222     KMP_DEBUG_ASSERT(team->t.t_threads[f]);  in __kmp_add_threads_to_team()
9223     KMP_COMPARE_AND_STORE_ACQ32(&(team->t.t_threads[f]->th.th_used_in_team), 0,  in __kmp_add_threads_to_team()
9226       __kmp_resume_32(team->t.t_threads[f]->th.th_info.ds.ds_gtid,  in __kmp_add_threads_to_team()
9233   int count = new_nthreads - 1;  in __kmp_add_threads_to_team()
9235     count = new_nthreads - 1;  in __kmp_add_threads_to_team()
9237       if (team->t.t_threads[f]->th.th_used_in_team.load() == 1) {  in __kmp_add_threads_to_team()
9238         count--;  in __kmp_add_threads_to_team()
9288   __kmp_hidden_helper_main_thread->th.th_set_nproc =  in __kmp_hidden_helper_threads_initz_routine()
9311       but starts with nesting OFF -- max-active-levels-var is 1 -- and requires
9342     for (loc = 0, hw_level = 0; hw_level < __kmp_topology->get_depth() &&  in __kmp_set_nesting_mode_threads()
9345       __kmp_nesting_nth_level[loc] = __kmp_topology->get_ratio(hw_level);  in __kmp_set_nesting_mode_threads()
9347         loc--;  in __kmp_set_nesting_mode_threads()
9351       int core_level = __kmp_topology->get_level(KMP_HW_CORE);  in __kmp_set_nesting_mode_threads()
9352       int num_cores = __kmp_topology->get_count(core_level);  in __kmp_set_nesting_mode_threads()
9354       for (int level = 0; level < loc - 1; ++level)  in __kmp_set_nesting_mode_threads()
9356       if (upper_levels * __kmp_nesting_nth_level[loc - 1] < num_cores)  in __kmp_set_nesting_mode_threads()
9357         __kmp_nesting_nth_level[loc - 1] =  in __kmp_set_nesting_mode_threads()
9358             num_cores / __kmp_nesting_nth_level[loc - 2];  in __kmp_set_nesting_mode_threads()