Lines Matching refs:team
88 static void __kmp_initialize_team(kmp_team_t *team, int new_nproc,
92 static void __kmp_partition_places(kmp_team_t *team,
98 void __kmp_setup_icv_copy(kmp_team_t *team, int new_nproc,
112 void __kmp_resize_dist_barrier(kmp_team_t *team, int old_nthreads,
114 void __kmp_add_threads_to_team(kmp_team_t *team, int new_nthreads);
536 static void __kmp_print_team_storage_map(const char *header, kmp_team_t *team, in __kmp_print_team_storage_map() argument
538 int num_disp_buff = team->t.t_max_nproc > 1 ? __kmp_dispatch_num_buffers : 2; in __kmp_print_team_storage_map()
539 __kmp_print_storage_map_gtid(-1, team, team + 1, sizeof(kmp_team_t), "%s_%d", in __kmp_print_team_storage_map()
542 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[0], in __kmp_print_team_storage_map()
543 &team->t.t_bar[bs_last_barrier], in __kmp_print_team_storage_map()
547 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_plain_barrier], in __kmp_print_team_storage_map()
548 &team->t.t_bar[bs_plain_barrier + 1], in __kmp_print_team_storage_map()
552 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_forkjoin_barrier], in __kmp_print_team_storage_map()
553 &team->t.t_bar[bs_forkjoin_barrier + 1], in __kmp_print_team_storage_map()
558 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_reduction_barrier], in __kmp_print_team_storage_map()
559 &team->t.t_bar[bs_reduction_barrier + 1], in __kmp_print_team_storage_map()
565 -1, &team->t.t_dispatch[0], &team->t.t_dispatch[num_thr], in __kmp_print_team_storage_map()
569 -1, &team->t.t_threads[0], &team->t.t_threads[num_thr], in __kmp_print_team_storage_map()
572 __kmp_print_storage_map_gtid(-1, &team->t.t_disp_buffer[0], in __kmp_print_team_storage_map()
573 &team->t.t_disp_buffer[num_disp_buff], in __kmp_print_team_storage_map()
650 kmp_team_t *team = __kmp_team_from_gtid(gtid); in __kmp_parallel_deo() local
662 if (!team->t.t_serialized) { in __kmp_parallel_deo()
664 KMP_WAIT(&team->t.t_ordered.dt.t_value, __kmp_tid_from_gtid(gtid), KMP_EQ, in __kmp_parallel_deo()
676 kmp_team_t *team = __kmp_team_from_gtid(gtid); in __kmp_parallel_dxo() local
684 if (!team->t.t_serialized) { in __kmp_parallel_dxo()
689 team->t.t_ordered.dt.t_value = ((tid + 1) % team->t.t_nproc); in __kmp_parallel_dxo()
702 kmp_team_t *team; in __kmp_enter_single() local
709 team = th->th.th_team; in __kmp_enter_single()
714 if (team->t.t_serialized) { in __kmp_enter_single()
723 if (team->t.t_construct == old_this) { in __kmp_enter_single()
724 status = __kmp_atomic_compare_store_acq(&team->t.t_construct, old_this, in __kmp_enter_single()
730 team->t.t_active_level == 1) { in __kmp_enter_single()
959 static void __kmp_fork_team_threads(kmp_root_t *root, kmp_team_t *team, in __kmp_fork_team_threads() argument
965 KA_TRACE(10, ("__kmp_fork_team_threads: new_nprocs = %d\n", team->t.t_nproc)); in __kmp_fork_team_threads()
971 master_th->th.th_team = team; in __kmp_fork_team_threads()
972 master_th->th.th_team_nproc = team->t.t_nproc; in __kmp_fork_team_threads()
975 master_th->th.th_dispatch = &team->t.t_dispatch[0]; in __kmp_fork_team_threads()
983 int level = team->t.t_active_level - 1; // index in array of hot teams in __kmp_fork_team_threads()
989 if (team->t.t_pkfn != (microtask_t)__kmp_teams_master && in __kmp_fork_team_threads()
990 master_th->th.th_teams_level == team->t.t_level) { in __kmp_fork_team_threads()
998 KMP_DEBUG_ASSERT(hot_teams[level].hot_team == team); in __kmp_fork_team_threads()
1002 hot_teams[level].hot_team = team; // remember new hot team in __kmp_fork_team_threads()
1003 hot_teams[level].hot_team_nth = team->t.t_nproc; in __kmp_fork_team_threads()
1010 use_hot_team = team == root->r.r_hot_team; in __kmp_fork_team_threads()
1015 team->t.t_threads[0] = master_th; in __kmp_fork_team_threads()
1016 __kmp_initialize_info(master_th, team, 0, master_gtid); in __kmp_fork_team_threads()
1019 for (i = 1; i < team->t.t_nproc; i++) { in __kmp_fork_team_threads()
1022 kmp_info_t *thr = __kmp_allocate_thread(root, team, i); in __kmp_fork_team_threads()
1023 team->t.t_threads[i] = thr; in __kmp_fork_team_threads()
1025 KMP_DEBUG_ASSERT(thr->th.th_team == team); in __kmp_fork_team_threads()
1029 __kmp_gtid_from_tid(0, team), team->t.t_id, 0, in __kmp_fork_team_threads()
1030 __kmp_gtid_from_tid(i, team), team->t.t_id, i, in __kmp_fork_team_threads()
1031 team->t.t_bar[bs_forkjoin_barrier].b_arrived, in __kmp_fork_team_threads()
1032 team->t.t_bar[bs_plain_barrier].b_arrived)); in __kmp_fork_team_threads()
1038 kmp_balign_t *balign = team->t.t_threads[i]->th.th_bar; in __kmp_fork_team_threads()
1040 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived; in __kmp_fork_team_threads()
1043 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived; in __kmp_fork_team_threads()
1054 __kmp_partition_places(team); in __kmp_fork_team_threads()
1058 if (team->t.t_nproc > 1 && in __kmp_fork_team_threads()
1060 team->t.b->update_num_threads(team->t.t_nproc); in __kmp_fork_team_threads()
1061 __kmp_add_threads_to_team(team, team->t.t_nproc); in __kmp_fork_team_threads()
1068 KMP_DEBUG_ASSERT_TASKTEAM_INVARIANT(team->t.t_parent, master_th); in __kmp_fork_team_threads()
1074 team->t.t_parent, team->t.t_task_team[master_th->th.th_task_state], in __kmp_fork_team_threads()
1075 team)); in __kmp_fork_team_threads()
1078 KMP_CHECK_UPDATE(team->t.t_primary_task_state, in __kmp_fork_team_threads()
1083 if (team->t.t_nproc > 1) { in __kmp_fork_team_threads()
1084 KMP_DEBUG_ASSERT(team->t.t_threads[1]->th.th_task_state == 0 || in __kmp_fork_team_threads()
1085 team->t.t_threads[1]->th.th_task_state == 1); in __kmp_fork_team_threads()
1087 team->t.t_threads[1]->th.th_task_state); in __kmp_fork_team_threads()
1093 KMP_CHECK_UPDATE(team->t.t_primary_task_state, in __kmp_fork_team_threads()
1100 if (__kmp_display_affinity && team->t.t_display_affinity != 1) { in __kmp_fork_team_threads()
1101 for (i = 0; i < team->t.t_nproc; i++) { in __kmp_fork_team_threads()
1102 kmp_info_t *thr = team->t.t_threads[i]; in __kmp_fork_team_threads()
1103 if (thr->th.th_prev_num_threads != team->t.t_nproc || in __kmp_fork_team_threads()
1104 thr->th.th_prev_level != team->t.t_level) { in __kmp_fork_team_threads()
1105 team->t.t_display_affinity = 1; in __kmp_fork_team_threads()
1118 inline static void propagateFPControl(kmp_team_t *team) { in propagateFPControl() argument
1137 KMP_CHECK_UPDATE(team->t.t_x87_fpu_control_word, x87_fpu_control_word); in propagateFPControl()
1138 KMP_CHECK_UPDATE(team->t.t_mxcsr, mxcsr); in propagateFPControl()
1141 KMP_CHECK_UPDATE(team->t.t_fp_control_saved, TRUE); in propagateFPControl()
1145 KMP_CHECK_UPDATE(team->t.t_fp_control_saved, FALSE); in propagateFPControl()
1151 inline static void updateHWFPControl(kmp_team_t *team) { in updateHWFPControl() argument
1152 if (__kmp_inherit_fp_control && team->t.t_fp_control_saved) { in updateHWFPControl()
1161 if (team->t.t_x87_fpu_control_word != x87_fpu_control_word) { in updateHWFPControl()
1163 __kmp_load_x87_fpu_control_word(&team->t.t_x87_fpu_control_word); in updateHWFPControl()
1166 if (team->t.t_mxcsr != mxcsr) { in updateHWFPControl()
1167 __kmp_load_mxcsr(&team->t.t_mxcsr); in updateHWFPControl()
1176 static void __kmp_alloc_argv_entries(int argc, kmp_team_t *team,
1710 kmp_team_t *team; in __kmp_serial_fork_call() local
1800 team = master_th->th.th_team; in __kmp_serial_fork_call()
1802 team->t.t_invoke = invoker; in __kmp_serial_fork_call()
1803 __kmp_alloc_argv_entries(argc, team, TRUE); in __kmp_serial_fork_call()
1804 team->t.t_argc = argc; in __kmp_serial_fork_call()
1805 argv = (void **)team->t.t_argv; in __kmp_serial_fork_call()
1810 team->t.t_level--; in __kmp_serial_fork_call()
1935 kmp_team_t *team; in __kmp_fork_call() local
2190 team = __kmp_allocate_team(root, nthreads, nthreads, in __kmp_fork_call()
2197 copy_icvs((kmp_internal_control_t *)team->t.b->team_icvs, &new_icvs); in __kmp_fork_call()
2201 team = __kmp_allocate_team(root, nthreads, nthreads, in __kmp_fork_call()
2209 copy_icvs((kmp_internal_control_t *)team->t.b->team_icvs, in __kmp_fork_call()
2213 10, ("__kmp_fork_call: after __kmp_allocate_team - team = %p\n", team)); in __kmp_fork_call()
2216 KMP_CHECK_UPDATE(team->t.t_master_tid, master_tid); in __kmp_fork_call()
2217 KMP_CHECK_UPDATE(team->t.t_master_this_cons, master_this_cons); in __kmp_fork_call()
2218 KMP_CHECK_UPDATE(team->t.t_ident, loc); in __kmp_fork_call()
2219 KMP_CHECK_UPDATE(team->t.t_parent, parent_team); in __kmp_fork_call()
2220 KMP_CHECK_UPDATE_SYNC(team->t.t_pkfn, microtask); in __kmp_fork_call()
2222 KMP_CHECK_UPDATE_SYNC(team->t.ompt_team_info.master_return_address, in __kmp_fork_call()
2225 KMP_CHECK_UPDATE(team->t.t_invoke, invoker); // TODO move to root, maybe in __kmp_fork_call()
2229 KMP_CHECK_UPDATE(team->t.t_level, new_level); in __kmp_fork_call()
2231 KMP_CHECK_UPDATE(team->t.t_active_level, new_level); in __kmp_fork_call()
2235 KMP_CHECK_UPDATE(team->t.t_level, new_level); in __kmp_fork_call()
2237 KMP_CHECK_UPDATE(team->t.t_active_level, new_level); in __kmp_fork_call()
2241 KMP_CHECK_UPDATE(team->t.t_sched.sched, new_sched.sched); in __kmp_fork_call()
2243 KMP_CHECK_UPDATE(team->t.t_cancel_request, cancel_noreq); in __kmp_fork_call()
2244 KMP_CHECK_UPDATE(team->t.t_def_allocator, master_th->th.th_def_allocator); in __kmp_fork_call()
2247 if (team->t.t_nested_nth && in __kmp_fork_call()
2248 team->t.t_nested_nth != parent_team->t.t_nested_nth) { in __kmp_fork_call()
2249 KMP_INTERNAL_FREE(team->t.t_nested_nth->nth); in __kmp_fork_call()
2250 KMP_INTERNAL_FREE(team->t.t_nested_nth); in __kmp_fork_call()
2251 team->t.t_nested_nth = NULL; in __kmp_fork_call()
2253 team->t.t_nested_nth = parent_team->t.t_nested_nth; in __kmp_fork_call()
2257 team->t.t_nested_nth = nested_nth; in __kmp_fork_call()
2265 propagateFPControl(team); in __kmp_fork_call()
2274 gtid, parent_team->t.t_id, team->t.t_master_tid, team->t.t_id, in __kmp_fork_call()
2275 team->t.t_nproc)); in __kmp_fork_call()
2276 KMP_DEBUG_ASSERT(team != root->r.r_hot_team || in __kmp_fork_call()
2277 (team->t.t_master_tid == 0 && in __kmp_fork_call()
2278 (team->t.t_parent == root->r.r_root_team || in __kmp_fork_call()
2279 team->t.t_parent->t.t_serialized))); in __kmp_fork_call()
2283 argv = (void **)team->t.t_argv; in __kmp_fork_call()
2293 KMP_CHECK_UPDATE(argv[i], team->t.t_parent->t.t_argv[i]); in __kmp_fork_call()
2298 KMP_CHECK_UPDATE(team->t.t_master_active, master_active); in __kmp_fork_call()
2302 __kmp_fork_team_threads(root, team, master_th, gtid, !ap); in __kmp_fork_call()
2303 __kmp_setup_icv_copy(team, nthreads, in __kmp_fork_call()
2313 if (team->t.t_active_level == 1 // only report frames at level 1 in __kmp_fork_call()
2325 team->t.t_region_time = tmp_time; in __kmp_fork_call()
2332 __kmp_itt_region_forking(gtid, team->t.t_nproc, 0); in __kmp_fork_call()
2338 KMP_DEBUG_ASSERT(team == __kmp_threads[gtid]->th.th_team); in __kmp_fork_call()
2342 root, team, master_th, gtid)); in __kmp_fork_call()
2348 KMP_DEBUG_ASSERT(team->t.t_stack_id == NULL); in __kmp_fork_call()
2349 team->t.t_stack_id = __kmp_itt_stack_caller_create(); in __kmp_fork_call()
2364 __kmp_internal_fork(loc, gtid, team); in __kmp_fork_call()
2367 root, team, master_th, gtid)); in __kmp_fork_call()
2377 team->t.t_id, team->t.t_pkfn)); in __kmp_fork_call()
2388 if (!team->t.t_invoke(gtid)) { in __kmp_fork_call()
2400 team->t.t_id, team->t.t_pkfn)); in __kmp_fork_call()
2415 kmp_team_t *team) { in __kmp_join_restore_state() argument
2418 ((team->t.t_serialized) ? ompt_state_work_serial in __kmp_join_restore_state()
2423 kmp_team_t *team, ompt_data_t *parallel_data, in __kmp_join_ompt() argument
2432 __kmp_join_restore_state(thread, team); in __kmp_join_ompt()
2444 kmp_team_t *team; in __kmp_join_call() local
2455 team = master_th->th.th_team; in __kmp_join_call()
2456 parent_team = team->t.t_parent; in __kmp_join_call()
2461 void *team_microtask = (void *)team->t.t_pkfn; in __kmp_join_call()
2466 !(team->t.t_serialized && fork_context == fork_context_gnu)) { in __kmp_join_call()
2475 __kmp_gtid_from_thread(master_th), team, in __kmp_join_call()
2476 team->t.t_task_team[master_th->th.th_task_state], in __kmp_join_call()
2478 KMP_DEBUG_ASSERT_TASKTEAM_INVARIANT(team, master_th); in __kmp_join_call()
2482 if (team->t.t_serialized) { in __kmp_join_call()
2485 int level = team->t.t_level; in __kmp_join_call()
2490 team->t.t_level++; in __kmp_join_call()
2495 team->t.t_serialized++; in __kmp_join_call()
2512 master_active = team->t.t_master_active; in __kmp_join_call()
2517 __kmp_internal_join(loc, gtid, team); in __kmp_join_call()
2520 KMP_DEBUG_ASSERT(team->t.t_stack_id != NULL); in __kmp_join_call()
2522 __kmp_itt_stack_caller_destroy((__itt_caller)team->t.t_stack_id); in __kmp_join_call()
2523 team->t.t_stack_id = NULL; in __kmp_join_call()
2544 ompt_data_t *parallel_data = &(team->t.ompt_team_info.parallel_data); in __kmp_join_call()
2545 void *codeptr = team->t.ompt_team_info.master_return_address; in __kmp_join_call()
2550 if (team->t.t_active_level == 1 && in __kmp_join_call()
2558 __kmp_itt_frame_submit(gtid, team->t.t_region_time, in __kmp_join_call()
2570 master_th->th.th_first_place = team->t.t_first_place; in __kmp_join_call()
2571 master_th->th.th_last_place = team->t.t_last_place; in __kmp_join_call()
2576 team->t.t_pkfn != (microtask_t)__kmp_teams_master && in __kmp_join_call()
2577 team->t.t_level == master_th->th.th_teams_level + 1) { in __kmp_join_call()
2586 int ompt_team_size = team->t.t_nproc; in __kmp_join_call()
2598 team->t.t_level--; in __kmp_join_call()
2599 team->t.t_active_level--; in __kmp_join_call()
2609 kmp_info_t **other_threads = team->t.t_threads; in __kmp_join_call()
2610 team->t.t_nproc = new_num; in __kmp_join_call()
2620 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived; in __kmp_join_call()
2623 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived; in __kmp_join_call()
2644 master_th->th.th_info.ds.ds_tid = team->t.t_master_tid; in __kmp_join_call()
2645 master_th->th.th_local.this_construct = team->t.t_master_this_cons; in __kmp_join_call()
2647 master_th->th.th_dispatch = &parent_team->t.t_dispatch[team->t.t_master_tid]; in __kmp_join_call()
2655 team->t.t_level > master_th->th.th_teams_level) { in __kmp_join_call()
2668 int ompt_team_size = (flags == ompt_task_initial) ? 0 : team->t.t_nproc; in __kmp_join_call()
2679 master_th, team)); in __kmp_join_call()
2682 master_th->th.th_def_allocator = team->t.t_def_allocator; in __kmp_join_call()
2688 updateHWFPControl(team); in __kmp_join_call()
2693 __kmp_free_team(root, team USE_NESTED_HOT_ARG( in __kmp_join_call()
2717 KMP_DEBUG_ASSERT(team->t.t_primary_task_state == 0 || in __kmp_join_call()
2718 team->t.t_primary_task_state == 1); in __kmp_join_call()
2719 master_th->th.th_task_state = (kmp_uint8)team->t.t_primary_task_state; in __kmp_join_call()
3052 kmp_team_t *team; in __kmp_get_ancestor_thread_num() local
3064 team = thr->th.th_team; in __kmp_get_ancestor_thread_num()
3065 ii = team->t.t_level; in __kmp_get_ancestor_thread_num()
3088 dd = team->t.t_serialized; in __kmp_get_ancestor_thread_num()
3091 for (dd = team->t.t_serialized; (dd > 0) && (ii > level); dd--, ii--) { in __kmp_get_ancestor_thread_num()
3093 if ((team->t.t_serialized) && (!dd)) { in __kmp_get_ancestor_thread_num()
3094 team = team->t.t_parent; in __kmp_get_ancestor_thread_num()
3098 team = team->t.t_parent; in __kmp_get_ancestor_thread_num()
3099 dd = team->t.t_serialized; in __kmp_get_ancestor_thread_num()
3104 return (dd > 1) ? (0) : (team->t.t_master_tid); in __kmp_get_ancestor_thread_num()
3110 kmp_team_t *team; in __kmp_get_team_size() local
3122 team = thr->th.th_team; in __kmp_get_team_size()
3123 ii = team->t.t_level; in __kmp_get_team_size()
3144 for (dd = team->t.t_serialized; (dd > 0) && (ii > level); dd--, ii--) { in __kmp_get_team_size()
3146 if (team->t.t_serialized && (!dd)) { in __kmp_get_team_size()
3147 team = team->t.t_parent; in __kmp_get_team_size()
3151 team = team->t.t_parent; in __kmp_get_team_size()
3156 return team->t.t_nproc; in __kmp_get_team_size()
3195 static void __kmp_alloc_argv_entries(int argc, kmp_team_t *team, int realloc) { in __kmp_alloc_argv_entries() argument
3197 KMP_DEBUG_ASSERT(team); in __kmp_alloc_argv_entries()
3198 if (!realloc || argc > team->t.t_max_argc) { in __kmp_alloc_argv_entries()
3202 team->t.t_id, argc, (realloc) ? team->t.t_max_argc : 0)); in __kmp_alloc_argv_entries()
3204 if (realloc && team->t.t_argv != &team->t.t_inline_argv[0]) in __kmp_alloc_argv_entries()
3205 __kmp_free((void *)team->t.t_argv); in __kmp_alloc_argv_entries()
3209 team->t.t_max_argc = KMP_INLINE_ARGV_ENTRIES; in __kmp_alloc_argv_entries()
3212 team->t.t_id, team->t.t_max_argc)); in __kmp_alloc_argv_entries()
3213 team->t.t_argv = &team->t.t_inline_argv[0]; in __kmp_alloc_argv_entries()
3216 -1, &team->t.t_inline_argv[0], in __kmp_alloc_argv_entries()
3217 &team->t.t_inline_argv[KMP_INLINE_ARGV_ENTRIES], in __kmp_alloc_argv_entries()
3219 team->t.t_id); in __kmp_alloc_argv_entries()
3223 team->t.t_max_argc = (argc <= (KMP_MIN_MALLOC_ARGV_ENTRIES >> 1)) in __kmp_alloc_argv_entries()
3228 team->t.t_id, team->t.t_max_argc)); in __kmp_alloc_argv_entries()
3229 team->t.t_argv = in __kmp_alloc_argv_entries()
3230 (void **)__kmp_page_allocate(sizeof(void *) * team->t.t_max_argc); in __kmp_alloc_argv_entries()
3232 __kmp_print_storage_map_gtid(-1, &team->t.t_argv[0], in __kmp_alloc_argv_entries()
3233 &team->t.t_argv[team->t.t_max_argc], in __kmp_alloc_argv_entries()
3234 sizeof(void *) * team->t.t_max_argc, in __kmp_alloc_argv_entries()
3235 "team_%d.t_argv", team->t.t_id); in __kmp_alloc_argv_entries()
3241 static void __kmp_allocate_team_arrays(kmp_team_t *team, int max_nth) { in __kmp_allocate_team_arrays() argument
3244 team->t.t_threads = in __kmp_allocate_team_arrays()
3246 team->t.t_disp_buffer = (dispatch_shared_info_t *)__kmp_allocate( in __kmp_allocate_team_arrays()
3248 team->t.t_dispatch = in __kmp_allocate_team_arrays()
3250 team->t.t_implicit_task_taskdata = in __kmp_allocate_team_arrays()
3252 team->t.t_max_nproc = max_nth; in __kmp_allocate_team_arrays()
3256 team->t.t_disp_buffer[i].buffer_index = i; in __kmp_allocate_team_arrays()
3257 team->t.t_disp_buffer[i].doacross_buf_idx = i; in __kmp_allocate_team_arrays()
3261 static void __kmp_free_team_arrays(kmp_team_t *team) { in __kmp_free_team_arrays() argument
3264 for (i = 0; i < team->t.t_max_nproc; ++i) { in __kmp_free_team_arrays()
3265 if (team->t.t_dispatch[i].th_disp_buffer != NULL) { in __kmp_free_team_arrays()
3266 __kmp_free(team->t.t_dispatch[i].th_disp_buffer); in __kmp_free_team_arrays()
3267 team->t.t_dispatch[i].th_disp_buffer = NULL; in __kmp_free_team_arrays()
3271 __kmp_dispatch_free_hierarchies(team); in __kmp_free_team_arrays()
3273 __kmp_free(team->t.t_threads); in __kmp_free_team_arrays()
3274 __kmp_free(team->t.t_disp_buffer); in __kmp_free_team_arrays()
3275 __kmp_free(team->t.t_dispatch); in __kmp_free_team_arrays()
3276 __kmp_free(team->t.t_implicit_task_taskdata); in __kmp_free_team_arrays()
3277 team->t.t_threads = NULL; in __kmp_free_team_arrays()
3278 team->t.t_disp_buffer = NULL; in __kmp_free_team_arrays()
3279 team->t.t_dispatch = NULL; in __kmp_free_team_arrays()
3280 team->t.t_implicit_task_taskdata = 0; in __kmp_free_team_arrays()
3283 static void __kmp_reallocate_team_arrays(kmp_team_t *team, int max_nth) { in __kmp_reallocate_team_arrays() argument
3284 kmp_info_t **oldThreads = team->t.t_threads; in __kmp_reallocate_team_arrays()
3286 __kmp_free(team->t.t_disp_buffer); in __kmp_reallocate_team_arrays()
3287 __kmp_free(team->t.t_dispatch); in __kmp_reallocate_team_arrays()
3288 __kmp_free(team->t.t_implicit_task_taskdata); in __kmp_reallocate_team_arrays()
3289 __kmp_allocate_team_arrays(team, max_nth); in __kmp_reallocate_team_arrays()
3291 KMP_MEMCPY(team->t.t_threads, oldThreads, in __kmp_reallocate_team_arrays()
3292 team->t.t_nproc * sizeof(kmp_info_t *)); in __kmp_reallocate_team_arrays()
3333 static kmp_internal_control_t __kmp_get_x_global_icvs(const kmp_team_t *team) { in __kmp_get_x_global_icvs() argument
3338 copy_icvs(&gx_icvs, &team->t.t_threads[0]->th.th_current_task->td_icvs); in __kmp_get_x_global_icvs()
3448 kmp_team_p const *team // Team to add. in __kmp_print_structure_team_accum() argument
3459 if (team == NULL) { in __kmp_print_structure_team_accum()
3463 __kmp_print_structure_team_accum(list, team->t.t_parent); in __kmp_print_structure_team_accum()
3464 __kmp_print_structure_team_accum(list, team->t.t_next_pool); in __kmp_print_structure_team_accum()
3468 while (l->next != NULL && l->entry != team) { in __kmp_print_structure_team_accum()
3477 while (l->next != NULL && l->entry->t.t_id <= team->t.t_id) { in __kmp_print_structure_team_accum()
3486 l->entry = team; in __kmp_print_structure_team_accum()
3491 static void __kmp_print_structure_team(char const *title, kmp_team_p const *team in __kmp_print_structure_team() argument
3495 if (team != NULL) { in __kmp_print_structure_team()
3496 __kmp_printf("%2x %p\n", team->t.t_id, team); in __kmp_print_structure_team()
3596 kmp_team_p const *team = list->entry; in __kmp_print_structure() local
3598 __kmp_printf("Team %2x %p:\n", team->t.t_id, team); in __kmp_print_structure()
3599 __kmp_print_structure_team(" Parent Team: ", team->t.t_parent); in __kmp_print_structure()
3600 __kmp_printf(" Primary TID: %2d\n", team->t.t_master_tid); in __kmp_print_structure()
3601 __kmp_printf(" Max threads: %2d\n", team->t.t_max_nproc); in __kmp_print_structure()
3602 __kmp_printf(" Levels of serial: %2d\n", team->t.t_serialized); in __kmp_print_structure()
3603 __kmp_printf(" Number threads: %2d\n", team->t.t_nproc); in __kmp_print_structure()
3604 for (i = 0; i < team->t.t_nproc; ++i) { in __kmp_print_structure()
3606 __kmp_print_structure_thread("", team->t.t_threads[i]); in __kmp_print_structure()
3608 __kmp_print_structure_team(" Next in pool: ", team->t.t_next_pool); in __kmp_print_structure()
4076 kmp_team_t *team = hot_teams[level].hot_team; in __kmp_free_hot_teams() local
4081 kmp_info_t *th = team->t.t_threads[i]; in __kmp_free_hot_teams()
4089 __kmp_free_team(root, team, NULL); in __kmp_free_hot_teams()
4212 kmp_team_t *team = thread->th.th_team; in __kmp_unregister_root_current_thread() local
4222 __kmp_task_team_wait(thread, team USE_ITT_BUILD_ARG(NULL)); in __kmp_unregister_root_current_thread()
4262 kmp_team_t *team = this_thr->th.th_team; in __kmp_task_info() local
4267 gtid, tid, this_thr, team, steam, this_thr->th.th_current_task, in __kmp_task_info()
4268 team->t.t_implicit_task_taskdata[tid].td_parent); in __kmp_task_info()
4275 static void __kmp_initialize_info(kmp_info_t *this_thr, kmp_team_t *team, in __kmp_initialize_info() argument
4282 KMP_DEBUG_ASSERT(team); in __kmp_initialize_info()
4283 KMP_DEBUG_ASSERT(team->t.t_threads); in __kmp_initialize_info()
4284 KMP_DEBUG_ASSERT(team->t.t_dispatch); in __kmp_initialize_info()
4285 kmp_info_t *master = team->t.t_threads[0]; in __kmp_initialize_info()
4291 TCW_SYNC_PTR(this_thr->th.th_team, team); in __kmp_initialize_info()
4309 this_thr->th.th_team_nproc = team->t.t_nproc; in __kmp_initialize_info()
4311 this_thr->th.th_team_serialized = team->t.t_serialized; in __kmp_initialize_info()
4313 KMP_DEBUG_ASSERT(team->t.t_implicit_task_taskdata); in __kmp_initialize_info()
4319 team, tid, TRUE); in __kmp_initialize_info()
4327 this_thr->th.th_dispatch = &team->t.t_dispatch[tid]; in __kmp_initialize_info()
4375 (team->t.t_max_nproc == 1 ? 1 : __kmp_dispatch_num_buffers); in __kmp_initialize_info()
4377 team->t.t_max_nproc)); in __kmp_initialize_info()
4379 KMP_DEBUG_ASSERT(team->t.t_dispatch); in __kmp_initialize_info()
4380 KMP_DEBUG_ASSERT(dispatch == &team->t.t_dispatch[tid]); in __kmp_initialize_info()
4391 &dispatch->th_disp_buffer[team->t.t_max_nproc == 1 in __kmp_initialize_info()
4397 gtid, team->t.t_id, gtid); in __kmp_initialize_info()
4423 kmp_info_t *__kmp_allocate_thread(kmp_root_t *root, kmp_team_t *team, in __kmp_allocate_thread() argument
4430 KMP_DEBUG_ASSERT(root && team); in __kmp_allocate_thread()
4439 if (__kmp_thread_pool && !KMP_HIDDEN_HELPER_TEAM(team)) { in __kmp_allocate_thread()
4461 __kmp_initialize_info(new_thr, team, new_tid, in __kmp_allocate_thread()
4502 KMP_ASSERT(KMP_HIDDEN_HELPER_TEAM(team) || __kmp_nth == __kmp_all_nth); in __kmp_allocate_thread()
4601 kmp_internal_control_t r_icvs = __kmp_get_x_global_icvs(team); in __kmp_allocate_thread()
4620 __kmp_initialize_info(new_thr, team, new_tid, new_gtid); in __kmp_allocate_thread()
4642 balign[b].bb.team = NULL; in __kmp_allocate_thread()
4724 static void __kmp_reinitialize_team(kmp_team_t *team, in __kmp_reinitialize_team() argument
4728 team->t.t_threads[0], team)); in __kmp_reinitialize_team()
4729 KMP_DEBUG_ASSERT(team && new_icvs); in __kmp_reinitialize_team()
4731 KMP_CHECK_UPDATE(team->t.t_ident, loc); in __kmp_reinitialize_team()
4733 KMP_CHECK_UPDATE(team->t.t_id, KMP_GEN_TEAM_ID()); in __kmp_reinitialize_team()
4735 __kmp_init_implicit_task(loc, team->t.t_threads[0], team, 0, FALSE); in __kmp_reinitialize_team()
4736 copy_icvs(&team->t.t_implicit_task_taskdata[0].td_icvs, new_icvs); in __kmp_reinitialize_team()
4739 team->t.t_threads[0], team)); in __kmp_reinitialize_team()
4745 static void __kmp_initialize_team(kmp_team_t *team, int new_nproc, in __kmp_initialize_team() argument
4748 KF_TRACE(10, ("__kmp_initialize_team: enter: team=%p\n", team)); in __kmp_initialize_team()
4751 KMP_DEBUG_ASSERT(team); in __kmp_initialize_team()
4752 KMP_DEBUG_ASSERT(new_nproc <= team->t.t_max_nproc); in __kmp_initialize_team()
4753 KMP_DEBUG_ASSERT(team->t.t_threads); in __kmp_initialize_team()
4756 team->t.t_master_tid = 0; /* not needed */ in __kmp_initialize_team()
4758 team->t.t_serialized = new_nproc > 1 ? 0 : 1; in __kmp_initialize_team()
4759 team->t.t_nproc = new_nproc; in __kmp_initialize_team()
4762 team->t.t_next_pool = NULL; in __kmp_initialize_team()
4766 TCW_SYNC_PTR(team->t.t_pkfn, NULL); /* not needed */ in __kmp_initialize_team()
4767 team->t.t_invoke = NULL; /* not needed */ in __kmp_initialize_team()
4770 team->t.t_sched.sched = new_icvs->sched.sched; in __kmp_initialize_team()
4773 team->t.t_fp_control_saved = FALSE; /* not needed */ in __kmp_initialize_team()
4774 team->t.t_x87_fpu_control_word = 0; /* not needed */ in __kmp_initialize_team()
4775 team->t.t_mxcsr = 0; /* not needed */ in __kmp_initialize_team()
4778 team->t.t_construct = 0; in __kmp_initialize_team()
4780 team->t.t_ordered.dt.t_value = 0; in __kmp_initialize_team()
4781 team->t.t_master_active = FALSE; in __kmp_initialize_team()
4784 team->t.t_copypriv_data = NULL; /* not necessary, but nice for debugging */ in __kmp_initialize_team()
4787 team->t.t_copyin_counter = 0; /* for barrier-free copyin implementation */ in __kmp_initialize_team()
4790 team->t.t_control_stack_top = NULL; in __kmp_initialize_team()
4792 __kmp_reinitialize_team(team, new_icvs, loc); in __kmp_initialize_team()
4795 KF_TRACE(10, ("__kmp_initialize_team: exit: team=%p\n", team)); in __kmp_initialize_team()
4799 static inline void __kmp_set_thread_place(kmp_team_t *team, kmp_info_t *th, in __kmp_set_thread_place() argument
4805 if (__kmp_display_affinity && team->t.t_display_affinity != 1) in __kmp_set_thread_place()
4806 team->t.t_display_affinity = 1; in __kmp_set_thread_place()
4817 static void __kmp_partition_places(kmp_team_t *team, int update_master_only) { in __kmp_partition_places() argument
4819 if (KMP_HIDDEN_HELPER_TEAM(team)) in __kmp_partition_places()
4822 kmp_info_t *master_th = team->t.t_threads[0]; in __kmp_partition_places()
4824 kmp_proc_bind_t proc_bind = team->t.t_proc_bind; in __kmp_partition_places()
4829 team->t.t_first_place = first_place; in __kmp_partition_places()
4830 team->t.t_last_place = last_place; in __kmp_partition_places()
4834 proc_bind, __kmp_gtid_from_thread(team->t.t_threads[0]), in __kmp_partition_places()
4835 team->t.t_id, masters_place, first_place, last_place)); in __kmp_partition_places()
4842 KMP_DEBUG_ASSERT(team->t.t_nproc == 1); in __kmp_partition_places()
4847 int n_th = team->t.t_nproc; in __kmp_partition_places()
4849 kmp_info_t *th = team->t.t_threads[f]; in __kmp_partition_places()
4851 __kmp_set_thread_place(team, th, first_place, last_place, masters_place); in __kmp_partition_places()
4855 __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id, in __kmp_partition_places()
4862 int n_th = team->t.t_nproc; in __kmp_partition_places()
4872 kmp_info_t *th = team->t.t_threads[f]; in __kmp_partition_places()
4882 __kmp_set_thread_place(team, th, first_place, last_place, place); in __kmp_partition_places()
4886 __kmp_gtid_from_thread(team->t.t_threads[f]), in __kmp_partition_places()
4887 team->t.t_id, f, place, first_place, last_place)); in __kmp_partition_places()
4898 kmp_info_t *th = team->t.t_threads[f]; in __kmp_partition_places()
4901 __kmp_set_thread_place(team, th, first_place, last_place, place); in __kmp_partition_places()
4933 __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id, f, in __kmp_partition_places()
4942 int n_th = team->t.t_nproc; in __kmp_partition_places()
4965 kmp_info_t *th = team->t.t_threads[f]; in __kmp_partition_places()
4991 __kmp_set_thread_place(team, th, fplace, place, nplace); in __kmp_partition_places()
5005 __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id, in __kmp_partition_places()
5055 th = team->t.t_threads[f]; in __kmp_partition_places()
5057 __kmp_set_thread_place(team, th, first, last, place); in __kmp_partition_places()
5061 __kmp_gtid_from_thread(team->t.t_threads[f]), in __kmp_partition_places()
5062 team->t.t_id, f, th->th.th_new_place, in __kmp_partition_places()
5080 kmp_info_t *th = team->t.t_threads[f]; in __kmp_partition_places()
5083 __kmp_set_thread_place(team, th, place, place, place); in __kmp_partition_places()
5114 __kmp_gtid_from_thread(team->t.t_threads[f]), in __kmp_partition_places()
5115 team->t.t_id, f, th->th.th_new_place, in __kmp_partition_places()
5126 KA_TRACE(20, ("__kmp_partition_places: exit T#%d\n", team->t.t_id)); in __kmp_partition_places()
5143 kmp_team_t *team; in __kmp_allocate_team() local
5156 team = master->th.th_team; in __kmp_allocate_team()
5157 level = team->t.t_active_level; in __kmp_allocate_team()
5161 team->t.t_pkfn == in __kmp_allocate_team()
5164 team->t.t_level)) { // or nested parallel inside the teams in __kmp_allocate_team()
5171 master->th.th_teams_level >= team->t.t_level) || in __kmp_allocate_team()
5172 (team->t.t_pkfn == (microtask_t)__kmp_teams_master)) in __kmp_allocate_team()
5192 team = hot_teams[level].hot_team; in __kmp_allocate_team()
5194 team = root->r.r_hot_team; in __kmp_allocate_team()
5200 team->t.t_task_team[0], team->t.t_task_team[1])); in __kmp_allocate_team()
5204 if (team->t.t_nproc != new_nproc && in __kmp_allocate_team()
5207 int old_nthr = team->t.t_nproc; in __kmp_allocate_team()
5208 __kmp_resize_dist_barrier(team, old_nthr, new_nproc); in __kmp_allocate_team()
5214 team->t.t_proc_bind = proc_bind_default; in __kmp_allocate_team()
5218 if (team->t.t_nproc == new_nproc) { // Check changes in number of threads in __kmp_allocate_team()
5222 if (team->t.t_size_changed == -1) { in __kmp_allocate_team()
5223 team->t.t_size_changed = 1; in __kmp_allocate_team()
5225 KMP_CHECK_UPDATE(team->t.t_size_changed, 0); in __kmp_allocate_team()
5231 KMP_CHECK_UPDATE(team->t.t_sched.sched, new_sched.sched); in __kmp_allocate_team()
5233 __kmp_reinitialize_team(team, new_icvs, in __kmp_allocate_team()
5237 team->t.t_threads[0], team)); in __kmp_allocate_team()
5238 __kmp_push_current_task_to_thread(team->t.t_threads[0], team, 0); in __kmp_allocate_team()
5241 if ((team->t.t_size_changed == 0) && in __kmp_allocate_team()
5242 (team->t.t_proc_bind == new_proc_bind)) { in __kmp_allocate_team()
5246 __kmp_partition_places(team, 1); in __kmp_allocate_team()
5251 team->t.t_id, new_proc_bind, team->t.t_first_place, in __kmp_allocate_team()
5252 team->t.t_last_place)); in __kmp_allocate_team()
5255 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind); in __kmp_allocate_team()
5256 __kmp_partition_places(team); in __kmp_allocate_team()
5260 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind); in __kmp_allocate_team()
5262 } else if (team->t.t_nproc > new_nproc) { in __kmp_allocate_team()
5267 team->t.t_size_changed = 1; in __kmp_allocate_team()
5271 __kmp_add_threads_to_team(team, new_nproc); in __kmp_allocate_team()
5276 for (f = new_nproc; f < team->t.t_nproc; f++) { in __kmp_allocate_team()
5277 kmp_info_t *th = team->t.t_threads[f]; in __kmp_allocate_team()
5286 KMP_DEBUG_ASSERT(hot_teams[level].hot_team_nth == team->t.t_nproc); in __kmp_allocate_team()
5290 for (f = new_nproc; f < team->t.t_nproc; f++) { in __kmp_allocate_team()
5291 KMP_DEBUG_ASSERT(team->t.t_threads[f]); in __kmp_allocate_team()
5292 __kmp_free_thread(team->t.t_threads[f]); in __kmp_allocate_team()
5293 team->t.t_threads[f] = NULL; in __kmp_allocate_team()
5300 for (f = new_nproc; f < team->t.t_nproc; ++f) { in __kmp_allocate_team()
5301 KMP_DEBUG_ASSERT(team->t.t_threads[f]); in __kmp_allocate_team()
5302 kmp_balign_t *balign = team->t.t_threads[f]->th.th_bar; in __kmp_allocate_team()
5312 team->t.t_nproc = new_nproc; in __kmp_allocate_team()
5314 KMP_CHECK_UPDATE(team->t.t_sched.sched, new_icvs->sched.sched); in __kmp_allocate_team()
5315 __kmp_reinitialize_team(team, new_icvs, in __kmp_allocate_team()
5320 team->t.t_threads[f]->th.th_team_nproc = new_nproc; in __kmp_allocate_team()
5326 team->t.t_threads[0], team)); in __kmp_allocate_team()
5328 __kmp_push_current_task_to_thread(team->t.t_threads[0], team, 0); in __kmp_allocate_team()
5331 for (f = 0; f < team->t.t_nproc; f++) { in __kmp_allocate_team()
5332 KMP_DEBUG_ASSERT(team->t.t_threads[f] && in __kmp_allocate_team()
5333 team->t.t_threads[f]->th.th_team_nproc == in __kmp_allocate_team()
5334 team->t.t_nproc); in __kmp_allocate_team()
5339 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind); in __kmp_allocate_team()
5341 __kmp_partition_places(team); in __kmp_allocate_team()
5349 int old_nproc = team->t.t_nproc; // save old value and use to update only in __kmp_allocate_team()
5350 team->t.t_size_changed = 1; in __kmp_allocate_team()
5356 kmp_info_t **other_threads = team->t.t_threads; in __kmp_allocate_team()
5357 for (f = team->t.t_nproc; f < avail_threads; ++f) { in __kmp_allocate_team()
5363 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived; in __kmp_allocate_team()
5366 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived; in __kmp_allocate_team()
5374 team->t.t_nproc = new_nproc; // just get reserved threads involved in __kmp_allocate_team()
5378 team->t.t_nproc = hot_teams[level].hot_team_nth; in __kmp_allocate_team()
5381 if (team->t.t_max_nproc < new_nproc) { in __kmp_allocate_team()
5383 __kmp_reallocate_team_arrays(team, new_nproc); in __kmp_allocate_team()
5384 __kmp_reinitialize_team(team, new_icvs, NULL); in __kmp_allocate_team()
5398 for (f = team->t.t_nproc; f < new_nproc; f++) { in __kmp_allocate_team()
5399 kmp_info_t *new_worker = __kmp_allocate_thread(root, team, f); in __kmp_allocate_team()
5401 team->t.t_threads[f] = new_worker; in __kmp_allocate_team()
5406 team->t.t_id, __kmp_gtid_from_tid(f, team), team->t.t_id, f, in __kmp_allocate_team()
5407 team->t.t_bar[bs_forkjoin_barrier].b_arrived, in __kmp_allocate_team()
5408 team->t.t_bar[bs_plain_barrier].b_arrived)); in __kmp_allocate_team()
5414 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived; in __kmp_allocate_team()
5418 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived; in __kmp_allocate_team()
5435 __kmp_add_threads_to_team(team, new_nproc); in __kmp_allocate_team()
5439 __kmp_initialize_team(team, new_nproc, new_icvs, in __kmp_allocate_team()
5443 KMP_DEBUG_ASSERT(team->t.t_nproc == new_nproc); in __kmp_allocate_team()
5444 for (f = 0; f < team->t.t_nproc; ++f) in __kmp_allocate_team()
5445 __kmp_initialize_info(team->t.t_threads[f], team, f, in __kmp_allocate_team()
5446 __kmp_gtid_from_tid(f, team)); in __kmp_allocate_team()
5449 kmp_uint8 old_state = team->t.t_threads[old_nproc - 1]->th.th_task_state; in __kmp_allocate_team()
5450 for (f = old_nproc; f < team->t.t_nproc; ++f) in __kmp_allocate_team()
5451 team->t.t_threads[f]->th.th_task_state = old_state; in __kmp_allocate_team()
5454 for (f = 0; f < team->t.t_nproc; ++f) { in __kmp_allocate_team()
5455 KMP_DEBUG_ASSERT(team->t.t_threads[f] && in __kmp_allocate_team()
5456 team->t.t_threads[f]->th.th_team_nproc == in __kmp_allocate_team()
5457 team->t.t_nproc); in __kmp_allocate_team()
5462 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind); in __kmp_allocate_team()
5464 __kmp_partition_places(team); in __kmp_allocate_team()
5472 kmp_info_t *thr = team->t.t_threads[f]; in __kmp_allocate_team()
5483 kmp_info_t *thr = team->t.t_threads[f]; in __kmp_allocate_team()
5487 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived; in __kmp_allocate_team()
5490 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived; in __kmp_allocate_team()
5498 __kmp_alloc_argv_entries(argc, team, TRUE); in __kmp_allocate_team()
5499 KMP_CHECK_UPDATE(team->t.t_argc, argc); in __kmp_allocate_team()
5503 KF_TRACE(10, (" hot_team = %p\n", team)); in __kmp_allocate_team()
5509 team->t.t_task_team[0], team->t.t_task_team[1])); in __kmp_allocate_team()
5514 __ompt_team_assign_id(team, ompt_parallel_data); in __kmp_allocate_team()
5519 return team; in __kmp_allocate_team()
5524 for (team = CCAST(kmp_team_t *, __kmp_team_pool); (team);) { in __kmp_allocate_team()
5527 if (team->t.t_max_nproc >= max_nproc) { in __kmp_allocate_team()
5529 __kmp_team_pool = team->t.t_next_pool; in __kmp_allocate_team()
5533 if (!team->t.b) { // Allocate barrier structure in __kmp_allocate_team()
5534 team->t.b = distributedBarrier::allocate(__kmp_dflt_team_nth_ub); in __kmp_allocate_team()
5539 __kmp_initialize_team(team, new_nproc, new_icvs, NULL); in __kmp_allocate_team()
5543 &team->t.t_task_team[0], &team->t.t_task_team[1])); in __kmp_allocate_team()
5544 team->t.t_task_team[0] = NULL; in __kmp_allocate_team()
5545 team->t.t_task_team[1] = NULL; in __kmp_allocate_team()
5548 __kmp_alloc_argv_entries(argc, team, TRUE); in __kmp_allocate_team()
5549 KMP_CHECK_UPDATE(team->t.t_argc, argc); in __kmp_allocate_team()
5553 team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE)); in __kmp_allocate_team()
5557 team->t.t_bar[b].b_arrived = KMP_INIT_BARRIER_STATE; in __kmp_allocate_team()
5559 team->t.t_bar[b].b_master_arrived = 0; in __kmp_allocate_team()
5560 team->t.t_bar[b].b_team_arrived = 0; in __kmp_allocate_team()
5565 team->t.t_proc_bind = new_proc_bind; in __kmp_allocate_team()
5568 team->t.t_id)); in __kmp_allocate_team()
5571 __ompt_team_assign_id(team, ompt_parallel_data); in __kmp_allocate_team()
5574 team->t.t_nested_nth = NULL; in __kmp_allocate_team()
5578 return team; in __kmp_allocate_team()
5585 team = __kmp_reap_team(team); in __kmp_allocate_team()
5586 __kmp_team_pool = team; in __kmp_allocate_team()
5591 team = (kmp_team_t *)__kmp_allocate(sizeof(kmp_team_t)); in __kmp_allocate_team()
5594 team->t.t_max_nproc = max_nproc; in __kmp_allocate_team()
5598 team->t.b = distributedBarrier::allocate(__kmp_dflt_team_nth_ub); in __kmp_allocate_team()
5603 __kmp_allocate_team_arrays(team, max_nproc); in __kmp_allocate_team()
5606 __kmp_initialize_team(team, new_nproc, new_icvs, NULL); in __kmp_allocate_team()
5610 &team->t.t_task_team[0], &team->t.t_task_team[1])); in __kmp_allocate_team()
5611 team->t.t_task_team[0] = NULL; // to be removed, as __kmp_allocate zeroes in __kmp_allocate_team()
5613 team->t.t_task_team[1] = NULL; // to be removed, as __kmp_allocate zeroes in __kmp_allocate_team()
5617 __kmp_print_team_storage_map("team", team, team->t.t_id, new_nproc); in __kmp_allocate_team()
5621 __kmp_alloc_argv_entries(argc, team, FALSE); in __kmp_allocate_team()
5622 team->t.t_argc = argc; in __kmp_allocate_team()
5626 team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE)); in __kmp_allocate_team()
5630 team->t.t_bar[b].b_arrived = KMP_INIT_BARRIER_STATE; in __kmp_allocate_team()
5632 team->t.t_bar[b].b_master_arrived = 0; in __kmp_allocate_team()
5633 team->t.t_bar[b].b_team_arrived = 0; in __kmp_allocate_team()
5638 team->t.t_proc_bind = new_proc_bind; in __kmp_allocate_team()
5641 __ompt_team_assign_id(team, ompt_parallel_data); in __kmp_allocate_team()
5642 team->t.ompt_serialized_team_info = NULL; in __kmp_allocate_team()
5647 team->t.t_nested_nth = NULL; in __kmp_allocate_team()
5650 team->t.t_id)); in __kmp_allocate_team()
5652 return team; in __kmp_allocate_team()
5661 kmp_team_t *team USE_NESTED_HOT_ARG(kmp_info_t *master)) { in __kmp_free_team()
5664 team->t.t_id)); in __kmp_free_team()
5668 KMP_DEBUG_ASSERT(team); in __kmp_free_team()
5669 KMP_DEBUG_ASSERT(team->t.t_nproc <= team->t.t_max_nproc); in __kmp_free_team()
5670 KMP_DEBUG_ASSERT(team->t.t_threads); in __kmp_free_team()
5672 int use_hot_team = team == root->r.r_hot_team; in __kmp_free_team()
5676 level = team->t.t_active_level - 1; in __kmp_free_team()
5682 if (team->t.t_pkfn != (microtask_t)__kmp_teams_master && in __kmp_free_team()
5683 master->th.th_teams_level == team->t.t_level) { in __kmp_free_team()
5692 KMP_DEBUG_ASSERT(team == hot_teams[level].hot_team); in __kmp_free_team()
5699 TCW_SYNC_PTR(team->t.t_pkfn, in __kmp_free_team()
5702 team->t.t_copyin_counter = 0; // init counter for possible reuse in __kmp_free_team()
5710 for (f = 1; f < team->t.t_nproc; ++f) { in __kmp_free_team()
5711 KMP_DEBUG_ASSERT(team->t.t_threads[f]); in __kmp_free_team()
5712 kmp_info_t *th = team->t.t_threads[f]; in __kmp_free_team()
5733 kmp_task_team_t *task_team = team->t.t_task_team[tt_idx]; in __kmp_free_team()
5735 for (f = 0; f < team->t.t_nproc; ++f) { // threads unref task teams in __kmp_free_team()
5736 KMP_DEBUG_ASSERT(team->t.t_threads[f]); in __kmp_free_team()
5737 team->t.t_threads[f]->th.th_task_team = NULL; in __kmp_free_team()
5742 __kmp_get_gtid(), task_team, team->t.t_id)); in __kmp_free_team()
5746 team->t.t_task_team[tt_idx] = NULL; in __kmp_free_team()
5752 if (team->t.t_nested_nth && team->t.t_nested_nth != &__kmp_nested_nth && in __kmp_free_team()
5753 team->t.t_nested_nth != team->t.t_parent->t.t_nested_nth) { in __kmp_free_team()
5754 KMP_INTERNAL_FREE(team->t.t_nested_nth->nth); in __kmp_free_team()
5755 KMP_INTERNAL_FREE(team->t.t_nested_nth); in __kmp_free_team()
5757 team->t.t_nested_nth = NULL; in __kmp_free_team()
5760 team->t.t_parent = NULL; in __kmp_free_team()
5761 team->t.t_level = 0; in __kmp_free_team()
5762 team->t.t_active_level = 0; in __kmp_free_team()
5765 for (f = 1; f < team->t.t_nproc; ++f) { in __kmp_free_team()
5766 KMP_DEBUG_ASSERT(team->t.t_threads[f]); in __kmp_free_team()
5768 KMP_COMPARE_AND_STORE_ACQ32(&(team->t.t_threads[f]->th.th_used_in_team), in __kmp_free_team()
5771 __kmp_free_thread(team->t.t_threads[f]); in __kmp_free_team()
5775 if (team->t.b) { in __kmp_free_team()
5777 team->t.b->go_release(); in __kmp_free_team()
5779 for (f = 1; f < team->t.t_nproc; ++f) { in __kmp_free_team()
5780 if (team->t.b->sleep[f].sleep) { in __kmp_free_team()
5782 team->t.t_threads[f]->th.th_info.ds.ds_gtid, in __kmp_free_team()
5788 for (int f = 1; f < team->t.t_nproc; ++f) { in __kmp_free_team()
5789 while (team->t.t_threads[f]->th.th_used_in_team.load() != 0) in __kmp_free_team()
5795 for (f = 1; f < team->t.t_nproc; ++f) { in __kmp_free_team()
5796 team->t.t_threads[f] = NULL; in __kmp_free_team()
5799 if (team->t.t_max_nproc > 1 && in __kmp_free_team()
5801 distributedBarrier::deallocate(team->t.b); in __kmp_free_team()
5802 team->t.b = NULL; in __kmp_free_team()
5806 team->t.t_next_pool = CCAST(kmp_team_t *, __kmp_team_pool); in __kmp_free_team()
5807 __kmp_team_pool = (volatile kmp_team_t *)team; in __kmp_free_team()
5810 KMP_DEBUG_ASSERT(team->t.t_threads[1] && in __kmp_free_team()
5811 team->t.t_threads[1]->th.th_cg_roots); in __kmp_free_team()
5812 if (team->t.t_threads[1]->th.th_cg_roots->cg_root == team->t.t_threads[1]) { in __kmp_free_team()
5814 for (f = 1; f < team->t.t_nproc; ++f) { in __kmp_free_team()
5815 kmp_info_t *thr = team->t.t_threads[f]; in __kmp_free_team()
5840 kmp_team_t *__kmp_reap_team(kmp_team_t *team) { in __kmp_reap_team() argument
5841 kmp_team_t *next_pool = team->t.t_next_pool; in __kmp_reap_team()
5843 KMP_DEBUG_ASSERT(team); in __kmp_reap_team()
5844 KMP_DEBUG_ASSERT(team->t.t_dispatch); in __kmp_reap_team()
5845 KMP_DEBUG_ASSERT(team->t.t_disp_buffer); in __kmp_reap_team()
5846 KMP_DEBUG_ASSERT(team->t.t_threads); in __kmp_reap_team()
5847 KMP_DEBUG_ASSERT(team->t.t_argv); in __kmp_reap_team()
5852 __kmp_free_team_arrays(team); in __kmp_reap_team()
5853 if (team->t.t_argv != &team->t.t_inline_argv[0]) in __kmp_reap_team()
5854 __kmp_free((void *)team->t.t_argv); in __kmp_reap_team()
5855 __kmp_free(team); in __kmp_reap_team()
5902 balign[b].bb.team = NULL; in __kmp_free_thread()
6401 kmp_team_t *team = CCAST(kmp_team_t *, __kmp_team_pool); in __kmp_internal_end() local
6402 __kmp_team_pool = team->t.t_next_pool; in __kmp_internal_end()
6404 team->t.t_next_pool = NULL; in __kmp_internal_end()
6405 __kmp_reap_team(team); in __kmp_internal_end()
7648 kmp_team_t *team) { in __kmp_run_before_invoked_task() argument
7660 KMP_DEBUG_ASSERT(team->t.t_dispatch); in __kmp_run_before_invoked_task()
7667 __kmp_push_parallel(gtid, team->t.t_ident); in __kmp_run_before_invoked_task()
7673 kmp_team_t *team) { in __kmp_run_after_invoked_task() argument
7675 __kmp_pop_parallel(gtid, team->t.t_ident); in __kmp_run_after_invoked_task()
7684 kmp_team_t *team = this_thr->th.th_team; in __kmp_invoke_task_func() local
7686 __kmp_run_before_invoked_task(gtid, tid, this_thr, team); in __kmp_invoke_task_func()
7690 if (team->t.t_stack_id != NULL) { in __kmp_invoke_task_func()
7691 __kmp_itt_stack_callee_enter((__itt_caller)team->t.t_stack_id); in __kmp_invoke_task_func()
7693 KMP_DEBUG_ASSERT(team->t.t_parent->t.t_stack_id != NULL); in __kmp_invoke_task_func()
7695 (__itt_caller)team->t.t_parent->t.t_stack_id); in __kmp_invoke_task_func()
7711 exit_frame_p = &(team->t.t_implicit_task_taskdata[tid] in __kmp_invoke_task_func()
7718 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data); in __kmp_invoke_task_func()
7719 my_parallel_data = &(team->t.ompt_team_info.parallel_data); in __kmp_invoke_task_func()
7721 ompt_team_size = team->t.t_nproc; in __kmp_invoke_task_func()
7739 rc = __kmp_invoke_microtask((microtask_t)TCR_SYNC_PTR(team->t.t_pkfn), gtid, in __kmp_invoke_task_func()
7740 tid, (int)team->t.t_argc, (void **)team->t.t_argv in __kmp_invoke_task_func()
7761 if (team->t.t_stack_id != NULL) { in __kmp_invoke_task_func()
7762 __kmp_itt_stack_callee_leave((__itt_caller)team->t.t_stack_id); in __kmp_invoke_task_func()
7764 KMP_DEBUG_ASSERT(team->t.t_parent->t.t_stack_id != NULL); in __kmp_invoke_task_func()
7766 (__itt_caller)team->t.t_parent->t.t_stack_id); in __kmp_invoke_task_func()
7770 __kmp_run_after_invoked_task(gtid, tid, this_thr, team); in __kmp_invoke_task_func()
7778 kmp_team_t *team = thr->th.th_team; in __kmp_teams_master() local
7779 ident_t *loc = team->t.t_ident; in __kmp_teams_master()
7803 __kmp_fork_call(loc, gtid, fork_context_intel, team->t.t_argc, in __kmp_teams_master()
7825 kmp_team_t *team = this_thr->th.th_team; in __kmp_invoke_teams_master() local
7831 __kmp_run_before_invoked_task(gtid, 0, this_thr, team); in __kmp_invoke_teams_master()
7835 &team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data; in __kmp_invoke_teams_master()
7836 ompt_data_t *parallel_data = &team->t.ompt_team_info.parallel_data; in __kmp_invoke_teams_master()
7839 ompt_scope_begin, parallel_data, task_data, team->t.t_nproc, tid, in __kmp_invoke_teams_master()
7848 __kmp_run_after_invoked_task(gtid, 0, this_thr, team); in __kmp_invoke_teams_master()
8059 void __kmp_internal_fork(ident_t *id, int gtid, kmp_team_t *team) { in __kmp_internal_fork() argument
8066 KMP_DEBUG_ASSERT(team); in __kmp_internal_fork()
8067 KMP_DEBUG_ASSERT(this_thr->th.th_team == team); in __kmp_internal_fork()
8071 team->t.t_construct = 0; /* no single directives seen yet */ in __kmp_internal_fork()
8072 team->t.t_ordered.dt.t_value = in __kmp_internal_fork()
8076 KMP_DEBUG_ASSERT(team->t.t_disp_buffer); in __kmp_internal_fork()
8077 if (team->t.t_max_nproc > 1) { in __kmp_internal_fork()
8080 team->t.t_disp_buffer[i].buffer_index = i; in __kmp_internal_fork()
8081 team->t.t_disp_buffer[i].doacross_buf_idx = i; in __kmp_internal_fork()
8084 team->t.t_disp_buffer[0].buffer_index = 0; in __kmp_internal_fork()
8085 team->t.t_disp_buffer[0].doacross_buf_idx = 0; in __kmp_internal_fork()
8089 KMP_ASSERT(this_thr->th.th_team == team); in __kmp_internal_fork()
8092 for (f = 0; f < team->t.t_nproc; f++) { in __kmp_internal_fork()
8093 KMP_DEBUG_ASSERT(team->t.t_threads[f] && in __kmp_internal_fork()
8094 team->t.t_threads[f]->th.th_team_nproc == team->t.t_nproc); in __kmp_internal_fork()
8102 void __kmp_internal_join(ident_t *id, int gtid, kmp_team_t *team) { in __kmp_internal_join() argument
8105 KMP_DEBUG_ASSERT(team); in __kmp_internal_join()
8106 KMP_DEBUG_ASSERT(this_thr->th.th_team == team); in __kmp_internal_join()
8114 __kmp_threads[gtid]->th.th_team_nproc != team->t.t_nproc) { in __kmp_internal_join()
8119 gtid, __kmp_threads[gtid]->th.th_team_nproc, team, in __kmp_internal_join()
8120 team->t.t_nproc); in __kmp_internal_join()
8124 __kmp_threads[gtid]->th.th_team_nproc == team->t.t_nproc); in __kmp_internal_join()
8164 KMP_ASSERT(this_thr->th.th_team == team); in __kmp_internal_join()
8527 kmp_team_t *team = thr->th.th_team; in __kmp_aux_get_team_info() local
8529 int ii = team->t.t_level; in __kmp_aux_get_team_info()
8530 teams_serialized = team->t.t_serialized; in __kmp_aux_get_team_info()
8534 for (teams_serialized = team->t.t_serialized; in __kmp_aux_get_team_info()
8537 if (team->t.t_serialized && (!teams_serialized)) { in __kmp_aux_get_team_info()
8538 team = team->t.t_parent; in __kmp_aux_get_team_info()
8542 team = team->t.t_parent; in __kmp_aux_get_team_info()
8546 return team; in __kmp_aux_get_team_info()
8553 kmp_team_t *team = __kmp_aux_get_team_info(serialized); in __kmp_aux_get_team_num() local
8554 if (team) { in __kmp_aux_get_team_num()
8558 return team->t.t_master_tid; in __kmp_aux_get_team_num()
8566 kmp_team_t *team = __kmp_aux_get_team_info(serialized); in __kmp_aux_get_num_teams() local
8567 if (team) { in __kmp_aux_get_num_teams()
8571 return team->t.t_parent->t.t_nproc; in __kmp_aux_get_num_teams()
9158 void __kmp_resize_dist_barrier(kmp_team_t *team, int old_nthreads, in __kmp_resize_dist_barrier() argument
9162 kmp_info_t **other_threads = team->t.t_threads; in __kmp_resize_dist_barrier()
9169 if (team->t.t_threads[f]->th.th_used_in_team.load() == 0) { in __kmp_resize_dist_barrier()
9175 if (team->t.t_threads[f]->th.th_used_in_team.load() == 3) { in __kmp_resize_dist_barrier()
9176 while (team->t.t_threads[f]->th.th_used_in_team.load() == 3) in __kmp_resize_dist_barrier()
9180 KMP_DEBUG_ASSERT(team->t.t_threads[f]->th.th_used_in_team.load() == 1); in __kmp_resize_dist_barrier()
9182 team->t.t_threads[f]->th.th_used_in_team.store(2); in __kmp_resize_dist_barrier()
9183 KMP_DEBUG_ASSERT(team->t.t_threads[f]->th.th_used_in_team.load() == 2); in __kmp_resize_dist_barrier()
9186 team->t.b->go_release(); in __kmp_resize_dist_barrier()
9203 KMP_DEBUG_ASSERT(team->t.t_threads[f]->th.th_used_in_team.load() == 0); in __kmp_resize_dist_barrier()
9209 team->t.b->update_num_threads(new_nthreads); in __kmp_resize_dist_barrier()
9210 team->t.b->go_reset(); in __kmp_resize_dist_barrier()
9213 void __kmp_add_threads_to_team(kmp_team_t *team, int new_nthreads) { in __kmp_add_threads_to_team() argument
9215 KMP_DEBUG_ASSERT(team); in __kmp_add_threads_to_team()
9222 KMP_DEBUG_ASSERT(team->t.t_threads[f]); in __kmp_add_threads_to_team()
9223 KMP_COMPARE_AND_STORE_ACQ32(&(team->t.t_threads[f]->th.th_used_in_team), 0, in __kmp_add_threads_to_team()
9226 __kmp_resume_32(team->t.t_threads[f]->th.th_info.ds.ds_gtid, in __kmp_add_threads_to_team()
9237 if (team->t.t_threads[f]->th.th_used_in_team.load() == 1) { in __kmp_add_threads_to_team()