Lines Matching refs:thr_bar
381 kmp_bstate_t *thr_bar; in __kmp_dist_barrier_release() local
389 thr_bar = &this_thr->th.th_bar[bt].bb; in __kmp_dist_barrier_release()
491 copy_icvs(&thr_bar->th_fixed_icvs, in __kmp_dist_barrier_release()
512 copy_icvs(&thr_bar->th_fixed_icvs, in __kmp_dist_barrier_release()
561 kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb; in __kmp_linear_barrier_gather_template() local
584 team->t.t_id, 0, &thr_bar->b_arrived, thr_bar->b_arrived, in __kmp_linear_barrier_gather_template()
585 thr_bar->b_arrived + KMP_BARRIER_STATE_BUMP)); in __kmp_linear_barrier_gather_template()
590 kmp_flag_64<> flag(&thr_bar->b_arrived, other_threads[0]); in __kmp_linear_barrier_gather_template()
662 kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb; in __kmp_linear_barrier_release_template() local
717 gtid, &thr_bar->b_go, KMP_BARRIER_STATE_BUMP)); in __kmp_linear_barrier_release_template()
719 kmp_flag_64<true, false> flag(&thr_bar->b_go, KMP_BARRIER_STATE_BUMP); in __kmp_linear_barrier_release_template()
723 kmp_flag_64<> flag(&thr_bar->b_go, KMP_BARRIER_STATE_BUMP); in __kmp_linear_barrier_release_template()
752 TCW_4(thr_bar->b_go, KMP_INIT_BARRIER_STATE); in __kmp_linear_barrier_release_template()
755 gtid, team->t.t_id, tid, &thr_bar->b_go, KMP_INIT_BARRIER_STATE)); in __kmp_linear_barrier_release_template()
799 kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb; in __kmp_tree_barrier_gather() local
875 team->t.t_id, parent_tid, &thr_bar->b_arrived, thr_bar->b_arrived, in __kmp_tree_barrier_gather()
876 thr_bar->b_arrived + KMP_BARRIER_STATE_BUMP)); in __kmp_tree_barrier_gather()
882 kmp_flag_64<> flag(&thr_bar->b_arrived, other_threads[parent_tid]); in __kmp_tree_barrier_gather()
905 kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb; in __kmp_tree_barrier_release() local
916 &thr_bar->b_go, KMP_BARRIER_STATE_BUMP)); in __kmp_tree_barrier_release()
918 kmp_flag_64<> flag(&thr_bar->b_go, KMP_BARRIER_STATE_BUMP); in __kmp_tree_barrier_release()
946 TCW_4(thr_bar->b_go, KMP_INIT_BARRIER_STATE); in __kmp_tree_barrier_release()
949 team->t.t_id, tid, &thr_bar->b_go, KMP_INIT_BARRIER_STATE)); in __kmp_tree_barrier_release()
1011 kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb; in __kmp_hyper_barrier_gather() local
1035 kmp_flag_64<> p_flag(&thr_bar->b_arrived); in __kmp_hyper_barrier_gather()
1049 team->t.t_id, parent_tid, &thr_bar->b_arrived, in __kmp_hyper_barrier_gather()
1050 thr_bar->b_arrived, in __kmp_hyper_barrier_gather()
1051 thr_bar->b_arrived + KMP_BARRIER_STATE_BUMP)); in __kmp_hyper_barrier_gather()
1130 kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb; in __kmp_hyper_barrier_release() local
1152 copy_icvs(&thr_bar->th_fixed_icvs, in __kmp_hyper_barrier_release()
1158 &thr_bar->b_go, KMP_BARRIER_STATE_BUMP)); in __kmp_hyper_barrier_release()
1160 kmp_flag_64<> flag(&thr_bar->b_go, KMP_BARRIER_STATE_BUMP); in __kmp_hyper_barrier_release()
1187 TCW_4(thr_bar->b_go, KMP_INIT_BARRIER_STATE); in __kmp_hyper_barrier_release()
1190 gtid, team->t.t_id, tid, &thr_bar->b_go, KMP_INIT_BARRIER_STATE)); in __kmp_hyper_barrier_release()
1249 copy_icvs(&child_bar->th_fixed_icvs, &thr_bar->th_fixed_icvs); in __kmp_hyper_barrier_release()
1271 &thr_bar->th_fixed_icvs); in __kmp_hyper_barrier_release()
1291 kmp_bstate_t *thr_bar, in __kmp_init_hierarchical_barrier_thread() argument
1295 bool uninitialized = thr_bar->team == NULL; in __kmp_init_hierarchical_barrier_thread()
1296 bool team_changed = team != thr_bar->team; in __kmp_init_hierarchical_barrier_thread()
1297 bool team_sz_changed = nproc != thr_bar->nproc; in __kmp_init_hierarchical_barrier_thread()
1298 bool tid_changed = tid != thr_bar->old_tid; in __kmp_init_hierarchical_barrier_thread()
1302 __kmp_get_hierarchy(nproc, thr_bar); in __kmp_init_hierarchical_barrier_thread()
1306 thr_bar->my_level = thr_bar->depth - 1; // default for primary thread in __kmp_init_hierarchical_barrier_thread()
1307 thr_bar->parent_tid = -1; // default for primary thread in __kmp_init_hierarchical_barrier_thread()
1311 while (d < thr_bar->depth) { // find parent based on level of thread in in __kmp_init_hierarchical_barrier_thread()
1314 if (d == thr_bar->depth - 2) { // reached level right below the primary in __kmp_init_hierarchical_barrier_thread()
1315 thr_bar->parent_tid = 0; in __kmp_init_hierarchical_barrier_thread()
1316 thr_bar->my_level = d; in __kmp_init_hierarchical_barrier_thread()
1318 } else if ((rem = tid % thr_bar->skip_per_level[d + 1]) != 0) { in __kmp_init_hierarchical_barrier_thread()
1321 thr_bar->parent_tid = tid - rem; in __kmp_init_hierarchical_barrier_thread()
1322 thr_bar->my_level = d; in __kmp_init_hierarchical_barrier_thread()
1328 __kmp_type_convert(7 - ((tid - thr_bar->parent_tid) / in __kmp_init_hierarchical_barrier_thread()
1329 (thr_bar->skip_per_level[thr_bar->my_level])), in __kmp_init_hierarchical_barrier_thread()
1330 &(thr_bar->offset)); in __kmp_init_hierarchical_barrier_thread()
1331 thr_bar->old_tid = tid; in __kmp_init_hierarchical_barrier_thread()
1332 thr_bar->wait_flag = KMP_BARRIER_NOT_WAITING; in __kmp_init_hierarchical_barrier_thread()
1333 thr_bar->team = team; in __kmp_init_hierarchical_barrier_thread()
1334 thr_bar->parent_bar = in __kmp_init_hierarchical_barrier_thread()
1335 &team->t.t_threads[thr_bar->parent_tid]->th.th_bar[bt].bb; in __kmp_init_hierarchical_barrier_thread()
1338 thr_bar->team = team; in __kmp_init_hierarchical_barrier_thread()
1339 thr_bar->parent_bar = in __kmp_init_hierarchical_barrier_thread()
1340 &team->t.t_threads[thr_bar->parent_tid]->th.th_bar[bt].bb; in __kmp_init_hierarchical_barrier_thread()
1344 thr_bar->nproc = nproc; in __kmp_init_hierarchical_barrier_thread()
1345 thr_bar->leaf_kids = thr_bar->base_leaf_kids; in __kmp_init_hierarchical_barrier_thread()
1346 if (thr_bar->my_level == 0) in __kmp_init_hierarchical_barrier_thread()
1347 thr_bar->leaf_kids = 0; in __kmp_init_hierarchical_barrier_thread()
1348 if (thr_bar->leaf_kids && (kmp_uint32)tid + thr_bar->leaf_kids + 1 > nproc) in __kmp_init_hierarchical_barrier_thread()
1349 __kmp_type_convert(nproc - tid - 1, &(thr_bar->leaf_kids)); in __kmp_init_hierarchical_barrier_thread()
1350 thr_bar->leaf_state = 0; in __kmp_init_hierarchical_barrier_thread()
1351 for (int i = 0; i < thr_bar->leaf_kids; ++i) in __kmp_init_hierarchical_barrier_thread()
1352 ((char *)&(thr_bar->leaf_state))[7 - i] = 1; in __kmp_init_hierarchical_barrier_thread()
1362 kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb; in __kmp_hierarchical_barrier_gather() local
1373 thr_bar->use_oncore_barrier = 1; in __kmp_hierarchical_barrier_gather()
1375 thr_bar->use_oncore_barrier = 0; // Do not use oncore barrier when nested in __kmp_hierarchical_barrier_gather()
1389 (void)__kmp_init_hierarchical_barrier_thread(bt, thr_bar, nproc, gtid, tid, in __kmp_hierarchical_barrier_gather()
1392 if (thr_bar->my_level) { // not a leaf (my_level==0 means leaf) in __kmp_hierarchical_barrier_gather()
1397 thr_bar->use_oncore_barrier) { in __kmp_hierarchical_barrier_gather()
1398 if (thr_bar->leaf_kids) { in __kmp_hierarchical_barrier_gather()
1402 ? thr_bar->b_arrived | thr_bar->leaf_state in __kmp_hierarchical_barrier_gather()
1403 : team->t.t_bar[bt].b_arrived | thr_bar->leaf_state; in __kmp_hierarchical_barrier_gather()
1407 kmp_flag_64<> flag(&thr_bar->b_arrived, leaf_state); in __kmp_hierarchical_barrier_gather()
1412 for (child_tid = tid + 1; child_tid <= tid + thr_bar->leaf_kids; in __kmp_hierarchical_barrier_gather()
1425 KMP_TEST_THEN_AND64(&thr_bar->b_arrived, ~(thr_bar->leaf_state)); in __kmp_hierarchical_barrier_gather()
1428 for (kmp_uint32 d = 1; d < thr_bar->my_level; in __kmp_hierarchical_barrier_gather()
1430 kmp_uint32 last = tid + thr_bar->skip_per_level[d + 1], in __kmp_hierarchical_barrier_gather()
1431 skip = thr_bar->skip_per_level[d]; in __kmp_hierarchical_barrier_gather()
1457 for (kmp_uint32 d = 0; d < thr_bar->my_level; in __kmp_hierarchical_barrier_gather()
1459 kmp_uint32 last = tid + thr_bar->skip_per_level[d + 1], in __kmp_hierarchical_barrier_gather()
1460 skip = thr_bar->skip_per_level[d]; in __kmp_hierarchical_barrier_gather()
1493 __kmp_gtid_from_tid(thr_bar->parent_tid, team), team->t.t_id, in __kmp_hierarchical_barrier_gather()
1494 thr_bar->parent_tid, &thr_bar->b_arrived, thr_bar->b_arrived, in __kmp_hierarchical_barrier_gather()
1495 thr_bar->b_arrived + KMP_BARRIER_STATE_BUMP)); in __kmp_hierarchical_barrier_gather()
1499 if (thr_bar->my_level || __kmp_dflt_blocktime != KMP_MAX_BLOCKTIME || in __kmp_hierarchical_barrier_gather()
1500 !thr_bar->use_oncore_barrier) { // Parent is waiting on my b_arrived in __kmp_hierarchical_barrier_gather()
1502 kmp_flag_64<> flag(&thr_bar->b_arrived, in __kmp_hierarchical_barrier_gather()
1503 other_threads[thr_bar->parent_tid]); in __kmp_hierarchical_barrier_gather()
1507 thr_bar->b_arrived = team->t.t_bar[bt].b_arrived + KMP_BARRIER_STATE_BUMP; in __kmp_hierarchical_barrier_gather()
1508 kmp_flag_oncore flag(&thr_bar->parent_bar->b_arrived, in __kmp_hierarchical_barrier_gather()
1509 thr_bar->offset + 1); in __kmp_hierarchical_barrier_gather()
1510 flag.set_waiter(other_threads[thr_bar->parent_tid]); in __kmp_hierarchical_barrier_gather()
1531 kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb; in __kmp_hierarchical_barrier_release() local
1543 if (!thr_bar->use_oncore_barrier || in __kmp_hierarchical_barrier_release()
1544 __kmp_dflt_blocktime != KMP_MAX_BLOCKTIME || thr_bar->my_level != 0 || in __kmp_hierarchical_barrier_release()
1545 thr_bar->team == NULL) { in __kmp_hierarchical_barrier_release()
1547 thr_bar->wait_flag = KMP_BARRIER_OWN_FLAG; in __kmp_hierarchical_barrier_release()
1548 kmp_flag_64<> flag(&thr_bar->b_go, KMP_BARRIER_STATE_BUMP); in __kmp_hierarchical_barrier_release()
1550 TCW_8(thr_bar->b_go, in __kmp_hierarchical_barrier_release()
1555 thr_bar->wait_flag = KMP_BARRIER_PARENT_FLAG; in __kmp_hierarchical_barrier_release()
1556 kmp_flag_oncore flag(&thr_bar->parent_bar->b_go, KMP_BARRIER_STATE_BUMP, in __kmp_hierarchical_barrier_release()
1557 thr_bar->offset + 1, bt, in __kmp_hierarchical_barrier_release()
1560 if (thr_bar->wait_flag == in __kmp_hierarchical_barrier_release()
1562 TCW_8(thr_bar->b_go, in __kmp_hierarchical_barrier_release()
1566 &(thr_bar->parent_bar->b_go)))[thr_bar->offset + 1] = 0; in __kmp_hierarchical_barrier_release()
1569 thr_bar->wait_flag = KMP_BARRIER_NOT_WAITING; in __kmp_hierarchical_barrier_release()
1581 gtid, team->t.t_id, tid, &thr_bar->b_go, KMP_INIT_BARRIER_STATE)); in __kmp_hierarchical_barrier_release()
1596 thr_bar->use_oncore_barrier = 1; in __kmp_hierarchical_barrier_release()
1598 thr_bar->use_oncore_barrier = 0; // Do not use oncore barrier when nested in __kmp_hierarchical_barrier_release()
1602 unsigned short int old_leaf_kids = thr_bar->leaf_kids; in __kmp_hierarchical_barrier_release()
1603 kmp_uint64 old_leaf_state = thr_bar->leaf_state; in __kmp_hierarchical_barrier_release()
1604 team_change = __kmp_init_hierarchical_barrier_thread(bt, thr_bar, nproc, gtid, in __kmp_hierarchical_barrier_release()
1616 copy_icvs(&thr_bar->th_fixed_icvs, in __kmp_hierarchical_barrier_release()
1619 thr_bar->use_oncore_barrier) { // optimization for inf blocktime in __kmp_hierarchical_barrier_release()
1620 if (!thr_bar->my_level) // I'm a leaf in the hierarchy (my_level==0) in __kmp_hierarchical_barrier_release()
1624 &thr_bar->parent_bar->th_fixed_icvs); in __kmp_hierarchical_barrier_release()
1627 if (thr_bar->my_level) // not a leaf; copy ICVs to my fixed ICVs child can in __kmp_hierarchical_barrier_release()
1629 copy_icvs(&thr_bar->th_fixed_icvs, &thr_bar->parent_bar->th_fixed_icvs); in __kmp_hierarchical_barrier_release()
1632 &thr_bar->parent_bar->th_fixed_icvs); in __kmp_hierarchical_barrier_release()
1638 if (thr_bar->my_level) { // not a leaf in __kmp_hierarchical_barrier_release()
1642 thr_bar->use_oncore_barrier) { in __kmp_hierarchical_barrier_release()
1646 thr_bar->b_go = KMP_BARRIER_STATE_BUMP; in __kmp_hierarchical_barrier_release()
1649 ngo_load(&thr_bar->th_fixed_icvs); in __kmp_hierarchical_barrier_release()
1652 for (child_tid = thr_bar->skip_per_level[1]; child_tid < (int)nproc; in __kmp_hierarchical_barrier_release()
1653 child_tid += thr_bar->skip_per_level[1]) { in __kmp_hierarchical_barrier_release()
1665 ngo_store_go(&child_bar->th_fixed_icvs, &thr_bar->th_fixed_icvs); in __kmp_hierarchical_barrier_release()
1669 TCW_8(thr_bar->b_go, in __kmp_hierarchical_barrier_release()
1672 if (thr_bar->leaf_kids) { // if there are any in __kmp_hierarchical_barrier_release()
1675 old_leaf_kids < thr_bar->leaf_kids) { // some old, some new in __kmp_hierarchical_barrier_release()
1677 thr_bar->b_go |= old_leaf_state; in __kmp_hierarchical_barrier_release()
1680 last = tid + thr_bar->skip_per_level[1]; in __kmp_hierarchical_barrier_release()
1700 thr_bar->b_go |= thr_bar->leaf_state; in __kmp_hierarchical_barrier_release()
1704 for (int d = thr_bar->my_level - 1; d >= 0; in __kmp_hierarchical_barrier_release()
1706 last = tid + thr_bar->skip_per_level[d + 1]; in __kmp_hierarchical_barrier_release()
1707 kmp_uint32 skip = thr_bar->skip_per_level[d]; in __kmp_hierarchical_barrier_release()
1729 &thr_bar->th_fixed_icvs); in __kmp_hierarchical_barrier_release()