Lines Matching refs:lck
70 static kmp_int32 __kmp_get_tas_lock_owner(kmp_tas_lock_t *lck) {
71 return KMP_LOCK_STRIP(KMP_ATOMIC_LD_RLX(&lck->lk.poll)) - 1;
74 static inline bool __kmp_is_tas_lock_nestable(kmp_tas_lock_t *lck) {
75 return lck->lk.depth_locked != -1;
79 __kmp_acquire_tas_lock_timed_template(kmp_tas_lock_t *lck, kmp_int32 gtid) {
83 kmp_uint32 curr = KMP_LOCK_STRIP(lck->lk.poll);
85 __kmp_printf("LOCK CONTENTION: %p\n", lck);
92 if (KMP_ATOMIC_LD_RLX(&lck->lk.poll) == tas_free &&
93 __kmp_atomic_compare_store_acq(&lck->lk.poll, tas_free, tas_busy)) {
94 KMP_FSYNC_ACQUIRED(lck);
100 KMP_FSYNC_PREPARE(lck);
112 } while (KMP_ATOMIC_LD_RLX(&lck->lk.poll) != tas_free ||
113 !__kmp_atomic_compare_store_acq(&lck->lk.poll, tas_free, tas_busy));
114 KMP_FSYNC_ACQUIRED(lck);
118 int __kmp_acquire_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) {
119 int retval = __kmp_acquire_tas_lock_timed_template(lck, gtid);
123 static int __kmp_acquire_tas_lock_with_checks(kmp_tas_lock_t *lck,
127 __kmp_is_tas_lock_nestable(lck)) {
130 if ((gtid >= 0) && (__kmp_get_tas_lock_owner(lck) == gtid)) {
133 return __kmp_acquire_tas_lock(lck, gtid);
136 int __kmp_test_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) {
139 if (KMP_ATOMIC_LD_RLX(&lck->lk.poll) == tas_free &&
140 __kmp_atomic_compare_store_acq(&lck->lk.poll, tas_free, tas_busy)) {
141 KMP_FSYNC_ACQUIRED(lck);
147 static int __kmp_test_tas_lock_with_checks(kmp_tas_lock_t *lck,
151 __kmp_is_tas_lock_nestable(lck)) {
154 return __kmp_test_tas_lock(lck, gtid);
157 int __kmp_release_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) {
160 KMP_FSYNC_RELEASING(lck);
161 KMP_ATOMIC_ST_REL(&lck->lk.poll, KMP_LOCK_FREE(tas));
168 static int __kmp_release_tas_lock_with_checks(kmp_tas_lock_t *lck,
173 __kmp_is_tas_lock_nestable(lck)) {
176 if (__kmp_get_tas_lock_owner(lck) == -1) {
179 if ((gtid >= 0) && (__kmp_get_tas_lock_owner(lck) >= 0) &&
180 (__kmp_get_tas_lock_owner(lck) != gtid)) {
183 return __kmp_release_tas_lock(lck, gtid);
186 void __kmp_init_tas_lock(kmp_tas_lock_t *lck) {
187 lck->lk.poll = KMP_LOCK_FREE(tas);
190 void __kmp_destroy_tas_lock(kmp_tas_lock_t *lck) { lck->lk.poll = 0; }
192 static void __kmp_destroy_tas_lock_with_checks(kmp_tas_lock_t *lck) {
195 __kmp_is_tas_lock_nestable(lck)) {
198 if (__kmp_get_tas_lock_owner(lck) != -1) {
201 __kmp_destroy_tas_lock(lck);
206 int __kmp_acquire_nested_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) {
209 if (__kmp_get_tas_lock_owner(lck) == gtid) {
210 lck->lk.depth_locked += 1;
213 __kmp_acquire_tas_lock_timed_template(lck, gtid);
214 lck->lk.depth_locked = 1;
219 static int __kmp_acquire_nested_tas_lock_with_checks(kmp_tas_lock_t *lck,
222 if (!__kmp_is_tas_lock_nestable(lck)) {
225 return __kmp_acquire_nested_tas_lock(lck, gtid);
228 int __kmp_test_nested_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) {
233 if (__kmp_get_tas_lock_owner(lck) == gtid) {
234 retval = ++lck->lk.depth_locked;
235 } else if (!__kmp_test_tas_lock(lck, gtid)) {
239 retval = lck->lk.depth_locked = 1;
244 static int __kmp_test_nested_tas_lock_with_checks(kmp_tas_lock_t *lck,
247 if (!__kmp_is_tas_lock_nestable(lck)) {
250 return __kmp_test_nested_tas_lock(lck, gtid);
253 int __kmp_release_nested_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) {
257 if (--(lck->lk.depth_locked) == 0) {
258 __kmp_release_tas_lock(lck, gtid);
264 static int __kmp_release_nested_tas_lock_with_checks(kmp_tas_lock_t *lck,
268 if (!__kmp_is_tas_lock_nestable(lck)) {
271 if (__kmp_get_tas_lock_owner(lck) == -1) {
274 if (__kmp_get_tas_lock_owner(lck) != gtid) {
277 return __kmp_release_nested_tas_lock(lck, gtid);
280 void __kmp_init_nested_tas_lock(kmp_tas_lock_t *lck) {
281 __kmp_init_tas_lock(lck);
282 lck->lk.depth_locked = 0; // >= 0 for nestable locks, -1 for simple locks
285 void __kmp_destroy_nested_tas_lock(kmp_tas_lock_t *lck) {
286 __kmp_destroy_tas_lock(lck);
287 lck->lk.depth_locked = 0;
290 static void __kmp_destroy_nested_tas_lock_with_checks(kmp_tas_lock_t *lck) {
292 if (!__kmp_is_tas_lock_nestable(lck)) {
295 if (__kmp_get_tas_lock_owner(lck) != -1) {
298 __kmp_destroy_nested_tas_lock(lck);
311 static kmp_int32 __kmp_get_futex_lock_owner(kmp_futex_lock_t *lck) {
312 return KMP_LOCK_STRIP((TCR_4(lck->lk.poll) >> 1)) - 1;
315 static inline bool __kmp_is_futex_lock_nestable(kmp_futex_lock_t *lck) {
316 return lck->lk.depth_locked != -1;
320 __kmp_acquire_futex_lock_timed_template(kmp_futex_lock_t *lck, kmp_int32 gtid) {
326 kmp_uint32 curr = KMP_LOCK_STRIP(TCR_4(lck->lk.poll));
328 __kmp_printf("LOCK CONTENTION: %p\n", lck);
332 KMP_FSYNC_PREPARE(lck);
333 KA_TRACE(1000, ("__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d entering\n",
334 lck, lck->lk.poll, gtid));
339 &(lck->lk.poll), KMP_LOCK_FREE(futex),
345 ("__kmp_acquire_futex_lock: lck:%p, T#%d poll_val = 0x%x cond = 0x%x\n",
346 lck, gtid, poll_val, cond));
357 if (!KMP_COMPARE_AND_STORE_REL32(&(lck->lk.poll), poll_val,
361 ("__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d can't set bit 0\n",
362 lck, lck->lk.poll, gtid));
368 ("__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d bit 0 set\n", lck,
369 lck->lk.poll, gtid));
374 ("__kmp_acquire_futex_lock: lck:%p, T#%d before futex_wait(0x%x)\n",
375 lck, gtid, poll_val));
378 if ((rc = syscall(__NR_futex, &(lck->lk.poll), FUTEX_WAIT, poll_val, NULL,
380 KA_TRACE(1000, ("__kmp_acquire_futex_lock: lck:%p, T#%d futex_wait(0x%x) "
382 lck, gtid, poll_val, rc, errno));
387 ("__kmp_acquire_futex_lock: lck:%p, T#%d after futex_wait(0x%x)\n",
388 lck, gtid, poll_val));
395 KMP_FSYNC_ACQUIRED(lck);
396 KA_TRACE(1000, ("__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d exiting\n", lck,
397 lck->lk.poll, gtid));
401 int __kmp_acquire_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) {
402 int retval = __kmp_acquire_futex_lock_timed_template(lck, gtid);
406 static int __kmp_acquire_futex_lock_with_checks(kmp_futex_lock_t *lck,
410 __kmp_is_futex_lock_nestable(lck)) {
413 if ((gtid >= 0) && (__kmp_get_futex_lock_owner(lck) == gtid)) {
416 return __kmp_acquire_futex_lock(lck, gtid);
419 int __kmp_test_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) {
420 if (KMP_COMPARE_AND_STORE_ACQ32(&(lck->lk.poll), KMP_LOCK_FREE(futex),
422 KMP_FSYNC_ACQUIRED(lck);
428 static int __kmp_test_futex_lock_with_checks(kmp_futex_lock_t *lck,
432 __kmp_is_futex_lock_nestable(lck)) {
435 return __kmp_test_futex_lock(lck, gtid);
438 int __kmp_release_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) {
441 KA_TRACE(1000, ("__kmp_release_futex_lock: lck:%p(0x%x), T#%d entering\n",
442 lck, lck->lk.poll, gtid));
444 KMP_FSYNC_RELEASING(lck);
446 kmp_int32 poll_val = KMP_XCHG_FIXED32(&(lck->lk.poll), KMP_LOCK_FREE(futex));
449 ("__kmp_release_futex_lock: lck:%p, T#%d released poll_val = 0x%x\n",
450 lck, gtid, poll_val));
454 ("__kmp_release_futex_lock: lck:%p, T#%d futex_wake 1 thread\n",
455 lck, gtid));
456 syscall(__NR_futex, &(lck->lk.poll), FUTEX_WAKE, KMP_LOCK_BUSY(1, futex),
462 KA_TRACE(1000, ("__kmp_release_futex_lock: lck:%p(0x%x), T#%d exiting\n", lck,
463 lck->lk.poll, gtid));
469 static int __kmp_release_futex_lock_with_checks(kmp_futex_lock_t *lck,
474 __kmp_is_futex_lock_nestable(lck)) {
477 if (__kmp_get_futex_lock_owner(lck) == -1) {
480 if ((gtid >= 0) && (__kmp_get_futex_lock_owner(lck) >= 0) &&
481 (__kmp_get_futex_lock_owner(lck) != gtid)) {
484 return __kmp_release_futex_lock(lck, gtid);
487 void __kmp_init_futex_lock(kmp_futex_lock_t *lck) {
488 TCW_4(lck->lk.poll, KMP_LOCK_FREE(futex));
491 void __kmp_destroy_futex_lock(kmp_futex_lock_t *lck) { lck->lk.poll = 0; }
493 static void __kmp_destroy_futex_lock_with_checks(kmp_futex_lock_t *lck) {
496 __kmp_is_futex_lock_nestable(lck)) {
499 if (__kmp_get_futex_lock_owner(lck) != -1) {
502 __kmp_destroy_futex_lock(lck);
507 int __kmp_acquire_nested_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) {
510 if (__kmp_get_futex_lock_owner(lck) == gtid) {
511 lck->lk.depth_locked += 1;
514 __kmp_acquire_futex_lock_timed_template(lck, gtid);
515 lck->lk.depth_locked = 1;
520 static int __kmp_acquire_nested_futex_lock_with_checks(kmp_futex_lock_t *lck,
523 if (!__kmp_is_futex_lock_nestable(lck)) {
526 return __kmp_acquire_nested_futex_lock(lck, gtid);
529 int __kmp_test_nested_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) {
534 if (__kmp_get_futex_lock_owner(lck) == gtid) {
535 retval = ++lck->lk.depth_locked;
536 } else if (!__kmp_test_futex_lock(lck, gtid)) {
540 retval = lck->lk.depth_locked = 1;
545 static int __kmp_test_nested_futex_lock_with_checks(kmp_futex_lock_t *lck,
548 if (!__kmp_is_futex_lock_nestable(lck)) {
551 return __kmp_test_nested_futex_lock(lck, gtid);
554 int __kmp_release_nested_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) {
558 if (--(lck->lk.depth_locked) == 0) {
559 __kmp_release_futex_lock(lck, gtid);
565 static int __kmp_release_nested_futex_lock_with_checks(kmp_futex_lock_t *lck,
569 if (!__kmp_is_futex_lock_nestable(lck)) {
572 if (__kmp_get_futex_lock_owner(lck) == -1) {
575 if (__kmp_get_futex_lock_owner(lck) != gtid) {
578 return __kmp_release_nested_futex_lock(lck, gtid);
581 void __kmp_init_nested_futex_lock(kmp_futex_lock_t *lck) {
582 __kmp_init_futex_lock(lck);
583 lck->lk.depth_locked = 0; // >= 0 for nestable locks, -1 for simple locks
586 void __kmp_destroy_nested_futex_lock(kmp_futex_lock_t *lck) {
587 __kmp_destroy_futex_lock(lck);
588 lck->lk.depth_locked = 0;
591 static void __kmp_destroy_nested_futex_lock_with_checks(kmp_futex_lock_t *lck) {
593 if (!__kmp_is_futex_lock_nestable(lck)) {
596 if (__kmp_get_futex_lock_owner(lck) != -1) {
599 __kmp_destroy_nested_futex_lock(lck);
607 static kmp_int32 __kmp_get_ticket_lock_owner(kmp_ticket_lock_t *lck) {
608 return std::atomic_load_explicit(&lck->lk.owner_id,
613 static inline bool __kmp_is_ticket_lock_nestable(kmp_ticket_lock_t *lck) {
614 return std::atomic_load_explicit(&lck->lk.depth_locked,
624 __kmp_acquire_ticket_lock_timed_template(kmp_ticket_lock_t *lck,
627 &lck->lk.next_ticket, 1U, std::memory_order_relaxed);
630 if (std::atomic_load_explicit(&lck->lk.now_serving,
632 __kmp_printf("LOCK CONTENTION: %p\n", lck);
636 if (std::atomic_load_explicit(&lck->lk.now_serving,
640 KMP_WAIT_PTR(&lck->lk.now_serving, my_ticket, __kmp_bakery_check, lck);
644 int __kmp_acquire_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) {
645 int retval = __kmp_acquire_ticket_lock_timed_template(lck, gtid);
649 static int __kmp_acquire_ticket_lock_with_checks(kmp_ticket_lock_t *lck,
653 if (!std::atomic_load_explicit(&lck->lk.initialized,
657 if (lck->lk.self != lck) {
660 if (__kmp_is_ticket_lock_nestable(lck)) {
663 if ((gtid >= 0) && (__kmp_get_ticket_lock_owner(lck) == gtid)) {
667 __kmp_acquire_ticket_lock(lck, gtid);
669 std::atomic_store_explicit(&lck->lk.owner_id, gtid + 1,
674 int __kmp_test_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) {
675 kmp_uint32 my_ticket = std::atomic_load_explicit(&lck->lk.next_ticket,
678 if (std::atomic_load_explicit(&lck->lk.now_serving,
682 &lck->lk.next_ticket, &my_ticket, next_ticket,
690 static int __kmp_test_ticket_lock_with_checks(kmp_ticket_lock_t *lck,
694 if (!std::atomic_load_explicit(&lck->lk.initialized,
698 if (lck->lk.self != lck) {
701 if (__kmp_is_ticket_lock_nestable(lck)) {
705 int retval = __kmp_test_ticket_lock(lck, gtid);
708 std::atomic_store_explicit(&lck->lk.owner_id, gtid + 1,
714 int __kmp_release_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) {
715 kmp_uint32 distance = std::atomic_load_explicit(&lck->lk.next_ticket,
717 std::atomic_load_explicit(&lck->lk.now_serving,
720 std::atomic_fetch_add_explicit(&lck->lk.now_serving, 1U,
728 static int __kmp_release_ticket_lock_with_checks(kmp_ticket_lock_t *lck,
732 if (!std::atomic_load_explicit(&lck->lk.initialized,
736 if (lck->lk.self != lck) {
739 if (__kmp_is_ticket_lock_nestable(lck)) {
742 if (__kmp_get_ticket_lock_owner(lck) == -1) {
745 if ((gtid >= 0) && (__kmp_get_ticket_lock_owner(lck) >= 0) &&
746 (__kmp_get_ticket_lock_owner(lck) != gtid)) {
749 std::atomic_store_explicit(&lck->lk.owner_id, 0, std::memory_order_relaxed);
750 return __kmp_release_ticket_lock(lck, gtid);
753 void __kmp_init_ticket_lock(kmp_ticket_lock_t *lck) {
754 lck->lk.location = NULL;
755 lck->lk.self = lck;
756 std::atomic_store_explicit(&lck->lk.next_ticket, 0U,
758 std::atomic_store_explicit(&lck->lk.now_serving, 0U,
761 &lck->lk.owner_id, 0,
764 &lck->lk.depth_locked, -1,
766 std::atomic_store_explicit(&lck->lk.initialized, true,
770 void __kmp_destroy_ticket_lock(kmp_ticket_lock_t *lck) {
771 std::atomic_store_explicit(&lck->lk.initialized, false,
773 lck->lk.self = NULL;
774 lck->lk.location = NULL;
775 std::atomic_store_explicit(&lck->lk.next_ticket, 0U,
777 std::atomic_store_explicit(&lck->lk.now_serving, 0U,
779 std::atomic_store_explicit(&lck->lk.owner_id, 0, std::memory_order_relaxed);
780 std::atomic_store_explicit(&lck->lk.depth_locked, -1,
784 static void __kmp_destroy_ticket_lock_with_checks(kmp_ticket_lock_t *lck) {
787 if (!std::atomic_load_explicit(&lck->lk.initialized,
791 if (lck->lk.self != lck) {
794 if (__kmp_is_ticket_lock_nestable(lck)) {
797 if (__kmp_get_ticket_lock_owner(lck) != -1) {
800 __kmp_destroy_ticket_lock(lck);
805 int __kmp_acquire_nested_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) {
808 if (__kmp_get_ticket_lock_owner(lck) == gtid) {
809 std::atomic_fetch_add_explicit(&lck->lk.depth_locked, 1,
813 __kmp_acquire_ticket_lock_timed_template(lck, gtid);
814 std::atomic_store_explicit(&lck->lk.depth_locked, 1,
816 std::atomic_store_explicit(&lck->lk.owner_id, gtid + 1,
822 static int __kmp_acquire_nested_ticket_lock_with_checks(kmp_ticket_lock_t *lck,
826 if (!std::atomic_load_explicit(&lck->lk.initialized,
830 if (lck->lk.self != lck) {
833 if (!__kmp_is_ticket_lock_nestable(lck)) {
836 return __kmp_acquire_nested_ticket_lock(lck, gtid);
839 int __kmp_test_nested_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) {
844 if (__kmp_get_ticket_lock_owner(lck) == gtid) {
845 retval = std::atomic_fetch_add_explicit(&lck->lk.depth_locked, 1,
848 } else if (!__kmp_test_ticket_lock(lck, gtid)) {
851 std::atomic_store_explicit(&lck->lk.depth_locked, 1,
853 std::atomic_store_explicit(&lck->lk.owner_id, gtid + 1,
860 static int __kmp_test_nested_ticket_lock_with_checks(kmp_ticket_lock_t *lck,
864 if (!std::atomic_load_explicit(&lck->lk.initialized,
868 if (lck->lk.self != lck) {
871 if (!__kmp_is_ticket_lock_nestable(lck)) {
874 return __kmp_test_nested_ticket_lock(lck, gtid);
877 int __kmp_release_nested_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) {
880 if ((std::atomic_fetch_add_explicit(&lck->lk.depth_locked, -1,
883 std::atomic_store_explicit(&lck->lk.owner_id, 0, std::memory_order_relaxed);
884 __kmp_release_ticket_lock(lck, gtid);
890 static int __kmp_release_nested_ticket_lock_with_checks(kmp_ticket_lock_t *lck,
894 if (!std::atomic_load_explicit(&lck->lk.initialized,
898 if (lck->lk.self != lck) {
901 if (!__kmp_is_ticket_lock_nestable(lck)) {
904 if (__kmp_get_ticket_lock_owner(lck) == -1) {
907 if (__kmp_get_ticket_lock_owner(lck) != gtid) {
910 return __kmp_release_nested_ticket_lock(lck, gtid);
913 void __kmp_init_nested_ticket_lock(kmp_ticket_lock_t *lck) {
914 __kmp_init_ticket_lock(lck);
915 std::atomic_store_explicit(&lck->lk.depth_locked, 0,
920 void __kmp_destroy_nested_ticket_lock(kmp_ticket_lock_t *lck) {
921 __kmp_destroy_ticket_lock(lck);
922 std::atomic_store_explicit(&lck->lk.depth_locked, 0,
927 __kmp_destroy_nested_ticket_lock_with_checks(kmp_ticket_lock_t *lck) {
930 if (!std::atomic_load_explicit(&lck->lk.initialized,
934 if (lck->lk.self != lck) {
937 if (!__kmp_is_ticket_lock_nestable(lck)) {
940 if (__kmp_get_ticket_lock_owner(lck) != -1) {
943 __kmp_destroy_nested_ticket_lock(lck);
948 static const ident_t *__kmp_get_ticket_lock_location(kmp_ticket_lock_t *lck) {
949 return lck->lk.location;
952 static void __kmp_set_ticket_lock_location(kmp_ticket_lock_t *lck,
954 lck->lk.location = loc;
957 static kmp_lock_flags_t __kmp_get_ticket_lock_flags(kmp_ticket_lock_t *lck) {
958 return lck->lk.flags;
961 static void __kmp_set_ticket_lock_flags(kmp_ticket_lock_t *lck,
963 lck->lk.flags = flags;
1036 kmp_queuing_lock_t *lck, kmp_int32 head_id,
1056 __kmp_printf_no_lock("\t\thead: %d ", lck->lk.head_id);
1058 if (lck->lk.head_id >= 1) {
1059 t = __kmp_threads[lck->lk.head_id - 1]->th.th_next_waiting;
1065 __kmp_printf_no_lock("; tail: %d ", lck->lk.tail_id);
1071 static kmp_int32 __kmp_get_queuing_lock_owner(kmp_queuing_lock_t *lck) {
1072 return TCR_4(lck->lk.owner_id) - 1;
1075 static inline bool __kmp_is_queuing_lock_nestable(kmp_queuing_lock_t *lck) {
1076 return lck->lk.depth_locked != -1;
1084 __kmp_acquire_queuing_lock_timed_template(kmp_queuing_lock_t *lck,
1087 volatile kmp_int32 *head_id_p = &lck->lk.head_id;
1088 volatile kmp_int32 *tail_id_p = &lck->lk.tail_id;
1096 ("__kmp_acquire_queuing_lock: lck:%p, T#%d entering\n", lck, gtid));
1098 KMP_FSYNC_PREPARE(lck);
1105 __kmp_dump_queuing_lock(this_thr, gtid, lck, *head_id_p, *tail_id_p);
1107 __kmp_dump_queuing_lock(this_thr, gtid, lck, *head_id_p, *tail_id_p);
1191 ("__kmp_acquire_queuing_lock: lck:%p, T#%d exiting: no queuing\n",
1192 lck, gtid));
1205 KMP_FSYNC_ACQUIRED(lck);
1216 this_thr->th.ompt_thread_info.wait_id = (uint64_t)lck;
1229 ("__kmp_acquire_queuing_lock: lck:%p, T#%d waiting for lock\n",
1230 lck, gtid));
1234 KMP_WAIT(spin_here_p, FALSE, KMP_EQ, lck);
1243 __kmp_dump_queuing_lock(this_thr, gtid, lck, *head_id_p, *tail_id_p);
1246 KA_TRACE(1000, ("__kmp_acquire_queuing_lock: lck:%p, T#%d exiting: after "
1248 lck, gtid));
1277 int __kmp_acquire_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
1280 int retval = __kmp_acquire_queuing_lock_timed_template<false>(lck, gtid);
1284 static int __kmp_acquire_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
1287 if (lck->lk.initialized != lck) {
1290 if (__kmp_is_queuing_lock_nestable(lck)) {
1293 if (__kmp_get_queuing_lock_owner(lck) == gtid) {
1297 __kmp_acquire_queuing_lock(lck, gtid);
1299 lck->lk.owner_id = gtid + 1;
1303 int __kmp_test_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
1304 volatile kmp_int32 *head_id_p = &lck->lk.head_id;
1325 KMP_FSYNC_ACQUIRED(lck);
1335 static int __kmp_test_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
1338 if (lck->lk.initialized != lck) {
1341 if (__kmp_is_queuing_lock_nestable(lck)) {
1345 int retval = __kmp_test_queuing_lock(lck, gtid);
1348 lck->lk.owner_id = gtid + 1;
1353 int __kmp_release_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
1354 volatile kmp_int32 *head_id_p = &lck->lk.head_id;
1355 volatile kmp_int32 *tail_id_p = &lck->lk.tail_id;
1358 ("__kmp_release_queuing_lock: lck:%p, T#%d entering\n", lck, gtid));
1368 __kmp_dump_queuing_lock(this_thr, gtid, lck, *head_id_p, *tail_id_p);
1370 __kmp_dump_queuing_lock(this_thr, gtid, lck, *head_id_p, *tail_id_p);
1375 KMP_FSYNC_RELEASING(lck);
1388 __kmp_dump_queuing_lock(this_thr, gtid, lck, head, tail);
1398 ("__kmp_release_queuing_lock: lck:%p, T#%d exiting: queue empty\n",
1399 lck, gtid));
1416 __kmp_dump_queuing_lock(this_thr, gtid, lck, head, tail);
1437 __kmp_dump_queuing_lock(this_thr, gtid, lck, head, tail);
1461 __kmp_dump_queuing_lock(this_thr, gtid, lck, head, tail);
1476 KA_TRACE(1000, ("__kmp_release_queuing_lock: lck:%p, T#%d exiting: after "
1478 lck, gtid));
1496 static int __kmp_release_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
1500 if (lck->lk.initialized != lck) {
1503 if (__kmp_is_queuing_lock_nestable(lck)) {
1506 if (__kmp_get_queuing_lock_owner(lck) == -1) {
1509 if (__kmp_get_queuing_lock_owner(lck) != gtid) {
1512 lck->lk.owner_id = 0;
1513 return __kmp_release_queuing_lock(lck, gtid);
1516 void __kmp_init_queuing_lock(kmp_queuing_lock_t *lck) {
1517 lck->lk.location = NULL;
1518 lck->lk.head_id = 0;
1519 lck->lk.tail_id = 0;
1520 lck->lk.next_ticket = 0;
1521 lck->lk.now_serving = 0;
1522 lck->lk.owner_id = 0; // no thread owns the lock.
1523 lck->lk.depth_locked = -1; // >= 0 for nestable locks, -1 for simple locks.
1524 lck->lk.initialized = lck;
1526 KA_TRACE(1000, ("__kmp_init_queuing_lock: lock %p initialized\n", lck));
1529 void __kmp_destroy_queuing_lock(kmp_queuing_lock_t *lck) {
1530 lck->lk.initialized = NULL;
1531 lck->lk.location = NULL;
1532 lck->lk.head_id = 0;
1533 lck->lk.tail_id = 0;
1534 lck->lk.next_ticket = 0;
1535 lck->lk.now_serving = 0;
1536 lck->lk.owner_id = 0;
1537 lck->lk.depth_locked = -1;
1540 static void __kmp_destroy_queuing_lock_with_checks(kmp_queuing_lock_t *lck) {
1542 if (lck->lk.initialized != lck) {
1545 if (__kmp_is_queuing_lock_nestable(lck)) {
1548 if (__kmp_get_queuing_lock_owner(lck) != -1) {
1551 __kmp_destroy_queuing_lock(lck);
1556 int __kmp_acquire_nested_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
1559 if (__kmp_get_queuing_lock_owner(lck) == gtid) {
1560 lck->lk.depth_locked += 1;
1563 __kmp_acquire_queuing_lock_timed_template<false>(lck, gtid);
1565 lck->lk.depth_locked = 1;
1567 lck->lk.owner_id = gtid + 1;
1573 __kmp_acquire_nested_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
1576 if (lck->lk.initialized != lck) {
1579 if (!__kmp_is_queuing_lock_nestable(lck)) {
1582 return __kmp_acquire_nested_queuing_lock(lck, gtid);
1585 int __kmp_test_nested_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
1590 if (__kmp_get_queuing_lock_owner(lck) == gtid) {
1591 retval = ++lck->lk.depth_locked;
1592 } else if (!__kmp_test_queuing_lock(lck, gtid)) {
1596 retval = lck->lk.depth_locked = 1;
1598 lck->lk.owner_id = gtid + 1;
1603 static int __kmp_test_nested_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
1606 if (lck->lk.initialized != lck) {
1609 if (!__kmp_is_queuing_lock_nestable(lck)) {
1612 return __kmp_test_nested_queuing_lock(lck, gtid);
1615 int __kmp_release_nested_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
1619 if (--(lck->lk.depth_locked) == 0) {
1621 lck->lk.owner_id = 0;
1622 __kmp_release_queuing_lock(lck, gtid);
1629 __kmp_release_nested_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
1633 if (lck->lk.initialized != lck) {
1636 if (!__kmp_is_queuing_lock_nestable(lck)) {
1639 if (__kmp_get_queuing_lock_owner(lck) == -1) {
1642 if (__kmp_get_queuing_lock_owner(lck) != gtid) {
1645 return __kmp_release_nested_queuing_lock(lck, gtid);
1648 void __kmp_init_nested_queuing_lock(kmp_queuing_lock_t *lck) {
1649 __kmp_init_queuing_lock(lck);
1650 lck->lk.depth_locked = 0; // >= 0 for nestable locks, -1 for simple locks
1653 void __kmp_destroy_nested_queuing_lock(kmp_queuing_lock_t *lck) {
1654 __kmp_destroy_queuing_lock(lck);
1655 lck->lk.depth_locked = 0;
1659 __kmp_destroy_nested_queuing_lock_with_checks(kmp_queuing_lock_t *lck) {
1661 if (lck->lk.initialized != lck) {
1664 if (!__kmp_is_queuing_lock_nestable(lck)) {
1667 if (__kmp_get_queuing_lock_owner(lck) != -1) {
1670 __kmp_destroy_nested_queuing_lock(lck);
1675 static const ident_t *__kmp_get_queuing_lock_location(kmp_queuing_lock_t *lck) {
1676 return lck->lk.location;
1679 static void __kmp_set_queuing_lock_location(kmp_queuing_lock_t *lck,
1681 lck->lk.location = loc;
1684 static kmp_lock_flags_t __kmp_get_queuing_lock_flags(kmp_queuing_lock_t *lck) {
1685 return lck->lk.flags;
1688 static void __kmp_set_queuing_lock_flags(kmp_queuing_lock_t *lck,
1690 lck->lk.flags = flags;
1813 kmp_adaptive_lock_info_t *lck = &liveLocks;
1815 memset(CCAST(kmp_adaptive_lock_statistics_t *, &(lck->stats)), 0,
1816 sizeof(lck->stats));
1817 lck->stats.next = lck;
1818 lck->stats.prev = lck;
1820 KMP_ASSERT(lck->stats.next->stats.prev == lck);
1821 KMP_ASSERT(lck->stats.prev->stats.next == lck);
1827 static void __kmp_remember_lock(kmp_adaptive_lock_info_t *lck) {
1830 lck->stats.next = liveLocks.stats.next;
1831 lck->stats.prev = &liveLocks;
1833 liveLocks.stats.next = lck;
1834 lck->stats.next->stats.prev = lck;
1836 KMP_ASSERT(lck->stats.next->stats.prev == lck);
1837 KMP_ASSERT(lck->stats.prev->stats.next == lck);
1842 static void __kmp_forget_lock(kmp_adaptive_lock_info_t *lck) {
1843 KMP_ASSERT(lck->stats.next->stats.prev == lck);
1844 KMP_ASSERT(lck->stats.prev->stats.next == lck);
1846 kmp_adaptive_lock_info_t *n = lck->stats.next;
1847 kmp_adaptive_lock_info_t *p = lck->stats.prev;
1853 static void __kmp_zero_speculative_stats(kmp_adaptive_lock_info_t *lck) {
1854 memset(CCAST(kmp_adaptive_lock_statistics_t *, &lck->stats), 0,
1855 sizeof(lck->stats));
1856 __kmp_remember_lock(lck);
1860 kmp_adaptive_lock_info_t *lck) {
1861 kmp_adaptive_lock_statistics_t volatile *s = &lck->stats;
1863 t->nonSpeculativeAcquireAttempts += lck->acquire_attempts;
1871 static void __kmp_accumulate_speculative_stats(kmp_adaptive_lock_info_t *lck) {
1874 __kmp_add_stats(&destroyedStats, lck);
1875 __kmp_forget_lock(lck);
1886 kmp_adaptive_lock_info_t *lck;
1888 for (lck = liveLocks.stats.next; lck != &liveLocks; lck = lck->stats.next) {
1889 __kmp_add_stats(&total, lck);
1944 #define KMP_INC_STAT(lck, stat) (lck->lk.adaptive.stats.stat++)
1946 #define KMP_INC_STAT(lck, stat)
1950 static inline bool __kmp_is_unlocked_queuing_lock(kmp_queuing_lock_t *lck) {
1953 bool res = lck->lk.head_id == 0;
1968 __kmp_update_badness_after_success(kmp_adaptive_lock_t *lck) {
1970 lck->lk.adaptive.badness = 0;
1971 KMP_INC_STAT(lck, successfulSpeculations);
1975 static __inline void __kmp_step_badness(kmp_adaptive_lock_t *lck) {
1976 kmp_uint32 newBadness = (lck->lk.adaptive.badness << 1) | 1;
1977 if (newBadness > lck->lk.adaptive.max_badness) {
1980 lck->lk.adaptive.badness = newBadness;
1986 static __inline int __kmp_should_speculate(kmp_adaptive_lock_t *lck,
1988 kmp_uint32 badness = lck->lk.adaptive.badness;
1989 kmp_uint32 attempts = lck->lk.adaptive.acquire_attempts;
1997 static int __kmp_test_adaptive_lock_only(kmp_adaptive_lock_t *lck,
1999 int retries = lck->lk.adaptive.max_soft_retries;
2016 if (!__kmp_is_unlocked_queuing_lock(GET_QLK_PTR(lck))) {
2026 KMP_INC_STAT(lck, softFailedSpeculations);
2029 KMP_INC_STAT(lck, hardFailedSpeculations);
2038 __kmp_step_badness(lck);
2045 static int __kmp_test_adaptive_lock(kmp_adaptive_lock_t *lck, kmp_int32 gtid) {
2047 if (__kmp_should_speculate(lck, gtid) &&
2048 __kmp_test_adaptive_lock_only(lck, gtid))
2053 lck->lk.adaptive.acquire_attempts++;
2056 if (__kmp_test_queuing_lock(GET_QLK_PTR(lck), gtid)) {
2057 KMP_INC_STAT(lck, nonSpeculativeAcquires);
2064 static int __kmp_test_adaptive_lock_with_checks(kmp_adaptive_lock_t *lck,
2067 if (lck->lk.qlk.initialized != GET_QLK_PTR(lck)) {
2071 int retval = __kmp_test_adaptive_lock(lck, gtid);
2074 lck->lk.qlk.owner_id = gtid + 1;
2090 static void __kmp_acquire_adaptive_lock(kmp_adaptive_lock_t *lck,
2092 if (__kmp_should_speculate(lck, gtid)) {
2093 if (__kmp_is_unlocked_queuing_lock(GET_QLK_PTR(lck))) {
2094 if (__kmp_test_adaptive_lock_only(lck, gtid))
2103 while (!__kmp_is_unlocked_queuing_lock(GET_QLK_PTR(lck))) {
2104 KMP_INC_STAT(lck, lemmingYields);
2108 if (__kmp_test_adaptive_lock_only(lck, gtid))
2115 lck->lk.adaptive.acquire_attempts++;
2117 __kmp_acquire_queuing_lock_timed_template<FALSE>(GET_QLK_PTR(lck), gtid);
2119 KMP_INC_STAT(lck, nonSpeculativeAcquires);
2122 static void __kmp_acquire_adaptive_lock_with_checks(kmp_adaptive_lock_t *lck,
2125 if (lck->lk.qlk.initialized != GET_QLK_PTR(lck)) {
2128 if (__kmp_get_queuing_lock_owner(GET_QLK_PTR(lck)) == gtid) {
2132 __kmp_acquire_adaptive_lock(lck, gtid);
2134 lck->lk.qlk.owner_id = gtid + 1;
2138 static int __kmp_release_adaptive_lock(kmp_adaptive_lock_t *lck,
2141 lck))) { // If the lock doesn't look claimed we must be speculating.
2145 __kmp_update_badness_after_success(lck);
2148 __kmp_release_queuing_lock(GET_QLK_PTR(lck), gtid);
2153 static int __kmp_release_adaptive_lock_with_checks(kmp_adaptive_lock_t *lck,
2157 if (lck->lk.qlk.initialized != GET_QLK_PTR(lck)) {
2160 if (__kmp_get_queuing_lock_owner(GET_QLK_PTR(lck)) == -1) {
2163 if (__kmp_get_queuing_lock_owner(GET_QLK_PTR(lck)) != gtid) {
2166 lck->lk.qlk.owner_id = 0;
2167 __kmp_release_adaptive_lock(lck, gtid);
2171 static void __kmp_init_adaptive_lock(kmp_adaptive_lock_t *lck) {
2172 __kmp_init_queuing_lock(GET_QLK_PTR(lck));
2173 lck->lk.adaptive.badness = 0;
2174 lck->lk.adaptive.acquire_attempts = 0; // nonSpeculativeAcquireAttempts = 0;
2175 lck->lk.adaptive.max_soft_retries =
2177 lck->lk.adaptive.max_badness = __kmp_adaptive_backoff_params.max_badness;
2179 __kmp_zero_speculative_stats(&lck->lk.adaptive);
2181 KA_TRACE(1000, ("__kmp_init_adaptive_lock: lock %p initialized\n", lck));
2184 static void __kmp_destroy_adaptive_lock(kmp_adaptive_lock_t *lck) {
2186 __kmp_accumulate_speculative_stats(&lck->lk.adaptive);
2188 __kmp_destroy_queuing_lock(GET_QLK_PTR(lck));
2192 static void __kmp_destroy_adaptive_lock_with_checks(kmp_adaptive_lock_t *lck) {
2194 if (lck->lk.qlk.initialized != GET_QLK_PTR(lck)) {
2197 if (__kmp_get_queuing_lock_owner(GET_QLK_PTR(lck)) != -1) {
2200 __kmp_destroy_adaptive_lock(lck);
2209 static kmp_int32 __kmp_get_drdpa_lock_owner(kmp_drdpa_lock_t *lck) {
2210 return lck->lk.owner_id - 1;
2213 static inline bool __kmp_is_drdpa_lock_nestable(kmp_drdpa_lock_t *lck) {
2214 return lck->lk.depth_locked != -1;
2218 __kmp_acquire_drdpa_lock_timed_template(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
2219 kmp_uint64 ticket = KMP_ATOMIC_INC(&lck->lk.next_ticket);
2220 kmp_uint64 mask = lck->lk.mask; // atomic load
2221 std::atomic<kmp_uint64> *polls = lck->lk.polls;
2225 __kmp_printf("LOCK CONTENTION: %p\n", lck);
2238 KMP_FSYNC_PREPARE(lck);
2250 mask = lck->lk.mask; // atomic load
2251 polls = lck->lk.polls; // atomic load
2255 KMP_FSYNC_ACQUIRED(lck);
2257 ticket, lck));
2258 lck->lk.now_serving = ticket; // non-volatile store
2265 if ((lck->lk.old_polls != NULL) && (ticket >= lck->lk.cleanup_ticket)) {
2266 __kmp_free(lck->lk.old_polls);
2267 lck->lk.old_polls = NULL;
2268 lck->lk.cleanup_ticket = 0;
2274 if (lck->lk.old_polls == NULL) {
2277 kmp_uint32 num_polls = TCR_4(lck->lk.num_polls);
2285 num_polls = TCR_4(lck->lk.num_polls);
2296 kmp_uint64 num_waiting = TCR_8(lck->lk.next_ticket) - ticket - 1;
2330 ticket, lck, num_polls));
2332 lck->lk.old_polls = old_polls;
2333 lck->lk.polls = polls; // atomic store
2337 lck->lk.num_polls = num_polls;
2338 lck->lk.mask = mask; // atomic store
2346 lck->lk.cleanup_ticket = lck->lk.next_ticket;
2352 int __kmp_acquire_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
2353 int retval = __kmp_acquire_drdpa_lock_timed_template(lck, gtid);
2357 static int __kmp_acquire_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck,
2360 if (lck->lk.initialized != lck) {
2363 if (__kmp_is_drdpa_lock_nestable(lck)) {
2366 if ((gtid >= 0) && (__kmp_get_drdpa_lock_owner(lck) == gtid)) {
2370 __kmp_acquire_drdpa_lock(lck, gtid);
2372 lck->lk.owner_id = gtid + 1;
2376 int __kmp_test_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
2379 kmp_uint64 ticket = lck->lk.next_ticket; // atomic load
2380 std::atomic<kmp_uint64> *polls = lck->lk.polls;
2381 kmp_uint64 mask = lck->lk.mask; // atomic load
2384 if (__kmp_atomic_compare_store_acq(&lck->lk.next_ticket, ticket,
2386 KMP_FSYNC_ACQUIRED(lck);
2388 ticket, lck));
2389 lck->lk.now_serving = ticket; // non-volatile store
2403 static int __kmp_test_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck,
2406 if (lck->lk.initialized != lck) {
2409 if (__kmp_is_drdpa_lock_nestable(lck)) {
2413 int retval = __kmp_test_drdpa_lock(lck, gtid);
2416 lck->lk.owner_id = gtid + 1;
2421 int __kmp_release_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
2424 kmp_uint64 ticket = lck->lk.now_serving + 1; // non-atomic load
2425 std::atomic<kmp_uint64> *polls = lck->lk.polls; // atomic load
2426 kmp_uint64 mask = lck->lk.mask; // atomic load
2428 ticket - 1, lck));
2429 KMP_FSYNC_RELEASING(lck);
2434 static int __kmp_release_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck,
2438 if (lck->lk.initialized != lck) {
2441 if (__kmp_is_drdpa_lock_nestable(lck)) {
2444 if (__kmp_get_drdpa_lock_owner(lck) == -1) {
2447 if ((gtid >= 0) && (__kmp_get_drdpa_lock_owner(lck) >= 0) &&
2448 (__kmp_get_drdpa_lock_owner(lck) != gtid)) {
2451 lck->lk.owner_id = 0;
2452 return __kmp_release_drdpa_lock(lck, gtid);
2455 void __kmp_init_drdpa_lock(kmp_drdpa_lock_t *lck) {
2456 lck->lk.location = NULL;
2457 lck->lk.mask = 0;
2458 lck->lk.num_polls = 1;
2459 lck->lk.polls = (std::atomic<kmp_uint64> *)__kmp_allocate(
2460 lck->lk.num_polls * sizeof(*(lck->lk.polls)));
2461 lck->lk.cleanup_ticket = 0;
2462 lck->lk.old_polls = NULL;
2463 lck->lk.next_ticket = 0;
2464 lck->lk.now_serving = 0;
2465 lck->lk.owner_id = 0; // no thread owns the lock.
2466 lck->lk.depth_locked = -1; // >= 0 for nestable locks, -1 for simple locks.
2467 lck->lk.initialized = lck;
2469 KA_TRACE(1000, ("__kmp_init_drdpa_lock: lock %p initialized\n", lck));
2472 void __kmp_destroy_drdpa_lock(kmp_drdpa_lock_t *lck) {
2473 lck->lk.initialized = NULL;
2474 lck->lk.location = NULL;
2475 if (lck->lk.polls.load() != NULL) {
2476 __kmp_free(lck->lk.polls.load());
2477 lck->lk.polls = NULL;
2479 if (lck->lk.old_polls != NULL) {
2480 __kmp_free(lck->lk.old_polls);
2481 lck->lk.old_polls = NULL;
2483 lck->lk.mask = 0;
2484 lck->lk.num_polls = 0;
2485 lck->lk.cleanup_ticket = 0;
2486 lck->lk.next_ticket = 0;
2487 lck->lk.now_serving = 0;
2488 lck->lk.owner_id = 0;
2489 lck->lk.depth_locked = -1;
2492 static void __kmp_destroy_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck) {
2494 if (lck->lk.initialized != lck) {
2497 if (__kmp_is_drdpa_lock_nestable(lck)) {
2500 if (__kmp_get_drdpa_lock_owner(lck) != -1) {
2503 __kmp_destroy_drdpa_lock(lck);
2508 int __kmp_acquire_nested_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
2511 if (__kmp_get_drdpa_lock_owner(lck) == gtid) {
2512 lck->lk.depth_locked += 1;
2515 __kmp_acquire_drdpa_lock_timed_template(lck, gtid);
2517 lck->lk.depth_locked = 1;
2519 lck->lk.owner_id = gtid + 1;
2524 static void __kmp_acquire_nested_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck,
2527 if (lck->lk.initialized != lck) {
2530 if (!__kmp_is_drdpa_lock_nestable(lck)) {
2533 __kmp_acquire_nested_drdpa_lock(lck, gtid);
2536 int __kmp_test_nested_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
2541 if (__kmp_get_drdpa_lock_owner(lck) == gtid) {
2542 retval = ++lck->lk.depth_locked;
2543 } else if (!__kmp_test_drdpa_lock(lck, gtid)) {
2547 retval = lck->lk.depth_locked = 1;
2549 lck->lk.owner_id = gtid + 1;
2554 static int __kmp_test_nested_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck,
2557 if (lck->lk.initialized != lck) {
2560 if (!__kmp_is_drdpa_lock_nestable(lck)) {
2563 return __kmp_test_nested_drdpa_lock(lck, gtid);
2566 int __kmp_release_nested_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
2570 if (--(lck->lk.depth_locked) == 0) {
2572 lck->lk.owner_id = 0;
2573 __kmp_release_drdpa_lock(lck, gtid);
2579 static int __kmp_release_nested_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck,
2583 if (lck->lk.initialized != lck) {
2586 if (!__kmp_is_drdpa_lock_nestable(lck)) {
2589 if (__kmp_get_drdpa_lock_owner(lck) == -1) {
2592 if (__kmp_get_drdpa_lock_owner(lck) != gtid) {
2595 return __kmp_release_nested_drdpa_lock(lck, gtid);
2598 void __kmp_init_nested_drdpa_lock(kmp_drdpa_lock_t *lck) {
2599 __kmp_init_drdpa_lock(lck);
2600 lck->lk.depth_locked = 0; // >= 0 for nestable locks, -1 for simple locks
2603 void __kmp_destroy_nested_drdpa_lock(kmp_drdpa_lock_t *lck) {
2604 __kmp_destroy_drdpa_lock(lck);
2605 lck->lk.depth_locked = 0;
2608 static void __kmp_destroy_nested_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck) {
2610 if (lck->lk.initialized != lck) {
2613 if (!__kmp_is_drdpa_lock_nestable(lck)) {
2616 if (__kmp_get_drdpa_lock_owner(lck) != -1) {
2619 __kmp_destroy_nested_drdpa_lock(lck);
2624 static const ident_t *__kmp_get_drdpa_lock_location(kmp_drdpa_lock_t *lck) {
2625 return lck->lk.location;
2628 static void __kmp_set_drdpa_lock_location(kmp_drdpa_lock_t *lck,
2630 lck->lk.location = loc;
2633 static kmp_lock_flags_t __kmp_get_drdpa_lock_flags(kmp_drdpa_lock_t *lck) {
2634 return lck->lk.flags;
2637 static void __kmp_set_drdpa_lock_flags(kmp_drdpa_lock_t *lck,
2639 lck->lk.flags = flags;
2690 static void __kmp_init_direct_lock(kmp_dyna_lock_t *lck,
2692 TCW_4(((kmp_base_tas_lock_t *)lck)->poll, KMP_GET_D_TAG(seq));
2709 static void __kmp_destroy_hle_lock(kmp_dyna_lock_t *lck) { TCW_4(*lck, 0); }
2711 static void __kmp_destroy_hle_lock_with_checks(kmp_dyna_lock_t *lck) {
2712 TCW_4(*lck, 0);
2715 static void __kmp_acquire_hle_lock(kmp_dyna_lock_t *lck, kmp_int32 gtid) {
2717 if (swap4(lck, KMP_LOCK_BUSY(1, hle)) != KMP_LOCK_FREE(hle)) {
2720 while (*(kmp_uint32 volatile *)lck != KMP_LOCK_FREE(hle)) {
2725 } while (swap4(lck, KMP_LOCK_BUSY(1, hle)) != KMP_LOCK_FREE(hle));
2729 static void __kmp_acquire_hle_lock_with_checks(kmp_dyna_lock_t *lck,
2731 __kmp_acquire_hle_lock(lck, gtid); // TODO: add checks
2734 static int __kmp_release_hle_lock(kmp_dyna_lock_t *lck, kmp_int32 gtid) {
2736 : "=m"(*lck)
2742 static int __kmp_release_hle_lock_with_checks(kmp_dyna_lock_t *lck,
2744 return __kmp_release_hle_lock(lck, gtid); // TODO: add checks
2747 static int __kmp_test_hle_lock(kmp_dyna_lock_t *lck, kmp_int32 gtid) {
2748 return swap4(lck, KMP_LOCK_BUSY(1, hle)) == KMP_LOCK_FREE(hle);
2751 static int __kmp_test_hle_lock_with_checks(kmp_dyna_lock_t *lck,
2753 return __kmp_test_hle_lock(lck, gtid); // TODO: add checks
2756 static void __kmp_init_rtm_queuing_lock(kmp_queuing_lock_t *lck) {
2757 __kmp_init_queuing_lock(lck);
2760 static void __kmp_destroy_rtm_queuing_lock(kmp_queuing_lock_t *lck) {
2761 __kmp_destroy_queuing_lock(lck);
2765 __kmp_destroy_rtm_queuing_lock_with_checks(kmp_queuing_lock_t *lck) {
2766 __kmp_destroy_queuing_lock_with_checks(lck);
2770 static void __kmp_acquire_rtm_queuing_lock(kmp_queuing_lock_t *lck,
2776 if (__kmp_is_unlocked_queuing_lock(lck))
2782 while (!__kmp_is_unlocked_queuing_lock(lck)) {
2790 __kmp_acquire_queuing_lock(lck, gtid);
2793 static void __kmp_acquire_rtm_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
2795 __kmp_acquire_rtm_queuing_lock(lck, gtid);
2799 static int __kmp_release_rtm_queuing_lock(kmp_queuing_lock_t *lck,
2801 if (__kmp_is_unlocked_queuing_lock(lck)) {
2806 __kmp_release_queuing_lock(lck, gtid);
2811 static int __kmp_release_rtm_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
2813 return __kmp_release_rtm_queuing_lock(lck, gtid);
2817 static int __kmp_test_rtm_queuing_lock(kmp_queuing_lock_t *lck,
2822 if (status == _XBEGIN_STARTED && __kmp_is_unlocked_queuing_lock(lck)) {
2829 return __kmp_test_queuing_lock(lck, gtid);
2832 static int __kmp_test_rtm_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
2834 return __kmp_test_rtm_queuing_lock(lck, gtid);
2840 static void __kmp_destroy_rtm_spin_lock(kmp_rtm_spin_lock_t *lck) {
2841 KMP_ATOMIC_ST_REL(&lck->lk.poll, 0);
2844 static void __kmp_destroy_rtm_spin_lock_with_checks(kmp_rtm_spin_lock_t *lck) {
2845 __kmp_destroy_rtm_spin_lock(lck);
2849 static int __kmp_acquire_rtm_spin_lock(kmp_rtm_spin_lock_t *lck,
2857 if (KMP_ATOMIC_LD_RLX(&lck->lk.poll) == lock_free)
2863 while (KMP_ATOMIC_LD_RLX(&lck->lk.poll) != lock_free) {
2871 KMP_FSYNC_PREPARE(lck);
2873 while (KMP_ATOMIC_LD_RLX(&lck->lk.poll) != lock_free ||
2874 !__kmp_atomic_compare_store_acq(&lck->lk.poll, lock_free, lock_busy)) {
2877 KMP_FSYNC_ACQUIRED(lck);
2881 static int __kmp_acquire_rtm_spin_lock_with_checks(kmp_rtm_spin_lock_t *lck,
2883 return __kmp_acquire_rtm_spin_lock(lck, gtid);
2887 static int __kmp_release_rtm_spin_lock(kmp_rtm_spin_lock_t *lck,
2889 if (KMP_ATOMIC_LD_RLX(&lck->lk.poll) == KMP_LOCK_FREE(rtm_spin)) {
2894 KMP_FSYNC_RELEASING(lck);
2895 KMP_ATOMIC_ST_REL(&lck->lk.poll, KMP_LOCK_FREE(rtm_spin));
2900 static int __kmp_release_rtm_spin_lock_with_checks(kmp_rtm_spin_lock_t *lck,
2902 return __kmp_release_rtm_spin_lock(lck, gtid);
2906 static int __kmp_test_rtm_spin_lock(kmp_rtm_spin_lock_t *lck, kmp_int32 gtid) {
2913 KMP_ATOMIC_LD_RLX(&lck->lk.poll) == lock_free) {
2920 if (KMP_ATOMIC_LD_RLX(&lck->lk.poll) == lock_free &&
2921 __kmp_atomic_compare_store_acq(&lck->lk.poll, lock_free, lock_busy)) {
2922 KMP_FSYNC_ACQUIRED(lck);
2928 static int __kmp_test_rtm_spin_lock_with_checks(kmp_rtm_spin_lock_t *lck,
2930 return __kmp_test_rtm_spin_lock(lck, gtid);
3123 kmp_indirect_lock_t *lck;
3130 lck = __kmp_indirect_lock_pool[tag];
3132 idx = lck->lock->pool.index;
3133 __kmp_indirect_lock_pool[tag] = (kmp_indirect_lock_t *)lck->lock->pool.next;
3135 lck));
3171 lck = &lock_table->table[row][col];
3173 lck->lock = (kmp_user_lock_p)__kmp_allocate(__kmp_indirect_lock_size[tag]);
3175 ("__kmp_allocate_indirect_lock: allocated a new lock %p\n", lck));
3180 lck->type = tag;
3186 *((kmp_indirect_lock_t **)user_lock) = lck;
3189 return lck;
3196 kmp_indirect_lock_t *lck = NULL;
3202 lck = __kmp_get_i_lock(idx);
3204 lck = *((kmp_indirect_lock_t **)user_lock);
3206 if (lck == NULL) {
3209 return lck;
3299 kmp_int32 __kmp_get_user_lock_owner(kmp_user_lock_p lck, kmp_uint32 seq) {
3303 return __kmp_get_tas_lock_owner((kmp_tas_lock_t *)lck);
3307 return __kmp_get_futex_lock_owner((kmp_futex_lock_t *)lck);
3311 return __kmp_get_ticket_lock_owner((kmp_ticket_lock_t *)lck);
3317 return __kmp_get_queuing_lock_owner((kmp_queuing_lock_t *)lck);
3320 return __kmp_get_drdpa_lock_owner((kmp_drdpa_lock_t *)lck);
3475 static void __kmp_init_tas_lock_with_checks(kmp_tas_lock_t *lck) {
3476 __kmp_init_tas_lock(lck);
3479 static void __kmp_init_nested_tas_lock_with_checks(kmp_tas_lock_t *lck) {
3480 __kmp_init_nested_tas_lock(lck);
3484 static void __kmp_init_futex_lock_with_checks(kmp_futex_lock_t *lck) {
3485 __kmp_init_futex_lock(lck);
3488 static void __kmp_init_nested_futex_lock_with_checks(kmp_futex_lock_t *lck) {
3489 __kmp_init_nested_futex_lock(lck);
3493 static int __kmp_is_ticket_lock_initialized(kmp_ticket_lock_t *lck) {
3494 return lck == lck->lk.self;
3497 static void __kmp_init_ticket_lock_with_checks(kmp_ticket_lock_t *lck) {
3498 __kmp_init_ticket_lock(lck);
3501 static void __kmp_init_nested_ticket_lock_with_checks(kmp_ticket_lock_t *lck) {
3502 __kmp_init_nested_ticket_lock(lck);
3505 static int __kmp_is_queuing_lock_initialized(kmp_queuing_lock_t *lck) {
3506 return lck == lck->lk.initialized;
3509 static void __kmp_init_queuing_lock_with_checks(kmp_queuing_lock_t *lck) {
3510 __kmp_init_queuing_lock(lck);
3514 __kmp_init_nested_queuing_lock_with_checks(kmp_queuing_lock_t *lck) {
3515 __kmp_init_nested_queuing_lock(lck);
3519 static void __kmp_init_adaptive_lock_with_checks(kmp_adaptive_lock_t *lck) {
3520 __kmp_init_adaptive_lock(lck);
3524 static int __kmp_is_drdpa_lock_initialized(kmp_drdpa_lock_t *lck) {
3525 return lck == lck->lk.initialized;
3528 static void __kmp_init_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck) {
3529 __kmp_init_drdpa_lock(lck);
3532 static void __kmp_init_nested_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck) {
3533 __kmp_init_nested_drdpa_lock(lck);
3545 kmp_int32 (*__kmp_get_user_lock_owner_)(kmp_user_lock_p lck) = NULL;
3546 int (*__kmp_acquire_user_lock_with_checks_)(kmp_user_lock_p lck,
3549 int (*__kmp_test_user_lock_with_checks_)(kmp_user_lock_p lck,
3551 int (*__kmp_release_user_lock_with_checks_)(kmp_user_lock_p lck,
3553 void (*__kmp_init_user_lock_with_checks_)(kmp_user_lock_p lck) = NULL;
3554 void (*__kmp_destroy_user_lock_)(kmp_user_lock_p lck) = NULL;
3555 void (*__kmp_destroy_user_lock_with_checks_)(kmp_user_lock_p lck) = NULL;
3556 int (*__kmp_acquire_nested_user_lock_with_checks_)(kmp_user_lock_p lck,
3559 int (*__kmp_test_nested_user_lock_with_checks_)(kmp_user_lock_p lck,
3561 int (*__kmp_release_nested_user_lock_with_checks_)(kmp_user_lock_p lck,
3563 void (*__kmp_init_nested_user_lock_with_checks_)(kmp_user_lock_p lck) = NULL;
3564 void (*__kmp_destroy_nested_user_lock_with_checks_)(kmp_user_lock_p lck) = NULL;
3566 int (*__kmp_is_user_lock_initialized_)(kmp_user_lock_p lck) = NULL;
3567 const ident_t *(*__kmp_get_user_lock_location_)(kmp_user_lock_p lck) = NULL;
3568 void (*__kmp_set_user_lock_location_)(kmp_user_lock_p lck,
3570 kmp_lock_flags_t (*__kmp_get_user_lock_flags_)(kmp_user_lock_p lck) = NULL;
3571 void (*__kmp_set_user_lock_flags_)(kmp_user_lock_p lck,
3795 static kmp_lock_index_t __kmp_lock_table_insert(kmp_user_lock_p lck) {
3821 __kmp_user_lock_table.table[index] = lck;
3856 kmp_user_lock_p lck;
3866 lck = (kmp_user_lock_p)__kmp_allocate(__kmp_user_lock_size);
3868 lck = __kmp_lock_block_allocate();
3873 index = __kmp_lock_table_insert(lck);
3876 lck = __kmp_lock_pool;
3886 *((kmp_user_lock_p *)user_lock) = lck;
3890 __kmp_set_user_lock_flags(lck, flags);
3894 return lck;
3899 kmp_user_lock_p lck) {
3901 KMP_DEBUG_ASSERT(lck != NULL);
3905 lck->pool.next = __kmp_lock_pool;
3906 __kmp_lock_pool = lck;
3910 lck->pool.index = index;
3917 kmp_user_lock_p lck = NULL;
3934 lck = __kmp_user_lock_table.table[index];
3936 lck = *((kmp_user_lock_p *)user_lock);
3940 if (lck == NULL) {
3945 return lck;
3953 #define IS_CRITICAL(lck) \
3955 ((*__kmp_get_user_lock_flags_)(lck)&kmp_lf_critical_section))
3971 // This means that the check (*__kmp_is_user_lock_initialized_)(lck) will fail
3985 kmp_user_lock_p lck =
3989 (*__kmp_is_user_lock_initialized_)(lck)) {
3993 if (__kmp_env_consistency_check && (!IS_CRITICAL(lck)) &&
3994 ((loc = __kmp_get_user_lock_location(lck)) != NULL) &&
4002 if (IS_CRITICAL(lck)) {
4006 lck, *(void **)lck));
4008 KA_TRACE(20, ("__kmp_cleanup_user_locks: free lock %p (%p)\n", lck,
4009 *(void **)lck));
4014 __kmp_destroy_user_lock(lck);
4019 __kmp_free(lck);