1*0b57cec5SDimitry Andric /* 2*0b57cec5SDimitry Andric * kmp_taskdeps.h 3*0b57cec5SDimitry Andric */ 4*0b57cec5SDimitry Andric 5*0b57cec5SDimitry Andric 6*0b57cec5SDimitry Andric //===----------------------------------------------------------------------===// 7*0b57cec5SDimitry Andric // 8*0b57cec5SDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 9*0b57cec5SDimitry Andric // See https://llvm.org/LICENSE.txt for license information. 10*0b57cec5SDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 11*0b57cec5SDimitry Andric // 12*0b57cec5SDimitry Andric //===----------------------------------------------------------------------===// 13*0b57cec5SDimitry Andric 14*0b57cec5SDimitry Andric 15*0b57cec5SDimitry Andric #ifndef KMP_TASKDEPS_H 16*0b57cec5SDimitry Andric #define KMP_TASKDEPS_H 17*0b57cec5SDimitry Andric 18*0b57cec5SDimitry Andric #include "kmp.h" 19*0b57cec5SDimitry Andric 20*0b57cec5SDimitry Andric #define KMP_ACQUIRE_DEPNODE(gtid, n) __kmp_acquire_lock(&(n)->dn.lock, (gtid)) 21*0b57cec5SDimitry Andric #define KMP_RELEASE_DEPNODE(gtid, n) __kmp_release_lock(&(n)->dn.lock, (gtid)) 22*0b57cec5SDimitry Andric 23*0b57cec5SDimitry Andric static inline void __kmp_node_deref(kmp_info_t *thread, kmp_depnode_t *node) { 24*0b57cec5SDimitry Andric if (!node) 25*0b57cec5SDimitry Andric return; 26*0b57cec5SDimitry Andric 27*0b57cec5SDimitry Andric kmp_int32 n = KMP_ATOMIC_DEC(&node->dn.nrefs) - 1; 28*0b57cec5SDimitry Andric if (n == 0) { 29*0b57cec5SDimitry Andric KMP_ASSERT(node->dn.nrefs == 0); 30*0b57cec5SDimitry Andric #if USE_FAST_MEMORY 31*0b57cec5SDimitry Andric __kmp_fast_free(thread, node); 32*0b57cec5SDimitry Andric #else 33*0b57cec5SDimitry Andric __kmp_thread_free(thread, node); 34*0b57cec5SDimitry Andric #endif 35*0b57cec5SDimitry Andric } 36*0b57cec5SDimitry Andric } 37*0b57cec5SDimitry Andric 38*0b57cec5SDimitry Andric static inline void __kmp_depnode_list_free(kmp_info_t *thread, 39*0b57cec5SDimitry Andric kmp_depnode_list *list) { 40*0b57cec5SDimitry Andric kmp_depnode_list *next; 41*0b57cec5SDimitry Andric 42*0b57cec5SDimitry Andric for (; list; list = next) { 43*0b57cec5SDimitry Andric next = list->next; 44*0b57cec5SDimitry Andric 45*0b57cec5SDimitry Andric __kmp_node_deref(thread, list->node); 46*0b57cec5SDimitry Andric #if USE_FAST_MEMORY 47*0b57cec5SDimitry Andric __kmp_fast_free(thread, list); 48*0b57cec5SDimitry Andric #else 49*0b57cec5SDimitry Andric __kmp_thread_free(thread, list); 50*0b57cec5SDimitry Andric #endif 51*0b57cec5SDimitry Andric } 52*0b57cec5SDimitry Andric } 53*0b57cec5SDimitry Andric 54*0b57cec5SDimitry Andric static inline void __kmp_dephash_free_entries(kmp_info_t *thread, 55*0b57cec5SDimitry Andric kmp_dephash_t *h) { 56*0b57cec5SDimitry Andric for (size_t i = 0; i < h->size; i++) { 57*0b57cec5SDimitry Andric if (h->buckets[i]) { 58*0b57cec5SDimitry Andric kmp_dephash_entry_t *next; 59*0b57cec5SDimitry Andric for (kmp_dephash_entry_t *entry = h->buckets[i]; entry; entry = next) { 60*0b57cec5SDimitry Andric next = entry->next_in_bucket; 61*0b57cec5SDimitry Andric __kmp_depnode_list_free(thread, entry->last_ins); 62*0b57cec5SDimitry Andric __kmp_depnode_list_free(thread, entry->last_mtxs); 63*0b57cec5SDimitry Andric __kmp_node_deref(thread, entry->last_out); 64*0b57cec5SDimitry Andric if (entry->mtx_lock) { 65*0b57cec5SDimitry Andric __kmp_destroy_lock(entry->mtx_lock); 66*0b57cec5SDimitry Andric __kmp_free(entry->mtx_lock); 67*0b57cec5SDimitry Andric } 68*0b57cec5SDimitry Andric #if USE_FAST_MEMORY 69*0b57cec5SDimitry Andric __kmp_fast_free(thread, entry); 70*0b57cec5SDimitry Andric #else 71*0b57cec5SDimitry Andric __kmp_thread_free(thread, entry); 72*0b57cec5SDimitry Andric #endif 73*0b57cec5SDimitry Andric } 74*0b57cec5SDimitry Andric h->buckets[i] = 0; 75*0b57cec5SDimitry Andric } 76*0b57cec5SDimitry Andric } 77*0b57cec5SDimitry Andric } 78*0b57cec5SDimitry Andric 79*0b57cec5SDimitry Andric static inline void __kmp_dephash_free(kmp_info_t *thread, kmp_dephash_t *h) { 80*0b57cec5SDimitry Andric __kmp_dephash_free_entries(thread, h); 81*0b57cec5SDimitry Andric #if USE_FAST_MEMORY 82*0b57cec5SDimitry Andric __kmp_fast_free(thread, h); 83*0b57cec5SDimitry Andric #else 84*0b57cec5SDimitry Andric __kmp_thread_free(thread, h); 85*0b57cec5SDimitry Andric #endif 86*0b57cec5SDimitry Andric } 87*0b57cec5SDimitry Andric 88*0b57cec5SDimitry Andric static inline void __kmp_release_deps(kmp_int32 gtid, kmp_taskdata_t *task) { 89*0b57cec5SDimitry Andric kmp_info_t *thread = __kmp_threads[gtid]; 90*0b57cec5SDimitry Andric kmp_depnode_t *node = task->td_depnode; 91*0b57cec5SDimitry Andric 92*0b57cec5SDimitry Andric if (task->td_dephash) { 93*0b57cec5SDimitry Andric KA_TRACE( 94*0b57cec5SDimitry Andric 40, ("__kmp_release_deps: T#%d freeing dependencies hash of task %p.\n", 95*0b57cec5SDimitry Andric gtid, task)); 96*0b57cec5SDimitry Andric __kmp_dephash_free(thread, task->td_dephash); 97*0b57cec5SDimitry Andric task->td_dephash = NULL; 98*0b57cec5SDimitry Andric } 99*0b57cec5SDimitry Andric 100*0b57cec5SDimitry Andric if (!node) 101*0b57cec5SDimitry Andric return; 102*0b57cec5SDimitry Andric 103*0b57cec5SDimitry Andric KA_TRACE(20, ("__kmp_release_deps: T#%d notifying successors of task %p.\n", 104*0b57cec5SDimitry Andric gtid, task)); 105*0b57cec5SDimitry Andric 106*0b57cec5SDimitry Andric KMP_ACQUIRE_DEPNODE(gtid, node); 107*0b57cec5SDimitry Andric node->dn.task = 108*0b57cec5SDimitry Andric NULL; // mark this task as finished, so no new dependencies are generated 109*0b57cec5SDimitry Andric KMP_RELEASE_DEPNODE(gtid, node); 110*0b57cec5SDimitry Andric 111*0b57cec5SDimitry Andric kmp_depnode_list_t *next; 112*0b57cec5SDimitry Andric for (kmp_depnode_list_t *p = node->dn.successors; p; p = next) { 113*0b57cec5SDimitry Andric kmp_depnode_t *successor = p->node; 114*0b57cec5SDimitry Andric kmp_int32 npredecessors = KMP_ATOMIC_DEC(&successor->dn.npredecessors) - 1; 115*0b57cec5SDimitry Andric 116*0b57cec5SDimitry Andric // successor task can be NULL for wait_depends or because deps are still 117*0b57cec5SDimitry Andric // being processed 118*0b57cec5SDimitry Andric if (npredecessors == 0) { 119*0b57cec5SDimitry Andric KMP_MB(); 120*0b57cec5SDimitry Andric if (successor->dn.task) { 121*0b57cec5SDimitry Andric KA_TRACE(20, ("__kmp_release_deps: T#%d successor %p of %p scheduled " 122*0b57cec5SDimitry Andric "for execution.\n", 123*0b57cec5SDimitry Andric gtid, successor->dn.task, task)); 124*0b57cec5SDimitry Andric __kmp_omp_task(gtid, successor->dn.task, false); 125*0b57cec5SDimitry Andric } 126*0b57cec5SDimitry Andric } 127*0b57cec5SDimitry Andric 128*0b57cec5SDimitry Andric next = p->next; 129*0b57cec5SDimitry Andric __kmp_node_deref(thread, p->node); 130*0b57cec5SDimitry Andric #if USE_FAST_MEMORY 131*0b57cec5SDimitry Andric __kmp_fast_free(thread, p); 132*0b57cec5SDimitry Andric #else 133*0b57cec5SDimitry Andric __kmp_thread_free(thread, p); 134*0b57cec5SDimitry Andric #endif 135*0b57cec5SDimitry Andric } 136*0b57cec5SDimitry Andric 137*0b57cec5SDimitry Andric __kmp_node_deref(thread, node); 138*0b57cec5SDimitry Andric 139*0b57cec5SDimitry Andric KA_TRACE( 140*0b57cec5SDimitry Andric 20, 141*0b57cec5SDimitry Andric ("__kmp_release_deps: T#%d all successors of %p notified of completion\n", 142*0b57cec5SDimitry Andric gtid, task)); 143*0b57cec5SDimitry Andric } 144*0b57cec5SDimitry Andric 145*0b57cec5SDimitry Andric #endif // KMP_TASKDEPS_H 146