Lines Matching +full:mmp +full:- +full:timer
1 // SPDX-License-Identifier: CDDL-1.0
10 * or https://opensource.org/licenses/CDDL-1.0.
42 * ----------------------
47 * assigned a 64-bit consecutive identifier. There are three active
53 * strokes, transactions -- operations that change in-memory structures -- are
61 * transactions -- updates to in-memory structures -- are assigned to the
81 * In the syncing state, the in-memory state built up during the open and (to
87 * data until it converges and all in-memory changes have been written out.
124 tx_state_t *tx = &dp->dp_tx; in txg_init()
128 tx->tx_cpu = vmem_zalloc(max_ncpus * sizeof (tx_cpu_t), KM_SLEEP); in txg_init()
133 mutex_init(&tx->tx_cpu[c].tc_lock, NULL, MUTEX_DEFAULT, NULL); in txg_init()
134 mutex_init(&tx->tx_cpu[c].tc_open_lock, NULL, MUTEX_NOLOCKDEP, in txg_init()
137 cv_init(&tx->tx_cpu[c].tc_cv[i], NULL, CV_DEFAULT, in txg_init()
139 list_create(&tx->tx_cpu[c].tc_callbacks[i], in txg_init()
145 mutex_init(&tx->tx_sync_lock, NULL, MUTEX_DEFAULT, NULL); in txg_init()
147 cv_init(&tx->tx_sync_more_cv, NULL, CV_DEFAULT, NULL); in txg_init()
148 cv_init(&tx->tx_sync_done_cv, NULL, CV_DEFAULT, NULL); in txg_init()
149 cv_init(&tx->tx_quiesce_more_cv, NULL, CV_DEFAULT, NULL); in txg_init()
150 cv_init(&tx->tx_quiesce_done_cv, NULL, CV_DEFAULT, NULL); in txg_init()
151 cv_init(&tx->tx_exit_cv, NULL, CV_DEFAULT, NULL); in txg_init()
153 tx->tx_open_txg = txg; in txg_init()
162 tx_state_t *tx = &dp->dp_tx; in txg_fini()
165 ASSERT0(tx->tx_threads); in txg_fini()
167 mutex_destroy(&tx->tx_sync_lock); in txg_fini()
169 cv_destroy(&tx->tx_sync_more_cv); in txg_fini()
170 cv_destroy(&tx->tx_sync_done_cv); in txg_fini()
171 cv_destroy(&tx->tx_quiesce_more_cv); in txg_fini()
172 cv_destroy(&tx->tx_quiesce_done_cv); in txg_fini()
173 cv_destroy(&tx->tx_exit_cv); in txg_fini()
178 mutex_destroy(&tx->tx_cpu[c].tc_open_lock); in txg_fini()
179 mutex_destroy(&tx->tx_cpu[c].tc_lock); in txg_fini()
181 cv_destroy(&tx->tx_cpu[c].tc_cv[i]); in txg_fini()
182 list_destroy(&tx->tx_cpu[c].tc_callbacks[i]); in txg_fini()
186 if (tx->tx_commit_cb_taskq != NULL) in txg_fini()
187 taskq_destroy(tx->tx_commit_cb_taskq); in txg_fini()
189 vmem_free(tx->tx_cpu, max_ncpus * sizeof (tx_cpu_t)); in txg_fini()
200 tx_state_t *tx = &dp->dp_tx; in txg_sync_start()
202 mutex_enter(&tx->tx_sync_lock); in txg_sync_start()
206 ASSERT0(tx->tx_threads); in txg_sync_start()
208 tx->tx_threads = 2; in txg_sync_start()
210 tx->tx_quiesce_thread = thread_create(NULL, 0, txg_quiesce_thread, in txg_sync_start()
214 * The sync thread can need a larger-than-default stack size on in txg_sync_start()
215 * 32-bit x86. This is due in part to nested pools and in txg_sync_start()
218 tx->tx_sync_thread = thread_create(NULL, 0, txg_sync_thread, in txg_sync_start()
221 mutex_exit(&tx->tx_sync_lock); in txg_sync_start()
227 CALLB_CPR_INIT(cpr, &tx->tx_sync_lock, callb_generic_cpr, FTAG); in txg_thread_enter()
228 mutex_enter(&tx->tx_sync_lock); in txg_thread_enter()
236 tx->tx_threads--; in txg_thread_exit()
237 cv_broadcast(&tx->tx_exit_cv); in txg_thread_exit()
238 CALLB_CPR_EXIT(cpr); /* drops &tx->tx_sync_lock */ in txg_thread_exit()
248 (void) cv_timedwait_idle(cv, &tx->tx_sync_lock, in txg_thread_wait()
251 cv_wait_idle(cv, &tx->tx_sync_lock); in txg_thread_wait()
254 CALLB_CPR_SAFE_END(cpr, &tx->tx_sync_lock); in txg_thread_wait()
263 tx_state_t *tx = &dp->dp_tx; in txg_sync_stop()
269 ASSERT3U(tx->tx_threads, ==, 2); in txg_sync_stop()
274 txg_wait_synced(dp, tx->tx_open_txg + TXG_DEFER_SIZE); in txg_sync_stop()
279 mutex_enter(&tx->tx_sync_lock); in txg_sync_stop()
281 ASSERT3U(tx->tx_threads, ==, 2); in txg_sync_stop()
283 tx->tx_exiting = 1; in txg_sync_stop()
285 cv_broadcast(&tx->tx_quiesce_more_cv); in txg_sync_stop()
286 cv_broadcast(&tx->tx_quiesce_done_cv); in txg_sync_stop()
287 cv_broadcast(&tx->tx_sync_more_cv); in txg_sync_stop()
289 while (tx->tx_threads != 0) in txg_sync_stop()
290 cv_wait(&tx->tx_exit_cv, &tx->tx_sync_lock); in txg_sync_stop()
292 tx->tx_exiting = 0; in txg_sync_stop()
294 mutex_exit(&tx->tx_sync_lock); in txg_sync_stop()
313 * 1 <- txg_hold_open(P, ...)
314 * 2 <- txg_hold_open(P, ...)
315 * 1 <- txg_hold_open(P, ...)
321 tx_state_t *tx = &dp->dp_tx; in txg_hold_open()
331 tc = &tx->tx_cpu[CPU_SEQID_UNSTABLE]; in txg_hold_open()
333 mutex_enter(&tc->tc_open_lock); in txg_hold_open()
334 txg = tx->tx_open_txg; in txg_hold_open()
336 mutex_enter(&tc->tc_lock); in txg_hold_open()
337 tc->tc_count[txg & TXG_MASK]++; in txg_hold_open()
338 mutex_exit(&tc->tc_lock); in txg_hold_open()
340 th->th_cpu = tc; in txg_hold_open()
341 th->th_txg = txg; in txg_hold_open()
349 tx_cpu_t *tc = th->th_cpu; in txg_rele_to_quiesce()
351 ASSERT(!MUTEX_HELD(&tc->tc_lock)); in txg_rele_to_quiesce()
352 mutex_exit(&tc->tc_open_lock); in txg_rele_to_quiesce()
358 tx_cpu_t *tc = th->th_cpu; in txg_register_callbacks()
359 int g = th->th_txg & TXG_MASK; in txg_register_callbacks()
361 mutex_enter(&tc->tc_lock); in txg_register_callbacks()
362 list_move_tail(&tc->tc_callbacks[g], tx_callbacks); in txg_register_callbacks()
363 mutex_exit(&tc->tc_lock); in txg_register_callbacks()
369 tx_cpu_t *tc = th->th_cpu; in txg_rele_to_sync()
370 int g = th->th_txg & TXG_MASK; in txg_rele_to_sync()
372 mutex_enter(&tc->tc_lock); in txg_rele_to_sync()
373 ASSERT(tc->tc_count[g] != 0); in txg_rele_to_sync()
374 if (--tc->tc_count[g] == 0) in txg_rele_to_sync()
375 cv_broadcast(&tc->tc_cv[g]); in txg_rele_to_sync()
376 mutex_exit(&tc->tc_lock); in txg_rele_to_sync()
378 th->th_cpu = NULL; /* defensive */ in txg_rele_to_sync()
390 tx_state_t *tx = &dp->dp_tx; in txg_quiesce()
399 mutex_enter(&tx->tx_cpu[c].tc_open_lock); in txg_quiesce()
401 ASSERT(txg == tx->tx_open_txg); in txg_quiesce()
402 tx->tx_open_txg++; in txg_quiesce()
403 tx->tx_open_time = tx_open_time = gethrtime(); in txg_quiesce()
406 DTRACE_PROBE2(txg__opened, dsl_pool_t *, dp, uint64_t, tx->tx_open_txg); in txg_quiesce()
413 mutex_exit(&tx->tx_cpu[c].tc_open_lock); in txg_quiesce()
415 spa_txg_history_set(dp->dp_spa, txg, TXG_STATE_OPEN, tx_open_time); in txg_quiesce()
416 spa_txg_history_add(dp->dp_spa, txg + 1, tx_open_time); in txg_quiesce()
423 tx_cpu_t *tc = &tx->tx_cpu[c]; in txg_quiesce()
424 mutex_enter(&tc->tc_lock); in txg_quiesce()
425 while (tc->tc_count[g] != 0) in txg_quiesce()
426 cv_wait(&tc->tc_cv[g], &tc->tc_lock); in txg_quiesce()
427 mutex_exit(&tc->tc_lock); in txg_quiesce()
430 spa_txg_history_set(dp->dp_spa, txg, TXG_STATE_QUIESCED, gethrtime()); in txg_quiesce()
453 tx_state_t *tx = &dp->dp_tx; in txg_dispatch_callbacks()
457 tx_cpu_t *tc = &tx->tx_cpu[c]; in txg_dispatch_callbacks()
465 if (list_is_empty(&tc->tc_callbacks[g])) in txg_dispatch_callbacks()
468 if (tx->tx_commit_cb_taskq == NULL) { in txg_dispatch_callbacks()
472 tx->tx_commit_cb_taskq = taskq_create("tx_commit_cb", in txg_dispatch_callbacks()
482 list_move_tail(cb_list, &tc->tc_callbacks[g]); in txg_dispatch_callbacks()
484 (void) taskq_dispatch(tx->tx_commit_cb_taskq, in txg_dispatch_callbacks()
490 * Wait for pending commit callbacks of already-synced transactions to finish
497 tx_state_t *tx = &dp->dp_tx; in txg_wait_callbacks()
499 if (tx->tx_commit_cb_taskq != NULL) in txg_wait_callbacks()
500 taskq_wait_outstanding(tx->tx_commit_cb_taskq, 0); in txg_wait_callbacks()
506 tx_state_t *tx = &dp->dp_tx; in txg_is_quiescing()
507 ASSERT(MUTEX_HELD(&tx->tx_sync_lock)); in txg_is_quiescing()
508 return (tx->tx_quiescing_txg != 0); in txg_is_quiescing()
514 tx_state_t *tx = &dp->dp_tx; in txg_has_quiesced_to_sync()
515 ASSERT(MUTEX_HELD(&tx->tx_sync_lock)); in txg_has_quiesced_to_sync()
516 return (tx->tx_quiesced_txg != 0); in txg_has_quiesced_to_sync()
523 spa_t *spa = dp->dp_spa; in txg_sync_thread()
524 tx_state_t *tx = &dp->dp_tx; in txg_sync_thread()
534 clock_t timer; in txg_sync_thread() local
542 timer = (delta >= timeout ? 0 : timeout - delta); in txg_sync_thread()
543 while (!dsl_scan_active(dp->dp_scan) && in txg_sync_thread()
544 !tx->tx_exiting && timer > 0 && in txg_sync_thread()
545 tx->tx_synced_txg >= tx->tx_sync_txg_waiting && in txg_sync_thread()
548 (u_longlong_t)tx->tx_synced_txg, in txg_sync_thread()
549 (u_longlong_t)tx->tx_sync_txg_waiting, dp); in txg_sync_thread()
550 txg_thread_wait(tx, &cpr, &tx->tx_sync_more_cv, timer); in txg_sync_thread()
551 delta = ddi_get_lbolt() - start; in txg_sync_thread()
552 timer = (delta > timeout ? 0 : timeout - delta); in txg_sync_thread()
557 * MMP we don't want to bump anything that would make it in txg_sync_thread()
559 * resuming after a MMP suspend. in txg_sync_thread()
568 while (!tx->tx_exiting && !txg_has_quiesced_to_sync(dp)) { in txg_sync_thread()
571 &tx->tx_quiesce_done_cv, 0); in txg_sync_thread()
574 if (tx->tx_quiesce_txg_waiting < tx->tx_open_txg+1) in txg_sync_thread()
575 tx->tx_quiesce_txg_waiting = tx->tx_open_txg+1; in txg_sync_thread()
576 cv_broadcast(&tx->tx_quiesce_more_cv); in txg_sync_thread()
577 txg_thread_wait(tx, &cpr, &tx->tx_quiesce_done_cv, 0); in txg_sync_thread()
580 if (tx->tx_exiting) in txg_sync_thread()
581 txg_thread_exit(tx, &cpr, &tx->tx_sync_thread); in txg_sync_thread()
588 ASSERT(tx->tx_quiesced_txg != 0); in txg_sync_thread()
589 txg = tx->tx_quiesced_txg; in txg_sync_thread()
590 tx->tx_quiesced_txg = 0; in txg_sync_thread()
591 tx->tx_syncing_txg = txg; in txg_sync_thread()
593 cv_broadcast(&tx->tx_quiesce_more_cv); in txg_sync_thread()
596 (u_longlong_t)txg, (u_longlong_t)tx->tx_quiesce_txg_waiting, in txg_sync_thread()
597 (u_longlong_t)tx->tx_sync_txg_waiting); in txg_sync_thread()
598 mutex_exit(&tx->tx_sync_lock); in txg_sync_thread()
603 delta = ddi_get_lbolt() - start; in txg_sync_thread()
606 mutex_enter(&tx->tx_sync_lock); in txg_sync_thread()
607 tx->tx_synced_txg = txg; in txg_sync_thread()
608 tx->tx_syncing_txg = 0; in txg_sync_thread()
610 cv_broadcast(&tx->tx_sync_done_cv); in txg_sync_thread()
623 tx_state_t *tx = &dp->dp_tx; in txg_quiesce_thread()
638 while (!tx->tx_exiting && in txg_quiesce_thread()
639 (tx->tx_open_txg >= tx->tx_quiesce_txg_waiting || in txg_quiesce_thread()
641 txg_thread_wait(tx, &cpr, &tx->tx_quiesce_more_cv, 0); in txg_quiesce_thread()
643 if (tx->tx_exiting) in txg_quiesce_thread()
644 txg_thread_exit(tx, &cpr, &tx->tx_quiesce_thread); in txg_quiesce_thread()
646 txg = tx->tx_open_txg; in txg_quiesce_thread()
649 (u_longlong_t)tx->tx_quiesce_txg_waiting, in txg_quiesce_thread()
650 (u_longlong_t)tx->tx_sync_txg_waiting); in txg_quiesce_thread()
651 tx->tx_quiescing_txg = txg; in txg_quiesce_thread()
653 mutex_exit(&tx->tx_sync_lock); in txg_quiesce_thread()
655 mutex_enter(&tx->tx_sync_lock); in txg_quiesce_thread()
662 tx->tx_quiescing_txg = 0; in txg_quiesce_thread()
663 tx->tx_quiesced_txg = txg; in txg_quiesce_thread()
665 cv_broadcast(&tx->tx_sync_more_cv); in txg_quiesce_thread()
666 cv_broadcast(&tx->tx_quiesce_done_cv); in txg_quiesce_thread()
678 tx_state_t *tx = &dp->dp_tx; in txg_delay()
682 if (tx->tx_open_txg > txg || in txg_delay()
683 tx->tx_syncing_txg == txg-1 || tx->tx_synced_txg == txg-1) in txg_delay()
686 mutex_enter(&tx->tx_sync_lock); in txg_delay()
687 if (tx->tx_open_txg > txg || tx->tx_synced_txg == txg-1) { in txg_delay()
688 mutex_exit(&tx->tx_sync_lock); in txg_delay()
692 while (gethrtime() - start < delay && in txg_delay()
693 tx->tx_syncing_txg < txg-1 && !txg_stalled(dp)) { in txg_delay()
694 (void) cv_timedwait_hires(&tx->tx_quiesce_more_cv, in txg_delay()
695 &tx->tx_sync_lock, delay, resolution, 0); in txg_delay()
700 mutex_exit(&tx->tx_sync_lock); in txg_delay()
707 tx_state_t *tx = &dp->dp_tx; in txg_wait_synced_flags()
712 mutex_enter(&tx->tx_sync_lock); in txg_wait_synced_flags()
713 ASSERT3U(tx->tx_threads, ==, 2); in txg_wait_synced_flags()
715 txg = tx->tx_open_txg + TXG_DEFER_SIZE; in txg_wait_synced_flags()
716 if (tx->tx_sync_txg_waiting < txg) in txg_wait_synced_flags()
717 tx->tx_sync_txg_waiting = txg; in txg_wait_synced_flags()
719 (u_longlong_t)txg, (u_longlong_t)tx->tx_quiesce_txg_waiting, in txg_wait_synced_flags()
720 (u_longlong_t)tx->tx_sync_txg_waiting); in txg_wait_synced_flags()
726 while (tx->tx_synced_txg < txg) { in txg_wait_synced_flags()
727 if ((flags & TXG_WAIT_SUSPEND) && spa_suspended(dp->dp_spa)) { in txg_wait_synced_flags()
738 (u_longlong_t)tx->tx_synced_txg, in txg_wait_synced_flags()
739 (u_longlong_t)tx->tx_sync_txg_waiting, dp); in txg_wait_synced_flags()
740 cv_broadcast(&tx->tx_sync_more_cv); in txg_wait_synced_flags()
748 if (cv_wait_io_sig(&tx->tx_sync_done_cv, in txg_wait_synced_flags()
749 &tx->tx_sync_lock) == 0) { in txg_wait_synced_flags()
755 cv_wait_io(&tx->tx_sync_done_cv, &tx->tx_sync_lock); in txg_wait_synced_flags()
759 mutex_exit(&tx->tx_sync_lock); in txg_wait_synced_flags()
772 tx_state_t *tx = &dp->dp_tx; in txg_wait_kick()
773 mutex_enter(&tx->tx_sync_lock); in txg_wait_kick()
774 cv_broadcast(&tx->tx_sync_done_cv); in txg_wait_kick()
775 mutex_exit(&tx->tx_sync_lock); in txg_wait_kick()
785 tx_state_t *tx = &dp->dp_tx; in txg_wait_open()
789 mutex_enter(&tx->tx_sync_lock); in txg_wait_open()
790 ASSERT3U(tx->tx_threads, ==, 2); in txg_wait_open()
792 txg = tx->tx_open_txg + 1; in txg_wait_open()
793 if (tx->tx_quiesce_txg_waiting < txg && should_quiesce) in txg_wait_open()
794 tx->tx_quiesce_txg_waiting = txg; in txg_wait_open()
796 (u_longlong_t)txg, (u_longlong_t)tx->tx_quiesce_txg_waiting, in txg_wait_open()
797 (u_longlong_t)tx->tx_sync_txg_waiting); in txg_wait_open()
798 while (tx->tx_open_txg < txg) { in txg_wait_open()
799 cv_broadcast(&tx->tx_quiesce_more_cv); in txg_wait_open()
807 cv_wait_io(&tx->tx_quiesce_done_cv, &tx->tx_sync_lock); in txg_wait_open()
809 cv_wait_idle(&tx->tx_quiesce_done_cv, in txg_wait_open()
810 &tx->tx_sync_lock); in txg_wait_open()
813 mutex_exit(&tx->tx_sync_lock); in txg_wait_open()
822 tx_state_t *tx = &dp->dp_tx; in txg_kick()
826 if (tx->tx_sync_txg_waiting >= txg) in txg_kick()
829 mutex_enter(&tx->tx_sync_lock); in txg_kick()
830 if (tx->tx_sync_txg_waiting < txg) { in txg_kick()
831 tx->tx_sync_txg_waiting = txg; in txg_kick()
832 cv_broadcast(&tx->tx_sync_more_cv); in txg_kick()
834 mutex_exit(&tx->tx_sync_lock); in txg_kick()
840 tx_state_t *tx = &dp->dp_tx; in txg_stalled()
841 return (tx->tx_quiesce_txg_waiting > tx->tx_open_txg); in txg_stalled()
847 tx_state_t *tx = &dp->dp_tx; in txg_sync_waiting()
849 return (tx->tx_syncing_txg <= tx->tx_sync_txg_waiting || in txg_sync_waiting()
850 tx->tx_quiesced_txg != 0); in txg_sync_waiting()
854 * Verify that this txg is active (open, quiescing, syncing). Non-active
864 ASSERT3U(txg, <=, dp->dp_tx.tx_open_txg); in txg_verify()
865 ASSERT3U(txg, >=, dp->dp_tx.tx_synced_txg); in txg_verify()
866 ASSERT3U(txg, >=, dp->dp_tx.tx_open_txg - TXG_CONCURRENT_STATES); in txg_verify()
871 * Per-txg object lists.
878 mutex_init(&tl->tl_lock, NULL, MUTEX_DEFAULT, NULL); in txg_list_create()
880 tl->tl_offset = offset; in txg_list_create()
881 tl->tl_spa = spa; in txg_list_create()
884 tl->tl_head[t] = NULL; in txg_list_create()
890 ASSERT(MUTEX_HELD(&tl->tl_lock)); in txg_list_empty_impl()
891 TXG_VERIFY(tl->tl_spa, txg); in txg_list_empty_impl()
892 return (tl->tl_head[txg & TXG_MASK] == NULL); in txg_list_empty_impl()
898 mutex_enter(&tl->tl_lock); in txg_list_empty()
900 mutex_exit(&tl->tl_lock); in txg_list_empty()
910 mutex_enter(&tl->tl_lock); in txg_list_destroy()
913 mutex_exit(&tl->tl_lock); in txg_list_destroy()
915 mutex_destroy(&tl->tl_lock); in txg_list_destroy()
929 res &= (tl->tl_head[i] == NULL); in txg_all_lists_empty()
941 txg_node_t *tn = (txg_node_t *)((char *)p + tl->tl_offset); in txg_list_add()
944 TXG_VERIFY(tl->tl_spa, txg); in txg_list_add()
945 mutex_enter(&tl->tl_lock); in txg_list_add()
946 add = (tn->tn_member[t] == 0); in txg_list_add()
948 tn->tn_member[t] = 1; in txg_list_add()
949 tn->tn_next[t] = tl->tl_head[t]; in txg_list_add()
950 tl->tl_head[t] = tn; in txg_list_add()
952 mutex_exit(&tl->tl_lock); in txg_list_add()
966 txg_node_t *tn = (txg_node_t *)((char *)p + tl->tl_offset); in txg_list_add_tail()
969 TXG_VERIFY(tl->tl_spa, txg); in txg_list_add_tail()
970 mutex_enter(&tl->tl_lock); in txg_list_add_tail()
971 add = (tn->tn_member[t] == 0); in txg_list_add_tail()
975 for (tp = &tl->tl_head[t]; *tp != NULL; tp = &(*tp)->tn_next[t]) in txg_list_add_tail()
978 tn->tn_member[t] = 1; in txg_list_add_tail()
979 tn->tn_next[t] = NULL; in txg_list_add_tail()
982 mutex_exit(&tl->tl_lock); in txg_list_add_tail()
997 TXG_VERIFY(tl->tl_spa, txg); in txg_list_remove()
998 mutex_enter(&tl->tl_lock); in txg_list_remove()
999 if ((tn = tl->tl_head[t]) != NULL) { in txg_list_remove()
1000 ASSERT(tn->tn_member[t]); in txg_list_remove()
1001 ASSERT(tn->tn_next[t] == NULL || tn->tn_next[t]->tn_member[t]); in txg_list_remove()
1002 p = (char *)tn - tl->tl_offset; in txg_list_remove()
1003 tl->tl_head[t] = tn->tn_next[t]; in txg_list_remove()
1004 tn->tn_next[t] = NULL; in txg_list_remove()
1005 tn->tn_member[t] = 0; in txg_list_remove()
1007 mutex_exit(&tl->tl_lock); in txg_list_remove()
1021 TXG_VERIFY(tl->tl_spa, txg); in txg_list_remove_this()
1022 mutex_enter(&tl->tl_lock); in txg_list_remove_this()
1024 for (tp = &tl->tl_head[t]; (tn = *tp) != NULL; tp = &tn->tn_next[t]) { in txg_list_remove_this()
1025 if ((char *)tn - tl->tl_offset == p) { in txg_list_remove_this()
1026 *tp = tn->tn_next[t]; in txg_list_remove_this()
1027 tn->tn_next[t] = NULL; in txg_list_remove_this()
1028 tn->tn_member[t] = 0; in txg_list_remove_this()
1029 mutex_exit(&tl->tl_lock); in txg_list_remove_this()
1034 mutex_exit(&tl->tl_lock); in txg_list_remove_this()
1043 txg_node_t *tn = (txg_node_t *)((char *)p + tl->tl_offset); in txg_list_member()
1045 TXG_VERIFY(tl->tl_spa, txg); in txg_list_member()
1046 return (tn->tn_member[t] != 0); in txg_list_member()
1058 mutex_enter(&tl->tl_lock); in txg_list_head()
1059 tn = tl->tl_head[t]; in txg_list_head()
1060 mutex_exit(&tl->tl_lock); in txg_list_head()
1062 TXG_VERIFY(tl->tl_spa, txg); in txg_list_head()
1063 return (tn == NULL ? NULL : (char *)tn - tl->tl_offset); in txg_list_head()
1070 txg_node_t *tn = (txg_node_t *)((char *)p + tl->tl_offset); in txg_list_next()
1072 TXG_VERIFY(tl->tl_spa, txg); in txg_list_next()
1074 mutex_enter(&tl->tl_lock); in txg_list_next()
1075 tn = tn->tn_next[t]; in txg_list_next()
1076 mutex_exit(&tl->tl_lock); in txg_list_next()
1078 return (tn == NULL ? NULL : (char *)tn - tl->tl_offset); in txg_list_next()