1 // SPDX-License-Identifier: CDDL-1.0 2 /* 3 * This file and its contents are supplied under the terms of the 4 * Common Development and Distribution License ("CDDL"), version 1.0. 5 * You may only use this file in accordance with the terms of version 6 * 1.0 of the CDDL. 7 * 8 * A full copy of the text of the CDDL should have accompanied this 9 * source. A copy of the CDDL is also available via the Internet at 10 * https://opensource.org/license/CDDL-1.0. 11 */ 12 13 /* 14 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 15 * Use is subject to license terms. 16 */ 17 18 /* 19 * Copyright (c) 2013, 2017 by Delphix. All rights reserved. 20 */ 21 22 #ifndef _SYS_TXG_IMPL_H 23 #define _SYS_TXG_IMPL_H 24 25 #include <sys/spa.h> 26 #include <sys/txg.h> 27 28 #ifdef __cplusplus 29 extern "C" { 30 #endif 31 32 /* 33 * The tx_cpu structure is a per-cpu structure that is used to track 34 * the number of active transaction holds (tc_count). As transactions 35 * are assigned into a transaction group the appropriate tc_count is 36 * incremented to indicate that there are pending changes that have yet 37 * to quiesce. Consumers eventually call txg_rele_to_sync() to decrement 38 * the tc_count. A transaction group is not considered quiesced until all 39 * tx_cpu structures have reached a tc_count of zero. 40 * 41 * This structure is a per-cpu structure by design. Updates to this structure 42 * are frequent and concurrent. Having a single structure would result in 43 * heavy lock contention so a per-cpu design was implemented. With the fanned 44 * out mutex design, consumers only need to lock the mutex associated with 45 * thread's cpu. 46 * 47 * The tx_cpu contains two locks, the tc_lock and tc_open_lock. 48 * The tc_lock is used to protect all members of the tx_cpu structure with 49 * the exception of the tc_open_lock. This lock should only be held for a 50 * short period of time, typically when updating the value of tc_count. 51 * 52 * The tc_open_lock protects the tx_open_txg member of the tx_state structure. 53 * This lock is used to ensure that transactions are only assigned into 54 * the current open transaction group. In order to move the current open 55 * transaction group to the quiesce phase, the txg_quiesce thread must 56 * grab all tc_open_locks, increment the tx_open_txg, and drop the locks. 57 * The tc_open_lock is held until the transaction is assigned into the 58 * transaction group. Typically, this is a short operation but if throttling 59 * is occurring it may be held for longer periods of time. 60 */ 61 struct tx_cpu { 62 kmutex_t tc_open_lock; /* protects tx_open_txg */ 63 kmutex_t tc_lock; /* protects the rest of this struct */ 64 kcondvar_t tc_cv[TXG_SIZE]; 65 uint64_t tc_count[TXG_SIZE]; /* tx hold count on each txg */ 66 list_t tc_callbacks[TXG_SIZE]; /* commit cb list */ 67 } ____cacheline_aligned; 68 69 /* 70 * The tx_state structure maintains the state information about the different 71 * stages of the pool's transaction groups. A per pool tx_state structure 72 * is used to track this information. The tx_state structure also points to 73 * an array of tx_cpu structures (described above). Although the tx_sync_lock 74 * is used to protect the members of this structure, it is not used to 75 * protect the tx_open_txg. Instead a special lock in the tx_cpu structure 76 * is used. Readers of tx_open_txg must grab the per-cpu tc_open_lock. 77 * Any thread wishing to update tx_open_txg must grab the tc_open_lock on 78 * every cpu (see txg_quiesce()). 79 */ 80 typedef struct tx_state { 81 tx_cpu_t *tx_cpu; /* protects access to tx_open_txg */ 82 kmutex_t tx_sync_lock; /* protects the rest of this struct */ 83 84 uint64_t tx_open_txg; /* currently open txg id */ 85 uint64_t tx_quiescing_txg; /* currently quiescing txg id */ 86 uint64_t tx_quiesced_txg; /* quiesced txg waiting for sync */ 87 uint64_t tx_syncing_txg; /* currently syncing txg id */ 88 uint64_t tx_synced_txg; /* last synced txg id */ 89 90 hrtime_t tx_open_time; /* start time of tx_open_txg */ 91 92 uint64_t tx_sync_txg_waiting; /* txg we're waiting to sync */ 93 uint64_t tx_quiesce_txg_waiting; /* txg we're waiting to open */ 94 95 kcondvar_t tx_sync_more_cv; 96 kcondvar_t tx_sync_done_cv; 97 kcondvar_t tx_quiesce_more_cv; 98 kcondvar_t tx_quiesce_done_cv; 99 kcondvar_t tx_timeout_cv; 100 kcondvar_t tx_exit_cv; /* wait for all threads to exit */ 101 102 uint8_t tx_threads; /* number of threads */ 103 uint8_t tx_exiting; /* set when we're exiting */ 104 105 kthread_t *tx_sync_thread; 106 kthread_t *tx_quiesce_thread; 107 108 taskq_t *tx_commit_cb_taskq; /* commit callback taskq */ 109 } tx_state_t; 110 111 #ifdef __cplusplus 112 } 113 #endif 114 115 #endif /* _SYS_TXG_IMPL_H */ 116