1 // SPDX-License-Identifier: CDDL-1.0 2 /* 3 * This file and its contents are supplied under the terms of the 4 * Common Development and Distribution License ("CDDL"), version 1.0. 5 * You may only use this file in accordance with the terms of version 6 * 1.0 of the CDDL. 7 * 8 * A full copy of the text of the CDDL should have accompanied this 9 * source. A copy of the CDDL is also available via the Internet at 10 * https://opensource.org/license/CDDL-1.0. 11 */ 12 /* 13 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 14 * Copyright (c) 2012, 2018 by Delphix. All rights reserved. 15 */ 16 17 /* Portions Copyright 2010 Robert Milkowski */ 18 19 #ifndef _SYS_ZIL_IMPL_H 20 #define _SYS_ZIL_IMPL_H 21 22 #include <sys/zil.h> 23 #include <sys/dmu_objset.h> 24 25 #ifdef __cplusplus 26 extern "C" { 27 #endif 28 29 /* 30 * Possible states for a given lwb structure. 31 * 32 * An lwb will start out in the "new" state, and transition to the "opened" 33 * state via a call to zil_lwb_write_open() on first itx assignment. When 34 * transitioning from "new" to "opened" the zilog's "zl_issuer_lock" and 35 * LWB's "lwb_lock" must be held. 36 * 37 * After the lwb is "opened", it can be assigned number of itxs and transition 38 * into the "closed" state via zil_lwb_write_close() when full or on timeout. 39 * When transitioning from "opened" to "closed" the zilog's "zl_issuer_lock" 40 * must be held. New lwb allocation also takes "zl_lock" to protect the list. 41 * 42 * After the lwb is "closed", it can transition into the "ready" state via 43 * zil_lwb_write_issue(). "zl_lock" must be held when making this transition. 44 * Since it is done by the same thread, "zl_issuer_lock" is not needed. 45 * 46 * When lwb in "ready" state receives its block pointer, it can transition to 47 * "issued". "zl_lock" must be held when making this transition. 48 * 49 * After the lwb's write zio completes, it transitions into the "write 50 * done" state via zil_lwb_write_done(); and then into the "flush done" 51 * state via zil_lwb_flush_vdevs_done(). When transitioning from 52 * "issued" to "write done", and then from "write done" to "flush done", 53 * the zilog's "zl_lock" must be held, *not* the "zl_issuer_lock". 54 * 55 * The zilog's "zl_issuer_lock" can become heavily contended in certain 56 * workloads, so we specifically avoid acquiring that lock when 57 * transitioning an lwb from "issued" to "done". This allows us to avoid 58 * having to acquire the "zl_issuer_lock" for each lwb ZIO completion, 59 * which would have added more lock contention on an already heavily 60 * contended lock. 61 * 62 * Additionally, correctness when reading an lwb's state is often 63 * achieved by exploiting the fact that these state transitions occur in 64 * this specific order; i.e. "new" to "opened" to "closed" to "ready" to 65 * "issued" to "write_done" and finally "flush_done". 66 * 67 * Thus, if an lwb is in the "new" or "opened" state, holding the 68 * "zl_issuer_lock" will prevent a concurrent thread from transitioning 69 * that lwb to the "closed" state. Likewise, if an lwb is already in the 70 * "ready" state, holding the "zl_lock" will prevent a concurrent thread 71 * from transitioning that lwb to the "issued" state. 72 */ 73 typedef enum { 74 LWB_STATE_NEW, 75 LWB_STATE_OPENED, 76 LWB_STATE_CLOSED, 77 LWB_STATE_READY, 78 LWB_STATE_ISSUED, 79 LWB_STATE_WRITE_DONE, 80 LWB_STATE_FLUSH_DONE, 81 LWB_NUM_STATES 82 } lwb_state_t; 83 84 /* 85 * Log write block (lwb) 86 * 87 * Prior to an lwb being issued to disk via zil_lwb_write_issue(), it 88 * will be protected by the zilog's "zl_issuer_lock". Basically, prior 89 * to it being issued, it will only be accessed by the thread that's 90 * holding the "zl_issuer_lock". After the lwb is issued, the zilog's 91 * "zl_lock" is used to protect the lwb against concurrent access. 92 */ 93 typedef enum { 94 LWB_FLAG_SLIM = (1<<0), /* log block has slim format */ 95 LWB_FLAG_SLOG = (1<<1), /* lwb_blk is on SLOG device */ 96 LWB_FLAG_CRASHED = (1<<2), /* lwb is on the crash list */ 97 } lwb_flag_t; 98 99 typedef struct lwb { 100 zilog_t *lwb_zilog; /* back pointer to log struct */ 101 blkptr_t lwb_blk; /* on disk address of this log blk */ 102 lwb_flag_t lwb_flags; /* extra info about this lwb */ 103 int lwb_error; /* log block allocation error */ 104 int lwb_nmax; /* max bytes in the buffer */ 105 int lwb_nused; /* # used bytes in buffer */ 106 int lwb_nfilled; /* # filled bytes in buffer */ 107 int lwb_sz; /* size of block and buffer */ 108 int lwb_min_sz; /* min size for range allocation */ 109 lwb_state_t lwb_state; /* the state of this lwb */ 110 char *lwb_buf; /* log write buffer */ 111 zio_t *lwb_child_zio; /* parent zio for children */ 112 zio_t *lwb_write_zio; /* zio for the lwb buffer */ 113 zio_t *lwb_root_zio; /* root zio for lwb write and flushes */ 114 hrtime_t lwb_issued_timestamp; /* when was the lwb issued? */ 115 uint64_t lwb_issued_txg; /* the txg when the write is issued */ 116 uint64_t lwb_alloc_txg; /* the txg when lwb_blk is allocated */ 117 uint64_t lwb_max_txg; /* highest txg in this lwb */ 118 list_node_t lwb_node; /* zilog->zl_lwb_list linkage */ 119 list_node_t lwb_issue_node; /* linkage of lwbs ready for issue */ 120 list_t lwb_itxs; /* list of itx's */ 121 list_t lwb_waiters; /* list of zil_commit_waiter's */ 122 avl_tree_t lwb_vdev_tree; /* vdevs to flush after lwb write */ 123 kmutex_t lwb_lock; /* protects lwb_vdev_tree and size */ 124 } lwb_t; 125 126 /* 127 * ZIL commit waiter. 128 * 129 * This structure is allocated each time zil_commit() is called, and is 130 * used by zil_commit() to communicate with other parts of the ZIL, such 131 * that zil_commit() can know when it safe for it return. For more 132 * details, see the comment above zil_commit(). 133 * 134 * The "zcw_lock" field is used to protect the commit waiter against 135 * concurrent access. This lock is often acquired while already holding 136 * the zilog's "zl_issuer_lock" or "zl_lock"; see the functions 137 * zil_process_commit_list() and zil_lwb_flush_vdevs_done() as examples 138 * of this. Thus, one must be careful not to acquire the 139 * "zl_issuer_lock" or "zl_lock" when already holding the "zcw_lock"; 140 * e.g. see the zil_commit_waiter_timeout() function. 141 */ 142 typedef struct zil_commit_waiter { 143 kcondvar_t zcw_cv; /* signalled when "done" */ 144 kmutex_t zcw_lock; /* protects fields of this struct */ 145 list_node_t zcw_node; /* linkage in lwb_t:lwb_waiter list */ 146 lwb_t *zcw_lwb; /* back pointer to lwb when linked */ 147 boolean_t zcw_done; /* B_TRUE when "done", else B_FALSE */ 148 int zcw_error; /* result to return from zil_commit() */ 149 } zil_commit_waiter_t; 150 151 /* 152 * Intent log transaction lists 153 */ 154 typedef struct itxs { 155 list_t i_sync_list; /* list of synchronous itxs */ 156 avl_tree_t i_async_tree; /* tree of foids for async itxs */ 157 } itxs_t; 158 159 typedef struct itxg { 160 kmutex_t itxg_lock; /* lock for this structure */ 161 uint64_t itxg_txg; /* txg for this chain */ 162 itxs_t *itxg_itxs; /* sync and async itxs */ 163 } itxg_t; 164 165 /* for async nodes we build up an AVL tree of lists of async itxs per file */ 166 typedef struct itx_async_node { 167 uint64_t ia_foid; /* file object id */ 168 list_t ia_list; /* list of async itxs for this foid */ 169 avl_node_t ia_node; /* AVL tree linkage */ 170 } itx_async_node_t; 171 172 /* 173 * Vdev flushing: during a zil_commit(), we build up an AVL tree of the vdevs 174 * we've touched so we know which ones need a write cache flush at the end. 175 */ 176 typedef struct zil_vdev_node { 177 uint64_t zv_vdev; /* vdev to be flushed */ 178 avl_node_t zv_node; /* AVL tree linkage */ 179 } zil_vdev_node_t; 180 181 #define ZIL_BURSTS 8 182 183 /* 184 * Stable storage intent log management structure. One per dataset. 185 */ 186 struct zilog { 187 kmutex_t zl_lock; /* protects most zilog_t fields */ 188 struct dsl_pool *zl_dmu_pool; /* DSL pool */ 189 spa_t *zl_spa; /* handle for read/write log */ 190 const zil_header_t *zl_header; /* log header buffer */ 191 objset_t *zl_os; /* object set we're logging */ 192 zil_get_data_t *zl_get_data; /* callback to get object content */ 193 lwb_t *zl_last_lwb_opened; /* most recent lwb opened */ 194 hrtime_t zl_last_lwb_latency; /* zio latency of last lwb done */ 195 uint64_t zl_lr_seq; /* on-disk log record sequence number */ 196 uint64_t zl_commit_lr_seq; /* last committed on-disk lr seq */ 197 uint64_t zl_destroy_txg; /* txg of last zil_destroy() */ 198 uint64_t zl_replayed_seq[TXG_SIZE]; /* last replayed rec seq */ 199 uint64_t zl_replaying_seq; /* current replay seq number */ 200 uint32_t zl_suspend; /* log suspend count */ 201 kcondvar_t zl_cv_suspend; /* log suspend completion */ 202 uint8_t zl_suspending; /* log is currently suspending */ 203 uint8_t zl_keep_first; /* keep first log block in destroy */ 204 uint8_t zl_replay; /* replaying records while set */ 205 uint8_t zl_stop_sync; /* for debugging */ 206 kmutex_t zl_issuer_lock; /* single writer, per ZIL, at a time */ 207 uint8_t zl_logbias; /* latency or throughput */ 208 uint8_t zl_sync; /* synchronous or asynchronous */ 209 int zl_parse_error; /* last zil_parse() error */ 210 uint64_t zl_parse_blk_seq; /* highest blk seq on last parse */ 211 uint64_t zl_parse_lr_seq; /* highest lr seq on last parse */ 212 uint64_t zl_parse_blk_count; /* number of blocks parsed */ 213 uint64_t zl_parse_lr_count; /* number of log records parsed */ 214 itxg_t zl_itxg[TXG_SIZE]; /* intent log txg chains */ 215 list_t zl_itx_commit_list; /* itx list to be committed */ 216 uint64_t zl_cur_size; /* current burst full size */ 217 uint64_t zl_cur_left; /* current burst remaining size */ 218 uint64_t zl_cur_max; /* biggest record in current burst */ 219 list_t zl_lwb_list; /* in-flight log write list */ 220 list_t zl_lwb_crash_list; /* log writes in-flight at crash */ 221 avl_tree_t zl_bp_tree; /* track bps during log parse */ 222 clock_t zl_replay_time; /* lbolt of when replay started */ 223 uint64_t zl_replay_blks; /* number of log blocks replayed */ 224 zil_header_t zl_old_header; /* debugging aid */ 225 uint_t zl_parallel; /* workload is multi-threaded */ 226 uint_t zl_prev_rotor; /* rotor for zl_prev[] */ 227 uint_t zl_prev_opt[ZIL_BURSTS]; /* optimal block size */ 228 uint_t zl_prev_min[ZIL_BURSTS]; /* minimal first block size */ 229 txg_node_t zl_dirty_link; /* protected by dp_dirty_zilogs list */ 230 uint64_t zl_dirty_max_txg; /* highest txg used to dirty zilog */ 231 232 kmutex_t zl_lwb_io_lock; /* protect following members */ 233 uint64_t zl_lwb_inflight[TXG_SIZE]; /* io issued, but not done */ 234 kcondvar_t zl_lwb_io_cv; /* signal when the flush is done */ 235 uint64_t zl_lwb_max_issued_txg; /* max txg when lwb io issued */ 236 237 /* 238 * Max block size for this ZIL. Note that this can not be changed 239 * while the ZIL is in use because consumers (ZPL/zvol) need to take 240 * this into account when deciding between WR_COPIED and WR_NEED_COPY 241 * (see zil_max_copied_data()). 242 */ 243 uint64_t zl_max_block_size; 244 245 /* After crash, txg to restart zil */ 246 uint64_t zl_restart_txg; 247 248 /* Pointer for per dataset zil sums */ 249 zil_sums_t *zl_sums; 250 }; 251 252 typedef struct zil_bp_node { 253 dva_t zn_dva; 254 avl_node_t zn_node; 255 } zil_bp_node_t; 256 257 #ifdef __cplusplus 258 } 259 #endif 260 261 #endif /* _SYS_ZIL_IMPL_H */ 262