1 /*- 2 * Copyright (c) 2003-2009 Silicon Graphics International Corp. 3 * Copyright (c) 2011 Spectra Logic Corporation 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions, and the following disclaimer, 11 * without modification. 12 * 2. Redistributions in binary form must reproduce at minimum a disclaimer 13 * substantially similar to the "NO WARRANTY" disclaimer below 14 * ("Disclaimer") and any redistribution must be conditioned upon 15 * including a substantially similar Disclaimer requirement for further 16 * binary redistribution. 17 * 18 * NO WARRANTY 19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR 22 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 27 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 28 * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGES. 30 * 31 * $Id: //depot/users/kenm/FreeBSD-test2/sys/cam/ctl/ctl_ha.h#1 $ 32 * $FreeBSD$ 33 */ 34 35 #ifndef _CTL_HA_H_ 36 #define _CTL_HA_H_ 37 38 /* 39 * CTL High Availability Modes: 40 * 41 * CTL_HA_MODE_ACT_STBY: One side is in Active state and processing commands, 42 * the other side is in Standby state, returning errors. 43 * CTL_HA_MODE_SER_ONLY: Commands are serialized to the other side. Write 44 * mirroring and read re-direction are assumed to 45 * happen in the back end. 46 * CTL_HA_MODE_XFER: Commands are serialized and data is transferred 47 * for write mirroring and read re-direction. 48 */ 49 50 typedef enum { 51 CTL_HA_MODE_ACT_STBY, 52 CTL_HA_MODE_SER_ONLY, 53 CTL_HA_MODE_XFER 54 } ctl_ha_mode; 55 56 57 /* 58 * This is a stubbed out High Availability interface. It assumes two nodes 59 * staying in sync. 60 * 61 * The reason this interface is here, and stubbed out, is that CTL was 62 * originally written with support for Copan's (now SGI) high availability 63 * framework. That framework was not released by SGI, and would not have 64 * been generally applicable to FreeBSD anyway. 65 * 66 * The idea here is to show the kind of API that would need to be in place 67 * in a HA framework to work with CTL's HA hooks. This API is very close 68 * to the Copan/SGI API, so that the code using it could stay in place 69 * as-is. 70 * 71 * So, in summary, this is a shell without real substance, and much more 72 * work would be needed to actually make HA work. The implementation 73 * inside CTL will also need to change to fit the eventual implementation. 74 * The additional pieces we would need are: 75 * 76 * - HA "Supervisor" framework that can startup the components of the 77 * system, and initiate failover (i.e. active/active to single mode) 78 * and failback (single to active/active mode) state transitions. 79 * This framework would be able to recognize when an event happens 80 * that requires it to initiate state transitions in the components it 81 * manages. 82 * 83 * - HA communication framework. This framework should have the following 84 * features: 85 * - Separate channels for separate system components. The CTL 86 * instance on one node should communicate with the CTL instance 87 * on another node. 88 * - Short message passing. These messages would be fixed length, so 89 * they could be preallocated and easily passed between the nodes. 90 * i.e. conceptually like an ethernet packet. 91 * - DMA/large buffer capability. This would require some negotiation 92 * with the other node to define the destination. It could 93 * allow for "push" (i.e. initiated by the requesting node) DMA or 94 * "pull" (i.e. initiated by the target controller) DMA or both. 95 * - Communication channel status change notification. 96 * - HA capability in other portions of the storage stack. Having two CTL 97 * instances communicate is just one part of an overall HA solution. 98 * State needs to be synchronized at multiple levels of the system in 99 * order for failover to actually work. For instance, if CTL is using a 100 * file on a ZFS filesystem as its backing store, the ZFS array state 101 * should be synchronized with the other node, so that the other node 102 * can immediately take over if the node that is primary for a particular 103 * array fails. 104 */ 105 106 /* 107 * Communication channel IDs for various system components. This is to 108 * make sure one CTL instance talks with another, one ZFS instance talks 109 * with another, etc. 110 */ 111 typedef enum { 112 CTL_HA_CHAN_NONE, 113 CTL_HA_CHAN_CTL, 114 CTL_HA_CHAN_ZFS, 115 CTL_HA_CHAN_MAX 116 } ctl_ha_channel; 117 118 /* 119 * HA communication event notification. These are events generated by the 120 * HA communication subsystem. 121 * 122 * CTL_HA_EVT_MSG_RECV: Message received by the other node. 123 * CTL_HA_EVT_MSG_SENT: Message sent to the other node. 124 * CTL_HA_EVT_DISCONNECT: Communication channel disconnected. 125 * CTL_HA_EVT_DMA_SENT: DMA successfully sent to other node (push). 126 * CTL_HA_EVT_DMA_RECEIVED: DMA successfully received by other node (pull). 127 */ 128 typedef enum { 129 CTL_HA_EVT_NONE, 130 CTL_HA_EVT_MSG_RECV, 131 CTL_HA_EVT_MSG_SENT, 132 CTL_HA_EVT_DISCONNECT, 133 CTL_HA_EVT_DMA_SENT, 134 CTL_HA_EVT_DMA_RECEIVED, 135 CTL_HA_EVT_MAX 136 } ctl_ha_event; 137 138 typedef enum { 139 CTL_HA_STATUS_WAIT, 140 CTL_HA_STATUS_SUCCESS, 141 CTL_HA_STATUS_ERROR, 142 CTL_HA_STATUS_INVALID, 143 CTL_HA_STATUS_DISCONNECT, 144 CTL_HA_STATUS_BUSY, 145 CTL_HA_STATUS_MAX 146 } ctl_ha_status; 147 148 typedef enum { 149 CTL_HA_DATA_CTL, 150 CTL_HA_DATA_ZFS, 151 CTL_HA_DATA_MAX 152 } ctl_ha_dtid; 153 154 typedef enum { 155 CTL_HA_DT_CMD_READ, 156 CTL_HA_DT_CMD_WRITE, 157 } ctl_ha_dt_cmd; 158 159 struct ctl_ha_dt_req; 160 161 typedef void (*ctl_ha_dt_cb)(struct ctl_ha_dt_req *); 162 163 struct ctl_ha_dt_req { 164 ctl_ha_dt_cmd command; 165 void *context; 166 ctl_ha_dt_cb callback; 167 ctl_ha_dtid id; 168 int ret; 169 uint32_t size; 170 uint8_t *local; 171 uint8_t *remote; 172 }; 173 174 typedef void (*ctl_evt_handler)(ctl_ha_channel channel, ctl_ha_event event, 175 int param); 176 void ctl_ha_register_evthandler(ctl_ha_channel channel, 177 ctl_evt_handler handler); 178 179 static inline ctl_ha_status 180 ctl_ha_msg_create(ctl_ha_channel channel, ctl_evt_handler handler) 181 { 182 return (CTL_HA_STATUS_SUCCESS); 183 } 184 185 /* 186 * Receive a message of the specified size. 187 */ 188 static inline ctl_ha_status 189 ctl_ha_msg_recv(ctl_ha_channel channel, void *buffer, unsigned int size, 190 int wait) 191 { 192 return (CTL_HA_STATUS_SUCCESS); 193 } 194 195 /* 196 * Send a message of the specified size. 197 */ 198 static inline ctl_ha_status 199 ctl_ha_msg_send(ctl_ha_channel channel, void *buffer, unsigned int size, 200 int wait) 201 { 202 return (CTL_HA_STATUS_SUCCESS); 203 } 204 205 /* 206 * Allocate a data transfer request structure. 207 */ 208 static inline struct ctl_ha_dt_req * 209 ctl_dt_req_alloc(void) 210 { 211 return (NULL); 212 } 213 214 /* 215 * Free a data transfer request structure. 216 */ 217 static inline void 218 ctl_dt_req_free(struct ctl_ha_dt_req *req) 219 { 220 return; 221 } 222 223 /* 224 * Issue a DMA request for a single buffer. 225 */ 226 static inline ctl_ha_status 227 ctl_dt_single(struct ctl_ha_dt_req *req) 228 { 229 return (CTL_HA_STATUS_WAIT); 230 } 231 232 /* 233 * SINGLE: One node 234 * HA: Two nodes (Active/Active implied) 235 * SLAVE/MASTER: The component can set these flags to indicate which side 236 * is in control. It has no effect on the HA framework. 237 */ 238 typedef enum { 239 CTL_HA_STATE_UNKNOWN = 0x00, 240 CTL_HA_STATE_SINGLE = 0x01, 241 CTL_HA_STATE_HA = 0x02, 242 CTL_HA_STATE_MASK = 0x0F, 243 CTL_HA_STATE_SLAVE = 0x10, 244 CTL_HA_STATE_MASTER = 0x20 245 } ctl_ha_state; 246 247 typedef enum { 248 CTL_HA_COMP_STATUS_OK, 249 CTL_HA_COMP_STATUS_FAILED, 250 CTL_HA_COMP_STATUS_ERROR 251 } ctl_ha_comp_status; 252 253 struct ctl_ha_component; 254 255 typedef ctl_ha_comp_status (*ctl_hacmp_init_t)(struct ctl_ha_component *); 256 typedef ctl_ha_comp_status (*ctl_hacmp_start_t)(struct ctl_ha_component *, 257 ctl_ha_state); 258 259 struct ctl_ha_component { 260 char *name; 261 ctl_ha_state state; 262 ctl_ha_comp_status status; 263 ctl_hacmp_init_t init; 264 ctl_hacmp_start_t start; 265 ctl_hacmp_init_t quiesce; 266 }; 267 268 #define CTL_HA_STATE_IS_SINGLE(state) ((state & CTL_HA_STATE_MASK) == \ 269 CTL_HA_STATE_SINGLE) 270 #define CTL_HA_STATE_IS_HA(state) ((state & CTL_HA_STATE_MASK) == \ 271 CTL_HA_STATE_HA) 272 273 #endif /* _CTL_HA_H_ */ 274