1 /*- 2 * Copyright (c) 2003-2009 Silicon Graphics International Corp. 3 * Copyright (c) 2011 Spectra Logic Corporation 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions, and the following disclaimer, 11 * without modification. 12 * 2. Redistributions in binary form must reproduce at minimum a disclaimer 13 * substantially similar to the "NO WARRANTY" disclaimer below 14 * ("Disclaimer") and any redistribution must be conditioned upon 15 * including a substantially similar Disclaimer requirement for further 16 * binary redistribution. 17 * 18 * NO WARRANTY 19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR 22 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 27 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 28 * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGES. 30 * 31 * $Id: //depot/users/kenm/FreeBSD-test2/sys/cam/ctl/ctl_ha.h#1 $ 32 * $FreeBSD$ 33 */ 34 35 #ifndef _CTL_HA_H_ 36 #define _CTL_HA_H_ 37 38 /* 39 * CTL High Availability Modes: 40 * 41 * CTL_HA_MODE_SER_ONLY: Commands are serialized to the other side. Write 42 * mirroring and read re-direction are assumed to 43 * happen in the back end. 44 * CTL_HA_MODE_XFER: Commands are serialized and data is transferred 45 * for write mirroring and read re-direction. 46 */ 47 48 typedef enum { 49 CTL_HA_MODE_SER_ONLY, 50 CTL_HA_MODE_XFER 51 } ctl_ha_mode; 52 53 54 /* 55 * This is a stubbed out High Availability interface. It assumes two nodes 56 * staying in sync. 57 * 58 * The reason this interface is here, and stubbed out, is that CTL was 59 * originally written with support for Copan's (now SGI) high availability 60 * framework. That framework was not released by SGI, and would not have 61 * been generally applicable to FreeBSD anyway. 62 * 63 * The idea here is to show the kind of API that would need to be in place 64 * in a HA framework to work with CTL's HA hooks. This API is very close 65 * to the Copan/SGI API, so that the code using it could stay in place 66 * as-is. 67 * 68 * So, in summary, this is a shell without real substance, and much more 69 * work would be needed to actually make HA work. The implementation 70 * inside CTL will also need to change to fit the eventual implementation. 71 * The additional pieces we would need are: 72 * 73 * - HA "Supervisor" framework that can startup the components of the 74 * system, and initiate failover (i.e. active/active to single mode) 75 * and failback (single to active/active mode) state transitions. 76 * This framework would be able to recognize when an event happens 77 * that requires it to initiate state transitions in the components it 78 * manages. 79 * 80 * - HA communication framework. This framework should have the following 81 * features: 82 * - Separate channels for separate system components. The CTL 83 * instance on one node should communicate with the CTL instance 84 * on another node. 85 * - Short message passing. These messages would be fixed length, so 86 * they could be preallocated and easily passed between the nodes. 87 * i.e. conceptually like an ethernet packet. 88 * - DMA/large buffer capability. This would require some negotiation 89 * with the other node to define the destination. It could 90 * allow for "push" (i.e. initiated by the requesting node) DMA or 91 * "pull" (i.e. initiated by the target controller) DMA or both. 92 * - Communication channel status change notification. 93 * - HA capability in other portions of the storage stack. Having two CTL 94 * instances communicate is just one part of an overall HA solution. 95 * State needs to be synchronized at multiple levels of the system in 96 * order for failover to actually work. For instance, if CTL is using a 97 * file on a ZFS filesystem as its backing store, the ZFS array state 98 * should be synchronized with the other node, so that the other node 99 * can immediately take over if the node that is primary for a particular 100 * array fails. 101 */ 102 103 /* 104 * Communication channel IDs for various system components. This is to 105 * make sure one CTL instance talks with another, one ZFS instance talks 106 * with another, etc. 107 */ 108 typedef enum { 109 CTL_HA_CHAN_NONE, 110 CTL_HA_CHAN_CTL, 111 CTL_HA_CHAN_ZFS, 112 CTL_HA_CHAN_MAX 113 } ctl_ha_channel; 114 115 /* 116 * HA communication event notification. These are events generated by the 117 * HA communication subsystem. 118 * 119 * CTL_HA_EVT_MSG_RECV: Message received by the other node. 120 * CTL_HA_EVT_MSG_SENT: Message sent to the other node. 121 * CTL_HA_EVT_DISCONNECT: Communication channel disconnected. 122 * CTL_HA_EVT_DMA_SENT: DMA successfully sent to other node (push). 123 * CTL_HA_EVT_DMA_RECEIVED: DMA successfully received by other node (pull). 124 */ 125 typedef enum { 126 CTL_HA_EVT_NONE, 127 CTL_HA_EVT_MSG_RECV, 128 CTL_HA_EVT_MSG_SENT, 129 CTL_HA_EVT_DISCONNECT, 130 CTL_HA_EVT_DMA_SENT, 131 CTL_HA_EVT_DMA_RECEIVED, 132 CTL_HA_EVT_MAX 133 } ctl_ha_event; 134 135 typedef enum { 136 CTL_HA_STATUS_WAIT, 137 CTL_HA_STATUS_SUCCESS, 138 CTL_HA_STATUS_ERROR, 139 CTL_HA_STATUS_INVALID, 140 CTL_HA_STATUS_DISCONNECT, 141 CTL_HA_STATUS_BUSY, 142 CTL_HA_STATUS_MAX 143 } ctl_ha_status; 144 145 typedef enum { 146 CTL_HA_DATA_CTL, 147 CTL_HA_DATA_ZFS, 148 CTL_HA_DATA_MAX 149 } ctl_ha_dtid; 150 151 typedef enum { 152 CTL_HA_DT_CMD_READ, 153 CTL_HA_DT_CMD_WRITE, 154 } ctl_ha_dt_cmd; 155 156 struct ctl_ha_dt_req; 157 158 typedef void (*ctl_ha_dt_cb)(struct ctl_ha_dt_req *); 159 160 struct ctl_ha_dt_req { 161 ctl_ha_dt_cmd command; 162 void *context; 163 ctl_ha_dt_cb callback; 164 ctl_ha_dtid id; 165 int ret; 166 uint32_t size; 167 uint8_t *local; 168 uint8_t *remote; 169 }; 170 171 typedef void (*ctl_evt_handler)(ctl_ha_channel channel, ctl_ha_event event, 172 int param); 173 void ctl_ha_register_evthandler(ctl_ha_channel channel, 174 ctl_evt_handler handler); 175 176 static inline ctl_ha_status 177 ctl_ha_msg_create(ctl_ha_channel channel, ctl_evt_handler handler) 178 { 179 return (CTL_HA_STATUS_SUCCESS); 180 } 181 182 /* 183 * Receive a message of the specified size. 184 */ 185 static inline ctl_ha_status 186 ctl_ha_msg_recv(ctl_ha_channel channel, void *buffer, unsigned int size, 187 int wait) 188 { 189 return (CTL_HA_STATUS_SUCCESS); 190 } 191 192 /* 193 * Send a message of the specified size. 194 */ 195 static inline ctl_ha_status 196 ctl_ha_msg_send(ctl_ha_channel channel, void *buffer, unsigned int size, 197 int wait) 198 { 199 return (CTL_HA_STATUS_SUCCESS); 200 } 201 202 /* 203 * Allocate a data transfer request structure. 204 */ 205 static inline struct ctl_ha_dt_req * 206 ctl_dt_req_alloc(void) 207 { 208 return (NULL); 209 } 210 211 /* 212 * Free a data transfer request structure. 213 */ 214 static inline void 215 ctl_dt_req_free(struct ctl_ha_dt_req *req) 216 { 217 return; 218 } 219 220 /* 221 * Issue a DMA request for a single buffer. 222 */ 223 static inline ctl_ha_status 224 ctl_dt_single(struct ctl_ha_dt_req *req) 225 { 226 return (CTL_HA_STATUS_WAIT); 227 } 228 229 /* 230 * SINGLE: One node 231 * HA: Two nodes (Active/Active implied) 232 * SLAVE/MASTER: The component can set these flags to indicate which side 233 * is in control. It has no effect on the HA framework. 234 */ 235 typedef enum { 236 CTL_HA_STATE_UNKNOWN = 0x00, 237 CTL_HA_STATE_SINGLE = 0x01, 238 CTL_HA_STATE_HA = 0x02, 239 CTL_HA_STATE_MASK = 0x0F, 240 CTL_HA_STATE_SLAVE = 0x10, 241 CTL_HA_STATE_MASTER = 0x20 242 } ctl_ha_state; 243 244 typedef enum { 245 CTL_HA_COMP_STATUS_OK, 246 CTL_HA_COMP_STATUS_FAILED, 247 CTL_HA_COMP_STATUS_ERROR 248 } ctl_ha_comp_status; 249 250 struct ctl_ha_component; 251 252 typedef ctl_ha_comp_status (*ctl_hacmp_init_t)(struct ctl_ha_component *); 253 typedef ctl_ha_comp_status (*ctl_hacmp_start_t)(struct ctl_ha_component *, 254 ctl_ha_state); 255 256 struct ctl_ha_component { 257 char *name; 258 ctl_ha_state state; 259 ctl_ha_comp_status status; 260 ctl_hacmp_init_t init; 261 ctl_hacmp_start_t start; 262 ctl_hacmp_init_t quiesce; 263 }; 264 265 #define CTL_HA_STATE_IS_SINGLE(state) ((state & CTL_HA_STATE_MASK) == \ 266 CTL_HA_STATE_SINGLE) 267 #define CTL_HA_STATE_IS_HA(state) ((state & CTL_HA_STATE_MASK) == \ 268 CTL_HA_STATE_HA) 269 270 #endif /* _CTL_HA_H_ */ 271