xref: /titanic_51/usr/src/uts/sun4v/sys/vdc.h (revision 23b5c241225a8ade2b6b9f06ebb891ee459e3b02)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #ifndef	_VDC_H
28 #define	_VDC_H
29 
30 #pragma ident	"%Z%%M%	%I%	%E% SMI"
31 
32 /*
33  * Virtual disk client implementation definitions
34  */
35 
36 #include <sys/sysmacros.h>
37 #include <sys/note.h>
38 
39 #include <sys/ldc.h>
40 #include <sys/vio_mailbox.h>
41 #include <sys/vdsk_mailbox.h>
42 #include <sys/vdsk_common.h>
43 
44 #ifdef	__cplusplus
45 extern "C" {
46 #endif
47 
48 #define	VDC_DRIVER_NAME		"vdc"
49 
50 /*
51  * Bit-field values to indicate if parts of the vdc driver are initialised.
52  */
53 #define	VDC_SOFT_STATE	0x0001
54 #define	VDC_LOCKS	0x0002
55 #define	VDC_MINOR	0x0004
56 #define	VDC_THREAD	0x0008
57 #define	VDC_LDC		0x0010
58 #define	VDC_LDC_INIT	0x0020
59 #define	VDC_LDC_CB	0x0040
60 #define	VDC_LDC_OPEN	0x0080
61 #define	VDC_DRING_INIT	0x0100	/* The DRing was created */
62 #define	VDC_DRING_BOUND	0x0200	/* The DRing was bound to an LDC channel */
63 #define	VDC_DRING_LOCAL	0x0400	/* The local private DRing was allocated */
64 #define	VDC_DRING_ENTRY	0x0800	/* At least one DRing entry was initialised */
65 #define	VDC_DRING	(VDC_DRING_INIT | VDC_DRING_BOUND |	\
66 				VDC_DRING_LOCAL | VDC_DRING_ENTRY)
67 #define	VDC_HANDSHAKE	0x1000	/* Indicates if a handshake is in progress */
68 #define	VDC_HANDSHAKE_STOP	0x2000	/* stop further handshakes */
69 
70 /*
71  * Definitions of strings to be used to create device node properties.
72  * (vdc uses the capitalised versions of these properties as they are 64-bit)
73  */
74 #define	VDC_NBLOCKS_PROP_NAME		"Nblocks"
75 #define	VDC_SIZE_PROP_NAME		"Size"
76 
77 /*
78  * Definitions of MD nodes/properties.
79  */
80 #define	VDC_MD_CHAN_NAME		"channel-endpoint"
81 #define	VDC_MD_VDEV_NAME		"virtual-device"
82 #define	VDC_MD_DISK_NAME		"disk"
83 #define	VDC_MD_CFG_HDL			"cfg-handle"
84 #define	VDC_ID_PROP			"id"
85 
86 /*
87  * Definition of actions to be carried out when processing the sequence ID
88  * of a message received from the vDisk server. The function verifying the
89  * sequence number checks the 'seq_num_xxx' fields in the soft state and
90  * returns whether the message should be processed (VDC_SEQ_NUM_TODO) or
91  * whether it was it was previously processed (VDC_SEQ_NUM_SKIP).
92  */
93 #define	VDC_SEQ_NUM_INVALID		-1	/* Error */
94 #define	VDC_SEQ_NUM_SKIP		0	/* Request already processed */
95 #define	VDC_SEQ_NUM_TODO		1	/* Request needs processing */
96 
97 /*
98  * Scheme to store the instance number and the slice number in the minor number.
99  * (Uses the same format and definitions as the sd(7D) driver)
100  */
101 #define	VD_MAKE_DEV(instance, minor)	((instance << SDUNIT_SHIFT) | minor)
102 
103 /*
104  * variables controlling how long to wait before timing out and how many
105  * retries to attempt before giving up when communicating with vds.
106  *
107  * These values need to be sufficiently large so that a guest can survive
108  * the reboot of the service domain.
109  */
110 #define	VDC_RETRIES	10
111 
112 #define	VDC_USEC_TIMEOUT_MIN	(30 * MICROSEC)		/* 30 sec */
113 
114 /*
115  * This macro returns the number of Hz that the vdc driver should wait before
116  * a timeout is triggered. The 'timeout' parameter specifiecs the wait
117  * time in Hz. The 'mul' parameter allows for a multiplier to be
118  * specified allowing for a backoff to be implemented (e.g. using the
119  * retry number as a multiplier) where the wait time will get longer if
120  * there is no response on the previous retry.
121  */
122 #define	VD_GET_TIMEOUT_HZ(timeout, mul)	\
123 	(ddi_get_lbolt() + ((timeout) * MAX(1, (mul))))
124 
125 /*
126  * Macros to manipulate Descriptor Ring variables in the soft state
127  * structure.
128  */
129 #define	VDC_GET_NEXT_REQ_ID(vdc)	((vdc)->req_id++)
130 
131 #define	VDC_GET_DRING_ENTRY_PTR(vdc, idx)	\
132 		(vd_dring_entry_t *)((vdc)->dring_mem_info.vaddr +	\
133 			(idx * (vdc)->dring_entry_size))
134 
135 #define	VDC_MARK_DRING_ENTRY_FREE(vdc, idx)			\
136 	{ \
137 		vd_dring_entry_t *dep = NULL;				\
138 		ASSERT(vdc != NULL);					\
139 		ASSERT((idx >= 0) && (idx < vdc->dring_len));		\
140 		ASSERT(vdc->dring_mem_info.vaddr != NULL);		\
141 		dep = (vd_dring_entry_t *)(vdc->dring_mem_info.vaddr +	\
142 			(idx * vdc->dring_entry_size));			\
143 		ASSERT(dep != NULL);					\
144 		dep->hdr.dstate = VIO_DESC_FREE;			\
145 	}
146 
147 /* Initialise the Session ID and Sequence Num in the DRing msg */
148 #define	VDC_INIT_DRING_DATA_MSG_IDS(dmsg, vdc)		\
149 		ASSERT(vdc != NULL);			\
150 		dmsg.tag.vio_sid = vdc->session_id;	\
151 		dmsg.seq_num = vdc->seq_num;
152 
153 /*
154  * The states that the read thread can be in.
155  */
156 typedef enum vdc_rd_state {
157 	VDC_READ_IDLE,			/* idling - conn is not up */
158 	VDC_READ_WAITING,		/* waiting for data */
159 	VDC_READ_PENDING,		/* pending data avail for read */
160 	VDC_READ_RESET			/* channel was reset - stop reads */
161 } vdc_rd_state_t;
162 
163 /*
164  * The states that the vdc-vds connection can be in.
165  */
166 typedef enum vdc_state {
167 	VDC_STATE_INIT,			/* device is initialized */
168 	VDC_STATE_INIT_WAITING,		/* waiting for ldc connection */
169 	VDC_STATE_NEGOTIATE,		/* doing handshake negotiation */
170 	VDC_STATE_HANDLE_PENDING,	/* handle requests in backup dring */
171 	VDC_STATE_RUNNING,		/* running and accepting requests */
172 	VDC_STATE_DETACH,		/* detaching */
173 	VDC_STATE_RESETTING		/* resetting connection with vds */
174 } vdc_state_t;
175 
176 /*
177  * The states that the vdc instance can be in.
178  */
179 typedef enum vdc_lc_state {
180 	VDC_LC_ATTACHING,	/* driver is attaching */
181 	VDC_LC_ONLINE,		/* driver is attached and online */
182 	VDC_LC_DETACHING	/* driver is detaching */
183 } vdc_lc_state_t;
184 
185 /*
186  * Local Descriptor Ring entry
187  *
188  * vdc creates a Local (private) descriptor ring the same size as the
189  * public descriptor ring it exports to vds.
190  */
191 
192 typedef enum {
193 	VIO_read_dir,		/* read data from server */
194 	VIO_write_dir,		/* write data to server */
195 	VIO_both_dir		/* transfer both in and out in same buffer */
196 } vio_desc_direction_t;
197 
198 typedef enum {
199 	CB_STRATEGY,		/* non-blocking strategy call */
200 	CB_SYNC			/* synchronous operation */
201 } vio_cb_type_t;
202 
203 typedef struct vdc_local_desc {
204 	boolean_t		is_free;	/* local state - inuse or not */
205 
206 	int			operation;	/* VD_OP_xxx to be performed */
207 	caddr_t			addr;		/* addr passed in by consumer */
208 	int			slice;
209 	diskaddr_t		offset;		/* disk offset */
210 	size_t			nbytes;
211 	vio_cb_type_t		cb_type;	/* operation type blk/nonblk */
212 	void			*cb_arg;	/* buf passed to strategy() */
213 	vio_desc_direction_t	dir;		/* direction of transfer */
214 
215 	caddr_t			align_addr;	/* used if addr non-aligned */
216 	ldc_mem_handle_t	desc_mhdl;	/* Mem handle of buf */
217 	vd_dring_entry_t	*dep;		/* public Dring Entry Pointer */
218 
219 } vdc_local_desc_t;
220 
221 /*
222  * vdc soft state structure
223  */
224 typedef struct vdc {
225 
226 	kmutex_t	lock;		/* protects next 2 sections of vars */
227 	kcondvar_t	running_cv;	/* signal when upper layers can send */
228 	kcondvar_t	initwait_cv;	/* signal when ldc conn is up */
229 	kcondvar_t	dring_free_cv;	/* signal when desc is avail */
230 	kcondvar_t	membind_cv;	/* signal when mem can be bound */
231 	boolean_t	self_reset;
232 
233 	int		initialized;	/* keeps track of what's init'ed */
234 	vdc_lc_state_t	lifecycle;	/* Current state of the vdc instance */
235 
236 	int		hshake_cnt;	/* number of failed handshakes */
237 	int		open_count;	/* count of outstanding opens */
238 	int		dkio_flush_pending; /* # outstanding DKIO flushes */
239 
240 	kthread_t	*msg_proc_thr;	/* main msg processing thread */
241 
242 	kmutex_t	read_lock;	/* lock to protect read */
243 	kcondvar_t	read_cv;	/* cv to wait for READ events */
244 	vdc_rd_state_t	read_state;	/* current read state */
245 
246 	uint32_t	sync_op_cnt;	/* num of active sync operations */
247 	boolean_t	sync_op_pending; /* sync operation is pending */
248 	boolean_t	sync_op_blocked; /* blocked waiting to do sync op */
249 	uint32_t	sync_op_status;	/* status of sync operation */
250 	kcondvar_t	sync_pending_cv; /* cv wait for sync op to finish */
251 	kcondvar_t	sync_blocked_cv; /* cv wait for other syncs to finish */
252 
253 	uint64_t	session_id;	/* common ID sent with all messages */
254 	uint64_t	seq_num;	/* most recent sequence num generated */
255 	uint64_t	seq_num_reply;	/* Last seq num ACK/NACK'ed by vds */
256 	uint64_t	req_id;		/* Most recent Request ID generated */
257 	uint64_t	req_id_proc;	/* Last request ID processed by vdc */
258 	vdc_state_t	state;		/* Current disk client-server state */
259 
260 	dev_info_t	*dip;		/* device info pointer */
261 	int		instance;	/* driver instance number */
262 
263 	vio_ver_t	ver;		/* version number agreed with server */
264 	vd_disk_type_t	vdisk_type;	/* type of device/disk being imported */
265 	vd_disk_label_t vdisk_label; 	/* label type of device/disk imported */
266 	uint64_t	vdisk_size;	/* device size in blocks */
267 	uint64_t	max_xfer_sz;	/* maximum block size of a descriptor */
268 	uint64_t	block_size;	/* device block size used */
269 	struct dk_label	*label;		/* structure to store disk label */
270 	struct dk_cinfo	*cinfo;		/* structure to store DKIOCINFO data */
271 	struct dk_minfo	*minfo;		/* structure for DKIOCGMEDIAINFO data */
272 	struct vtoc	*vtoc;		/* structure to store VTOC data */
273 	ddi_devid_t	devid;		/* device id */
274 
275 	ldc_mem_info_t		dring_mem_info;		/* dring information */
276 	uint_t			dring_curr_idx;		/* current index */
277 	uint32_t		dring_len;		/* dring length */
278 	uint32_t		dring_max_cookies;	/* dring max cookies */
279 	uint32_t		dring_cookie_count;	/* num cookies */
280 	uint32_t		dring_entry_size;	/* descriptor size */
281 	ldc_mem_cookie_t 	*dring_cookie;		/* dring cookies */
282 	uint64_t		dring_ident;		/* dring ident */
283 
284 	uint64_t		threads_pending; 	/* num of threads */
285 
286 	vdc_local_desc_t	*local_dring;		/* local dring */
287 	vdc_local_desc_t	*local_dring_backup;	/* local dring backup */
288 	int			local_dring_backup_tail; /* backup dring tail */
289 	int			local_dring_backup_len;	/* backup dring len */
290 
291 	uint64_t		ldc_id;			/* LDC channel id */
292 	ldc_status_t		ldc_state;		/* LDC channel state */
293 	ldc_handle_t		ldc_handle;		/* LDC handle */
294 	ldc_dring_handle_t	ldc_dring_hdl;		/* LDC dring handle */
295 } vdc_t;
296 
297 /*
298  * Debugging macros
299  */
300 #ifdef DEBUG
301 extern int	vdc_msglevel;
302 extern uint64_t	vdc_matchinst;
303 
304 #define	DMSG(_vdc, err_level, format, ...)				\
305 	do {								\
306 		if (vdc_msglevel > err_level &&				\
307 		(vdc_matchinst & (1ull << (_vdc)->instance)))		\
308 			cmn_err(CE_CONT, "?[%d,t@%p] %s: "format,	\
309 			(_vdc)->instance, (void *)curthread,		\
310 			__func__, __VA_ARGS__);				\
311 		_NOTE(CONSTANTCONDITION)				\
312 	} while (0);
313 
314 #define	DMSGX(err_level, format, ...)					\
315 	do {								\
316 		if (vdc_msglevel > err_level)				\
317 			cmn_err(CE_CONT, "?%s: "format, __func__, __VA_ARGS__);\
318 		_NOTE(CONSTANTCONDITION)				\
319 	} while (0);
320 
321 #define	VDC_DUMP_DRING_MSG(dmsgp)					\
322 		DMSGX(0, "sq:%lu start:%d end:%d ident:%lu\n",		\
323 			dmsgp->seq_num, dmsgp->start_idx,		\
324 			dmsgp->end_idx, dmsgp->dring_ident);
325 
326 #else	/* !DEBUG */
327 #define	DMSG(err_level, ...)
328 #define	DMSGX(err_level, format, ...)
329 #define	VDC_DUMP_DRING_MSG(dmsgp)
330 
331 #endif	/* !DEBUG */
332 
333 #ifdef	__cplusplus
334 }
335 #endif
336 
337 #endif	/* _VDC_H */
338