xref: /illumos-gate/usr/src/uts/common/sys/ib/mgt/ibdm/ibdm_impl.h (revision 8e55d2638713a256e1e595d733e9ea6900834744)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #ifndef _SYS_IB_MGT_IBDM_IBDM_IMPL_H
27 #define	_SYS_IB_MGT_IBDM_IBDM_IMPL_H
28 
29 /*
30  * ibdm_impl.h
31  *
32  *	This file contains definitions of the data structures, macros etc
33  *	related to the IBDM module.
34  */
35 
36 #include <sys/ib/mgt/ibdm/ibdm_ibnex.h>
37 #include <sys/ib/ibtl/impl/ibtl_util.h>
38 
39 #ifdef __cplusplus
40 extern "C" {
41 #endif
42 
43 /* values for "cb_req_type" */
44 #define	IBDM_REQ_TYPE_INVALID		0x0
45 #define	IBDM_REQ_TYPE_CLASSPORTINFO	0x1
46 #define	IBDM_REQ_TYPE_IOUINFO		0x2
47 #define	IBDM_REQ_TYPE_IOCINFO		0x4
48 #define	IBDM_REQ_TYPE_SRVENTS		0x8
49 #define	IBDM_REQ_TYPE_IOU_DIAGCODE	0x10
50 #define	IBDM_REQ_TYPE_IOC_DIAGCODE	0x20
51 
52 typedef struct ibdm_taskq_args_s {
53 	ibmf_handle_t		tq_ibmf_handle;
54 	ibmf_msg_t		*tq_ibmf_msg;
55 	void			*tq_args;
56 } ibdm_taskq_args_t;
57 _NOTE(SCHEME_PROTECTS_DATA("unique per call", ibdm_taskq_args_t))
58 _NOTE(SCHEME_PROTECTS_DATA("unique per call", ib_mad_hdr_t))
59 _NOTE(SCHEME_PROTECTS_DATA("unique per call", ibmf_msg_t))
60 
61 #define	IBDM_GID_PRESENT		0x1
62 #define	IBDM_GID_NOT_PRESENT		0x0
63 
64 #define	IBDM_IBMF_PKT_DUP_RESP		0x1
65 #define	IBDM_IBMF_PKT_REUSED		0x2
66 #define	IBDM_IBMF_PKT_UNEXP_RESP	0x4
67 
68 #define	IBDM_MAX_SERV_ENTRIES_PER_REQ	4
69 
70 typedef struct ibdm_gid_s {
71 	uint64_t		gid_dgid_hi;
72 	uint64_t		gid_dgid_lo;
73 	struct ibdm_gid_s	*gid_next;
74 } ibdm_gid_t;
75 
76 #define	IBDM_GID_PROBE_NOT_DONE		0x00
77 #define	IBDM_GET_CLASSPORTINFO		0x01
78 #define	IBDM_GET_IOUNITINFO		0x02
79 #define	IBDM_GET_IOC_DETAILS		0x04
80 #define	IBDM_GID_PROBING_COMPLETE	0x08
81 #define	IBDM_GID_PROBING_SKIPPED	0x10
82 #define	IBDM_GID_PROBING_FAILED		0x20
83 #define	IBDM_SET_CLASSPORTINFO		0x40
84 
85 /*
86  * Identifiers to distinguish a Cisco FC GW from others.
87  * Used to filter a setclassportinfo request.
88  */
89 #define	IBDM_CISCO_COMPANY_ID		(0x5ad)
90 #define	IBDM_CISCO_DEVICE_ID		(0xa87c)
91 
92 /*
93  * the bit-shift value for OUI in GUID
94  * A 64 bit globally unique identifier (GUID) composed of a 24 bit company id
95  * and an 48 bit extension identifier, and this value is used to extract
96  * the company id from the GUID.
97  */
98 #define	IBDM_OUI_GUID_SHIFT		(40)
99 
100 /*
101  * The state diagram for the gl_state
102  *
103  *                          (in case of Cisco FC GW)
104  * IBDM_GID_PROBE_NOT_DONE  ---------- 40 -> IBDM_SET_CLASSPORTINFO
105  *                          ----.	        |
106  *    |      |			| (others)      |
107  *    |      |			1		|
108  *    |      |			|               1
109  *    |      |			`-------------.	|
110  *    |      |                                v v
111  *    |	     |				     IBDM_GET_CLASSPORTINFO
112  *    |      |
113  *    |      |                                  |
114  *    |      2                                  3
115  *    |      |                                  |
116  *    |      v                                  v
117  *    |     IBDM_GID_PROBING_FAILED          IBDM_GET_IOUNITINFO
118  *    |                                         |
119  *    6                                         4
120  *    |                                         |
121  *    v                                         v
122  *  IBDM_GID_PROBING_SKIPPLED                IBDM_GET_IOC_DETAILS
123  *                                              |
124  *                                              5
125  *                                              |
126  *                                              v
127  *                                           IBDM_GID_PROBE_COMPLETE
128  *
129  * Initial state : IBDM_GID_PROBE_NOT_DONE
130  *     40 = Port sends setClassPortInfo to activate Cisco FC GW
131  *	1 = Port supports DM MAD's and a request to ClassportInfo is sent
132  *	3 = Received ClassPortInfo and sent IOUnitInfo
133  *	4 = Recevied IOUunitInfo and sent IOC profile, diagcodes, and
134  *		service entries requests
135  *	5 = Received all the IOC information
136  *	2 = Failed to probe the GID
137  *		Port does not support DM MAD's
138  *		Port did not respond property
139  *	6 = A different GID for the same port, skip the probe
140  *
141  * Reprobe state transition :
142  *
143  * IBDM_GID_PROBE_COMPLETE
144  *	|
145  *	7
146  *	|
147  *	v
148  * IBDM_GET_IOC_DETAILS
149  *	|
150  *	8
151  *	|
152  *	v
153  * IBDM_GID_PROBE_COMPLETE
154  *
155  *	7 = Reprobe request for one or more IOCs initiated.
156  *	8 = Reprobe done(IOC COntroller Profile & Service entries)
157  */
158 
159 typedef struct ibdm_dp_gidinfo_s {
160 	kmutex_t		gl_mutex;
161 	uint_t			gl_state;
162 	int			gl_reprobe_flag;	/* pass this to taskq */
163 	struct ibdm_dp_gidinfo_s *gl_next;
164 	struct ibdm_dp_gidinfo_s *gl_prev;
165 	ibdm_iou_info_t		*gl_iou;
166 	int			gl_pending_cmds;
167 	ibmf_qp_handle_t	gl_qp_hdl;
168 	uint64_t		gl_transactionID;
169 	ibdm_timeout_cb_args_t	gl_iou_cb_args;
170 	ib_lid_t		gl_dlid;
171 	ib_lid_t		gl_slid;
172 	uint64_t		gl_dgid_hi;
173 	uint64_t		gl_dgid_lo;
174 	uint64_t		gl_sgid_hi;
175 	uint64_t		gl_sgid_lo;
176 	ib_guid_t		gl_nodeguid;
177 	ib_guid_t		gl_portguid;
178 	ib_pkey_t		gl_p_key;
179 	boolean_t		gl_is_dm_capable;
180 	boolean_t		gl_redirected;
181 	uint32_t		gl_redirect_dlid;
182 	uint32_t		gl_redirect_QP;
183 	ib_pkey_t		gl_redirect_pkey;
184 	ib_qkey_t		gl_redirect_qkey;
185 	uint64_t		gl_redirectGID_hi;
186 	uint64_t		gl_redirectGID_lo;
187 	ibmf_handle_t		gl_ibmf_hdl;
188 	ibmf_saa_handle_t	gl_sa_hdl;
189 	timeout_id_t		gl_timeout_id;
190 	ibdm_timeout_cb_args_t	gl_cpi_cb_args;
191 	uint32_t		gl_ngids;
192 	ibdm_gid_t		*gl_gid;
193 	uint32_t		gl_resp_timeout;
194 	int			gl_num_iocs;
195 	ibdm_hca_list_t		*gl_hca_list;
196 	int			gl_disconnected;
197 	uint64_t		gl_min_transactionID;
198 	uint64_t		gl_max_transactionID;
199 	ibdm_iou_info_t		*gl_prev_iou;
200 	uint16_t		gl_devid;	/* device ID info */
201 	kcondvar_t		gl_probe_cv;	/* sync for Cisco FC GW */
202 	uint32_t		gl_flag;
203 	uint8_t			gl_SL:4; 	/* SL from path_record */
204 	uint8_t			gl_redirectSL:4; /* SL from redirection */
205 } ibdm_dp_gidinfo_t;
206 _NOTE(MUTEX_PROTECTS_DATA(ibdm_dp_gidinfo_s::gl_mutex,
207 	ibdm_dp_gidinfo_s::{gl_state gl_timeout_id gl_pending_cmds}))
208 _NOTE(SCHEME_PROTECTS_DATA("Serialized access by cv", ibdm_dp_gidinfo_s))
209 _NOTE(DATA_READABLE_WITHOUT_LOCK(ibdm_dp_gidinfo_s::{gl_ibmf_hdl gl_sa_hdl}))
210 _NOTE(MUTEX_PROTECTS_DATA(ibdm_dp_gidinfo_s::gl_mutex,
211 	ibdm_ioc_info_s::{ioc_timeout_id ioc_dc_timeout_id}))
212 _NOTE(MUTEX_PROTECTS_DATA(ibdm_dp_gidinfo_s::gl_mutex,
213 	ibdm_srvents_info_s::se_timeout_id))
214 
215 /*
216  * The transaction ID for the GID contains of two parts :
217  *	1. Upper 32 bits which is unique for each GID.
218  *	2. Lower 32 bits which is unique for each MAD.
219  * The assumptions are :
220  *	1. At most 2 power 32 DM capable GIDs on the IB fabric
221  *	2. IBDM sends maximum of 2 power 32 MADs to the same DM GID
222  * The limits are sufficient for practical configurations.
223  */
224 #define	IBDM_GID_TRANSACTIONID_SHIFT	((ulong_t)32)
225 #define	IBDM_GID_TRANSACTIONID_MASK	0xFFFFFFFF00000000ULL
226 
227 typedef struct ibdm_s {
228 	/* Protects IBDM's critical data */
229 	kmutex_t		ibdm_mutex;
230 	uint32_t		ibdm_hca_count;
231 	kmutex_t		ibdm_hl_mutex;
232 	kmutex_t		ibdm_ibnex_mutex;
233 	ibdm_hca_list_t		*ibdm_hca_list_head;
234 	ibdm_hca_list_t		*ibdm_hca_list_tail;
235 
236 	ibdm_dp_gidinfo_t	*ibdm_dp_gidlist_head;
237 	ibdm_dp_gidinfo_t	*ibdm_dp_gidlist_tail;
238 
239 	kcondvar_t		ibdm_probe_cv;
240 	kcondvar_t		ibdm_busy_cv;
241 	kcondvar_t		ibdm_port_settle_cv;
242 	uint32_t		ibdm_ngid_probes_in_progress;
243 	uint64_t		ibdm_transactionID;
244 	uint32_t		ibdm_ngids;
245 	uint32_t		ibdm_busy;
246 	int			ibdm_state;
247 	ibt_clnt_hdl_t		ibdm_ibt_clnt_hdl;
248 
249 	/*
250 	 * These are callback routines registered by the IB nexus driver.
251 	 * These callbacks are used to inform the IB nexus driver about
252 	 * the arrival/removal of HCA and IOC's
253 	 */
254 	ibdm_callback_t		ibdm_ibnex_callback;
255 
256 	/* Flag indicating - prev_iou during sweep */
257 	int			ibdm_prev_iou;
258 } ibdm_t;
259 _NOTE(MUTEX_PROTECTS_DATA(ibdm_s::ibdm_mutex, ibdm_s::{ibdm_ibt_clnt_hdl
260 	ibdm_busy ibdm_state}))
261 _NOTE(DATA_READABLE_WITHOUT_LOCK(ibdm_s::ibdm_ibt_clnt_hdl))
262 _NOTE(MUTEX_PROTECTS_DATA(ibdm_s::ibdm_hl_mutex,
263 	ibdm_s::{ibdm_hca_list_head ibdm_hca_list_tail}))
264 _NOTE(MUTEX_PROTECTS_DATA(ibdm_s::ibdm_ibnex_mutex,
265 	ibdm_s::ibdm_ibnex_callback))
266 _NOTE(SCHEME_PROTECTS_DATA("Serialized access by cv", ibdm_s))
267 _NOTE(LOCK_ORDER(ibdm_s::ibdm_mutex ibdm_dp_gidinfo_s::gl_mutex))
268 
269 /* valid values for ibdm_state */
270 #define	IBDM_LOCKS_ALLOCED	0x01		/* global locks alloced */
271 #define	IBDM_CVS_ALLOCED	0x02		/* global "cv"s alloced */
272 #define	IBDM_IBT_ATTACHED	0x04		/* ibt_attach() called */
273 #define	IBDM_HCA_ATTACHED	0x08		/* ibdm_handle_hca() called */
274 
275 #define	IBDM_8_BIT_MASK		0x00FF
276 #define	IBDM_16_BIT_MASK	0xFFFF
277 #define	IBDM_RETRY_COUNT	0x2
278 
279 #define	IBDM_BUSY		0x1
280 #define	IBDM_PROBE_IN_PROGRESS	0x2
281 #define	IBDM_CISCO_PROBE	0x4
282 #define	IBDM_CISCO_PROBE_DONE	0x8
283 
284 /*
285  * Device Management MAD packet format
286  * +--------+------------+------------+------------+------------+
287  * | offset |   byte 0   |   byte 1   |   byte 2   |   byte 3   |
288  * +--------+------------+------------+------------+------------+ --
289  * |   0    |                                                   |  ^
290  * +--------+                                                   | sizeof(
291  * |   ...  |              Common MAD Header                    | ib_mad_hdr_t)
292  * +--------+                                                   |  | (A)
293  * |   20   |                                                   |  v
294  * +--------+------------+------------+------------+------------+ --
295  * |   24   |                                                   |  ^
296  * +--------+                                                   |  |
297  * |   ...  |              RMPP Header                          |  |
298  * +--------+                                                   |  |
299  * |   32   |                                                   |  |
300  * +--------+------------+------------+------------+------------+  |
301  * |   36   |                                                   |  |
302  * +--------+              Access_Key                           |
303  * |   40   |                                                   | IBDM_DM_MAD_
304  * +--------+------------+------------+------------+------------+ HDR_SZ
305  * |   44   |  KeyType   |              reserved                |    (B)
306  * +--------+------------+------------+------------+------------+  |
307  * |   48   |                                                   |  |
308  * +--------+                                                   |  |
309  * |   52   |              Reserved                             |  |
310  * +--------+                                                   |  |
311  * |   56   |                                                   |  |
312  * +--------+------------+------------+------------+------------+  |
313  * |   60   |       Change_ID         |     ComponentMask       |  v
314  * +--------+------------+------------+------------+------------+ --
315  * |   64   |                                                   |  ^
316  * +--------+                                                   | IBDM_MAD_SIZE
317  * |   ...  |              Device Management Data               | - (A) - (B)
318  * +--------+                                                   |  |
319  * |  252   |                                                   |  v
320  * +--------+------------+------------+------------+------------+ --
321  */
322 #define	IBDM_MAD_SIZE		256
323 #define	IBDM_DM_MAD_HDR_SZ	40
324 
325 #define	IBDM_DFT_TIMEOUT	4
326 #define	IBDM_DFT_NRETRIES	3
327 
328 #define	IBDM_ENABLE_TASKQ_HANDLING	1
329 #define	IBDM_DISABLE_TASKQ_HANLDING	0
330 
331 typedef struct ibdm_saa_event_arg_s {
332 	ibmf_saa_handle_t ibmf_saa_handle;
333 	ibmf_saa_subnet_event_t ibmf_saa_event;
334 	ibmf_saa_event_details_t event_details;
335 	void *callback_arg;
336 } ibdm_saa_event_arg_t;
337 
338 #define	IBDM_TIMEOUT_VALUE(t)	(drv_usectohz(t * 1000000))
339 
340 #define	IBDM_OUT_IBMFMSG_MADHDR(msg)\
341 		(msg->im_msgbufs_send.im_bufs_mad_hdr)
342 
343 #define	IBDM_IN_IBMFMSG_MADHDR(msg)\
344 		(msg->im_msgbufs_recv.im_bufs_mad_hdr)
345 
346 #define	IBDM_IN_IBMFMSG_STATUS(msg)\
347 		b2h16(msg->im_msgbufs_recv.im_bufs_mad_hdr->Status)
348 
349 #define	IBDM_IN_IBMFMSG_ATTR(msg)\
350 		b2h16(msg->im_msgbufs_recv.im_bufs_mad_hdr->AttributeID)
351 
352 #define	IBDM_IN_IBMFMSG_ATTRMOD(msg)\
353 		b2h32(msg->im_msgbufs_recv.im_bufs_mad_hdr->AttributeModifier)
354 
355 #define	IBDM_IN_IBMFMSG2IOU(msg)	(ib_dm_io_unitinfo_t *)\
356 		(msg->im_msgbufs_recv.im_bufs_cl_data)
357 
358 #define	IBDM_IN_IBMFMSG2IOC(msg)	(ib_dm_ioc_ctrl_profile_t *)\
359 		(msg->im_msgbufs_recv.im_bufs_cl_data)
360 
361 #define	IBDM_IN_IBMFMSG2SRVENT(msg)	(ib_dm_srv_t *)\
362 		(msg->im_msgbufs_recv.im_bufs_cl_data)
363 
364 #define	IBDM_IN_IBMFMSG2DIAGCODE(msg)	(uint32_t *)\
365 		(msg->im_msgbufs_recv.im_bufs_cl_data)
366 
367 #define	IBDM_GIDINFO2IOCINFO(gid_info, idx) \
368 		(ibdm_ioc_info_t *)&gid_info->gl_iou->iou_ioc_info[idx];
369 
370 #define	IBDM_IS_IOC_NUM_INVALID(ioc_no, gid_info)\
371 		((ioc_no < 1) || (ioc_no >\
372 			gid_info->gl_iou->iou_info.iou_num_ctrl_slots))
373 
374 #define	IBDM_INVALID_PKEY(pkey)	\
375 		(((pkey) == IB_PKEY_INVALID_FULL) || \
376 		((pkey) == IB_PKEY_INVALID_LIMITED))
377 
378 #ifdef DEBUG
379 
380 void	ibdm_dump_mad_hdr(ib_mad_hdr_t *);
381 void	ibdm_dump_ibmf_msg(ibmf_msg_t *, int);
382 void	ibdm_dump_path_info(sa_path_record_t *);
383 void	ibdm_dump_classportinfo(ib_mad_classportinfo_t *);
384 void	ibdm_dump_iounitinfo(ib_dm_io_unitinfo_t *);
385 void	ibdm_dump_ioc_profile(ib_dm_ioc_ctrl_profile_t *);
386 void	ibdm_dump_service_entries(ib_dm_srv_t *);
387 void	ibdm_dump_sweep_fabric_timestamp(int);
388 
389 #define	ibdm_dump_mad_hdr(a)		ibdm_dump_mad_hdr(a)
390 #define	ibdm_dump_ibmf_msg(a, b)	ibdm_dump_ibmf_msg(a, b)
391 #define	ibdm_dump_path_info(a)		ibdm_dump_path_info(a)
392 #define	ibdm_dump_classportinfo(a)	ibdm_dump_classportinfo(a)
393 #define	ibdm_dump_iounitinfo(a)		ibdm_dump_iounitinfo(a)
394 #define	ibdm_dump_ioc_profile(a)	ibdm_dump_ioc_profile(a)
395 #define	ibdm_dump_service_entries(a)	ibdm_dump_service_entries(a)
396 
397 #else
398 
399 #define	ibdm_dump_mad_hdr(a)
400 #define	ibdm_dump_ibmf_msg(a, b)
401 #define	ibdm_dump_path_info(a)
402 #define	ibdm_dump_classportinfo(a)
403 #define	ibdm_dump_iounitinfo(a)
404 #define	ibdm_dump_ioc_profile(a)
405 #define	ibdm_dump_service_entries(a)
406 #define	ibdm_dump_sweep_fabric_timestamp(a)
407 
408 #endif
409 
410 #ifdef __cplusplus
411 }
412 #endif
413 
414 #endif	/* _SYS_IB_MGT_IBDM_IBDM_IMPL_H */
415