1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25
26 #include <sys/types.h>
27 #include <sys/ddi.h>
28 #include <sys/types.h>
29 #include <sys/socket.h>
30 #include <netinet/in.h>
31 #include <sys/sunddi.h>
32 #include <sys/ib/ibtl/ibti.h>
33 #include <sys/ib/ibtl/ibtl_types.h>
34
35 #include <sys/ib/clients/iser/iser.h>
36
37 extern idm_transport_ops_t iser_transport_ops;
38
39 /*
40 * iser_cm.c
41 * InfiniBand Communication Manager routines for iSER
42 */
43 static ibt_cm_status_t iser_ib_handle_cm_req(idm_svc_t *svc_hdl,
44 ibt_cm_event_t *evp, ibt_cm_return_args_t *rargsp, void *rcmp,
45 ibt_priv_data_len_t rcmp_len);
46
47 static ibt_cm_status_t iser_ib_handle_cm_rep(iser_state_t *statep,
48 ibt_cm_event_t *evp, ibt_cm_return_args_t *rargsp, void *rcmp,
49 ibt_priv_data_len_t rcmp_len);
50
51 static ibt_cm_status_t iser_handle_cm_conn_est(ibt_cm_event_t *evp);
52 static ibt_cm_status_t iser_handle_cm_conn_closed(ibt_cm_event_t *evp);
53 static ibt_cm_status_t iser_handle_cm_event_failure(ibt_cm_event_t *evp);
54
55 /*
56 * iser_ib_cm_handler()
57 */
58 ibt_cm_status_t
iser_ib_cm_handler(void * cm_private,ibt_cm_event_t * eventp,ibt_cm_return_args_t * ret_args,void * ret_priv_data,ibt_priv_data_len_t ret_len_max)59 iser_ib_cm_handler(void *cm_private, ibt_cm_event_t *eventp,
60 ibt_cm_return_args_t *ret_args, void *ret_priv_data,
61 ibt_priv_data_len_t ret_len_max)
62 {
63 ibt_cm_status_t ret = IBT_CM_REJECT;
64
65 switch (eventp->cm_type) {
66
67 case IBT_CM_EVENT_REQ_RCV:
68 ISER_LOG(CE_NOTE, "iser_ib_cm_handler: IBT_CM_EVENT_REQ_RCV");
69 ret = iser_ib_handle_cm_req((idm_svc_t *)cm_private, eventp,
70 ret_args, ret_priv_data, ret_len_max);
71 break;
72
73 case IBT_CM_EVENT_REP_RCV:
74 ISER_LOG(CE_NOTE, "iser_ib_cm_handler: IBT_CM_EVENT_REP_RCV");
75 ret = iser_ib_handle_cm_rep((iser_state_t *)cm_private,
76 eventp, ret_args, ret_priv_data, ret_len_max);
77 break;
78
79 case IBT_CM_EVENT_CONN_EST:
80 ISER_LOG(CE_NOTE, "iser_ib_cm_handler: IBT_CM_EVENT_CONN_EST");
81 ret = iser_handle_cm_conn_est(eventp);
82 break;
83
84 case IBT_CM_EVENT_CONN_CLOSED:
85 ISER_LOG(CE_NOTE, "iser_ib_cm_handler: "
86 "IBT_CM_EVENT_CONN_CLOSED");
87 ret = iser_handle_cm_conn_closed(eventp);
88 break;
89
90 case IBT_CM_EVENT_FAILURE:
91 ISER_LOG(CE_NOTE, "iser_ib_cm_handler: Event failure");
92 ret = iser_handle_cm_event_failure(eventp);
93 break;
94
95 case IBT_CM_EVENT_MRA_RCV:
96 /* Not supported */
97 ISER_LOG(CE_NOTE, "iser_ib_cm_handler: MRA message received");
98 break;
99
100 case IBT_CM_EVENT_LAP_RCV:
101 /* Not supported */
102 ISER_LOG(CE_NOTE, "iser_ib_cm_handler: LAP message received");
103 break;
104
105 case IBT_CM_EVENT_APR_RCV:
106 /* Not supported */
107 ISER_LOG(CE_NOTE, "iser_ib_cm_handler: APR message received");
108 break;
109
110 default:
111 ISER_LOG(CE_NOTE, "iser_ib_cm_handler: unknown event (0x%x)",
112 eventp->cm_type);
113 break;
114 }
115
116 return (ret);
117 }
118
119 /* ARGSUSED */
120 static ibt_cm_status_t
iser_ib_handle_cm_req(idm_svc_t * svc_hdl,ibt_cm_event_t * evp,ibt_cm_return_args_t * rargsp,void * rcmp,ibt_priv_data_len_t rcmp_len)121 iser_ib_handle_cm_req(idm_svc_t *svc_hdl, ibt_cm_event_t *evp,
122 ibt_cm_return_args_t *rargsp, void *rcmp, ibt_priv_data_len_t rcmp_len)
123 {
124
125 iser_private_data_t iser_priv_data;
126 ibt_ip_cm_info_t ipcm_info;
127 iser_chan_t *chan;
128 iser_conn_t *iser_conn;
129 int status;
130
131 /*
132 * CM private data brings IP information
133 * Private data received is a stream of bytes and may not be properly
134 * aligned. So, bcopy the data onto the stack before accessing it.
135 */
136 bcopy((uint8_t *)evp->cm_priv_data, &iser_priv_data,
137 sizeof (iser_private_data_t));
138
139 /* extract the CM IP info */
140 status = ibt_get_ip_data(evp->cm_priv_data_len, evp->cm_priv_data,
141 &ipcm_info);
142 if (status != IBT_SUCCESS) {
143 return (IBT_CM_REJECT);
144 }
145
146 ISER_LOG(CE_NOTE, "iser_ib_handle_cm_req: ipcm_info (0x%p): src IP "
147 "(0x%08x) src port (0x%04x) dst IP: (0x%08x)", (void *)&ipcm_info,
148 ipcm_info.src_addr.un.ip4addr, ipcm_info.src_port,
149 ipcm_info.dst_addr.un.ip4addr);
150
151 /* Allocate a channel to establish the new connection */
152 chan = iser_ib_alloc_channel_nopathlookup(
153 evp->cm_event.req.req_hca_guid,
154 evp->cm_event.req.req_prim_hca_port);
155 if (chan == NULL) {
156 ISER_LOG(CE_NOTE, "iser_ib_handle_cm_req: failed to allocate "
157 "a channel from src IP (0x%08x) src port (0x%04x) "
158 "to dst IP: (0x%08x) on hca(%llx %d)",
159 ipcm_info.src_addr.un.ip4addr, ipcm_info.src_port,
160 ipcm_info.dst_addr.un.ip4addr,
161 (longlong_t)evp->cm_event.req.req_hca_guid,
162 evp->cm_event.req.req_prim_hca_port);
163 return (IBT_CM_REJECT);
164 }
165
166 /* Set the local and remote ip */
167 chan->ic_localip = ipcm_info.dst_addr;
168 chan->ic_remoteip = ipcm_info.src_addr;
169
170 /* Set the local and remote port numbers on the channel handle */
171 chan->ic_lport = svc_hdl->is_svc_req.sr_port;
172 chan->ic_rport = ipcm_info.src_port;
173
174 /* Allocate the iser_conn_t for the IDM svc binding */
175 iser_conn = kmem_zalloc(sizeof (iser_conn_t), KM_SLEEP);
176
177 /* Set up the iser_conn attributes */
178 mutex_init(&iser_conn->ic_lock, NULL, MUTEX_DRIVER, NULL);
179 cv_init(&iser_conn->ic_stage_cv, NULL, CV_DEFAULT, NULL);
180 iser_conn->ic_type = ISER_CONN_TYPE_TGT;
181 iser_conn->ic_chan = chan;
182 iser_conn->ic_stage = ISER_CONN_STAGE_ALLOCATED;
183
184 /* Hold a reference to the iSER service handle */
185 iser_tgt_svc_hold((iser_svc_t *)svc_hdl->is_iser_svc);
186
187 iser_conn->ic_idms = svc_hdl;
188
189 /*
190 * Now set a pointer to the iser_conn in the iser_chan for
191 * access during CM event handling
192 */
193 chan->ic_conn = iser_conn;
194
195 rargsp->cm_ret.rep.cm_channel = chan->ic_chanhdl;
196
197 return (IBT_CM_ACCEPT);
198 }
199
200 /* ARGSUSED */
201 static ibt_cm_status_t
iser_ib_handle_cm_rep(iser_state_t * statep,ibt_cm_event_t * evp,ibt_cm_return_args_t * rargsp,void * rcmp,ibt_priv_data_len_t rcmp_len)202 iser_ib_handle_cm_rep(iser_state_t *statep, ibt_cm_event_t *evp,
203 ibt_cm_return_args_t *rargsp, void *rcmp, ibt_priv_data_len_t rcmp_len)
204 {
205 /* pre-post work requests into the receive queue */
206 iser_ib_post_recv(evp->cm_channel);
207
208 /* It looks like the RTU need not be send specifically */
209 return (IBT_CM_ACCEPT);
210 }
211
212 static ibt_cm_status_t
iser_handle_cm_conn_est(ibt_cm_event_t * evp)213 iser_handle_cm_conn_est(ibt_cm_event_t *evp)
214 {
215 iser_chan_t *iser_chan;
216 iser_conn_t *iser_conn;
217 iser_svc_t *iser_svc;
218 idm_status_t status;
219 idm_conn_t *ic;
220
221 iser_chan = (iser_chan_t *)ibt_get_chan_private(evp->cm_channel);
222
223 /*
224 * An ibt_open_rc_channel() comes in as a IBT_CM_EVENT_REQ_RCV on the
225 * iSER-IB target, upon which the target sends a Response, accepting
226 * the request. This comes in as a IBT_CM_EVENT_REP_RCV on the iSER-IB
227 * initiator, which then sends an RTU. Upon getting this RTU from the
228 * iSER-IB initiator, the IBT_CM_EVENT_CONN_EST event is generated on
229 * the target. Then subsequently an IBT_CM_EVENT_CONN_EST event is
230 * generated on the initiator.
231 *
232 * Our new connection has been established on the target. If we are
233 * receiving this event on the target side, the iser_channel can be
234 * used as it is already populated. On the target side, an IDM
235 * connection is then allocated and the IDM layer is notified.
236 * If we are on the initiator we needn't do anything, since we
237 * already have the IDM linkage in place for this connection.
238 */
239 if (iser_chan->ic_conn->ic_type == ISER_CONN_TYPE_TGT) {
240
241 iser_conn = iser_chan->ic_conn;
242 iser_svc = (iser_svc_t *)iser_conn->ic_idms->is_iser_svc;
243
244 mutex_enter(&iser_conn->ic_lock);
245
246 status = idm_svc_conn_create(iser_conn->ic_idms,
247 IDM_TRANSPORT_TYPE_ISER, &ic);
248 if (status != IDM_STATUS_SUCCESS) {
249 /*
250 * No IDM rsrcs or something equally Bad.
251 * Return non-SUCCESS to IBCM. He'll give
252 * us a CONN_CLOSED, which we'll handle
253 * below.
254 */
255 ISER_LOG(CE_NOTE, "iser_handle_cm_conn_est: "
256 "idm_svc_conn_create_failed");
257 mutex_exit(&iser_conn->ic_lock);
258 return (IBT_CM_NO_RESOURCE);
259 }
260
261 /* We no longer need the hold on the iSER service handle */
262 iser_tgt_svc_rele(iser_svc);
263
264 /* Hold a reference on the IDM connection handle */
265 idm_conn_hold(ic);
266
267 /* Set the transport ops and conn on the idm_conn handle */
268 ic->ic_transport_ops = &iser_transport_ops;
269 ic->ic_transport_private = (void *)iser_conn;
270 ic->ic_transport_hdrlen = ISER_HEADER_LENGTH;
271 iser_conn->ic_idmc = ic;
272
273 /*
274 * Set the local and remote addresses in the idm conn handle.
275 */
276 iser_ib_conv_ibtaddr2sockaddr(&ic->ic_laddr,
277 &iser_conn->ic_chan->ic_localip, iser_chan->ic_lport);
278 iser_ib_conv_ibtaddr2sockaddr(&ic->ic_raddr,
279 &iser_conn->ic_chan->ic_remoteip, iser_chan->ic_rport);
280
281 /*
282 * Kick the state machine. At CS_S3_XPT_UP the state machine
283 * will notify the client (target) about the new connection.
284 */
285 idm_conn_event(ic, CE_CONNECT_ACCEPT, NULL);
286 iser_conn->ic_stage = ISER_CONN_STAGE_IC_CONNECTED;
287 mutex_exit(&iser_conn->ic_lock);
288
289 /*
290 * Post work requests on the receive queue
291 */
292 iser_ib_post_recv(iser_chan->ic_chanhdl);
293
294 }
295
296 return (IBT_CM_ACCEPT);
297 }
298
299 static ibt_cm_status_t
iser_handle_cm_conn_closed(ibt_cm_event_t * evp)300 iser_handle_cm_conn_closed(ibt_cm_event_t *evp)
301 {
302
303 iser_chan_t *chan;
304
305 chan = (iser_chan_t *)ibt_get_chan_private(evp->cm_channel);
306
307 ISER_LOG(CE_NOTE, "iser_handle_cm_conn_closed: chan (0x%p) "
308 "reason (0x%x)", (void *)chan, evp->cm_event.closed);
309
310 switch (evp->cm_event.closed) {
311 case IBT_CM_CLOSED_DREP_RCVD: /* we requested a disconnect */
312 case IBT_CM_CLOSED_ALREADY: /* duplicate close */
313 /* ignore these */
314 return (IBT_CM_ACCEPT);
315
316 case IBT_CM_CLOSED_DREQ_RCVD: /* request to close the channel */
317 case IBT_CM_CLOSED_REJ_RCVD: /* reject after conn establishment */
318 case IBT_CM_CLOSED_DREQ_TIMEOUT: /* our close request timed out */
319 case IBT_CM_CLOSED_DUP: /* duplicate close request */
320 case IBT_CM_CLOSED_ABORT: /* aborted connection establishment */
321 case IBT_CM_CLOSED_STALE: /* stale / unref connection */
322 /* handle these depending upon our connection state */
323 mutex_enter(&chan->ic_conn->ic_lock);
324 switch (chan->ic_conn->ic_stage) {
325 case ISER_CONN_STAGE_UNDEFINED:
326 case ISER_CONN_STAGE_CLOSED:
327 /* do nothing, just drop the lock */
328 mutex_exit(&chan->ic_conn->ic_lock);
329 break;
330
331 case ISER_CONN_STAGE_ALLOCATED:
332 /*
333 * We blew up or were offlined during connection
334 * establishment. Teardown the iSER conn and chan
335 * handles.
336 */
337 mutex_exit(&chan->ic_conn->ic_lock);
338 iser_internal_conn_destroy(chan->ic_conn);
339 break;
340
341 case ISER_CONN_STAGE_IC_DISCONNECTED:
342 case ISER_CONN_STAGE_IC_FREED:
343 case ISER_CONN_STAGE_CLOSING:
344 /* we're down, set CLOSED */
345 chan->ic_conn->ic_stage = ISER_CONN_STAGE_CLOSED;
346 mutex_exit(&chan->ic_conn->ic_lock);
347 break;
348
349 case ISER_CONN_STAGE_IC_CONNECTED:
350 case ISER_CONN_STAGE_HELLO_SENT:
351 case ISER_CONN_STAGE_HELLO_SENT_FAIL:
352 case ISER_CONN_STAGE_HELLO_WAIT:
353 case ISER_CONN_STAGE_HELLO_RCV:
354 case ISER_CONN_STAGE_HELLO_RCV_FAIL:
355 case ISER_CONN_STAGE_HELLOREPLY_SENT:
356 case ISER_CONN_STAGE_HELLOREPLY_SENT_FAIL:
357 case ISER_CONN_STAGE_HELLOREPLY_RCV:
358 case ISER_CONN_STAGE_HELLOREPLY_RCV_FAIL:
359 case ISER_CONN_STAGE_LOGGED_IN:
360 /* for all other stages, fail the transport */
361 idm_conn_event(chan->ic_conn->ic_idmc,
362 CE_TRANSPORT_FAIL, IDM_STATUS_FAIL);
363 chan->ic_conn->ic_stage = ISER_CONN_STAGE_CLOSING;
364 mutex_exit(&chan->ic_conn->ic_lock);
365 break;
366
367 default:
368 mutex_exit(&chan->ic_conn->ic_lock);
369 ASSERT(0);
370
371 }
372
373 /* accept the event */
374 return (IBT_CM_ACCEPT);
375
376 default:
377 /* unknown event */
378 ISER_LOG(CE_NOTE, "iser_handle_cm_conn_closed: unknown closed "
379 "event: (0x%x)", evp->cm_event.closed);
380 return (IBT_CM_REJECT);
381 }
382 }
383
384 /*
385 * Handle EVENT FAILURE
386 */
387 static ibt_cm_status_t
iser_handle_cm_event_failure(ibt_cm_event_t * evp)388 iser_handle_cm_event_failure(ibt_cm_event_t *evp)
389 {
390 iser_chan_t *chan;
391
392 chan = (iser_chan_t *)ibt_get_chan_private(evp->cm_channel);
393
394 ISER_LOG(CE_NOTE, "iser_handle_cm_event_failure: chan (0x%p): "
395 "code: %d msg: %d reason: %d", (void *)chan,
396 evp->cm_event.failed.cf_code, evp->cm_event.failed.cf_msg,
397 evp->cm_event.failed.cf_reason);
398
399 if ((evp->cm_channel == NULL) || (chan == NULL)) {
400 /* channel not established yet */
401 return (IBT_CM_ACCEPT);
402 }
403
404 if ((evp->cm_event.failed.cf_code != IBT_CM_FAILURE_STALE) &&
405 (evp->cm_event.failed.cf_msg == IBT_CM_FAILURE_REQ)) {
406 /*
407 * This end is active, just ignore, ibt_open_rc_channel()
408 * caller will take care of cleanup.
409 */
410 return (IBT_CM_ACCEPT);
411 }
412
413 /* handle depending upon our connection state */
414 mutex_enter(&chan->ic_conn->ic_lock);
415 switch (chan->ic_conn->ic_stage) {
416 case ISER_CONN_STAGE_UNDEFINED:
417 case ISER_CONN_STAGE_CLOSED:
418 /* do nothing, just drop the lock */
419 mutex_exit(&chan->ic_conn->ic_lock);
420 break;
421
422 case ISER_CONN_STAGE_ALLOCATED:
423 /*
424 * We blew up or were offlined during connection
425 * establishment. Teardown the iSER conn and chan
426 * handles.
427 */
428 mutex_exit(&chan->ic_conn->ic_lock);
429 iser_internal_conn_destroy(chan->ic_conn);
430 break;
431
432 case ISER_CONN_STAGE_IC_DISCONNECTED:
433 case ISER_CONN_STAGE_IC_FREED:
434 case ISER_CONN_STAGE_CLOSING:
435 /* update to CLOSED, then drop the lock */
436 chan->ic_conn->ic_stage = ISER_CONN_STAGE_CLOSED;
437 mutex_exit(&chan->ic_conn->ic_lock);
438 break;
439
440 case ISER_CONN_STAGE_IC_CONNECTED:
441 case ISER_CONN_STAGE_HELLO_SENT:
442 case ISER_CONN_STAGE_HELLO_SENT_FAIL:
443 case ISER_CONN_STAGE_HELLO_WAIT:
444 case ISER_CONN_STAGE_HELLO_RCV:
445 case ISER_CONN_STAGE_HELLO_RCV_FAIL:
446 case ISER_CONN_STAGE_HELLOREPLY_SENT:
447 case ISER_CONN_STAGE_HELLOREPLY_SENT_FAIL:
448 case ISER_CONN_STAGE_HELLOREPLY_RCV:
449 case ISER_CONN_STAGE_HELLOREPLY_RCV_FAIL:
450 case ISER_CONN_STAGE_LOGGED_IN:
451 /* fail the transport and move the conn to CLOSING */
452 idm_conn_event(chan->ic_conn->ic_idmc, CE_TRANSPORT_FAIL,
453 IDM_STATUS_FAIL);
454 chan->ic_conn->ic_stage = ISER_CONN_STAGE_CLOSING;
455 mutex_exit(&chan->ic_conn->ic_lock);
456 break;
457
458 default:
459 mutex_exit(&chan->ic_conn->ic_lock);
460 ASSERT(0);
461 }
462
463 /* accept the event */
464 return (IBT_CM_ACCEPT);
465 }
466