1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #include <sys/types.h> 27 #include <sys/ddi.h> 28 #include <sys/types.h> 29 #include <sys/socket.h> 30 #include <netinet/in.h> 31 #include <sys/sunddi.h> 32 #include <sys/ib/ibtl/ibti.h> 33 #include <sys/ib/ibtl/ibtl_types.h> 34 35 #include <sys/ib/clients/iser/iser.h> 36 37 extern idm_transport_ops_t iser_transport_ops; 38 39 /* 40 * iser_cm.c 41 * InfiniBand Communication Manager routines for iSER 42 */ 43 static ibt_cm_status_t iser_ib_handle_cm_req(idm_svc_t *svc_hdl, 44 ibt_cm_event_t *evp, ibt_cm_return_args_t *rargsp, void *rcmp, 45 ibt_priv_data_len_t rcmp_len); 46 47 static ibt_cm_status_t iser_ib_handle_cm_rep(iser_state_t *statep, 48 ibt_cm_event_t *evp, ibt_cm_return_args_t *rargsp, void *rcmp, 49 ibt_priv_data_len_t rcmp_len); 50 51 static ibt_cm_status_t iser_handle_cm_conn_est(ibt_cm_event_t *evp); 52 static ibt_cm_status_t iser_handle_cm_conn_closed(ibt_cm_event_t *evp); 53 static ibt_cm_status_t iser_handle_cm_event_failure(ibt_cm_event_t *evp); 54 55 /* 56 * iser_ib_cm_handler() 57 */ 58 ibt_cm_status_t 59 iser_ib_cm_handler(void *cm_private, ibt_cm_event_t *eventp, 60 ibt_cm_return_args_t *ret_args, void *ret_priv_data, 61 ibt_priv_data_len_t ret_len_max) 62 { 63 ibt_cm_status_t ret = IBT_CM_REJECT; 64 65 switch (eventp->cm_type) { 66 67 case IBT_CM_EVENT_REQ_RCV: 68 ISER_LOG(CE_NOTE, "iser_ib_cm_handler: IBT_CM_EVENT_REQ_RCV"); 69 ret = iser_ib_handle_cm_req((idm_svc_t *)cm_private, eventp, 70 ret_args, ret_priv_data, ret_len_max); 71 break; 72 73 case IBT_CM_EVENT_REP_RCV: 74 ISER_LOG(CE_NOTE, "iser_ib_cm_handler: IBT_CM_EVENT_REP_RCV"); 75 ret = iser_ib_handle_cm_rep((iser_state_t *)cm_private, 76 eventp, ret_args, ret_priv_data, ret_len_max); 77 break; 78 79 case IBT_CM_EVENT_CONN_EST: 80 ISER_LOG(CE_NOTE, "iser_ib_cm_handler: IBT_CM_EVENT_CONN_EST"); 81 ret = iser_handle_cm_conn_est(eventp); 82 break; 83 84 case IBT_CM_EVENT_CONN_CLOSED: 85 ISER_LOG(CE_NOTE, "iser_ib_cm_handler: " 86 "IBT_CM_EVENT_CONN_CLOSED"); 87 ret = iser_handle_cm_conn_closed(eventp); 88 break; 89 90 case IBT_CM_EVENT_FAILURE: 91 ISER_LOG(CE_NOTE, "iser_ib_cm_handler: Event failure"); 92 ret = iser_handle_cm_event_failure(eventp); 93 break; 94 95 case IBT_CM_EVENT_MRA_RCV: 96 /* Not supported */ 97 ISER_LOG(CE_NOTE, "iser_ib_cm_handler: MRA message received"); 98 break; 99 100 case IBT_CM_EVENT_LAP_RCV: 101 /* Not supported */ 102 ISER_LOG(CE_NOTE, "iser_ib_cm_handler: LAP message received"); 103 break; 104 105 case IBT_CM_EVENT_APR_RCV: 106 /* Not supported */ 107 ISER_LOG(CE_NOTE, "iser_ib_cm_handler: APR message received"); 108 break; 109 110 default: 111 ISER_LOG(CE_NOTE, "iser_ib_cm_handler: unknown event (0x%x)", 112 eventp->cm_type); 113 break; 114 } 115 116 return (ret); 117 } 118 119 /* ARGSUSED */ 120 static ibt_cm_status_t 121 iser_ib_handle_cm_req(idm_svc_t *svc_hdl, ibt_cm_event_t *evp, 122 ibt_cm_return_args_t *rargsp, void *rcmp, ibt_priv_data_len_t rcmp_len) 123 { 124 125 iser_private_data_t iser_priv_data; 126 ibt_ip_cm_info_t ipcm_info; 127 iser_chan_t *chan; 128 iser_conn_t *iser_conn; 129 int status; 130 131 /* 132 * CM private data brings IP information 133 * Private data received is a stream of bytes and may not be properly 134 * aligned. So, bcopy the data onto the stack before accessing it. 135 */ 136 bcopy((uint8_t *)evp->cm_priv_data, &iser_priv_data, 137 sizeof (iser_private_data_t)); 138 139 /* extract the CM IP info */ 140 status = ibt_get_ip_data(evp->cm_priv_data_len, evp->cm_priv_data, 141 &ipcm_info); 142 if (status != IBT_SUCCESS) { 143 return (IBT_CM_REJECT); 144 } 145 146 ISER_LOG(CE_NOTE, "iser_ib_handle_cm_req: ipcm_info (0x%p): src IP " 147 "(0x%08x) src port (0x%04x) dst IP: (0x%08x)", (void *)&ipcm_info, 148 ipcm_info.src_addr.un.ip4addr, ipcm_info.src_port, 149 ipcm_info.dst_addr.un.ip4addr); 150 151 /* Allocate a channel to establish the new connection */ 152 chan = iser_ib_alloc_channel_nopathlookup( 153 evp->cm_event.req.req_hca_guid, 154 evp->cm_event.req.req_prim_hca_port); 155 if (chan == NULL) { 156 ISER_LOG(CE_NOTE, "iser_ib_handle_cm_req: failed to allocate " 157 "a channel from src IP (0x%08x) src port (0x%04x) " 158 "to dst IP: (0x%08x) on hca(%llx %d)", 159 ipcm_info.src_addr.un.ip4addr, ipcm_info.src_port, 160 ipcm_info.dst_addr.un.ip4addr, 161 (longlong_t)evp->cm_event.req.req_hca_guid, 162 evp->cm_event.req.req_prim_hca_port); 163 return (IBT_CM_REJECT); 164 } 165 166 /* Set the local and remote ip */ 167 chan->ic_localip = ipcm_info.dst_addr; 168 chan->ic_remoteip = ipcm_info.src_addr; 169 170 /* Set the local and remote port numbers on the channel handle */ 171 chan->ic_lport = svc_hdl->is_svc_req.sr_port; 172 chan->ic_rport = ipcm_info.src_port; 173 174 /* Allocate the iser_conn_t for the IDM svc binding */ 175 iser_conn = kmem_zalloc(sizeof (iser_conn_t), KM_SLEEP); 176 177 /* Set up the iser_conn attributes */ 178 mutex_init(&iser_conn->ic_lock, NULL, MUTEX_DRIVER, NULL); 179 cv_init(&iser_conn->ic_stage_cv, NULL, CV_DEFAULT, NULL); 180 iser_conn->ic_type = ISER_CONN_TYPE_TGT; 181 iser_conn->ic_chan = chan; 182 iser_conn->ic_stage = ISER_CONN_STAGE_ALLOCATED; 183 184 /* Hold a reference to the iSER service handle */ 185 iser_tgt_svc_hold((iser_svc_t *)svc_hdl->is_iser_svc); 186 187 iser_conn->ic_idms = svc_hdl; 188 189 /* 190 * Now set a pointer to the iser_conn in the iser_chan for 191 * access during CM event handling 192 */ 193 chan->ic_conn = iser_conn; 194 195 rargsp->cm_ret.rep.cm_channel = chan->ic_chanhdl; 196 197 return (IBT_CM_ACCEPT); 198 } 199 200 /* ARGSUSED */ 201 static ibt_cm_status_t 202 iser_ib_handle_cm_rep(iser_state_t *statep, ibt_cm_event_t *evp, 203 ibt_cm_return_args_t *rargsp, void *rcmp, ibt_priv_data_len_t rcmp_len) 204 { 205 /* pre-post work requests into the receive queue */ 206 iser_ib_post_recv(evp->cm_channel); 207 208 /* It looks like the RTU need not be send specifically */ 209 return (IBT_CM_ACCEPT); 210 } 211 212 static ibt_cm_status_t 213 iser_handle_cm_conn_est(ibt_cm_event_t *evp) 214 { 215 iser_chan_t *iser_chan; 216 iser_conn_t *iser_conn; 217 iser_svc_t *iser_svc; 218 idm_status_t status; 219 idm_conn_t *ic; 220 221 iser_chan = (iser_chan_t *)ibt_get_chan_private(evp->cm_channel); 222 223 /* 224 * An ibt_open_rc_channel() comes in as a IBT_CM_EVENT_REQ_RCV on the 225 * iSER-IB target, upon which the target sends a Response, accepting 226 * the request. This comes in as a IBT_CM_EVENT_REP_RCV on the iSER-IB 227 * initiator, which then sends an RTU. Upon getting this RTU from the 228 * iSER-IB initiator, the IBT_CM_EVENT_CONN_EST event is generated on 229 * the target. Then subsequently an IBT_CM_EVENT_CONN_EST event is 230 * generated on the initiator. 231 * 232 * Our new connection has been established on the target. If we are 233 * receiving this event on the target side, the iser_channel can be 234 * used as it is already populated. On the target side, an IDM 235 * connection is then allocated and the IDM layer is notified. 236 * If we are on the initiator we needn't do anything, since we 237 * already have the IDM linkage in place for this connection. 238 */ 239 if (iser_chan->ic_conn->ic_type == ISER_CONN_TYPE_TGT) { 240 241 iser_conn = iser_chan->ic_conn; 242 iser_svc = (iser_svc_t *)iser_conn->ic_idms->is_iser_svc; 243 244 mutex_enter(&iser_conn->ic_lock); 245 246 status = idm_svc_conn_create(iser_conn->ic_idms, 247 IDM_TRANSPORT_TYPE_ISER, &ic); 248 if (status != IDM_STATUS_SUCCESS) { 249 /* 250 * No IDM rsrcs or something equally Bad. 251 * Return non-SUCCESS to IBCM. He'll give 252 * us a CONN_CLOSED, which we'll handle 253 * below. 254 */ 255 ISER_LOG(CE_NOTE, "iser_handle_cm_conn_est: " 256 "idm_svc_conn_create_failed"); 257 mutex_exit(&iser_conn->ic_lock); 258 return (IBT_CM_NO_RESOURCE); 259 } 260 261 /* We no longer need the hold on the iSER service handle */ 262 iser_tgt_svc_rele(iser_svc); 263 264 /* Hold a reference on the IDM connection handle */ 265 idm_conn_hold(ic); 266 267 /* Set the transport ops and conn on the idm_conn handle */ 268 ic->ic_transport_ops = &iser_transport_ops; 269 ic->ic_transport_private = (void *)iser_conn; 270 ic->ic_transport_hdrlen = ISER_HEADER_LENGTH; 271 iser_conn->ic_idmc = ic; 272 273 /* 274 * Set the local and remote addresses in the idm conn handle. 275 */ 276 iser_ib_conv_ibtaddr2sockaddr(&ic->ic_laddr, 277 &iser_conn->ic_chan->ic_localip, iser_chan->ic_lport); 278 iser_ib_conv_ibtaddr2sockaddr(&ic->ic_raddr, 279 &iser_conn->ic_chan->ic_remoteip, iser_chan->ic_rport); 280 281 /* 282 * Kick the state machine. At CS_S3_XPT_UP the state machine 283 * will notify the client (target) about the new connection. 284 */ 285 idm_conn_event(ic, CE_CONNECT_ACCEPT, NULL); 286 iser_conn->ic_stage = ISER_CONN_STAGE_IC_CONNECTED; 287 mutex_exit(&iser_conn->ic_lock); 288 289 /* 290 * Post work requests on the receive queue 291 */ 292 iser_ib_post_recv(iser_chan->ic_chanhdl); 293 294 } 295 296 return (IBT_CM_ACCEPT); 297 } 298 299 static ibt_cm_status_t 300 iser_handle_cm_conn_closed(ibt_cm_event_t *evp) 301 { 302 303 iser_chan_t *chan; 304 305 chan = (iser_chan_t *)ibt_get_chan_private(evp->cm_channel); 306 307 ISER_LOG(CE_NOTE, "iser_handle_cm_conn_closed: chan (0x%p) " 308 "reason (0x%x)", (void *)chan, evp->cm_event.closed); 309 310 switch (evp->cm_event.closed) { 311 case IBT_CM_CLOSED_DREP_RCVD: /* we requested a disconnect */ 312 case IBT_CM_CLOSED_ALREADY: /* duplicate close */ 313 /* ignore these */ 314 return (IBT_CM_ACCEPT); 315 316 case IBT_CM_CLOSED_DREQ_RCVD: /* request to close the channel */ 317 case IBT_CM_CLOSED_REJ_RCVD: /* reject after conn establishment */ 318 case IBT_CM_CLOSED_DREQ_TIMEOUT: /* our close request timed out */ 319 case IBT_CM_CLOSED_DUP: /* duplicate close request */ 320 case IBT_CM_CLOSED_ABORT: /* aborted connection establishment */ 321 case IBT_CM_CLOSED_STALE: /* stale / unref connection */ 322 /* handle these depending upon our connection state */ 323 mutex_enter(&chan->ic_conn->ic_lock); 324 switch (chan->ic_conn->ic_stage) { 325 case ISER_CONN_STAGE_UNDEFINED: 326 case ISER_CONN_STAGE_CLOSED: 327 /* do nothing, just drop the lock */ 328 mutex_exit(&chan->ic_conn->ic_lock); 329 break; 330 331 case ISER_CONN_STAGE_ALLOCATED: 332 /* 333 * We blew up or were offlined during connection 334 * establishment. Teardown the iSER conn and chan 335 * handles. 336 */ 337 mutex_exit(&chan->ic_conn->ic_lock); 338 iser_internal_conn_destroy(chan->ic_conn); 339 break; 340 341 case ISER_CONN_STAGE_IC_DISCONNECTED: 342 case ISER_CONN_STAGE_IC_FREED: 343 case ISER_CONN_STAGE_CLOSING: 344 /* we're down, set CLOSED */ 345 chan->ic_conn->ic_stage = ISER_CONN_STAGE_CLOSED; 346 mutex_exit(&chan->ic_conn->ic_lock); 347 break; 348 349 case ISER_CONN_STAGE_IC_CONNECTED: 350 case ISER_CONN_STAGE_HELLO_SENT: 351 case ISER_CONN_STAGE_HELLO_SENT_FAIL: 352 case ISER_CONN_STAGE_HELLO_WAIT: 353 case ISER_CONN_STAGE_HELLO_RCV: 354 case ISER_CONN_STAGE_HELLO_RCV_FAIL: 355 case ISER_CONN_STAGE_HELLOREPLY_SENT: 356 case ISER_CONN_STAGE_HELLOREPLY_SENT_FAIL: 357 case ISER_CONN_STAGE_HELLOREPLY_RCV: 358 case ISER_CONN_STAGE_HELLOREPLY_RCV_FAIL: 359 case ISER_CONN_STAGE_LOGGED_IN: 360 /* for all other stages, fail the transport */ 361 idm_conn_event(chan->ic_conn->ic_idmc, 362 CE_TRANSPORT_FAIL, IDM_STATUS_FAIL); 363 chan->ic_conn->ic_stage = ISER_CONN_STAGE_CLOSING; 364 mutex_exit(&chan->ic_conn->ic_lock); 365 break; 366 367 default: 368 mutex_exit(&chan->ic_conn->ic_lock); 369 ASSERT(0); 370 371 } 372 373 /* accept the event */ 374 return (IBT_CM_ACCEPT); 375 376 default: 377 /* unknown event */ 378 ISER_LOG(CE_NOTE, "iser_handle_cm_conn_closed: unknown closed " 379 "event: (0x%x)", evp->cm_event.closed); 380 return (IBT_CM_REJECT); 381 } 382 } 383 384 /* 385 * Handle EVENT FAILURE 386 */ 387 static ibt_cm_status_t 388 iser_handle_cm_event_failure(ibt_cm_event_t *evp) 389 { 390 iser_chan_t *chan; 391 392 chan = (iser_chan_t *)ibt_get_chan_private(evp->cm_channel); 393 394 ISER_LOG(CE_NOTE, "iser_handle_cm_event_failure: chan (0x%p): " 395 "code: %d msg: %d reason: %d", (void *)chan, 396 evp->cm_event.failed.cf_code, evp->cm_event.failed.cf_msg, 397 evp->cm_event.failed.cf_reason); 398 399 if ((evp->cm_channel == NULL) || (chan == NULL)) { 400 /* channel not established yet */ 401 return (IBT_CM_ACCEPT); 402 } 403 404 if ((evp->cm_event.failed.cf_code != IBT_CM_FAILURE_STALE) && 405 (evp->cm_event.failed.cf_msg == IBT_CM_FAILURE_REQ)) { 406 /* 407 * This end is active, just ignore, ibt_open_rc_channel() 408 * caller will take care of cleanup. 409 */ 410 return (IBT_CM_ACCEPT); 411 } 412 413 /* handle depending upon our connection state */ 414 mutex_enter(&chan->ic_conn->ic_lock); 415 switch (chan->ic_conn->ic_stage) { 416 case ISER_CONN_STAGE_UNDEFINED: 417 case ISER_CONN_STAGE_CLOSED: 418 /* do nothing, just drop the lock */ 419 mutex_exit(&chan->ic_conn->ic_lock); 420 break; 421 422 case ISER_CONN_STAGE_ALLOCATED: 423 /* 424 * We blew up or were offlined during connection 425 * establishment. Teardown the iSER conn and chan 426 * handles. 427 */ 428 mutex_exit(&chan->ic_conn->ic_lock); 429 iser_internal_conn_destroy(chan->ic_conn); 430 break; 431 432 case ISER_CONN_STAGE_IC_DISCONNECTED: 433 case ISER_CONN_STAGE_IC_FREED: 434 case ISER_CONN_STAGE_CLOSING: 435 /* update to CLOSED, then drop the lock */ 436 chan->ic_conn->ic_stage = ISER_CONN_STAGE_CLOSED; 437 mutex_exit(&chan->ic_conn->ic_lock); 438 break; 439 440 case ISER_CONN_STAGE_IC_CONNECTED: 441 case ISER_CONN_STAGE_HELLO_SENT: 442 case ISER_CONN_STAGE_HELLO_SENT_FAIL: 443 case ISER_CONN_STAGE_HELLO_WAIT: 444 case ISER_CONN_STAGE_HELLO_RCV: 445 case ISER_CONN_STAGE_HELLO_RCV_FAIL: 446 case ISER_CONN_STAGE_HELLOREPLY_SENT: 447 case ISER_CONN_STAGE_HELLOREPLY_SENT_FAIL: 448 case ISER_CONN_STAGE_HELLOREPLY_RCV: 449 case ISER_CONN_STAGE_HELLOREPLY_RCV_FAIL: 450 case ISER_CONN_STAGE_LOGGED_IN: 451 /* fail the transport and move the conn to CLOSING */ 452 idm_conn_event(chan->ic_conn->ic_idmc, CE_TRANSPORT_FAIL, 453 IDM_STATUS_FAIL); 454 chan->ic_conn->ic_stage = ISER_CONN_STAGE_CLOSING; 455 mutex_exit(&chan->ic_conn->ic_lock); 456 break; 457 458 default: 459 mutex_exit(&chan->ic_conn->ic_lock); 460 ASSERT(0); 461 } 462 463 /* accept the event */ 464 return (IBT_CM_ACCEPT); 465 } 466