1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. 24 */ 25 26 /* 27 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 28 * Use is subject to license terms. 29 */ 30 31 /* 32 * 33 * MODULE: dapls_cr_callback.c 34 * 35 * PURPOSE: implements passive side connection callbacks 36 * 37 * Description: Accepts asynchronous callbacks from the Communications Manager 38 * for EVDs that have been specified as the connection_evd. 39 * 40 * $Id: dapl_cr_callback.c,v 1.58 2003/08/20 14:55:39 sjs2 Exp $ 41 */ 42 43 #include "dapl.h" 44 #include "dapl_evd_util.h" 45 #include "dapl_cr_util.h" 46 #include "dapl_ia_util.h" 47 #include "dapl_sp_util.h" 48 #include "dapl_ep_util.h" 49 #include "dapl_adapter_util.h" 50 51 52 /* 53 * Prototypes 54 */ 55 DAT_RETURN dapli_connection_request( 56 IN ib_cm_handle_t ib_cm_handle, 57 IN DAPL_SP *sp_ptr, 58 IN DAPL_PRIVATE *prd_ptr, 59 IN DAPL_EVD *evd_ptr); 60 61 DAPL_EP * dapli_get_sp_ep( 62 IN ib_cm_handle_t ib_cm_handle, 63 IN DAPL_SP *sp_ptr, 64 IN const ib_cm_events_t ib_cm_event); 65 66 /* 67 * dapls_cr_callback 68 * 69 * The callback function registered with verbs for passive side of 70 * connection requests. The interface is specified by cm_api.h 71 * 72 * 73 * Input: 74 * ib_cm_handle, Handle to CM 75 * ib_cm_event Specific CM event 76 * instant_data Private data with DAT ADDRESS header 77 * context SP pointer 78 * 79 * Output: 80 * None 81 * 82 */ 83 void 84 dapls_cr_callback( 85 IN ib_cm_handle_t ib_cm_handle, 86 IN const ib_cm_events_t ib_cm_event, 87 IN const void *private_data_ptr, /* event data */ 88 IN const void *context) 89 { 90 DAPL_EP *ep_ptr; 91 DAPL_EVD *evd_ptr; 92 DAPL_SP *sp_ptr; 93 DAPL_PRIVATE *prd_ptr; 94 DAT_EVENT_NUMBER event_type; 95 DAT_RETURN dat_status; 96 97 dapl_dbg_log(DAPL_DBG_TYPE_CM, 98 "--> dapls_cr_callback! context: 0x%p " 99 "event: %d cm_handle 0x%llx magic 0x%x\n", 100 context, ib_cm_event, ib_cm_handle, 101 ((DAPL_HEADER *)context)->magic); 102 103 if (((DAPL_HEADER *)context)->magic == DAPL_MAGIC_INVALID) { 104 return; 105 } 106 /* 107 * Passive side of the connection, context is a SP and 108 * we need to look up the EP. 109 */ 110 dapl_os_assert(((DAPL_HEADER *)context)->magic == DAPL_MAGIC_PSP || 111 ((DAPL_HEADER *)context)->magic == DAPL_MAGIC_RSP); 112 sp_ptr = (DAPL_SP *) context; 113 114 /* 115 * CONNECT_REQUEST events create an event on the PSP 116 * EVD, which will trigger connection processing. The 117 * sequence is: 118 * CONNECT_REQUEST Event to SP 119 * CONNECTED Event to EP 120 * DISCONNECT Event to EP 121 * 122 * Obtain the EP if required and set an event up on the correct EVD. 123 */ 124 if (ib_cm_event == IB_CME_CONNECTION_REQUEST_PENDING || 125 ib_cm_event == IB_CME_CONNECTION_REQUEST_PENDING_PRIVATE_DATA) { 126 ep_ptr = NULL; 127 evd_ptr = sp_ptr->evd_handle; 128 } else { 129 ep_ptr = dapli_get_sp_ep(ib_cm_handle, sp_ptr, ib_cm_event); 130 dapl_os_assert(ep_ptr != NULL); 131 evd_ptr = (DAPL_EVD *) ep_ptr->param.connect_evd_handle; 132 dapl_dbg_log(DAPL_DBG_TYPE_CM, 133 " dapls_cr_callback cont: ep 0x%p evd 0x%p\n", 134 ep_ptr, evd_ptr); 135 } 136 137 prd_ptr = (DAPL_PRIVATE *)private_data_ptr; 138 dat_status = DAT_INTERNAL_ERROR; /* init to ERR */ 139 140 switch (ib_cm_event) { 141 case IB_CME_CONNECTION_REQUEST_PENDING: 142 case IB_CME_CONNECTION_REQUEST_PENDING_PRIVATE_DATA: { 143 /* 144 * Requests arriving on a disabled SP are immediatly rejected 145 */ 146 147 dapl_os_lock(&sp_ptr->header.lock); 148 if (sp_ptr->listening == DAT_FALSE) { 149 dapl_os_unlock(&sp_ptr->header.lock); 150 dapl_dbg_log(DAPL_DBG_TYPE_CM, 151 "---> dapls_cr_callback: conn event on down SP\n"); 152 return; 153 } 154 155 if (sp_ptr->header.handle_type == DAT_HANDLE_TYPE_RSP) { 156 /* 157 * RSP connections only allow a single connection. Close 158 * it down NOW so we reject any further connections. 159 */ 160 sp_ptr->listening = DAT_FALSE; 161 } 162 dapl_os_unlock(&sp_ptr->header.lock); 163 164 /* 165 * Only occurs on the passive side of a connection 166 * dapli_connection_request will post the connection 167 * event if appropriate. 168 */ 169 dat_status = dapli_connection_request(ib_cm_handle, 170 sp_ptr, prd_ptr, evd_ptr); 171 break; 172 } 173 case IB_CME_CONNECTED: { 174 /* 175 * This is just a notification the connection is now 176 * established, there isn't any private data to deal with. 177 * 178 * Update the EP state and cache a copy of the cm handle, 179 * then let the user know we are ready to go. 180 */ 181 dapl_os_lock(&ep_ptr->header.lock); 182 if (ep_ptr->param.ep_state == DAT_EP_STATE_DISCONNECT_PENDING) { 183 /* 184 * If someone pulled the plug on the connection, just 185 * exit 186 */ 187 dapl_os_unlock(&ep_ptr->header.lock); 188 dat_status = DAT_SUCCESS; 189 break; 190 } 191 dapls_ib_connected(ep_ptr); 192 ep_ptr->param.ep_state = DAT_EP_STATE_CONNECTED; 193 ep_ptr->cm_handle = ib_cm_handle; 194 dapl_os_unlock(&ep_ptr->header.lock); 195 196 dat_status = dapls_evd_post_connection_event( 197 evd_ptr, 198 DAT_CONNECTION_EVENT_ESTABLISHED, 199 (DAT_HANDLE) ep_ptr, 200 ((DAPL_CR *)ep_ptr->cr_ptr)->param.private_data_size, 201 ((DAPL_CR *)ep_ptr->cr_ptr)->param.private_data); 202 /* 203 * post them to the recv evd now. 204 * there is a race here - if events arrive after we change 205 * the ep state to connected and before we process premature 206 * events 207 */ 208 dapls_evd_post_premature_events(ep_ptr); 209 break; 210 } 211 case IB_CME_DISCONNECTED: 212 case IB_CME_DISCONNECTED_ON_LINK_DOWN: { 213 /* 214 * EP is now fully disconnected; initiate any post processing 215 * to reset the underlying QP and get the EP ready for 216 * another connection 217 */ 218 dapl_os_lock(&ep_ptr->header.lock); 219 if (ep_ptr->param.ep_state == DAT_EP_STATE_DISCONNECTED) { 220 /* DTO error caused this */ 221 event_type = DAT_CONNECTION_EVENT_BROKEN; 222 } else { 223 ep_ptr->param.ep_state = DAT_EP_STATE_DISCONNECTED; 224 dapls_ib_disconnect_clean(ep_ptr, DAT_FALSE, 225 ib_cm_event); 226 event_type = DAT_CONNECTION_EVENT_DISCONNECTED; 227 } 228 dapls_evd_post_premature_events(ep_ptr); 229 230 ep_ptr->cr_ptr = NULL; 231 dapl_os_unlock(&ep_ptr->header.lock); 232 233 /* 234 * If the user has done an ep_free of the EP, we have been 235 * waiting for the disconnect event; just clean it up now. 236 */ 237 if (ep_ptr->header.magic == DAPL_MAGIC_EP_EXIT) { 238 (void) dapl_ep_free(ep_ptr); 239 } 240 241 /* If the EP has been freed, the evd_ptr will be NULL */ 242 if (evd_ptr != NULL) { 243 dat_status = dapls_evd_post_connection_event( 244 evd_ptr, event_type, (DAT_HANDLE) ep_ptr, 0, 0); 245 } 246 247 break; 248 } 249 case IB_CME_DESTINATION_REJECT: 250 case IB_CME_DESTINATION_REJECT_PRIVATE_DATA: 251 case IB_CME_DESTINATION_UNREACHABLE: { 252 /* 253 * After posting an accept the requesting node has 254 * stopped talking. 255 */ 256 dapl_os_lock(&ep_ptr->header.lock); 257 ep_ptr->param.ep_state = DAT_EP_STATE_DISCONNECTED; 258 ep_ptr->cm_handle = IB_INVALID_HANDLE; 259 dapls_ib_disconnect_clean(ep_ptr, DAT_FALSE, ib_cm_event); 260 dapl_os_unlock(&ep_ptr->header.lock); 261 dat_status = dapls_evd_post_connection_event( 262 evd_ptr, 263 DAT_CONNECTION_EVENT_ACCEPT_COMPLETION_ERROR, 264 (DAT_HANDLE) ep_ptr, 0, 0); 265 266 break; 267 } 268 case IB_CME_TOO_MANY_CONNECTION_REQUESTS: { 269 /* 270 * DAPL does not deal with this IB error. There is a 271 * separate OVERFLOW event error if we try to post too many 272 * events, but we don't propagate this provider error. Not 273 * all providers generate this error. 274 */ 275 break; 276 } 277 case IB_CME_LOCAL_FAILURE: { 278 ep_ptr->param.ep_state = DAT_EP_STATE_DISCONNECTED; 279 dapls_ib_disconnect_clean(ep_ptr, DAT_FALSE, ib_cm_event); 280 dat_status = dapls_evd_post_connection_event( 281 evd_ptr, 282 DAT_CONNECTION_EVENT_BROKEN, 283 (DAT_HANDLE) ep_ptr, 0, 0); 284 285 break; 286 } 287 case IB_CME_TIMED_OUT: { 288 ep_ptr->param.ep_state = DAT_EP_STATE_DISCONNECTED; 289 dapls_ib_disconnect_clean(ep_ptr, DAT_FALSE, ib_cm_event); 290 dat_status = dapls_evd_post_connection_event( 291 evd_ptr, 292 DAT_CONNECTION_EVENT_TIMED_OUT, 293 (DAT_HANDLE) ep_ptr, 0, 0); 294 295 break; 296 } 297 default: 298 dapl_os_assert(0); /* shouldn't happen */ 299 break; 300 } 301 302 if (dat_status != DAT_SUCCESS) { 303 /* The event post failed; take appropriate action. */ 304 (void) dapls_ib_reject_connection(ib_cm_handle, 305 IB_CME_LOCAL_FAILURE, sp_ptr); 306 return; 307 } 308 } 309 310 311 /* 312 * dapli_connection_request 313 * 314 * Process a connection request on the Passive side of a connection. 315 * Create a CR record and link it on to the SP so we can update it 316 * and free it later. Create an EP if specified by the PSP flags. 317 * 318 * Input: 319 * ib_cm_handle, 320 * sp_ptr 321 * event_ptr 322 * prd_ptr 323 * 324 * Output: 325 * None 326 * 327 * Returns 328 * DAT_INSUFFICIENT_RESOURCES 329 * DAT_SUCCESS 330 * 331 */ 332 DAT_RETURN 333 dapli_connection_request( 334 IN ib_cm_handle_t ib_cm_handle, 335 IN DAPL_SP *sp_ptr, 336 IN DAPL_PRIVATE *prd_ptr, 337 IN DAPL_EVD *evd_ptr) 338 { 339 DAT_RETURN dat_status; 340 DAPL_CR *cr_ptr; 341 DAPL_EP *ep_ptr; 342 DAPL_IA *ia_ptr; 343 DAT_SP_HANDLE sp_handle; 344 struct sockaddr_in *sv4; 345 struct sockaddr_in6 *sv6; 346 uint8_t *sadata; 347 DAT_COUNT length; 348 349 cr_ptr = dapls_cr_alloc(sp_ptr->header.owner_ia); 350 if (cr_ptr == NULL) { 351 /* Invoking function will call dapls_ib_cm_reject() */ 352 return (DAT_INSUFFICIENT_RESOURCES); 353 } 354 355 /* 356 * Set up the CR 357 */ 358 cr_ptr->sp_ptr = sp_ptr; /* maintain sp_ptr in case of reject */ 359 cr_ptr->ib_cm_handle = ib_cm_handle; 360 /* 361 * Copy the remote address and private data out of the private_data 362 * payload and put them in a local structure 363 */ 364 cr_ptr->param.private_data = cr_ptr->private_data; 365 cr_ptr->param.remote_ia_address_ptr = 366 (DAT_IA_ADDRESS_PTR)&cr_ptr->remote_ia_address; 367 cr_ptr->param.remote_port_qual = 368 (DAT_PORT_QUAL) prd_ptr->hello_msg.hi_port; 369 length = (DAT_COUNT) prd_ptr->hello_msg.hi_clen; 370 cr_ptr->param.private_data_size = length; 371 (void) dapl_os_memcpy(cr_ptr->private_data, 372 prd_ptr->private_data, length); 373 switch (prd_ptr->hello_msg.hi_ipv) { 374 case AF_INET: 375 sv4 = (struct sockaddr_in *)&cr_ptr->remote_ia_address; 376 sv4->sin_family = AF_INET; 377 sv4->sin_port = prd_ptr->hello_msg.hi_port; 378 sv4->sin_addr = prd_ptr->hello_msg.hi_v4ipaddr; 379 break; 380 case AF_INET6: 381 sv6 = (struct sockaddr_in6 *)&cr_ptr->remote_ia_address; 382 sv6->sin6_family = AF_INET6; 383 sv6->sin6_port = prd_ptr->hello_msg.hi_port; 384 sv6->sin6_addr = prd_ptr->hello_msg.hi_v6ipaddr; 385 break; 386 default: 387 sadata = (uint8_t *)&cr_ptr->remote_ia_address; 388 (void) dapl_os_memcpy(sadata, prd_ptr->hello_msg.hi_saaddr, 389 DAPL_ATS_NBYTES); 390 break; 391 } 392 393 /* EP will be NULL unless RSP service point */ 394 ep_ptr = (DAPL_EP *) sp_ptr->ep_handle; 395 396 if (sp_ptr->psp_flags == DAT_PSP_PROVIDER_FLAG) { 397 /* 398 * Never true for RSP connections 399 * 400 * Create an EP for the user. If we can't allocate an 401 * EP we are out of resources and need to tell the 402 * requestor that we cant help them. 403 */ 404 ia_ptr = sp_ptr->header.owner_ia; 405 ep_ptr = dapl_ep_alloc(ia_ptr, NULL, DAT_FALSE); 406 if (ep_ptr == NULL) { 407 dapls_cr_free(cr_ptr); 408 /* Invoking function will call dapls_ib_cm_reject() */ 409 return (DAT_INSUFFICIENT_RESOURCES); 410 } 411 /* Link the EP onto the IA */ 412 dapl_ia_link_ep(ia_ptr, ep_ptr); 413 } 414 415 cr_ptr->param.local_ep_handle = ep_ptr; 416 417 if (ep_ptr != NULL) { 418 /* Assign valid EP fields: RSP and PSP_PROVIDER_FLAG only */ 419 if (sp_ptr->psp_flags == DAT_PSP_PROVIDER_FLAG) { 420 ep_ptr->param.ep_state = 421 DAT_EP_STATE_TENTATIVE_CONNECTION_PENDING; 422 } else { /* RSP */ 423 dapl_os_assert(sp_ptr->header.handle_type == 424 DAT_HANDLE_TYPE_RSP); 425 ep_ptr->param.ep_state = 426 DAT_EP_STATE_PASSIVE_CONNECTION_PENDING; 427 } 428 ep_ptr->cm_handle = ib_cm_handle; 429 } 430 431 /* Post the event. */ 432 /* assign sp_ptr to union to avoid typecast errors from compilers */ 433 sp_handle.psp_handle = (DAT_PSP_HANDLE)sp_ptr; 434 dat_status = dapls_evd_post_cr_arrival_event( 435 evd_ptr, 436 DAT_CONNECTION_REQUEST_EVENT, 437 sp_handle, 438 (DAT_IA_ADDRESS_PTR)&sp_ptr->header.owner_ia->hca_ptr->hca_address, 439 sp_ptr->conn_qual, 440 (DAT_CR_HANDLE)cr_ptr); 441 if (dat_status != DAT_SUCCESS) { 442 dapls_cr_free(cr_ptr); 443 (void) dapls_ib_reject_connection(ib_cm_handle, 444 IB_CME_LOCAL_FAILURE, sp_ptr); 445 return (DAT_INSUFFICIENT_RESOURCES); 446 } 447 448 /* link the CR onto the SP so we can pick it up later */ 449 dapl_sp_link_cr(sp_ptr, cr_ptr); 450 451 return (DAT_SUCCESS); 452 } 453 454 455 /* 456 * dapli_get_sp_ep 457 * 458 * Passive side of a connection is now fully established. Clean 459 * up resources and obtain the EP pointer associated with a CR in 460 * the SP 461 * 462 * Input: 463 * ib_cm_handle, 464 * sp_ptr 465 * 466 * Output: 467 * none 468 * 469 * Returns 470 * ep_ptr 471 * 472 */ 473 DAPL_EP * 474 dapli_get_sp_ep( 475 IN ib_cm_handle_t ib_cm_handle, 476 IN DAPL_SP *sp_ptr, 477 IN const ib_cm_events_t ib_cm_event) 478 { 479 DAPL_CR *cr_ptr; 480 DAPL_EP *ep_ptr; 481 482 /* 483 * There are potentially multiple connections in progress. Need to 484 * go through the list and find the one we are interested 485 * in. There is no guarantee of order. dapl_sp_search_cr 486 * leaves the CR on the SP queue. 487 */ 488 cr_ptr = dapl_sp_search_cr(sp_ptr, ib_cm_handle); 489 if (cr_ptr == NULL) { 490 dapl_os_assert(0); 491 return (NULL); 492 } 493 494 ep_ptr = (DAPL_EP *)cr_ptr->param.local_ep_handle; 495 496 dapl_os_assert(!(DAPL_BAD_HANDLE(ep_ptr, DAPL_MAGIC_EP)) || 497 ep_ptr->header.magic == DAPL_MAGIC_EP_EXIT); 498 499 if (ib_cm_event == IB_CME_DISCONNECTED || 500 ib_cm_event == IB_CME_DISCONNECTED_ON_LINK_DOWN) { 501 /* Remove the CR from the queue */ 502 dapl_sp_remove_cr(sp_ptr, cr_ptr); 503 /* 504 * Last event, time to clean up and dispose of the resource 505 */ 506 dapls_cr_free(cr_ptr); 507 508 /* 509 * If this SP has been removed from service, free it 510 * up after the last CR is removed 511 */ 512 dapl_os_lock(&sp_ptr->header.lock); 513 if (sp_ptr->listening != DAT_TRUE && 514 sp_ptr->cr_list_count == 0 && 515 sp_ptr->state != DAPL_SP_STATE_FREE) { 516 dapl_dbg_log(DAPL_DBG_TYPE_CM, 517 "--> dapli_get_sp_ep! disconnect dump sp: %p \n", 518 sp_ptr); 519 sp_ptr->state = DAPL_SP_STATE_FREE; 520 dapl_os_unlock(&sp_ptr->header.lock); 521 (void) dapls_ib_remove_conn_listener(sp_ptr-> 522 header.owner_ia, sp_ptr); 523 dapls_ia_unlink_sp((DAPL_IA *)sp_ptr->header.owner_ia, 524 sp_ptr); 525 dapls_sp_free_sp(sp_ptr); 526 } else { 527 dapl_os_unlock(&sp_ptr->header.lock); 528 } 529 } 530 return (ep_ptr); 531 } 532