1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved.
24 */
25
26 /*
27 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
28 * Use is subject to license terms.
29 */
30
31 /*
32 *
33 * MODULE: dapls_cr_callback.c
34 *
35 * PURPOSE: implements passive side connection callbacks
36 *
37 * Description: Accepts asynchronous callbacks from the Communications Manager
38 * for EVDs that have been specified as the connection_evd.
39 *
40 * $Id: dapl_cr_callback.c,v 1.58 2003/08/20 14:55:39 sjs2 Exp $
41 */
42
43 #include "dapl.h"
44 #include "dapl_evd_util.h"
45 #include "dapl_cr_util.h"
46 #include "dapl_ia_util.h"
47 #include "dapl_sp_util.h"
48 #include "dapl_ep_util.h"
49 #include "dapl_adapter_util.h"
50
51
52 /*
53 * Prototypes
54 */
55 DAT_RETURN dapli_connection_request(
56 IN ib_cm_handle_t ib_cm_handle,
57 IN DAPL_SP *sp_ptr,
58 IN DAPL_PRIVATE *prd_ptr,
59 IN DAPL_EVD *evd_ptr);
60
61 DAPL_EP * dapli_get_sp_ep(
62 IN ib_cm_handle_t ib_cm_handle,
63 IN DAPL_SP *sp_ptr,
64 IN const ib_cm_events_t ib_cm_event);
65
66 /*
67 * dapls_cr_callback
68 *
69 * The callback function registered with verbs for passive side of
70 * connection requests. The interface is specified by cm_api.h
71 *
72 *
73 * Input:
74 * ib_cm_handle, Handle to CM
75 * ib_cm_event Specific CM event
76 * instant_data Private data with DAT ADDRESS header
77 * context SP pointer
78 *
79 * Output:
80 * None
81 *
82 */
83 void
dapls_cr_callback(IN ib_cm_handle_t ib_cm_handle,IN const ib_cm_events_t ib_cm_event,IN const void * private_data_ptr,IN const void * context)84 dapls_cr_callback(
85 IN ib_cm_handle_t ib_cm_handle,
86 IN const ib_cm_events_t ib_cm_event,
87 IN const void *private_data_ptr, /* event data */
88 IN const void *context)
89 {
90 DAPL_EP *ep_ptr;
91 DAPL_EVD *evd_ptr;
92 DAPL_SP *sp_ptr;
93 DAPL_PRIVATE *prd_ptr;
94 DAT_EVENT_NUMBER event_type;
95 DAT_RETURN dat_status;
96
97 dapl_dbg_log(DAPL_DBG_TYPE_CM,
98 "--> dapls_cr_callback! context: 0x%p "
99 "event: %d cm_handle 0x%llx magic 0x%x\n",
100 context, ib_cm_event, ib_cm_handle,
101 ((DAPL_HEADER *)context)->magic);
102
103 if (((DAPL_HEADER *)context)->magic == DAPL_MAGIC_INVALID) {
104 return;
105 }
106 /*
107 * Passive side of the connection, context is a SP and
108 * we need to look up the EP.
109 */
110 dapl_os_assert(((DAPL_HEADER *)context)->magic == DAPL_MAGIC_PSP ||
111 ((DAPL_HEADER *)context)->magic == DAPL_MAGIC_RSP);
112 sp_ptr = (DAPL_SP *) context;
113
114 /*
115 * CONNECT_REQUEST events create an event on the PSP
116 * EVD, which will trigger connection processing. The
117 * sequence is:
118 * CONNECT_REQUEST Event to SP
119 * CONNECTED Event to EP
120 * DISCONNECT Event to EP
121 *
122 * Obtain the EP if required and set an event up on the correct EVD.
123 */
124 if (ib_cm_event == IB_CME_CONNECTION_REQUEST_PENDING ||
125 ib_cm_event == IB_CME_CONNECTION_REQUEST_PENDING_PRIVATE_DATA) {
126 ep_ptr = NULL;
127 evd_ptr = sp_ptr->evd_handle;
128 } else {
129 ep_ptr = dapli_get_sp_ep(ib_cm_handle, sp_ptr, ib_cm_event);
130 dapl_os_assert(ep_ptr != NULL);
131 evd_ptr = (DAPL_EVD *) ep_ptr->param.connect_evd_handle;
132 dapl_dbg_log(DAPL_DBG_TYPE_CM,
133 " dapls_cr_callback cont: ep 0x%p evd 0x%p\n",
134 ep_ptr, evd_ptr);
135 }
136
137 prd_ptr = (DAPL_PRIVATE *)private_data_ptr;
138 dat_status = DAT_INTERNAL_ERROR; /* init to ERR */
139
140 switch (ib_cm_event) {
141 case IB_CME_CONNECTION_REQUEST_PENDING:
142 case IB_CME_CONNECTION_REQUEST_PENDING_PRIVATE_DATA: {
143 /*
144 * Requests arriving on a disabled SP are immediatly rejected
145 */
146
147 dapl_os_lock(&sp_ptr->header.lock);
148 if (sp_ptr->listening == DAT_FALSE) {
149 dapl_os_unlock(&sp_ptr->header.lock);
150 dapl_dbg_log(DAPL_DBG_TYPE_CM,
151 "---> dapls_cr_callback: conn event on down SP\n");
152 return;
153 }
154
155 if (sp_ptr->header.handle_type == DAT_HANDLE_TYPE_RSP) {
156 /*
157 * RSP connections only allow a single connection. Close
158 * it down NOW so we reject any further connections.
159 */
160 sp_ptr->listening = DAT_FALSE;
161 }
162 dapl_os_unlock(&sp_ptr->header.lock);
163
164 /*
165 * Only occurs on the passive side of a connection
166 * dapli_connection_request will post the connection
167 * event if appropriate.
168 */
169 dat_status = dapli_connection_request(ib_cm_handle,
170 sp_ptr, prd_ptr, evd_ptr);
171 break;
172 }
173 case IB_CME_CONNECTED: {
174 /*
175 * This is just a notification the connection is now
176 * established, there isn't any private data to deal with.
177 *
178 * Update the EP state and cache a copy of the cm handle,
179 * then let the user know we are ready to go.
180 */
181 dapl_os_lock(&ep_ptr->header.lock);
182 if (ep_ptr->param.ep_state == DAT_EP_STATE_DISCONNECT_PENDING) {
183 /*
184 * If someone pulled the plug on the connection, just
185 * exit
186 */
187 dapl_os_unlock(&ep_ptr->header.lock);
188 dat_status = DAT_SUCCESS;
189 break;
190 }
191 dapls_ib_connected(ep_ptr);
192 ep_ptr->param.ep_state = DAT_EP_STATE_CONNECTED;
193 ep_ptr->cm_handle = ib_cm_handle;
194 dapl_os_unlock(&ep_ptr->header.lock);
195
196 dat_status = dapls_evd_post_connection_event(
197 evd_ptr,
198 DAT_CONNECTION_EVENT_ESTABLISHED,
199 (DAT_HANDLE) ep_ptr,
200 ((DAPL_CR *)ep_ptr->cr_ptr)->param.private_data_size,
201 ((DAPL_CR *)ep_ptr->cr_ptr)->param.private_data);
202 /*
203 * post them to the recv evd now.
204 * there is a race here - if events arrive after we change
205 * the ep state to connected and before we process premature
206 * events
207 */
208 dapls_evd_post_premature_events(ep_ptr);
209 break;
210 }
211 case IB_CME_DISCONNECTED:
212 case IB_CME_DISCONNECTED_ON_LINK_DOWN: {
213 /*
214 * EP is now fully disconnected; initiate any post processing
215 * to reset the underlying QP and get the EP ready for
216 * another connection
217 */
218 dapl_os_lock(&ep_ptr->header.lock);
219 if (ep_ptr->param.ep_state == DAT_EP_STATE_DISCONNECTED) {
220 /* DTO error caused this */
221 event_type = DAT_CONNECTION_EVENT_BROKEN;
222 } else {
223 ep_ptr->param.ep_state = DAT_EP_STATE_DISCONNECTED;
224 dapls_ib_disconnect_clean(ep_ptr, DAT_FALSE,
225 ib_cm_event);
226 event_type = DAT_CONNECTION_EVENT_DISCONNECTED;
227 }
228 dapls_evd_post_premature_events(ep_ptr);
229
230 ep_ptr->cr_ptr = NULL;
231 dapl_os_unlock(&ep_ptr->header.lock);
232
233 /*
234 * If the user has done an ep_free of the EP, we have been
235 * waiting for the disconnect event; just clean it up now.
236 */
237 if (ep_ptr->header.magic == DAPL_MAGIC_EP_EXIT) {
238 (void) dapl_ep_free(ep_ptr);
239 }
240
241 /* If the EP has been freed, the evd_ptr will be NULL */
242 if (evd_ptr != NULL) {
243 dat_status = dapls_evd_post_connection_event(
244 evd_ptr, event_type, (DAT_HANDLE) ep_ptr, 0, 0);
245 }
246
247 break;
248 }
249 case IB_CME_DESTINATION_REJECT:
250 case IB_CME_DESTINATION_REJECT_PRIVATE_DATA:
251 case IB_CME_DESTINATION_UNREACHABLE: {
252 /*
253 * After posting an accept the requesting node has
254 * stopped talking.
255 */
256 dapl_os_lock(&ep_ptr->header.lock);
257 ep_ptr->param.ep_state = DAT_EP_STATE_DISCONNECTED;
258 ep_ptr->cm_handle = IB_INVALID_HANDLE;
259 dapls_ib_disconnect_clean(ep_ptr, DAT_FALSE, ib_cm_event);
260 dapl_os_unlock(&ep_ptr->header.lock);
261 dat_status = dapls_evd_post_connection_event(
262 evd_ptr,
263 DAT_CONNECTION_EVENT_ACCEPT_COMPLETION_ERROR,
264 (DAT_HANDLE) ep_ptr, 0, 0);
265
266 break;
267 }
268 case IB_CME_TOO_MANY_CONNECTION_REQUESTS: {
269 /*
270 * DAPL does not deal with this IB error. There is a
271 * separate OVERFLOW event error if we try to post too many
272 * events, but we don't propagate this provider error. Not
273 * all providers generate this error.
274 */
275 break;
276 }
277 case IB_CME_LOCAL_FAILURE: {
278 ep_ptr->param.ep_state = DAT_EP_STATE_DISCONNECTED;
279 dapls_ib_disconnect_clean(ep_ptr, DAT_FALSE, ib_cm_event);
280 dat_status = dapls_evd_post_connection_event(
281 evd_ptr,
282 DAT_CONNECTION_EVENT_BROKEN,
283 (DAT_HANDLE) ep_ptr, 0, 0);
284
285 break;
286 }
287 case IB_CME_TIMED_OUT: {
288 ep_ptr->param.ep_state = DAT_EP_STATE_DISCONNECTED;
289 dapls_ib_disconnect_clean(ep_ptr, DAT_FALSE, ib_cm_event);
290 dat_status = dapls_evd_post_connection_event(
291 evd_ptr,
292 DAT_CONNECTION_EVENT_TIMED_OUT,
293 (DAT_HANDLE) ep_ptr, 0, 0);
294
295 break;
296 }
297 default:
298 dapl_os_assert(0); /* shouldn't happen */
299 break;
300 }
301
302 if (dat_status != DAT_SUCCESS) {
303 /* The event post failed; take appropriate action. */
304 (void) dapls_ib_reject_connection(ib_cm_handle,
305 IB_CME_LOCAL_FAILURE, sp_ptr);
306 return;
307 }
308 }
309
310
311 /*
312 * dapli_connection_request
313 *
314 * Process a connection request on the Passive side of a connection.
315 * Create a CR record and link it on to the SP so we can update it
316 * and free it later. Create an EP if specified by the PSP flags.
317 *
318 * Input:
319 * ib_cm_handle,
320 * sp_ptr
321 * event_ptr
322 * prd_ptr
323 *
324 * Output:
325 * None
326 *
327 * Returns
328 * DAT_INSUFFICIENT_RESOURCES
329 * DAT_SUCCESS
330 *
331 */
332 DAT_RETURN
dapli_connection_request(IN ib_cm_handle_t ib_cm_handle,IN DAPL_SP * sp_ptr,IN DAPL_PRIVATE * prd_ptr,IN DAPL_EVD * evd_ptr)333 dapli_connection_request(
334 IN ib_cm_handle_t ib_cm_handle,
335 IN DAPL_SP *sp_ptr,
336 IN DAPL_PRIVATE *prd_ptr,
337 IN DAPL_EVD *evd_ptr)
338 {
339 DAT_RETURN dat_status;
340 DAPL_CR *cr_ptr;
341 DAPL_EP *ep_ptr;
342 DAPL_IA *ia_ptr;
343 DAT_SP_HANDLE sp_handle;
344 struct sockaddr_in *sv4;
345 struct sockaddr_in6 *sv6;
346 uint8_t *sadata;
347 DAT_COUNT length;
348
349 cr_ptr = dapls_cr_alloc(sp_ptr->header.owner_ia);
350 if (cr_ptr == NULL) {
351 /* Invoking function will call dapls_ib_cm_reject() */
352 return (DAT_INSUFFICIENT_RESOURCES);
353 }
354
355 /*
356 * Set up the CR
357 */
358 cr_ptr->sp_ptr = sp_ptr; /* maintain sp_ptr in case of reject */
359 cr_ptr->ib_cm_handle = ib_cm_handle;
360 /*
361 * Copy the remote address and private data out of the private_data
362 * payload and put them in a local structure
363 */
364 cr_ptr->param.private_data = cr_ptr->private_data;
365 cr_ptr->param.remote_ia_address_ptr =
366 (DAT_IA_ADDRESS_PTR)&cr_ptr->remote_ia_address;
367 cr_ptr->param.remote_port_qual =
368 (DAT_PORT_QUAL) prd_ptr->hello_msg.hi_port;
369 length = (DAT_COUNT) prd_ptr->hello_msg.hi_clen;
370 cr_ptr->param.private_data_size = length;
371 (void) dapl_os_memcpy(cr_ptr->private_data,
372 prd_ptr->private_data, length);
373 switch (prd_ptr->hello_msg.hi_ipv) {
374 case AF_INET:
375 sv4 = (struct sockaddr_in *)&cr_ptr->remote_ia_address;
376 sv4->sin_family = AF_INET;
377 sv4->sin_port = prd_ptr->hello_msg.hi_port;
378 sv4->sin_addr = prd_ptr->hello_msg.hi_v4ipaddr;
379 break;
380 case AF_INET6:
381 sv6 = (struct sockaddr_in6 *)&cr_ptr->remote_ia_address;
382 sv6->sin6_family = AF_INET6;
383 sv6->sin6_port = prd_ptr->hello_msg.hi_port;
384 sv6->sin6_addr = prd_ptr->hello_msg.hi_v6ipaddr;
385 break;
386 default:
387 sadata = (uint8_t *)&cr_ptr->remote_ia_address;
388 (void) dapl_os_memcpy(sadata, prd_ptr->hello_msg.hi_saaddr,
389 DAPL_ATS_NBYTES);
390 break;
391 }
392
393 /* EP will be NULL unless RSP service point */
394 ep_ptr = (DAPL_EP *) sp_ptr->ep_handle;
395
396 if (sp_ptr->psp_flags == DAT_PSP_PROVIDER_FLAG) {
397 /*
398 * Never true for RSP connections
399 *
400 * Create an EP for the user. If we can't allocate an
401 * EP we are out of resources and need to tell the
402 * requestor that we cant help them.
403 */
404 ia_ptr = sp_ptr->header.owner_ia;
405 ep_ptr = dapl_ep_alloc(ia_ptr, NULL, DAT_FALSE);
406 if (ep_ptr == NULL) {
407 dapls_cr_free(cr_ptr);
408 /* Invoking function will call dapls_ib_cm_reject() */
409 return (DAT_INSUFFICIENT_RESOURCES);
410 }
411 /* Link the EP onto the IA */
412 dapl_ia_link_ep(ia_ptr, ep_ptr);
413 }
414
415 cr_ptr->param.local_ep_handle = ep_ptr;
416
417 if (ep_ptr != NULL) {
418 /* Assign valid EP fields: RSP and PSP_PROVIDER_FLAG only */
419 if (sp_ptr->psp_flags == DAT_PSP_PROVIDER_FLAG) {
420 ep_ptr->param.ep_state =
421 DAT_EP_STATE_TENTATIVE_CONNECTION_PENDING;
422 } else { /* RSP */
423 dapl_os_assert(sp_ptr->header.handle_type ==
424 DAT_HANDLE_TYPE_RSP);
425 ep_ptr->param.ep_state =
426 DAT_EP_STATE_PASSIVE_CONNECTION_PENDING;
427 }
428 ep_ptr->cm_handle = ib_cm_handle;
429 }
430
431 /* Post the event. */
432 /* assign sp_ptr to union to avoid typecast errors from compilers */
433 sp_handle.psp_handle = (DAT_PSP_HANDLE)sp_ptr;
434 dat_status = dapls_evd_post_cr_arrival_event(
435 evd_ptr,
436 DAT_CONNECTION_REQUEST_EVENT,
437 sp_handle,
438 (DAT_IA_ADDRESS_PTR)&sp_ptr->header.owner_ia->hca_ptr->hca_address,
439 sp_ptr->conn_qual,
440 (DAT_CR_HANDLE)cr_ptr);
441 if (dat_status != DAT_SUCCESS) {
442 dapls_cr_free(cr_ptr);
443 (void) dapls_ib_reject_connection(ib_cm_handle,
444 IB_CME_LOCAL_FAILURE, sp_ptr);
445 return (DAT_INSUFFICIENT_RESOURCES);
446 }
447
448 /* link the CR onto the SP so we can pick it up later */
449 dapl_sp_link_cr(sp_ptr, cr_ptr);
450
451 return (DAT_SUCCESS);
452 }
453
454
455 /*
456 * dapli_get_sp_ep
457 *
458 * Passive side of a connection is now fully established. Clean
459 * up resources and obtain the EP pointer associated with a CR in
460 * the SP
461 *
462 * Input:
463 * ib_cm_handle,
464 * sp_ptr
465 *
466 * Output:
467 * none
468 *
469 * Returns
470 * ep_ptr
471 *
472 */
473 DAPL_EP *
dapli_get_sp_ep(IN ib_cm_handle_t ib_cm_handle,IN DAPL_SP * sp_ptr,IN const ib_cm_events_t ib_cm_event)474 dapli_get_sp_ep(
475 IN ib_cm_handle_t ib_cm_handle,
476 IN DAPL_SP *sp_ptr,
477 IN const ib_cm_events_t ib_cm_event)
478 {
479 DAPL_CR *cr_ptr;
480 DAPL_EP *ep_ptr;
481
482 /*
483 * There are potentially multiple connections in progress. Need to
484 * go through the list and find the one we are interested
485 * in. There is no guarantee of order. dapl_sp_search_cr
486 * leaves the CR on the SP queue.
487 */
488 cr_ptr = dapl_sp_search_cr(sp_ptr, ib_cm_handle);
489 if (cr_ptr == NULL) {
490 dapl_os_assert(0);
491 return (NULL);
492 }
493
494 ep_ptr = (DAPL_EP *)cr_ptr->param.local_ep_handle;
495
496 dapl_os_assert(!(DAPL_BAD_HANDLE(ep_ptr, DAPL_MAGIC_EP)) ||
497 ep_ptr->header.magic == DAPL_MAGIC_EP_EXIT);
498
499 if (ib_cm_event == IB_CME_DISCONNECTED ||
500 ib_cm_event == IB_CME_DISCONNECTED_ON_LINK_DOWN) {
501 /* Remove the CR from the queue */
502 dapl_sp_remove_cr(sp_ptr, cr_ptr);
503 /*
504 * Last event, time to clean up and dispose of the resource
505 */
506 dapls_cr_free(cr_ptr);
507
508 /*
509 * If this SP has been removed from service, free it
510 * up after the last CR is removed
511 */
512 dapl_os_lock(&sp_ptr->header.lock);
513 if (sp_ptr->listening != DAT_TRUE &&
514 sp_ptr->cr_list_count == 0 &&
515 sp_ptr->state != DAPL_SP_STATE_FREE) {
516 dapl_dbg_log(DAPL_DBG_TYPE_CM,
517 "--> dapli_get_sp_ep! disconnect dump sp: %p \n",
518 sp_ptr);
519 sp_ptr->state = DAPL_SP_STATE_FREE;
520 dapl_os_unlock(&sp_ptr->header.lock);
521 (void) dapls_ib_remove_conn_listener(sp_ptr->
522 header.owner_ia, sp_ptr);
523 dapls_ia_unlink_sp((DAPL_IA *)sp_ptr->header.owner_ia,
524 sp_ptr);
525 dapls_sp_free_sp(sp_ptr);
526 } else {
527 dapl_os_unlock(&sp_ptr->header.lock);
528 }
529 }
530 return (ep_ptr);
531 }
532