1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25
26 #include <sys/ib/mgt/ibmf/ibmf_saa_impl.h>
27 #include <sys/ib/mgt/ibmf/ibmf_saa_utils.h>
28
29 /* Global sa_access State Pointer */
30 saa_state_t *saa_statep;
31 _NOTE(READ_ONLY_DATA(saa_statep))
32
33 extern int ibmf_trace_level;
34
35 extern int ibmf_taskq_max_tasks;
36
37 static int
38 ibmf_saa_impl_new_smlid_retry(saa_port_t *saa_portp, ibmf_msg_t *msgp,
39 ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_arg, int transport_flags);
40 static int
41 ibmf_saa_impl_revert_to_qp1(saa_port_t *saa_portp, ibmf_msg_t *msgp,
42 ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_args, int transport_flags);
43 static int
44 ibmf_saa_check_sa_and_retry(saa_port_t *saa_portp, ibmf_msg_t *msgp,
45 ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_arg,
46 hrtime_t trans_send_time, int transport_flags);
47 static int ibmf_saa_impl_init_msg(saa_impl_trans_info_t *trans_info,
48 boolean_t sleep_flag, ibmf_msg_t **msgp, uint32_t *transport_flagsp,
49 ibmf_retrans_t *ibmf_retransp);
50 static int ibmf_saa_must_purge(saa_port_t *saa_portp);
51 static void ibmf_saa_impl_invalidate_port(saa_port_t *saa_portp);
52 static void ibmf_saa_impl_destroy_port(saa_port_t *saa_portp);
53 static void ibmf_saa_impl_uninit_kstats(saa_port_t *saa_portp);
54 static void ibmf_saa_impl_get_cpi_cb(void *arg, size_t length, char *buffer,
55 int status);
56 static void ibmf_saa_impl_async_event_cb(ibmf_handle_t ibmf_handle,
57 void *clnt_private, ibmf_async_event_t event_type);
58 static void ibmf_saa_impl_port_up(ib_guid_t ci_guid, uint8_t port_num);
59 static void ibmf_saa_impl_port_down(ib_guid_t ci_guid, uint8_t port_num);
60 static void ibmf_saa_impl_port_chg(ibt_async_event_t *event);
61 static void ibmf_saa_impl_client_rereg(ib_guid_t ci_guid, uint8_t port_num);
62 static void ibmf_saa_impl_hca_detach(saa_port_t *saa_removed);
63 static void ibmf_saa_impl_prepare_response(ibmf_handle_t ibmf_handle,
64 ibmf_msg_t *msgp, boolean_t ignore_data, int *status, void **result,
65 size_t *length, boolean_t sleep_flag);
66 static int ibmf_saa_impl_check_sa_support(uint16_t cap_mask, uint16_t attr_id);
67 static uint_t ibmf_saa_impl_get_attr_id_length(uint16_t attr_id);
68 static void ibmf_saa_impl_free_msg(ibmf_handle_t ibmf_hdl, ibmf_msg_t *msgp);
69 static int ibmf_saa_impl_get_port_guid(ibt_hca_portinfo_t *ibt_portinfop,
70 ib_guid_t *guid_ret);
71 static void ibmf_saa_impl_set_transaction_params(saa_port_t *saa_portp,
72 ibt_hca_portinfo_t *portinfop);
73 static void ibmf_saa_impl_update_sa_address_info(saa_port_t *saa_portp,
74 ibmf_msg_t *msgp);
75 static int ibmf_saa_impl_ibmf_unreg(saa_port_t *saa_portp);
76
77 int ibmf_saa_max_wait_time = IBMF_SAA_MAX_WAIT_TIME_IN_SECS;
78 int ibmf_saa_trans_wait_time = IBMF_SAA_TRANS_WAIT_TIME_IN_SECS;
79 int ibmf_saa_max_resp_time = IBMF_SAA_MAX_RESP_TIME;
80 int ibmf_saa_max_subnet_timeout = IBMF_SAA_MAX_SUBNET_TIMEOUT;
81 int ibmf_saa_retrans_retries = IBMF_SAA_RETRANS_RETRIES;
82
83 /*
84 * ibmf_saa_impl_init:
85 * Allocates memory for the ibmf_saa state structure and initializes the taskq.
86 * Called from the modules init() routine.
87 *
88 * Input Arguments
89 * none
90 *
91 * Output Arguments
92 * none
93 *
94 * Returns
95 * IBMF_NO_RESOURCES if taskq could not be created.
96 * IBMF_SUCCESS on success
97 *
98 */
99 int
ibmf_saa_impl_init()100 ibmf_saa_impl_init()
101 {
102 int res;
103
104 /* CONSTCOND */
105 ASSERT(NO_COMPETING_THREADS);
106
107 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_init_start,
108 IBMF_TNF_TRACE, "", "ibmf_saa_impl_init() enter\n");
109
110 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*saa_statep))
111
112 saa_statep = kmem_zalloc(sizeof (saa_state_t), KM_SLEEP);
113
114 /* create taskq for notifying event subscribers */
115 saa_statep->saa_event_taskq = taskq_create(
116 "ibmf_saa_event_taskq", IBMF_TASKQ_NTHREADS,
117 MINCLSYSPRI, 1, ibmf_taskq_max_tasks, TASKQ_DYNAMIC |
118 TASKQ_PREPOPULATE);
119 if (saa_statep->saa_event_taskq == NULL) {
120
121 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L4,
122 ibmf_saa_impl_init_end_err,
123 IBMF_TNF_TRACE, "", "ibmf_saa_impl_init(): %s\n",
124 tnf_string, msg, "event taskq create failed");
125
126 kmem_free(saa_statep, sizeof (saa_state_t));
127
128 res = IBMF_NO_RESOURCES;
129
130 goto bail;
131 }
132
133 mutex_init(&saa_statep->saa_port_list_mutex, NULL, MUTEX_DRIVER,
134 NULL);
135
136 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*saa_statep))
137
138 res = IBMF_SUCCESS;
139 bail:
140
141 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_init_end,
142 IBMF_TNF_TRACE, "", "ibmf_saa_impl_init() exit: status = %d\n",
143 tnf_int, res, res);
144
145 return (res);
146 }
147
148 /*
149 * ibmf_saa_impl_fini:
150 * If there are no registered clients, cleans up all memory associated with the
151 * state, including each of the port list entries.
152 * Called from the modules fini() routine.
153 *
154 * Input Arguments
155 * none
156 *
157 * Output Arguments
158 * none
159 *
160 * Returns
161 * EBUSY if there are outstanding transactions or registered clients
162 * 0 if cleanup was sucessfull
163 *
164 */
165 int
ibmf_saa_impl_fini()166 ibmf_saa_impl_fini()
167 {
168 int ret = 0;
169 saa_port_t *saa_portp;
170 saa_port_t *next;
171
172 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_fini_start,
173 IBMF_TNF_TRACE, "", "ibmf_saa_impl_fini() enter\n");
174
175 /* make sure there are no registered clients */
176 mutex_enter(&saa_statep->saa_port_list_mutex);
177
178 saa_portp = saa_statep->saa_port_list;
179 while (saa_portp != NULL) {
180
181 mutex_enter(&saa_portp->saa_pt_mutex);
182
183 if (saa_portp->saa_pt_reference_count > 0) {
184
185 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
186 ibmf_saa_impl_fini_err, IBMF_TNF_ERROR, "",
187 "ibmf_saa_impl_fini: %s, port %016" PRIx64 "\n",
188 tnf_string, msg,
189 "cannot unload ibmf_saa. Client on port still"
190 " registered", tnf_opaque, port,
191 saa_portp->saa_pt_port_guid);
192
193 mutex_exit(&saa_portp->saa_pt_mutex);
194
195 mutex_exit(&saa_statep->saa_port_list_mutex);
196
197 ret = EBUSY;
198 goto bail;
199 }
200
201 /* make sure there are no outstanding transactions */
202
203 if (saa_portp->saa_pt_num_outstanding_trans > 0) {
204
205 IBMF_TRACE_3(IBMF_TNF_NODEBUG, DPRINT_L1,
206 ibmf_saa_impl_fini_err, IBMF_TNF_ERROR, "",
207 "ibmf_saa_impl_fini: %s, port = %016" PRIx64
208 ", num transactions = %d\n",
209 tnf_string, msg, "Cannot unload ibmf_saa."
210 " Outstanding transactions on port.",
211 tnf_opaque, port,
212 saa_portp->saa_pt_port_guid,
213 tnf_uint, outstanding_transactions,
214 saa_portp->saa_pt_num_outstanding_trans);
215
216 mutex_exit(&saa_portp->saa_pt_mutex);
217
218 mutex_exit(&saa_statep->saa_port_list_mutex);
219
220 ret = EBUSY;
221 goto bail;
222 }
223
224 mutex_exit(&saa_portp->saa_pt_mutex);
225
226 saa_portp = saa_portp->next;
227 }
228
229 mutex_exit(&saa_statep->saa_port_list_mutex);
230
231 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(saa_statep->saa_port_list,
232 *saa_portp))
233
234 /*
235 * no more clients nor pending transaction:
236 * unregister ibmf and destroy port entries
237 */
238 while (saa_statep->saa_port_list != NULL) {
239
240 saa_portp = saa_statep->saa_port_list;
241 next = saa_portp->next;
242
243 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
244 ibmf_saa_impl_fini, IBMF_TNF_TRACE, "",
245 "ibmf_saa_impl_fini: %s, prefix = %016" PRIx64 "\n",
246 tnf_string, msg, "deinitializing port",
247 tnf_opaque, port_guid, saa_portp->saa_pt_port_guid);
248
249 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*saa_portp))
250
251 mutex_enter(&saa_portp->saa_pt_mutex);
252
253 /* unregister from ibmf */
254 if (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_READY) {
255
256 mutex_exit(&saa_portp->saa_pt_mutex);
257
258 if (ibmf_saa_impl_ibmf_unreg(saa_portp)
259 != IBMF_SUCCESS) {
260 ret = EBUSY;
261 goto bail;
262 }
263 } else
264 mutex_exit(&saa_portp->saa_pt_mutex);
265
266 ibmf_saa_impl_destroy_port(saa_portp);
267
268 saa_statep->saa_port_list = next;
269 }
270
271 taskq_destroy(saa_statep->saa_event_taskq);
272
273 mutex_destroy(&saa_statep->saa_port_list_mutex);
274
275 kmem_free(saa_statep, sizeof (saa_state_t));
276
277 bail:
278 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_fini_end,
279 IBMF_TNF_TRACE, "", "ibmf_saa_impl_fini() exit\n");
280
281 return (ret);
282 }
283
284 /*
285 * ibmf_saa_is_valid
286 * Returns true the entry is valid.
287 *
288 * Input Arguments
289 * saa_portp pointer to state structure
290 * add_ref if B_TRUE ref count is incremented on a valid portp
291 *
292 * Output Arguments
293 * none
294 *
295 * Returns
296 * B_TRUE if entry was in a valid state, B_FALSE otherwise
297 */
298 boolean_t
ibmf_saa_is_valid(saa_port_t * saa_portp,int add_ref)299 ibmf_saa_is_valid(saa_port_t *saa_portp, int add_ref)
300 {
301 boolean_t is_valid = B_TRUE;
302
303 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_is_valid_start,
304 IBMF_TNF_TRACE, "", "ibmf_saa_is_valid() enter\n");
305
306 mutex_enter(&saa_portp->saa_pt_mutex);
307
308 if (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_INVALID ||
309 saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_PURGING) {
310
311 is_valid = B_FALSE;
312
313 } else if (add_ref == B_TRUE) {
314 /*
315 * increment reference count here to ensure that
316 * entry does not get purged behind our backs
317 */
318 saa_portp->saa_pt_reference_count++;
319 }
320 mutex_exit(&saa_portp->saa_pt_mutex);
321
322 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_is_valid_end,
323 IBMF_TNF_TRACE, "", "ibmf_saa_is_valid() exit\n");
324
325 return (is_valid);
326 }
327
328 /*
329 * ibmf_saa_must_purge
330 * Determines if we can purge a portp (remove it from the list) based on the
331 * state and number of clients
332 *
333 * Input Arguments
334 * saa_portp pointer to state structure
335 *
336 * Output Arguments
337 * none
338 *
339 * Returns
340 * B_TRUE if the entry can be removed, B_FALSE otherwise
341 */
342 static int
ibmf_saa_must_purge(saa_port_t * saa_portp)343 ibmf_saa_must_purge(saa_port_t *saa_portp)
344 {
345 int must_purge = B_FALSE;
346
347 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_must_purge_start,
348 IBMF_TNF_TRACE, "", "ibmf_saa_must_purge() enter\n");
349
350 mutex_enter(&saa_portp->saa_pt_mutex);
351
352 if (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_INVALID &&
353 saa_portp->saa_pt_reference_count == 0) {
354
355 saa_portp->saa_pt_state = IBMF_SAA_PORT_STATE_PURGING;
356 must_purge = B_TRUE;
357 }
358
359 mutex_exit(&saa_portp->saa_pt_mutex);
360
361 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_must_purge_end,
362 IBMF_TNF_TRACE, "", "ibmf_saa_must_purge() exit\n");
363
364 return (must_purge);
365 }
366
367
368 /*
369 * ibmf_saa_impl_purge:
370 * Removes invalid port state entries from the list
371 *
372 * Input Arguments
373 * none
374 *
375 * Output Arguments
376 * none
377 *
378 * Returns
379 * void
380 */
381 void
ibmf_saa_impl_purge()382 ibmf_saa_impl_purge()
383 {
384 saa_port_t *cur_portp = NULL;
385 saa_port_t *prev_portp = NULL;
386 saa_port_t *rem_portp = NULL;
387
388 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_purge_start,
389 IBMF_TNF_TRACE, "", "ibmf_saa_impl_purge() enter\n");
390
391 mutex_enter(&saa_statep->saa_port_list_mutex);
392
393 cur_portp = saa_statep->saa_port_list;
394 prev_portp = cur_portp;
395
396 while (cur_portp != NULL) {
397
398 if (ibmf_saa_must_purge(cur_portp) == B_TRUE) {
399
400 rem_portp = cur_portp;
401
402 /* unlink entry */
403 if (cur_portp == saa_statep->saa_port_list) {
404
405 saa_statep->saa_port_list = cur_portp->next;
406 cur_portp = saa_statep->saa_port_list;
407 prev_portp = cur_portp;
408
409 } else {
410
411 prev_portp->next = cur_portp->next;
412 cur_portp = cur_portp->next;
413 }
414
415 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*rem_portp))
416
417 /* destroy entry */
418 ASSERT(rem_portp != NULL);
419 ibmf_saa_impl_destroy_port(rem_portp);
420
421 } else {
422
423 prev_portp = cur_portp;
424 cur_portp = cur_portp->next;
425 }
426 }
427
428 mutex_exit(&saa_statep->saa_port_list_mutex);
429
430 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_purge_end,
431 IBMF_TNF_TRACE, "", "ibmf_saa_impl_purge() exit\n");
432 }
433
434 /*
435 * saa_impl_add_client:
436 * Adds a client for a particular portp. Reference count has been incremented
437 * before this call. It is decremented by saa_impl_add_client() if the call
438 * fails.
439 *
440 * Input Arguments
441 * none
442 *
443 * Output Arguments
444 * none
445 *
446 * Returns
447 * IBMF_BUSY if there are already too many clients registered,
448 * IBMF_BAD_PORT_STATE if the port is invalid (generally because a previous
449 * client failed during registration for this port)
450 * IBMF_SUCCESS otherwise
451 */
452 int
ibmf_saa_impl_add_client(saa_port_t * saa_portp)453 ibmf_saa_impl_add_client(saa_port_t *saa_portp)
454 {
455 int status = IBMF_SUCCESS;
456
457 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_add_client_start,
458 IBMF_TNF_TRACE, "", "ibmf_saa_impl_add_client() enter\n");
459
460 mutex_enter(&saa_portp->saa_pt_mutex);
461
462 /*
463 * check that we don't exceed max clients
464 */
465 if (saa_portp->saa_pt_reference_count >
466 SAA_MAX_CLIENTS_PER_PORT) {
467
468 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
469 ibmf_saa_impl_add_client_err, IBMF_TNF_ERROR, "",
470 "ibmf_saa_impl_add_client: %s, num_reg_clients %d\n",
471 tnf_string, msg, "too many clients registered for"
472 " port", tnf_uint, num_reg_clients,
473 saa_portp->saa_pt_reference_count);
474
475 status = IBMF_BUSY;
476 goto bail;
477 }
478
479 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
480 ibmf_saa_impl_add_client, IBMF_TNF_TRACE, "",
481 "ibmf_saa_impl_add_client: num_registered_clients %d\n",
482 tnf_uint, num_registered_clients,
483 saa_portp->saa_pt_reference_count);
484
485 /*
486 * wait until anyone who is currently registering
487 * this port with ibmf is done
488 */
489 while (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_REGISTERING) {
490
491 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
492 ibmf_saa_impl_add_client, IBMF_TNF_TRACE, "",
493 "ibmf_saa_impl_add_client: %s\n",
494 tnf_string, msg, "someone is registering. waiting"
495 " for them to finish");
496
497 cv_wait(&saa_portp->saa_pt_ibmf_reg_cv,
498 &saa_portp->saa_pt_mutex);
499
500 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
501 ibmf_saa_impl_add_client,
502 IBMF_TNF_TRACE, "", "ibmf_saa_impl_add_client: %s\n",
503 tnf_string, msg, "done waiting");
504 }
505
506 /*
507 * if port isn't ready here, fail.
508 */
509 if (saa_portp->saa_pt_state != IBMF_SAA_PORT_STATE_READY) {
510
511 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
512 ibmf_saa_impl_add_client_err, IBMF_TNF_ERROR,
513 "", "ibmf_saa_impl_add_client: %s\n",
514 tnf_string, msg, "port state not ready,"
515 " removing client.");
516
517 status = IBMF_BAD_PORT_STATE;
518 goto bail;
519 }
520
521 bail:
522 mutex_exit(&saa_portp->saa_pt_mutex);
523
524 if (status != IBMF_SUCCESS) {
525
526 mutex_enter(&saa_portp->saa_pt_kstat_mutex);
527
528 IBMF_SAA_ADD32_KSTATS(saa_portp,
529 clients_reg_failed, 1);
530
531 mutex_exit(&saa_portp->saa_pt_kstat_mutex);
532
533 /* decrementing refcount is last thing we do on entry */
534
535 mutex_enter(&saa_portp->saa_pt_mutex);
536
537 ASSERT(saa_portp->saa_pt_reference_count > 0);
538 saa_portp->saa_pt_reference_count--;
539
540 mutex_exit(&saa_portp->saa_pt_mutex);
541 }
542
543 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
544 ibmf_saa_impl_add_client_end, IBMF_TNF_TRACE, "",
545 "ibmf_saa_impl_add_client() exit\n");
546
547 return (status);
548 }
549
550 /*
551 * ibmf_saa_impl_create_port()
552 * Create port entry with mimimal inits because
553 * we're holding the list mutex: NO BLOCKING CALLS HERE, please.
554 *
555 * Initialize port state to "registering", so that clients accessing
556 * same port concurrently will wait for the end of the ibmf registration.
557 * Note: this thread will access port members without locking mutex.
558 *
559 * Input Arguments
560 * pt_guid guid of port
561 *
562 * Output Arguments
563 * saa_portpp pointer to new saa_portp structure
564 *
565 * Returns
566 * IBMF_NO_MEMORY if memory could not be allocated
567 * IBMF_SUCCESS otherwise
568 */
569 int
ibmf_saa_impl_create_port(ib_guid_t pt_guid,saa_port_t ** saa_portpp)570 ibmf_saa_impl_create_port(ib_guid_t pt_guid, saa_port_t **saa_portpp)
571 {
572 int status = IBMF_SUCCESS;
573 saa_port_t *saa_portp = NULL;
574
575 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_create_port_start,
576 IBMF_TNF_TRACE, "", "ibmf_saa_impl_create_port:"
577 " guid %016" PRIx64 "\n",
578 tnf_opaque, port_guid, pt_guid);
579
580 ASSERT(MUTEX_HELD(&saa_statep->saa_port_list_mutex));
581
582 /* create & initialize new port */
583 saa_portp = kmem_zalloc(sizeof (saa_port_t), KM_NOSLEEP);
584
585 if (saa_portp == NULL) {
586
587 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
588 ibmf_sa_session_open_err, IBMF_TNF_ERROR, "",
589 "ibmf_saa_impl_create_port: %s\n",
590 tnf_string, msg, "could not allocate memory for "
591 "new port");
592
593 status = IBMF_NO_MEMORY;
594 goto bail;
595 }
596
597 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_sa_session_open,
598 IBMF_TNF_TRACE, "", "ibmf_saa_impl_create_port: %s\n",
599 tnf_string, msg, "first client registering, initializing");
600
601 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*saa_portp))
602
603 /* tell everyone that kstats are not initialized */
604 saa_portp->saa_pt_kstatp = NULL;
605
606 /*
607 * set up mutexe and state variable to indicate to
608 * other clients that were currently in the process of
609 * setting up the port data. This will prevent a subsequent
610 * client from trying to to register with ibmf before the
611 * port data has been initialized.
612 */
613 mutex_init(&saa_portp->saa_pt_mutex, NULL, MUTEX_DRIVER, NULL);
614 cv_init(&saa_portp->saa_pt_ibmf_reg_cv, NULL, CV_DRIVER, NULL);
615
616 saa_portp->saa_pt_state = IBMF_SAA_PORT_STATE_REGISTERING;
617
618 /* create other mutexes */
619 mutex_init(&saa_portp->saa_pt_kstat_mutex, NULL, MUTEX_DRIVER, NULL);
620
621 mutex_init(&saa_portp->saa_pt_event_sub_mutex, NULL, MUTEX_DRIVER,
622 NULL);
623
624 /*
625 * clients assume all arrive; set mask to this so we only notify
626 * if something failed
627 */
628 saa_portp->saa_pt_event_sub_last_success_mask =
629 IBMF_SAA_PORT_EVENT_SUB_ALL_ARRIVE;
630
631 /*
632 * set port_guid now so any immediately subsequent clients
633 * registering on this port, guid will know we're already here
634 */
635 saa_portp->saa_pt_port_guid = pt_guid;
636 saa_portp->saa_pt_reference_count = 1;
637 saa_portp->saa_pt_current_tid = pt_guid << 32;
638
639 saa_portp->saa_pt_redirect_active = B_FALSE;
640
641 /* set sa_uptime now in case we never receive anything from SA */
642 saa_portp->saa_pt_sa_uptime = gethrtime();
643
644 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*saa_portp))
645
646 /* Set new pointer in caller's */
647 *saa_portpp = saa_portp;
648
649 bail:
650 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_create_port_end,
651 IBMF_TNF_TRACE, "", "ibmf_saa_impl_create_port() exit\n");
652
653 return (status);
654 }
655
656 /*
657 * ibmf_saa_impl_invalidate_port:
658 * invalidates port entry (assumes exist) and deletes kstat object
659 * kstat object is destroyed in order to allow creating port entry
660 * even if this entry is not purged
661 */
662 static void
ibmf_saa_impl_invalidate_port(saa_port_t * saa_portp)663 ibmf_saa_impl_invalidate_port(saa_port_t *saa_portp)
664 {
665 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
666 ibmf_saa_impl_invalidate_port_start,
667 IBMF_TNF_TRACE, "", "ibmf_saa_impl_invalidate_port() enter\n");
668
669 ASSERT(saa_portp != NULL);
670 ASSERT(MUTEX_HELD(&saa_portp->saa_pt_mutex));
671
672 saa_portp->saa_pt_state = IBMF_SAA_PORT_STATE_INVALID;
673 ibmf_saa_impl_uninit_kstats(saa_portp);
674
675 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
676 ibmf_saa_impl_invalidate_port_end,
677 IBMF_TNF_TRACE, "", "ibmf_saa_impl_invalidate_port() exit\n");
678 }
679
680 /*
681 * ibmf_saa_impl_destroy_port:
682 * Frees the resources associated with an saa_portp structure. Assumes the
683 * saa_portp exists
684 *
685 * Input Arguments
686 * saa_portp pointer to saa_portp structure
687 *
688 * Output Arguments
689 * none
690 *
691 * Returns
692 * void
693 */
694 static void
ibmf_saa_impl_destroy_port(saa_port_t * saa_portp)695 ibmf_saa_impl_destroy_port(saa_port_t *saa_portp)
696 {
697 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_destroy_start,
698 IBMF_TNF_TRACE, "", "ibmf_saa_impl_destroy() enter\n");
699
700 ASSERT(saa_portp != NULL);
701
702 _NOTE(ASSUMING_PROTECTED(*saa_portp))
703
704 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
705 ibmf_saa_impl_destroy, IBMF_TNF_TRACE, "",
706 "ibmf_saa_impl_destroy(): destroying port_guid %016" PRIx64 "\n",
707 tnf_opaque, port_guid, saa_portp->saa_pt_port_guid);
708
709 ibmf_saa_impl_uninit_kstats(saa_portp);
710
711 /* uninit synchronization variables used for registration */
712 mutex_destroy(&saa_portp->saa_pt_mutex);
713 cv_destroy(&saa_portp->saa_pt_ibmf_reg_cv);
714
715 mutex_destroy(&saa_portp->saa_pt_event_sub_mutex);
716 mutex_destroy(&saa_portp->saa_pt_kstat_mutex);
717
718 kmem_free(saa_portp, sizeof (saa_port_t));
719
720 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_destroy_end,
721 IBMF_TNF_TRACE, "", "ibmf_saa_impl_destroy() exit\n");
722 }
723
724 /*
725 * ibmf_saa_impl_init_kstats:
726 * Create kstats structure. Should be called when memory is alloced for a new
727 * port entry.
728 */
729 int
ibmf_saa_impl_init_kstats(saa_port_t * saa_portp)730 ibmf_saa_impl_init_kstats(saa_port_t *saa_portp)
731 {
732 char buf[128];
733 ibmf_saa_kstat_t *ksp;
734
735 _NOTE(ASSUMING_PROTECTED(saa_portp->saa_pt_kstatp))
736
737 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
738 ibmf_saa_impl_init_kstats_start,
739 IBMF_TNF_TRACE, "", "ibmf_saa_impl_init_kstats() enter\n");
740
741 /* set up kstats structure */
742 (void) sprintf(buf, "ibmf_saa_%016" PRIx64 "_stat",
743 saa_portp->saa_pt_port_guid);
744
745 saa_portp->saa_pt_kstatp = kstat_create("ibmf_saa",
746 0, buf, "misc", KSTAT_TYPE_NAMED,
747 sizeof (ibmf_saa_kstat_t) / sizeof (kstat_named_t),
748 KSTAT_FLAG_WRITABLE);
749
750 if (saa_portp->saa_pt_kstatp == NULL)
751 return (IBMF_NO_RESOURCES);
752
753 ksp = (ibmf_saa_kstat_t *)saa_portp->saa_pt_kstatp->ks_data;
754
755 kstat_named_init(&ksp->clients_registered,
756 "clients_registered", KSTAT_DATA_UINT32);
757
758 kstat_named_init(&ksp->clients_reg_failed,
759 "clients_reg_failed", KSTAT_DATA_UINT32);
760
761 kstat_named_init(&ksp->outstanding_requests,
762 "outstanding_requests", KSTAT_DATA_UINT32);
763
764 kstat_named_init(&ksp->total_requests,
765 "total_requests", KSTAT_DATA_UINT32);
766
767 kstat_named_init(&ksp->failed_requests,
768 "failed_requests", KSTAT_DATA_UINT32);
769
770 kstat_named_init(&ksp->requests_timedout,
771 "requests_timedout", KSTAT_DATA_UINT32);
772
773 kstat_install(saa_portp->saa_pt_kstatp);
774
775 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
776 ibmf_saa_impl_init_kstats_end,
777 IBMF_TNF_TRACE, "", "ibmf_saa_impl_init_kstats() exit\n");
778
779 return (IBMF_SUCCESS);
780 }
781
782 /*
783 * ibmf_saa_impl_uninit_kstats:
784 * Free kstats context. Should be called when port is either destroyed
785 * or invalidated.
786 */
787 static void
ibmf_saa_impl_uninit_kstats(saa_port_t * saa_portp)788 ibmf_saa_impl_uninit_kstats(saa_port_t *saa_portp)
789 {
790 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
791 ibmf_saa_impl_uninit_kstats_start,
792 IBMF_TNF_TRACE, "", "ibmf_saa_impl_uninit_kstats() enter\n");
793
794 mutex_enter(&saa_portp->saa_pt_kstat_mutex);
795
796 if (saa_portp->saa_pt_kstatp != NULL) {
797 kstat_delete(saa_portp->saa_pt_kstatp);
798 }
799 saa_portp->saa_pt_kstatp = NULL;
800
801 mutex_exit(&saa_portp->saa_pt_kstat_mutex);
802
803 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
804 ibmf_saa_impl_uninit_kstats_end,
805 IBMF_TNF_TRACE, "", "ibmf_saa_impl_uninit_kstats() exit\n");
806 }
807
808 /*
809 * ibmf_saa_impl_register_failed:
810 * invalidate entry and kick waiters
811 */
812 void
ibmf_saa_impl_register_failed(saa_port_t * saa_portp)813 ibmf_saa_impl_register_failed(saa_port_t *saa_portp)
814 {
815 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
816 ibmf_saa_impl_register_failed_start,
817 IBMF_TNF_TRACE, "", "ibmf_saa_impl_register_failed() enter\n");
818
819 mutex_enter(&saa_portp->saa_pt_mutex);
820
821 ibmf_saa_impl_invalidate_port(saa_portp);
822
823 cv_broadcast(&saa_portp->saa_pt_ibmf_reg_cv);
824
825 /* decrementing refcount is last thing we do on entry */
826
827 ASSERT(saa_portp->saa_pt_reference_count > 0);
828 saa_portp->saa_pt_reference_count--;
829
830 mutex_exit(&saa_portp->saa_pt_mutex);
831
832 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
833 ibmf_saa_impl_register_failed_end,
834 IBMF_TNF_TRACE, "", "ibmf_saa_impl_register_failed() exit\n");
835 }
836
837 static int
ibmf_saa_impl_setup_qp_async_cb(saa_port_t * saa_portp,int setup_async_cb_only)838 ibmf_saa_impl_setup_qp_async_cb(saa_port_t *saa_portp, int setup_async_cb_only)
839 {
840 int status;
841 int unreg_status;
842 ib_pkey_t p_key;
843 ib_qkey_t q_key;
844 uint8_t portnum;
845 boolean_t qp_alloced = B_FALSE;
846
847 if (setup_async_cb_only == 0) {
848
849 /* allocate a qp through ibmf */
850 status = ibmf_alloc_qp(saa_portp->saa_pt_ibmf_handle,
851 IB_PKEY_DEFAULT_LIMITED, IB_GSI_QKEY,
852 IBMF_ALT_QP_MAD_RMPP, &saa_portp->saa_pt_qp_handle);
853
854 if (status != IBMF_SUCCESS) {
855
856 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
857 ibmf_saa_impl_setup_qp_async_cb, IBMF_TNF_ERROR, "",
858 "ibmf_saa_impl_setup_qp_async_cb: %s, "
859 "ibmf_status = %d\n",
860 tnf_string, msg, "Cannot alloc qp with ibmf",
861 tnf_int, status, status);
862
863 return (status);
864 }
865
866 qp_alloced = B_TRUE;
867
868 /*
869 * query the queue pair number; we will need it to unsubscribe
870 * from notice reports
871 */
872 status = ibmf_query_qp(saa_portp->saa_pt_ibmf_handle,
873 saa_portp->saa_pt_qp_handle, &saa_portp->saa_pt_qpn,
874 &p_key, &q_key, &portnum, 0);
875
876 if (status != IBMF_SUCCESS) {
877
878 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
879 ibmf_saa_impl_setup_qp_async_cb, IBMF_TNF_ERROR, "",
880 "ibmf_saa_impl_setup_qp_async_cb: %s, "
881 "ibmf_status = %d\n",
882 tnf_string, msg,
883 "Cannot query alt qp to get qp num",
884 tnf_int, status, status);
885
886 goto bail;
887 }
888 }
889
890 /*
891 * core ibmf is taking advantage of the fact that saa_portp is our
892 * callback arg. If this changes, the code in ibmf_recv would need to
893 * change as well
894 */
895 status = ibmf_setup_async_cb(saa_portp->saa_pt_ibmf_handle,
896 saa_portp->saa_pt_qp_handle, ibmf_saa_report_cb, saa_portp, 0);
897 if (status != IBMF_SUCCESS) {
898
899 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
900 ibmf_saa_impl_setup_qp_async_cb, IBMF_TNF_ERROR, "",
901 "ibmf_saa_impl_setup_qp_async_cb: %s, ibmf_status = %d\n",
902 tnf_string, msg, "Cannot register async cb with ibmf",
903 tnf_int, status, status);
904
905 goto bail;
906 }
907
908 return (IBMF_SUCCESS);
909
910 bail:
911 if (qp_alloced == B_TRUE) {
912 /* free alternate qp */
913 unreg_status = ibmf_free_qp(saa_portp->saa_pt_ibmf_handle,
914 &saa_portp->saa_pt_qp_handle, 0);
915 if (unreg_status != IBMF_SUCCESS) {
916
917 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
918 ibmf_saa_impl_setup_qp_async_cb, IBMF_TNF_ERROR, "",
919 "ibmf_saa_impl_setup_qp_async_cb: %s, ibmf_status ="
920 " %d\n", tnf_string, msg,
921 "Cannot free alternate queue pair with ibmf",
922 tnf_int, unreg_status, unreg_status);
923 }
924 }
925
926 return (status);
927 }
928
929 /*
930 * ibmf_saa_impl_register_port:
931 */
932 int
ibmf_saa_impl_register_port(saa_port_t * saa_portp)933 ibmf_saa_impl_register_port(
934 saa_port_t *saa_portp)
935 {
936 uint_t hca_count = 0;
937 ib_guid_t *hca_list = NULL;
938 int status = IBMF_SUCCESS;
939 int unreg_status = IBMF_SUCCESS;
940 int ibt_status = IBT_SUCCESS;
941 ibt_hca_portinfo_t *port_info_list = NULL;
942 uint_t port_count = 0;
943 uint_t port_size = 0;
944 int ihca, iport;
945 ib_guid_t port_guid;
946 boolean_t ibmf_reg = B_FALSE;
947
948 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
949 ibmf_saa_impl_register_port_start, IBMF_TNF_TRACE, "",
950 "ibmf_saa_impl_register_port() enter\n");
951
952 ASSERT(saa_portp != NULL);
953
954 _NOTE(ASSUMING_PROTECTED(*saa_portp))
955
956 /* get the HCA list */
957
958 hca_count = ibt_get_hca_list(&hca_list);
959
960 if (hca_count == 0) {
961
962 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
963 ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "",
964 "ibmf_saa_impl_register_port: %s\n",
965 tnf_string, msg, "cannot register port (no HCAs).\n");
966
967 status = IBMF_BAD_PORT;
968 goto bail;
969 }
970
971 /* lookup requested port guid in hca list */
972 for (ihca = 0; ihca != hca_count; ihca++) {
973
974 ibt_status = ibt_query_hca_ports_byguid(hca_list[ihca],
975 0 /* all ports */, &port_info_list,
976 &port_count, &port_size);
977
978 if (ibt_status != IBT_SUCCESS) {
979
980 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
981 ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "",
982 "ibmf_saa_impl_register_port: %s, %016" PRIx64 "\n",
983 tnf_string, msg, "Could not query hca. Exiting.",
984 tnf_opaque, guid, hca_list[ihca]);
985
986 status = IBMF_TRANSPORT_FAILURE;
987 break;
988 }
989
990 for (iport = 0; iport < port_count; iport++) {
991
992 /* get port guid associated with hca guid, port num */
993 if (ibmf_saa_impl_get_port_guid(
994 port_info_list + iport, &port_guid) != IBMF_SUCCESS)
995 continue;
996
997 if (saa_portp->saa_pt_port_guid != port_guid)
998 continue;
999
1000 IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L3,
1001 ibmf_saa_impl_register_port,
1002 IBMF_TNF_TRACE, "",
1003 "ibmf_saa_impl_register_port: %s, hca_guid = %016"
1004 PRIx64 ", port_guid = %016" PRIx64
1005 ", number = %d\n",
1006 tnf_string, msg, "found port",
1007 tnf_opaque, hca_guid, hca_list[ihca],
1008 tnf_opaque, port_guid, port_guid,
1009 tnf_uint, port, iport + 1);
1010
1011 /*
1012 * we're here? then we found our port:
1013 * fill in ibmf registration info
1014 * and address parameters from the portinfo
1015 */
1016
1017 saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid
1018 = hca_list[ihca];
1019 saa_portp->saa_pt_ibmf_reginfo.ir_port_num = iport+1;
1020 saa_portp->saa_pt_ibmf_reginfo.ir_client_class
1021 = SUBN_ADM_MANAGER;
1022
1023 saa_portp->saa_pt_node_guid = hca_list[ihca];
1024 saa_portp->saa_pt_port_num = iport + 1;
1025
1026 ibmf_saa_impl_set_transaction_params(
1027 saa_portp, port_info_list + iport);
1028 break;
1029 }
1030
1031 ibt_free_portinfo(port_info_list, port_size);
1032
1033 if (iport != port_count)
1034 break; /* found our port */
1035 }
1036
1037 ibt_free_hca_list(hca_list, hca_count);
1038
1039 if (ihca == hca_count) {
1040
1041 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
1042 ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "",
1043 "ibmf_saa_impl_register_port: %s, port_guid %016"
1044 PRIx64 "\n",
1045 tnf_string, msg, "Could not find port, exiting",
1046 tnf_opaque, port_guid, saa_portp->saa_pt_port_guid);
1047
1048 status = IBMF_BAD_PORT;
1049 }
1050
1051 if (status != IBMF_SUCCESS) {
1052
1053 goto bail;
1054 }
1055
1056 /*
1057 * Now we found the port we searched for,
1058 * and open an ibmf session on that port.
1059 */
1060
1061 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L3,
1062 ibmf_saa_impl_register_port, IBMF_TNF_TRACE, "",
1063 "ibmf_saa_impl_register_port: %s, port_guid = %016" PRIx64
1064 ", port = %d\n", tnf_string, msg, "Registering with ibmf",
1065 tnf_opaque, port_guid, saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid,
1066 tnf_uint, port, saa_portp->saa_pt_ibmf_reginfo.ir_port_num);
1067
1068 status = ibmf_register(&saa_portp->saa_pt_ibmf_reginfo,
1069 IBMF_VERSION, IBMF_REG_FLAG_RMPP,
1070 ibmf_saa_impl_async_event_cb, saa_portp,
1071 &saa_portp->saa_pt_ibmf_handle,
1072 &saa_portp->saa_pt_ibmf_impl_features);
1073
1074 if (status != IBMF_SUCCESS) {
1075
1076 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
1077 ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "",
1078 "ibmf_saa_impl_register_port: %s, ibmf_status = %d\n",
1079 tnf_string, msg, "Could not register with ibmf",
1080 tnf_int, status, status);
1081
1082 goto bail;
1083 }
1084
1085 ibmf_reg = B_TRUE;
1086
1087 if (ibmf_saa_impl_setup_qp_async_cb(saa_portp, 0) == IBMF_SUCCESS)
1088 return (IBMF_SUCCESS);
1089
1090 bail:
1091 if (ibmf_reg == B_TRUE) {
1092 /* unregister from ibmf */
1093 unreg_status = ibmf_unregister(
1094 &saa_portp->saa_pt_ibmf_handle, 0);
1095
1096 if (unreg_status != IBMF_SUCCESS) {
1097
1098 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
1099 ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "",
1100 "ibmf_saa_impl_register_port: %s, ibmf_status ="
1101 " %d\n", tnf_string, msg,
1102 "Cannot unregister from ibmf",
1103 tnf_int, unreg_status, unreg_status);
1104 }
1105 }
1106
1107 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_register_port_end,
1108 IBMF_TNF_TRACE, "", "ibmf_saa_impl_register_port() exit\n");
1109
1110 return (status);
1111 }
1112
1113 /*
1114 * ibmf_saa_impl_getclassportinfo:
1115 */
1116 void
ibmf_saa_impl_get_classportinfo(saa_port_t * saa_portp)1117 ibmf_saa_impl_get_classportinfo(saa_port_t *saa_portp)
1118 {
1119 int res;
1120 saa_impl_trans_info_t *trans_info;
1121
1122 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
1123 ibmf_saa_impl_get_classportinfo_start,
1124 IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_classportinfo() enter\n");
1125
1126 /*
1127 * allocate memory for trans_info; send_request's callback will free up
1128 * memory since request is asynchronous
1129 */
1130 trans_info = kmem_zalloc(sizeof (saa_impl_trans_info_t), KM_NOSLEEP);
1131 if (trans_info == NULL) {
1132
1133 mutex_enter(&saa_portp->saa_pt_mutex);
1134
1135 /* cpi transaction is handled as a client, decrement refcount */
1136 ASSERT(saa_portp->saa_pt_reference_count > 0);
1137 saa_portp->saa_pt_reference_count--;
1138
1139 mutex_exit(&saa_portp->saa_pt_mutex);
1140
1141 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
1142 ibmf_saa_impl_get_classportinfo_err, IBMF_TNF_ERROR, "",
1143 "ibmf_saa_impl_get_classportinfo: %s\n", tnf_string, msg,
1144 "Could not allocate memory for classportinfo trans_info");
1145
1146 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
1147 ibmf_saa_impl_get_classportinfo_end, IBMF_TNF_TRACE, "",
1148 "ibmf_saa_impl_get_classportinfo() exiting\n");
1149
1150 return;
1151 }
1152
1153 /* no specific client associated with this transaction */
1154 trans_info->si_trans_client_data = NULL;
1155 trans_info->si_trans_port = saa_portp;
1156 trans_info->si_trans_method = SA_SUBN_ADM_GET;
1157 trans_info->si_trans_attr_id = MAD_ATTR_ID_CLASSPORTINFO;
1158
1159 trans_info->si_trans_callback = ibmf_saa_impl_get_cpi_cb;
1160 trans_info->si_trans_callback_arg = saa_portp;
1161
1162 mutex_enter(&saa_portp->saa_pt_kstat_mutex);
1163
1164 IBMF_SAA_ADD32_KSTATS(saa_portp, outstanding_requests, 1);
1165 IBMF_SAA_ADD32_KSTATS(saa_portp, total_requests, 1);
1166
1167 mutex_exit(&saa_portp->saa_pt_kstat_mutex);
1168
1169 res = ibmf_saa_impl_send_request(trans_info);
1170
1171 if (res != IBMF_SUCCESS) {
1172
1173 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2,
1174 ibmf_saa_impl_get_classportinfo_err, IBMF_TNF_TRACE, "",
1175 "ibmf_saa_impl_get_classportinfo: %s, res = 0x%x\n",
1176 tnf_string, msg, "ibmf_saa_impl_send_request failed",
1177 tnf_opaque, res, res);
1178
1179 mutex_enter(&saa_portp->saa_pt_kstat_mutex);
1180
1181 IBMF_SAA_SUB32_KSTATS(saa_portp, outstanding_requests, 1);
1182 IBMF_SAA_ADD32_KSTATS(saa_portp, failed_requests, 1);
1183
1184 mutex_exit(&saa_portp->saa_pt_kstat_mutex);
1185
1186 mutex_enter(&saa_portp->saa_pt_mutex);
1187
1188 /* cpi transaction is handled as a client, decrement refcount */
1189 ASSERT(saa_portp->saa_pt_reference_count > 0);
1190 saa_portp->saa_pt_reference_count--;
1191
1192 mutex_exit(&saa_portp->saa_pt_mutex);
1193
1194 }
1195
1196 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
1197 ibmf_saa_impl_get_classportinfo_end,
1198 IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_classportinfo() exit\n");
1199 }
1200
1201 /*
1202 * ibmf_saa_impl_get_cpi_cb:
1203 *
1204 * Called when the asynchronous getportinfo request receives its response.
1205 * Checks the status. If success, updates the times in the port's
1206 * ibmf_retrans structure that is used in ibmf_msg_transport calls. If failure,
1207 * just use default values.
1208 *
1209 * Input Arguments
1210 * arg user-specified pointer (points to the current port data)
1211 * length length of payload returned (should be size of classportinfo_t)
1212 * buffer pointer to classportinfo returned (should not be null)
1213 * status status of sa access request
1214 *
1215 * Output Arguments
1216 * none
1217 *
1218 * Returns void
1219 */
1220 static void
ibmf_saa_impl_get_cpi_cb(void * arg,size_t length,char * buffer,int status)1221 ibmf_saa_impl_get_cpi_cb(void *arg, size_t length, char *buffer, int status)
1222 {
1223 saa_port_t *saa_portp;
1224 uint64_t base_time, resp_timeout, rttv_timeout;
1225 ib_mad_classportinfo_t *classportinfo;
1226 int resp_time_value;
1227 uint16_t sa_cap_mask;
1228
1229 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_get_cpi_cb_start,
1230 IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_cpi_cb() enter\n");
1231
1232 /*
1233 * access port entry: note that it may have become invalid
1234 * but we hold a ref count for cpi and the interactions on
1235 * the entry are harmless
1236 */
1237 saa_portp = (saa_port_t *)arg;
1238
1239 /* process response */
1240
1241 if ((status != IBMF_SUCCESS) || (buffer == NULL)) {
1242
1243 IBMF_TRACE_4(IBMF_TNF_NODEBUG, DPRINT_L1,
1244 ibmf_saa_impl_get_cpi_cb, IBMF_TNF_ERROR, "",
1245 "ibmf_saa_impl_get_cpi_cb: %s, status = %d, buffer = "
1246 " 0x%p, length = %d\n", tnf_string, msg,
1247 "could not get classportinfo. Check node and path to sm"
1248 " lid", tnf_int, status, status,
1249 tnf_opaque, buffer, buffer, tnf_uint, length, length);
1250
1251 /*
1252 * IB spec (C13-13) indicates 20 can be used as default or
1253 * intial value for classportinfo->resptimeout value
1254 */
1255 resp_time_value = 20;
1256
1257 sa_cap_mask = 0xFFFF;
1258
1259 } else if (buffer != NULL) {
1260
1261 classportinfo = (ib_mad_classportinfo_t *)buffer;
1262
1263 resp_time_value = classportinfo->RespTimeValue & 0x1f;
1264
1265 /*
1266 * Because some subnet managers might not provide sane
1267 * value for "resp_time_value", we limit it here. In
1268 * case this limit is too restrictive (very large fabric),
1269 * we allow the limit to be raised (/etc/system).
1270 */
1271 if (resp_time_value > ibmf_saa_max_resp_time) {
1272 cmn_err(CE_CONT, "!ibmf_saa_max_resp_time (%d) "
1273 "exceeded.", ibmf_saa_max_resp_time);
1274 cmn_err(CE_CONT, "!Reducing subnet administrator "
1275 "resp_time value from %d to %d.",
1276 resp_time_value, ibmf_saa_max_resp_time);
1277 resp_time_value = ibmf_saa_max_resp_time;
1278 }
1279
1280 sa_cap_mask = classportinfo->CapabilityMask;
1281
1282 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L3,
1283 ibmf_saa_impl_get_cpi_cb, IBMF_TNF_TRACE, "",
1284 "ibmf_saa_impl_get_cpi_cb: %s, timeout = 0x%x,"
1285 " cap_mask = 0x%x\n",
1286 tnf_string, msg, "got classportinfo",
1287 tnf_opaque, timeout, resp_time_value,
1288 tnf_opaque, cap_mask, sa_cap_mask);
1289
1290 kmem_free(buffer, length);
1291 }
1292
1293 /*
1294 * using IB spec calculation from 13.4.6.2
1295 * use bit shifting for 2^x.
1296 */
1297 base_time = (1 << resp_time_value);
1298
1299 resp_timeout = (4 * base_time * 1000 + 96 * base_time) / 1000;
1300
1301 mutex_enter(&saa_portp->saa_pt_mutex);
1302
1303 base_time = 2 * (1 << saa_portp->saa_pt_timeout);
1304
1305 rttv_timeout = (4 * base_time * 1000 + 96 * base_time) / 1000;
1306
1307 saa_portp->saa_pt_ibmf_retrans.retrans_rtv = resp_timeout;
1308 saa_portp->saa_pt_ibmf_retrans.retrans_rttv = rttv_timeout;
1309 saa_portp->saa_pt_sa_cap_mask = sa_cap_mask;
1310
1311 /*
1312 * cpi transaction is handled as a client,
1313 * decrement refcount; make sure it's the last
1314 * thing we do on this entry
1315 */
1316 ASSERT(saa_portp->saa_pt_reference_count > 0);
1317 saa_portp->saa_pt_reference_count--;
1318
1319 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L3,
1320 ibmf_saa_impl_get_cpi_cb, IBMF_TNF_TRACE, "",
1321 "ibmf_saa_impl_get_cpi_cb: %s, subnet_timeout = 0x%x, "
1322 "resp_time_value = 0x%x\n",
1323 tnf_string, msg, "updated resp timeout",
1324 tnf_opaque, subnet_timeout, saa_portp->saa_pt_timeout,
1325 tnf_opaque, resp_time_value, resp_time_value);
1326
1327 mutex_exit(&saa_portp->saa_pt_mutex);
1328
1329 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_get_cpi_cb_end,
1330 IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_cpi_cb() exit\n");
1331 }
1332
1333 /*
1334 * ibmf_saa_impl_send_request:
1335 * Sends a request to the sa. Can be used for both classportinfo and record
1336 * requests. Will set up all data structures for using the multi-packet
1337 * protocol, create the mad, and send it. Returns SA_SUCCESS if msg transport
1338 * worked, meaning succesful send for the async case and a succesful send and
1339 * recv for the sync case.
1340 */
1341 int
ibmf_saa_impl_send_request(saa_impl_trans_info_t * trans_info)1342 ibmf_saa_impl_send_request(saa_impl_trans_info_t *trans_info)
1343 {
1344 uint16_t attr_id;
1345 saa_client_data_t *client_data;
1346 saa_port_t *saa_portp;
1347 uint32_t transport_flags;
1348 ibmf_msg_cb_t ibmf_callback;
1349 void *ibmf_callback_arg;
1350 ibmf_msg_t *msgp;
1351 ibmf_retrans_t ibmf_retrans;
1352 uint16_t sa_cap_mask;
1353 boolean_t sleep_flag;
1354 int ibmf_status = IBMF_SUCCESS;
1355 int retry_count;
1356 uint16_t mad_status;
1357 boolean_t sa_is_redirected = B_FALSE;
1358
1359 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
1360 ibmf_saa_impl_send_request_start,
1361 IBMF_TNF_TRACE, "", "ibmf_saa_impl_send_request() enter\n");
1362
1363 attr_id = trans_info->si_trans_attr_id;
1364 client_data = trans_info->si_trans_client_data;
1365 saa_portp = trans_info->si_trans_port;
1366
1367 /*
1368 * don't send on invalid entry
1369 * Note that there is a window where it could become
1370 * invalid after this test is done, but we'd rely on ibmf errors...
1371 */
1372 if (ibmf_saa_is_valid(saa_portp, B_FALSE) == B_FALSE) {
1373
1374 IBMF_TRACE_4(IBMF_TNF_NODEBUG, DPRINT_L1,
1375 ibmf_saa_impl_send_request,
1376 IBMF_TNF_ERROR, "",
1377 "ibmf_saa_impl_send_request: %s, hca_guid = %016"
1378 PRIx64 ", port_guid = %016" PRIx64
1379 ", number = %d\n",
1380 tnf_string, msg, "sending on invalid port",
1381 tnf_opaque, hca_guid,
1382 saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid,
1383 tnf_opaque, port_guid,
1384 saa_portp->saa_pt_port_guid,
1385 tnf_uint, port,
1386 saa_portp->saa_pt_ibmf_reginfo.ir_port_num);
1387
1388 ibmf_status = IBMF_REQ_INVALID;
1389 goto bail;
1390 }
1391
1392 /* check whether SA supports this attribute */
1393 mutex_enter(&saa_portp->saa_pt_mutex);
1394
1395 sa_cap_mask = saa_portp->saa_pt_sa_cap_mask;
1396 sa_is_redirected = saa_portp->saa_pt_redirect_active;
1397
1398 mutex_exit(&saa_portp->saa_pt_mutex);
1399
1400 ibmf_status = ibmf_saa_impl_check_sa_support(sa_cap_mask, attr_id);
1401
1402 if (ibmf_status != IBMF_SUCCESS) {
1403
1404 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
1405 ibmf_saa_impl_send_request_err, IBMF_TNF_ERROR, "",
1406 "ibmf_saa_impl_send_request: %s, ibmf_status = %d\n",
1407 tnf_string, msg, "SA does not support attribute",
1408 tnf_int, ibmf_status, ibmf_status);
1409
1410 goto bail;
1411 }
1412
1413 /* make only non-blocking calls if this is an async request */
1414 if ((trans_info->si_trans_callback == NULL) &&
1415 (trans_info->si_trans_sub_callback == NULL)) {
1416 ibmf_callback = NULL;
1417 ibmf_callback_arg = NULL;
1418 sleep_flag = B_TRUE;
1419 } else {
1420 ibmf_callback = ibmf_saa_async_cb;
1421 ibmf_callback_arg = (void *)trans_info;
1422 sleep_flag = B_FALSE;
1423 }
1424
1425 ibmf_status = ibmf_saa_impl_init_msg(trans_info, sleep_flag, &msgp,
1426 &transport_flags, &ibmf_retrans);
1427 if (ibmf_status != IBMF_SUCCESS) {
1428
1429 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
1430 ibmf_saa_impl_send_request_err, IBMF_TNF_ERROR, "",
1431 "ibmf_saa_impl_send_request: %s, ibmf_status = %d\n",
1432 tnf_string, msg, "init_msg() failed",
1433 tnf_int, ibmf_status, ibmf_status);
1434
1435 goto bail;
1436 }
1437
1438 mutex_enter(&saa_portp->saa_pt_mutex);
1439
1440 saa_portp->saa_pt_num_outstanding_trans++;
1441
1442 mutex_exit(&saa_portp->saa_pt_mutex);
1443
1444 /*
1445 * increment the number of outstanding transaction so
1446 * ibmf_close_sa_session() will wait. classportinfo requests
1447 * don't have associated clients so check for valid clientp
1448 */
1449 if (client_data != NULL) {
1450
1451 mutex_enter(&client_data->saa_client_mutex);
1452
1453 client_data->saa_client_num_pending_trans++;
1454
1455 mutex_exit(&client_data->saa_client_mutex);
1456 }
1457
1458 /*
1459 * make the call to msg_transport. If synchronous and success,
1460 * check that the response mad isn't status busy. If so, repeat the
1461 * call
1462 */
1463 retry_count = 0;
1464
1465 /*
1466 * set the send time here. We only set this once at the beginning of
1467 * the transaction. Retrying because of busys or mastersmlid changes
1468 * does not change the original send time. It is meant to be an
1469 * absolute time out value and will only be used if there are other
1470 * problems (i.e. a buggy SA)
1471 */
1472 trans_info->si_trans_send_time = gethrtime();
1473
1474 for (;;) {
1475
1476 ibmf_status = ibmf_msg_transport(saa_portp->saa_pt_ibmf_handle,
1477 saa_portp->saa_pt_qp_handle, msgp, &ibmf_retrans,
1478 ibmf_callback, ibmf_callback_arg, transport_flags);
1479
1480 if (ibmf_callback != NULL)
1481 break;
1482
1483 /*
1484 * stop here for non-sequenced transactions since they wouldn't
1485 * receive a timeout or busy response
1486 */
1487 if (!(transport_flags & IBMF_MSG_TRANS_FLAG_SEQ))
1488 break;
1489
1490 /*
1491 * if the transaction timed out and this was a synchronous
1492 * request there's a possiblity we were talking to the wrong
1493 * master smlid or that the SA has stopped responding on the
1494 * redirected desination (if redirect is active).
1495 * Check this and retry if necessary.
1496 */
1497 if ((ibmf_status == IBMF_TRANS_TIMEOUT) &&
1498 (sleep_flag == B_TRUE)) {
1499 if (sa_is_redirected == B_TRUE) {
1500 ibmf_status = ibmf_saa_impl_revert_to_qp1(
1501 saa_portp, msgp, ibmf_callback,
1502 ibmf_callback_arg, transport_flags);
1503 } else {
1504 ibmf_status = ibmf_saa_impl_new_smlid_retry(
1505 saa_portp, msgp, ibmf_callback,
1506 ibmf_callback_arg, transport_flags);
1507 }
1508 }
1509
1510 /*
1511 * if the transaction timed out (and retrying with a new SM LID
1512 * didn't help) check how long it's been since we received an SA
1513 * packet. If it hasn't been max_wait_time then retry the
1514 * request.
1515 */
1516 if ((ibmf_status == IBMF_TRANS_TIMEOUT) &&
1517 (sleep_flag == B_TRUE)) {
1518
1519 ibmf_status = ibmf_saa_check_sa_and_retry(
1520 saa_portp, msgp, ibmf_callback, ibmf_callback_arg,
1521 trans_info->si_trans_send_time, transport_flags);
1522 }
1523
1524 if (ibmf_status != IBMF_SUCCESS)
1525 break;
1526
1527 if (retry_count >= IBMF_SAA_MAX_BUSY_RETRY_COUNT)
1528 break;
1529
1530 /* sync transaction with status SUCCESS should have response */
1531 ASSERT(msgp->im_msgbufs_recv.im_bufs_mad_hdr != NULL);
1532
1533 mad_status = b2h16(msgp->im_msgbufs_recv.
1534 im_bufs_mad_hdr->Status);
1535
1536 if ((mad_status != MAD_STATUS_BUSY) &&
1537 (mad_status != MAD_STATUS_REDIRECT_REQUIRED))
1538 break;
1539
1540 if (mad_status == MAD_STATUS_REDIRECT_REQUIRED) {
1541
1542 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2,
1543 ibmf_saa_impl_send_request, IBMF_TNF_TRACE, "",
1544 "ibmf_saa_impl_send_request: %s, retry_count %d\n",
1545 tnf_string, msg,
1546 "response returned redirect status",
1547 tnf_int, retry_count, retry_count);
1548
1549 /* update address info and copy it into msgp */
1550 ibmf_saa_impl_update_sa_address_info(saa_portp, msgp);
1551 } else {
1552 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2,
1553 ibmf_saa_impl_send_request, IBMF_TNF_TRACE, "",
1554 "ibmf_saa_impl_send_request: %s, retry_count %d\n",
1555 tnf_string, msg, "response returned busy status",
1556 tnf_int, retry_count, retry_count);
1557 }
1558
1559 retry_count++;
1560
1561 /*
1562 * since this is a blocking call, sleep for some time
1563 * to allow SA to transition from busy state (if busy)
1564 */
1565 if (mad_status == MAD_STATUS_BUSY)
1566 delay(drv_usectohz(
1567 IBMF_SAA_BUSY_RETRY_SLEEP_SECS * 1000000));
1568 }
1569
1570 if (ibmf_status != IBMF_SUCCESS) {
1571
1572 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2,
1573 ibmf_saa_impl_send_request, IBMF_TNF_TRACE, "",
1574 "ibmf_saa_impl_send_request: %s, ibmf_status = %d\n",
1575 tnf_string, msg, "ibmf_msg_transport() failed",
1576 tnf_int, ibmf_status, ibmf_status);
1577
1578 ibmf_saa_impl_free_msg(saa_portp->saa_pt_ibmf_handle, msgp);
1579
1580 mutex_enter(&saa_portp->saa_pt_mutex);
1581
1582 ASSERT(saa_portp->saa_pt_num_outstanding_trans > 0);
1583 saa_portp->saa_pt_num_outstanding_trans--;
1584
1585 mutex_exit(&saa_portp->saa_pt_mutex);
1586
1587 if (client_data != NULL) {
1588
1589 mutex_enter(&client_data->saa_client_mutex);
1590
1591 ASSERT(client_data->saa_client_num_pending_trans > 0);
1592 client_data->saa_client_num_pending_trans--;
1593
1594 if ((client_data->saa_client_num_pending_trans == 0) &&
1595 (client_data->saa_client_state ==
1596 SAA_CLIENT_STATE_WAITING))
1597 cv_signal(&client_data->saa_client_state_cv);
1598
1599 mutex_exit(&client_data->saa_client_mutex);
1600 }
1601
1602 } else if (sleep_flag == B_TRUE) {
1603
1604 mutex_enter(&saa_portp->saa_pt_mutex);
1605
1606 ASSERT(saa_portp->saa_pt_num_outstanding_trans > 0);
1607 saa_portp->saa_pt_num_outstanding_trans--;
1608
1609 mutex_exit(&saa_portp->saa_pt_mutex);
1610
1611 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
1612 ibmf_saa_impl_send_request, IBMF_TNF_TRACE, "",
1613 "ibmf_saa_impl_send_request: %s\n",
1614 tnf_string, msg, "Message sent and received successfully");
1615
1616 /* fill in response values and free the message */
1617 ibmf_saa_impl_prepare_response(saa_portp->saa_pt_ibmf_handle,
1618 msgp, B_FALSE, &trans_info->si_trans_status,
1619 &trans_info->si_trans_result,
1620 &trans_info->si_trans_length, sleep_flag);
1621
1622 if (client_data != NULL) {
1623 mutex_enter(&client_data->saa_client_mutex);
1624
1625 ASSERT(client_data->saa_client_num_pending_trans > 0);
1626 client_data->saa_client_num_pending_trans--;
1627
1628 if ((client_data->saa_client_num_pending_trans == 0) &&
1629 (client_data->saa_client_state ==
1630 SAA_CLIENT_STATE_WAITING))
1631 cv_signal(&client_data->saa_client_state_cv);
1632
1633 mutex_exit(&client_data->saa_client_mutex);
1634 }
1635 } else {
1636
1637 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
1638 ibmf_saa_impl_send_request, IBMF_TNF_TRACE, "",
1639 "ibmf_saa_impl_send_request: %s\n",
1640 tnf_string, msg, "Message sent successfully");
1641 }
1642
1643 bail:
1644 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
1645 ibmf_saa_impl_send_request_end,
1646 IBMF_TNF_TRACE, "", "ibmf_saa_impl_send_request() exiting"
1647 " ibmf_status = %d\n", tnf_int, result, ibmf_status);
1648
1649 return (ibmf_status);
1650 }
1651
1652 /*
1653 * ibmf_saa_impl_init_msg:
1654 * Allocates an ibmf message and fills out the header fields and formatted data
1655 * fields. Also sets up the correct transport_flags and retrans argument for
1656 * the message transport call based on the request information.
1657 *
1658 * Input Arguments
1659 * trans_info saa_trans_info structure passed to send_request
1660 * sleep_flag B_TRUE if init_msg can sleep in function calls
1661 *
1662 * Output Arguments
1663 * msgp ibmf message that should be given to msg_transport
1664 * transport_flagsp transport flags that should be given to msg_transport
1665 * ibmf_retrans_t retrans parameter that should be given to msg_transport
1666 *
1667 * Returns
1668 * ibmf_status
1669 */
1670 static int
ibmf_saa_impl_init_msg(saa_impl_trans_info_t * trans_info,boolean_t sleep_flag,ibmf_msg_t ** msgp,uint32_t * transport_flagsp,ibmf_retrans_t * ibmf_retransp)1671 ibmf_saa_impl_init_msg(saa_impl_trans_info_t *trans_info, boolean_t sleep_flag,
1672 ibmf_msg_t **msgp, uint32_t *transport_flagsp,
1673 ibmf_retrans_t *ibmf_retransp)
1674 {
1675 int ibmf_status;
1676 ibmf_msg_bufs_t *req_mad;
1677 ib_mad_hdr_t *mad_hdr;
1678 int ibmf_sleep_flag, km_sleep_flag;
1679 int free_res;
1680 ib_sa_hdr_t sa_hdr;
1681 ibmf_msg_t *ibmf_msg;
1682 uint16_t attr_id, pack_attr_id;
1683 uint8_t method;
1684 saa_client_data_t *client_data;
1685 saa_port_t *saa_portp;
1686 sa_multipath_record_t *multipath_template;
1687 size_t payload_length;
1688 uint32_t transport_flags;
1689
1690 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
1691 ibmf_saa_impl_init_msg_start,
1692 IBMF_TNF_TRACE, "", "ibmf_saa_impl_init_msg() entering\n");
1693
1694 attr_id = trans_info->si_trans_attr_id;
1695 method = trans_info->si_trans_method;
1696 client_data = trans_info->si_trans_client_data;
1697 saa_portp = trans_info->si_trans_port;
1698
1699 if (sleep_flag == B_TRUE) {
1700 ibmf_sleep_flag = IBMF_ALLOC_SLEEP;
1701 km_sleep_flag = KM_SLEEP;
1702 } else {
1703 ibmf_sleep_flag = IBMF_ALLOC_NOSLEEP;
1704 km_sleep_flag = KM_NOSLEEP;
1705 }
1706
1707 ibmf_status = ibmf_alloc_msg(saa_portp->saa_pt_ibmf_handle,
1708 ibmf_sleep_flag, &ibmf_msg);
1709 if (ibmf_status != IBMF_SUCCESS) {
1710
1711 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
1712 ibmf_saa_impl_init_msg_err, IBMF_TNF_ERROR, "",
1713 "ibmf_saa_impl_init_msg: %s, ibmf_status = %d\n",
1714 tnf_string, msg, "Cannot allocate msg_buf.",
1715 tnf_int, ibmf_status, ibmf_status);
1716
1717 goto bail;
1718 }
1719
1720 req_mad = &ibmf_msg->im_msgbufs_send;
1721
1722 /* create a template (SA MAD) */
1723 mad_hdr = kmem_zalloc(sizeof (ib_mad_hdr_t), km_sleep_flag);
1724
1725 if (mad_hdr == NULL) {
1726
1727 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
1728 ibmf_saa_impl_init_msg_err, IBMF_TNF_ERROR, "",
1729 "ibmf_saa_impl_init_msg: %s\n",
1730 tnf_string, msg, "Cannot allocate mad header.");
1731
1732 free_res = ibmf_free_msg(saa_portp->saa_pt_ibmf_handle,
1733 &ibmf_msg);
1734 ASSERT(free_res == IBMF_SUCCESS);
1735
1736 ibmf_status = IBMF_NO_MEMORY;
1737 goto bail;
1738 }
1739
1740 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mad_hdr,
1741 *req_mad))
1742
1743 bzero(mad_hdr, sizeof (ib_mad_hdr_t));
1744 mad_hdr->BaseVersion = SAA_MAD_BASE_VERSION;
1745 mad_hdr->MgmtClass = MAD_MGMT_CLASS_SUBN_ADM;
1746 mad_hdr->ClassVersion = SAA_MAD_CLASS_VERSION;
1747 mad_hdr->R_Method = method;
1748 mad_hdr->AttributeID = h2b16(attr_id);
1749
1750 /* attribute modifier is all Fs since RIDs are no longer used */
1751 mad_hdr->AttributeModifier = h2b32(0xffffffff);
1752
1753 IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L3,
1754 ibmf_saa_impl_init_msg, IBMF_TNF_TRACE, "",
1755 "ibmf_saa_impl_init_msg: %s, class = 0x%x, method = 0x%x,"
1756 " attr_id = 0x%x\n", tnf_string, msg, "Sending MAD",
1757 tnf_opaque, class, mad_hdr->MgmtClass,
1758 tnf_opaque, method, mad_hdr->R_Method,
1759 tnf_opaque, attr_id, attr_id);
1760
1761 bzero(&sa_hdr, sizeof (ib_sa_hdr_t));
1762 sa_hdr.ComponentMask = trans_info->si_trans_component_mask;
1763
1764 if (client_data != NULL)
1765 sa_hdr.SM_KEY = client_data->saa_client_sm_key;
1766
1767 /*
1768 * pack data for IB wire format; req_mad will have different pointers to
1769 * sa header and payload, mad_hdr will be the same
1770 */
1771 req_mad->im_bufs_mad_hdr = mad_hdr;
1772
1773 ibmf_status = ibmf_saa_utils_pack_sa_hdr(&sa_hdr,
1774 &req_mad->im_bufs_cl_hdr, &req_mad->im_bufs_cl_hdr_len,
1775 km_sleep_flag);
1776
1777 if (ibmf_status != IBMF_SUCCESS) {
1778
1779 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
1780 ibmf_saa_impl_init_msg, IBMF_TNF_ERROR, "",
1781 "ibmf_saa_impl_init_msg: %s, ibmf_status = %d\n",
1782 tnf_string, msg, "ibmf_saa_utils_pack_sa_hdr() failed",
1783 tnf_int, ibmf_status, ibmf_status);
1784
1785 kmem_free(mad_hdr, sizeof (ib_mad_hdr_t));
1786
1787 free_res = ibmf_free_msg(saa_portp->saa_pt_ibmf_handle,
1788 &ibmf_msg);
1789 ASSERT(free_res == IBMF_SUCCESS);
1790
1791 goto bail;
1792 }
1793
1794 if (attr_id == SA_MULTIPATHRECORD_ATTRID) {
1795
1796 multipath_template =
1797 (sa_multipath_record_t *)trans_info->si_trans_template;
1798
1799 payload_length = sizeof (sa_multipath_record_t) +
1800 ((multipath_template->SGIDCount +
1801 multipath_template->DGIDCount) * sizeof (ib_gid_t));
1802
1803 pack_attr_id = attr_id;
1804 } else {
1805
1806 /* trace record template is a path record */
1807 pack_attr_id = (attr_id == SA_TRACERECORD_ATTRID) ?
1808 SA_PATHRECORD_ATTRID : attr_id;
1809
1810 payload_length = ibmf_saa_impl_get_attr_id_length(pack_attr_id);
1811
1812 if (payload_length == 0) {
1813 payload_length = trans_info->si_trans_template_length;
1814
1815 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
1816 ibmf_saa_impl_init_msg, IBMF_TNF_TRACE, "",
1817 "ibmf_saa_impl_init_msg: %s, length = %d\n",
1818 tnf_string, msg,
1819 "Unknown attribute. Using user-defined length.",
1820 tnf_uint, length, payload_length)
1821 }
1822 }
1823
1824 /* transport type depends on method */
1825 switch (method) {
1826
1827 case SA_SUBN_ADM_GET:
1828 case SA_SUBN_ADM_DELETE:
1829 case SA_SUBN_ADM_GET_TABLE:
1830 case SA_SUBN_ADM_GET_TRACE_TABLE:
1831 transport_flags = IBMF_MSG_TRANS_FLAG_SEQ;
1832 break;
1833 case SA_SUBN_ADM_SET:
1834 /* unsubscribes can be sequenced or unsequenced */
1835 if (trans_info->si_trans_unseq_unsubscribe == B_TRUE) {
1836 transport_flags = 0;
1837 } else {
1838 transport_flags = IBMF_MSG_TRANS_FLAG_SEQ;
1839 }
1840 break;
1841 case SA_SUBN_ADM_GET_MULTI:
1842 transport_flags = IBMF_MSG_TRANS_FLAG_SEQ |
1843 IBMF_MSG_TRANS_FLAG_RMPP;
1844 break;
1845 default :
1846 ibmf_status = IBMF_UNSUPP_METHOD;
1847 goto bail;
1848 }
1849
1850 trans_info->si_trans_transport_flags = transport_flags;
1851
1852 if (trans_info->si_trans_template != NULL) {
1853
1854 ibmf_status = ibmf_saa_utils_pack_payload(
1855 trans_info->si_trans_template, payload_length, pack_attr_id,
1856 &req_mad->im_bufs_cl_data, &req_mad->im_bufs_cl_data_len,
1857 km_sleep_flag);
1858 if (ibmf_status != IBMF_SUCCESS) {
1859
1860 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
1861 ibmf_saa_impl_init_msg_err, IBMF_TNF_ERROR, "",
1862 "ibmf_saa_impl_init_msg: %s, ibmf_status ="
1863 " %d\n", tnf_string, msg,
1864 "ibmf_saa_utils_pack_payload() failed",
1865 tnf_int, ibmf_status, ibmf_status);
1866
1867 kmem_free(mad_hdr, sizeof (ib_mad_hdr_t));
1868
1869 kmem_free(req_mad->im_bufs_cl_hdr,
1870 req_mad->im_bufs_cl_hdr_len);
1871
1872 free_res = ibmf_free_msg(saa_portp->saa_pt_ibmf_handle,
1873 &ibmf_msg);
1874 ASSERT(free_res == IBMF_SUCCESS);
1875
1876 goto bail;
1877 }
1878
1879 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L3,
1880 ibmf_saa_impl_init_msg, IBMF_TNF_TRACE, "",
1881 "ibmf_saa_impl_init_msg: %s, attr_id = 0x%x, length ="
1882 " %d\n", tnf_string, msg, "Packed payload successfully",
1883 tnf_opaque, attr_id, attr_id,
1884 tnf_uint, length, req_mad->im_bufs_cl_data_len);
1885
1886 /* non-RMPP transactions have template size limit */
1887 if (((transport_flags & IBMF_MSG_TRANS_FLAG_RMPP) == 0) &&
1888 ((req_mad->im_bufs_cl_data_len + req_mad->im_bufs_cl_hdr_len
1889 + sizeof (ib_mad_hdr_t)) > IBMF_MAD_SIZE)) {
1890
1891 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
1892 ibmf_saa_impl_init_msg_err, IBMF_TNF_ERROR, "",
1893 "ibmf_saa_impl_init_msg: %s\n", tnf_string, msg,
1894 "Template too large to fit in single packet");
1895
1896 kmem_free(mad_hdr, sizeof (ib_mad_hdr_t));
1897
1898 kmem_free(req_mad->im_bufs_cl_hdr,
1899 req_mad->im_bufs_cl_hdr_len);
1900
1901 kmem_free(req_mad->im_bufs_cl_data,
1902 req_mad->im_bufs_cl_data_len);
1903
1904 free_res = ibmf_free_msg(saa_portp->saa_pt_ibmf_handle,
1905 &ibmf_msg);
1906 ASSERT(free_res == IBMF_SUCCESS);
1907
1908 ibmf_status = IBMF_REQ_INVALID;
1909 goto bail;
1910 }
1911 }
1912
1913 mutex_enter(&saa_portp->saa_pt_mutex);
1914
1915 mad_hdr->TransactionID = h2b64(saa_portp->saa_pt_current_tid++);
1916
1917 bcopy(&saa_portp->saa_pt_ibmf_retrans, ibmf_retransp,
1918 sizeof (ibmf_retrans_t));
1919
1920 /* copy local addressing information to message */
1921 bcopy(&saa_portp->saa_pt_ibmf_addr_info, &ibmf_msg->im_local_addr,
1922 sizeof (ibmf_addr_info_t));
1923
1924 /* copy global addressing information to message if in use */
1925 if (saa_portp->saa_pt_ibmf_msg_flags & IBMF_MSG_FLAGS_GLOBAL_ADDRESS) {
1926
1927 ibmf_msg->im_msg_flags = IBMF_MSG_FLAGS_GLOBAL_ADDRESS;
1928
1929 bcopy(&saa_portp->saa_pt_ibmf_global_addr,
1930 &ibmf_msg->im_global_addr,
1931 sizeof (ibmf_global_addr_info_t));
1932 } else {
1933 ibmf_msg->im_msg_flags = 0;
1934 }
1935
1936 mutex_exit(&saa_portp->saa_pt_mutex);
1937
1938 *msgp = ibmf_msg;
1939 *transport_flagsp = transport_flags;
1940 bail:
1941 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
1942 ibmf_saa_impl_init_msg_end,
1943 IBMF_TNF_TRACE, "", "ibmf_saa_impl_init_msg() exiting"
1944 " ibmf_status = %d\n", tnf_int, result, ibmf_status);
1945
1946 return (ibmf_status);
1947
1948 }
1949
1950 /*
1951 * ibmf_saa_impl_new_smlid_retry:
1952 *
1953 * It's possible for the MasterSMLID to change while ibmf_saa is running. The
1954 * MasterSMLID is set when we first register with ibmf_saa. If a request
1955 * timesout, this function should be called to check whether the SM LID changed.
1956 * If so, it will call msg_transport again with the request.
1957 *
1958 * msgp, ibmf_callback, ibmf_callback_arg, and transport flags should be the
1959 * same values passed to the original ibmf_msg_transport that timedout. The
1960 * ibmf_retrans parameter will be re-retrieved from the saa_portp structure.
1961 *
1962 * If the lid did not change then this function returns IBMF_TRANS_TIMEOUT.
1963 * That way, callers can simply return the result of this function.
1964 *
1965 * Input Arguments
1966 * saa_portp pointer to saa_port structure
1967 * msgp ibmf message that timedout
1968 * ibmf_callback callback that should be called by msg_transport
1969 * ibmf_callback_arg args for ibmf_callback
1970 * transport_flags flags for ibmf_msg_transport
1971 *
1972 * Output Arguments
1973 * none
1974 *
1975 * Returns
1976 * IBMF_SUCCESS if lid changed and request was resent successfully,
1977 * IBMF_TRANS_TIMEOUT if lid did not change,
1978 * same values as ibmf_msg_transport() if lid changed but request could not be
1979 * resent.
1980 */
1981 static int
ibmf_saa_impl_new_smlid_retry(saa_port_t * saa_portp,ibmf_msg_t * msgp,ibmf_msg_cb_t ibmf_callback,void * ibmf_callback_arg,int transport_flags)1982 ibmf_saa_impl_new_smlid_retry(saa_port_t *saa_portp, ibmf_msg_t *msgp,
1983 ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_arg, int transport_flags)
1984 {
1985 ibt_hca_portinfo_t *ibt_portinfop;
1986 ib_lid_t master_sm_lid;
1987 int subnet_timeout;
1988 uint_t nports, size;
1989 ibmf_retrans_t ibmf_retrans;
1990 int ibmf_status;
1991 ibt_status_t ibt_status;
1992
1993 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
1994 ibmf_saa_impl_new_smlid_retry_start,
1995 IBMF_TNF_TRACE, "", "ibmf_saa_impl_new_smlid_retry() enter\n");
1996
1997 _NOTE(ASSUMING_PROTECTED(*msgp))
1998 _NOTE(ASSUMING_PROTECTED(*msgp->im_msgbufs_send.im_bufs_mad_hdr))
1999
2000 /* first query the portinfo to see if the lid changed */
2001 ibt_status = ibt_query_hca_ports_byguid(saa_portp->saa_pt_node_guid,
2002 saa_portp->saa_pt_port_num, &ibt_portinfop, &nports, &size);
2003
2004 if (ibt_status != IBT_SUCCESS) {
2005
2006 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
2007 ibmf_saa_impl_new_smlid_retry_err, IBMF_TNF_ERROR, "",
2008 "ibmf_saa_impl_new_smlid_retry: %s, ibmf_status ="
2009 " %d\n", tnf_string, msg,
2010 "ibt_query_hca_ports_byguid() failed",
2011 tnf_int, ibt_status, ibt_status);
2012
2013 ibmf_status = IBMF_TRANSPORT_FAILURE;
2014
2015 goto bail;
2016 }
2017
2018 master_sm_lid = ibt_portinfop->p_sm_lid;
2019 subnet_timeout = ibt_portinfop->p_subnet_timeout;
2020
2021 ibt_free_portinfo(ibt_portinfop, size);
2022
2023 /* if master smlid is different than the remote lid we sent to */
2024 if (master_sm_lid != msgp->im_local_addr.ia_remote_lid) {
2025
2026 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L2,
2027 ibmf_saa_impl_new_smlid_retry, IBMF_TNF_TRACE, "",
2028 "ibmf_saa_impl_new_smlid_retry: %s, new_lid 0x%x,"
2029 " old_lid 0x%x\n", tnf_string, msg,
2030 "master smlid has changed. retrying msg_transport",
2031 tnf_opaque, new_lid, master_sm_lid,
2032 tnf_opaque, old_lid, msgp->im_local_addr.ia_remote_lid);
2033
2034 mutex_enter(&saa_portp->saa_pt_mutex);
2035
2036 /* update the master sm lid value in ibmf_saa */
2037 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid =
2038 master_sm_lid;
2039
2040 /* new tid needed */
2041 msgp->im_msgbufs_send.im_bufs_mad_hdr->TransactionID =
2042 h2b64(saa_portp->saa_pt_current_tid++);
2043
2044 bcopy(&saa_portp->saa_pt_ibmf_retrans, &ibmf_retrans,
2045 sizeof (ibmf_retrans_t));
2046
2047 /* update the subnet timeout since this may be a new sm/sa */
2048 saa_portp->saa_pt_timeout = subnet_timeout;
2049
2050 /* place upper bound on subnet timeout in case of faulty SM */
2051 if (saa_portp->saa_pt_timeout > ibmf_saa_max_subnet_timeout)
2052 saa_portp->saa_pt_timeout = ibmf_saa_max_subnet_timeout;
2053
2054 /* increment the reference count to account for the cpi call */
2055 saa_portp->saa_pt_reference_count++;
2056
2057 mutex_exit(&saa_portp->saa_pt_mutex);
2058
2059 /* update the remote lid for this particular message */
2060 msgp->im_local_addr.ia_remote_lid = master_sm_lid;
2061
2062 /* get the classportinfo again since this may be a new sm/sa */
2063 ibmf_saa_impl_get_classportinfo(saa_portp);
2064
2065 ibmf_status = ibmf_msg_transport(saa_portp->saa_pt_ibmf_handle,
2066 saa_portp->saa_pt_qp_handle, msgp, &ibmf_retrans,
2067 ibmf_callback, ibmf_callback_arg, transport_flags);
2068
2069 if (ibmf_status != IBMF_SUCCESS) {
2070
2071 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2,
2072 ibmf_saa_impl_new_smlid_retry, IBMF_TNF_TRACE, "",
2073 "ibmf_saa_impl_new_smlid_retry: %s, ibmf_status = "
2074 "%d\n", tnf_string, msg,
2075 "ibmf_msg_transport() failed",
2076 tnf_int, ibmf_status, ibmf_status);
2077 }
2078
2079 goto bail;
2080 }
2081
2082 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
2083 ibmf_saa_impl_new_smlid_retry, IBMF_TNF_TRACE, "",
2084 "ibmf_saa_impl_new_smlid_retry: %s, master_smlid = 0x%x\n",
2085 tnf_string, msg,
2086 "master smlid did not change. returning failure",
2087 tnf_opaque, master_smlid, master_sm_lid);
2088
2089 /* mark status as timeout since that was original failure */
2090 ibmf_status = IBMF_TRANS_TIMEOUT;
2091
2092 bail:
2093 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
2094 ibmf_saa_impl_new_smlid_retry_end,
2095 IBMF_TNF_TRACE, "", "ibmf_saa_impl_new_smlid_retry() exiting"
2096 " ibmf_status = %d\n", tnf_int, result, ibmf_status);
2097
2098 return (ibmf_status);
2099 }
2100
2101 /*
2102 * ibmf_saa_impl_revert_to_qp1()
2103 *
2104 * The SA that we had contact with via redirect may fail to respond. If this
2105 * occurs SA should revert back to qp1 and the SMLID set in the port.
2106 * msg_transport for the message that timed out will be retried with
2107 * these new parameters.
2108 *
2109 * msgp, ibmf_callback, ibmf_callback_arg, and transport flags should be the
2110 * same values passed to the original ibmf_msg_transport that timedout. The
2111 * ibmf_retrans parameter will be re-retrieved from the saa_portp structure.
2112 *
2113 * Input Arguments
2114 * saa_portp pointer to saa_port structure
2115 * msgp ibmf message that timedout
2116 * ibmf_callback callback that should be called by msg_transport
2117 * ibmf_callback_arg args for ibmf_callback
2118 * transport_flags flags for ibmf_msg_transport
2119 *
2120 * Output Arguments
2121 * none
2122 *
2123 * Returns
2124 * none
2125 */
2126 static int
ibmf_saa_impl_revert_to_qp1(saa_port_t * saa_portp,ibmf_msg_t * msgp,ibmf_msg_cb_t ibmf_callback,void * ibmf_callback_args,int transport_flags)2127 ibmf_saa_impl_revert_to_qp1(saa_port_t *saa_portp, ibmf_msg_t *msgp,
2128 ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_args, int transport_flags)
2129 {
2130 ibt_hca_portinfo_t *ibt_portinfop;
2131 ib_lid_t master_sm_lid, base_lid;
2132 uint8_t sm_sl;
2133 int subnet_timeout;
2134 uint_t nports, size;
2135 ibmf_retrans_t ibmf_retrans;
2136 int ibmf_status;
2137 ibt_status_t ibt_status;
2138
2139 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
2140 ibmf_saa_impl_revert_to_qp1_start,
2141 IBMF_TNF_TRACE, "", "ibmf_saa_impl_revert_to_qp1() enter\n");
2142
2143 _NOTE(ASSUMING_PROTECTED(*msgp))
2144 _NOTE(ASSUMING_PROTECTED(*msgp->im_msgbufs_send.im_bufs_mad_hdr))
2145
2146 /* first query the portinfo to see if the lid changed */
2147 ibt_status = ibt_query_hca_ports_byguid(saa_portp->saa_pt_node_guid,
2148 saa_portp->saa_pt_port_num, &ibt_portinfop, &nports, &size);
2149
2150 if (ibt_status != IBT_SUCCESS) {
2151
2152 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
2153 ibmf_saa_impl_revert_to_qp1_err, IBMF_TNF_ERROR, "",
2154 "ibmf_saa_impl_revert_to_qp1: %s, ibmf_status ="
2155 " %d\n", tnf_string, msg,
2156 "ibt_query_hca_ports_byguid() failed",
2157 tnf_int, ibt_status, ibt_status);
2158
2159 ibmf_status = IBMF_TRANSPORT_FAILURE;
2160
2161 goto bail;
2162 }
2163
2164 master_sm_lid = ibt_portinfop->p_sm_lid;
2165 base_lid = ibt_portinfop->p_base_lid;
2166 sm_sl = ibt_portinfop->p_sm_sl;
2167 subnet_timeout = ibt_portinfop->p_subnet_timeout;
2168
2169 ibt_free_portinfo(ibt_portinfop, size);
2170
2171
2172 mutex_enter(&saa_portp->saa_pt_mutex);
2173
2174 saa_portp->saa_pt_redirect_active = B_FALSE;
2175
2176 /* update the address info in ibmf_saa */
2177 saa_portp->saa_pt_ibmf_addr_info.ia_local_lid = base_lid;
2178 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid = master_sm_lid;
2179 saa_portp->saa_pt_ibmf_addr_info.ia_service_level = sm_sl;
2180 saa_portp->saa_pt_ibmf_addr_info.ia_remote_qno = 1;
2181 saa_portp->saa_pt_ibmf_addr_info.ia_p_key = IB_PKEY_DEFAULT_LIMITED;
2182 saa_portp->saa_pt_ibmf_addr_info.ia_q_key = IB_GSI_QKEY;
2183 saa_portp->saa_pt_ibmf_msg_flags = 0;
2184
2185 /* new tid needed */
2186 msgp->im_msgbufs_send.im_bufs_mad_hdr->TransactionID =
2187 h2b64(saa_portp->saa_pt_current_tid++);
2188
2189 bcopy(&saa_portp->saa_pt_ibmf_retrans, &ibmf_retrans,
2190 sizeof (ibmf_retrans_t));
2191
2192 /* update the subnet timeout since this may be a new sm/sa */
2193 saa_portp->saa_pt_timeout = subnet_timeout;
2194
2195 /* place upper bound on subnet timeout in case of faulty SM */
2196 if (saa_portp->saa_pt_timeout > ibmf_saa_max_subnet_timeout)
2197 saa_portp->saa_pt_timeout = ibmf_saa_max_subnet_timeout;
2198
2199 /* increment the reference count to account for the cpi call */
2200 saa_portp->saa_pt_reference_count++;
2201
2202 mutex_exit(&saa_portp->saa_pt_mutex);
2203
2204 /* update the address info for this particular message */
2205 bcopy(&saa_portp->saa_pt_ibmf_addr_info, &msgp->im_local_addr,
2206 sizeof (ibmf_addr_info_t));
2207 msgp->im_msg_flags = 0; /* No GRH */
2208
2209 /* get the classportinfo again since this may be a new sm/sa */
2210 ibmf_saa_impl_get_classportinfo(saa_portp);
2211
2212 ibmf_status = ibmf_msg_transport(saa_portp->saa_pt_ibmf_handle,
2213 saa_portp->saa_pt_qp_handle, msgp, &ibmf_retrans,
2214 ibmf_callback, ibmf_callback_args, transport_flags);
2215
2216 if (ibmf_status != IBMF_SUCCESS) {
2217
2218 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2,
2219 ibmf_saa_impl_revert_to_qp1, IBMF_TNF_TRACE, "",
2220 "ibmf_saa_impl_revert_to_qp1: %s, ibmf_status = "
2221 "%d\n", tnf_string, msg,
2222 "ibmf_msg_transport() failed",
2223 tnf_int, ibmf_status, ibmf_status);
2224 }
2225
2226 bail:
2227
2228 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
2229 ibmf_saa_impl_revert_to_qp1_end,
2230 IBMF_TNF_TRACE, "", "ibmf_saa_impl_revert_to_qp1() exiting"
2231 " ibmf_status = %d\n", tnf_int, result, ibmf_status);
2232
2233 return (ibmf_status);
2234 }
2235
2236 /*
2237 * ibmf_saa_impl_async_event_cb:
2238 * ibmf event callback, argument to ibmf_register
2239 * ibmf_handle is unused
2240 */
2241 /* ARGSUSED */
2242 static void
ibmf_saa_impl_async_event_cb(ibmf_handle_t ibmf_handle,void * clnt_private,ibmf_async_event_t event_type)2243 ibmf_saa_impl_async_event_cb(
2244 ibmf_handle_t ibmf_handle,
2245 void *clnt_private,
2246 ibmf_async_event_t event_type)
2247 {
2248 saa_port_t *saa_portp;
2249
2250 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
2251 ibmf_saa_impl_async_event_cb_start, IBMF_TNF_TRACE, "",
2252 "ibmf_saa_impl_async_event_cb: Handling event type 0x%x\n",
2253 tnf_opaque, event_type, event_type);
2254
2255 saa_portp = (saa_port_t *)clnt_private;
2256 ASSERT(saa_portp != NULL);
2257
2258 switch (event_type) {
2259
2260 case IBMF_CI_OFFLINE:
2261 ibmf_saa_impl_hca_detach(saa_portp);
2262 break;
2263 default:
2264 break;
2265 }
2266
2267 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L3,
2268 ibmf_saa_impl_async_event_cb_end,
2269 IBMF_TNF_TRACE, "", "ibmf_saa_impl_async_event_cb() exit\n");
2270 }
2271
2272
2273 /*
2274 * ibmf_saa_impl_ibt_async_handler:
2275 * MUST NOT BE STATIC (referred from within IBMF)
2276 */
2277 void
ibmf_saa_impl_ibt_async_handler(ibt_async_code_t code,ibt_async_event_t * event)2278 ibmf_saa_impl_ibt_async_handler(ibt_async_code_t code, ibt_async_event_t *event)
2279 {
2280 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
2281 ibmf_saa_impl_ibt_async_handler_start, IBMF_TNF_TRACE, "",
2282 "ibmf_saa_impl_ibt_async_handler: Handling event code 0x%x\n",
2283 tnf_opaque, code, code);
2284
2285 switch (code) {
2286
2287 case IBT_EVENT_PORT_UP:
2288 ibmf_saa_impl_port_up(event->ev_hca_guid, event->ev_port);
2289 break;
2290 case IBT_ERROR_PORT_DOWN:
2291 ibmf_saa_impl_port_down(event->ev_hca_guid, event->ev_port);
2292 break;
2293 case IBT_PORT_CHANGE_EVENT:
2294 ibmf_saa_impl_port_chg(event);
2295 break;
2296 case IBT_CLNT_REREG_EVENT:
2297 ibmf_saa_impl_client_rereg(event->ev_hca_guid, event->ev_port);
2298 break;
2299 default:
2300 break;
2301 }
2302
2303 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_async_handler_end,
2304 IBMF_TNF_TRACE, "", "ibmf_saa_impl_ibt_async_handler() exit\n");
2305 }
2306
2307 /*
2308 * ibmf_saa_impl_port_chg:
2309 */
2310 static void
ibmf_saa_impl_port_chg(ibt_async_event_t * event)2311 ibmf_saa_impl_port_chg(ibt_async_event_t *event)
2312 {
2313 saa_port_t *saa_portp = NULL;
2314 boolean_t is_ready = B_FALSE;
2315 ibt_hca_portinfo_t *ibt_portinfop;
2316 uint_t nports, size;
2317 ibt_status_t ibt_status;
2318 ib_guid_t ci_guid;
2319 int port_num;
2320
2321 ci_guid = event->ev_hca_guid;
2322 port_num = event->ev_port;
2323
2324 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_port_chg_start,
2325 IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_chg: Handling port chg"
2326 " guid %016" PRIx64 " port %d\n",
2327 tnf_opaque, hca_guid, ci_guid, tnf_uint, port, port_num);
2328
2329 /* Get classportinfo of corresponding entry */
2330 mutex_enter(&saa_statep->saa_port_list_mutex);
2331
2332 saa_portp = saa_statep->saa_port_list;
2333 while (saa_portp != NULL) {
2334 if (saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid == ci_guid &&
2335 saa_portp->saa_pt_ibmf_reginfo.ir_port_num == port_num) {
2336 mutex_enter(&saa_portp->saa_pt_mutex);
2337
2338 is_ready = (saa_portp->saa_pt_state
2339 == IBMF_SAA_PORT_STATE_READY) ? B_TRUE : B_FALSE;
2340
2341 /*
2342 * increment reference count to account for cpi and
2343 * informinfos. All 4 informinfo's sent are treated as
2344 * one port client reference
2345 */
2346 if (is_ready)
2347 saa_portp->saa_pt_reference_count ++;
2348
2349 mutex_exit(&saa_portp->saa_pt_mutex);
2350
2351 if (is_ready)
2352 break; /* normally, only 1 port entry */
2353 }
2354 saa_portp = saa_portp->next;
2355 }
2356
2357 mutex_exit(&saa_statep->saa_port_list_mutex);
2358
2359 if (saa_portp != NULL) {
2360 /* first query the portinfo to see if the lid changed */
2361 ibt_status = ibt_query_hca_ports_byguid(ci_guid, port_num,
2362 &ibt_portinfop, &nports, &size);
2363
2364 if (ibt_status != IBT_SUCCESS) {
2365
2366 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
2367 ibmf_saa_impl_port_chg_err, IBMF_TNF_ERROR, "",
2368 "ibmf_saa_impl_port_chg: %s, ibmf_status ="
2369 " %d\n", tnf_string, msg,
2370 "ibt_query_hca_ports_byguid() failed",
2371 tnf_int, ibt_status, ibt_status);
2372
2373 goto bail;
2374 }
2375
2376 mutex_enter(&saa_portp->saa_pt_mutex);
2377 if (event->ev_port_flags & IBT_PORT_CHANGE_SM_LID) {
2378 /* update the Master SM Lid value in ibmf_saa */
2379 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid =
2380 ibt_portinfop->p_sm_lid;
2381 }
2382 if (event->ev_port_flags & IBT_PORT_CHANGE_SM_SL) {
2383 /* update the Master SM SL value in ibmf_saa */
2384 saa_portp->saa_pt_ibmf_addr_info.ia_service_level =
2385 ibt_portinfop->p_sm_sl;
2386 }
2387 if (event->ev_port_flags & IBT_PORT_CHANGE_SUB_TIMEOUT) {
2388 /* update the Subnet timeout value in ibmf_saa */
2389 saa_portp->saa_pt_timeout =
2390 ibt_portinfop->p_subnet_timeout;
2391 }
2392 mutex_exit(&saa_portp->saa_pt_mutex);
2393
2394 ibt_free_portinfo(ibt_portinfop, size);
2395
2396 /* get the classportinfo again */
2397 ibmf_saa_impl_get_classportinfo(saa_portp);
2398 }
2399 bail:
2400
2401 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_port_chg_end,
2402 IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_chg() exit\n");
2403 }
2404 /*
2405 * ibmf_saa_impl_client_rereg:
2406 */
2407 static void
ibmf_saa_impl_client_rereg(ib_guid_t ci_guid,uint8_t port_num)2408 ibmf_saa_impl_client_rereg(ib_guid_t ci_guid, uint8_t port_num)
2409 {
2410 saa_port_t *saa_portp = NULL;
2411 boolean_t is_ready = B_FALSE;
2412 ibt_hca_portinfo_t *ibt_portinfop;
2413 ib_lid_t master_sm_lid;
2414 uint_t nports, size;
2415 ibt_status_t ibt_status;
2416 boolean_t event_subs = B_FALSE;
2417
2418 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_port_rereg_start,
2419 IBMF_TNF_TRACE, "", "ibmf_saa_impl_client_rereg: Handling clnt "
2420 "rereg guid %016" PRIx64 " port %d\n",
2421 tnf_opaque, hca_guid, ci_guid, tnf_uint, port, port_num);
2422
2423 /* Get classportinfo of corresponding entry */
2424 mutex_enter(&saa_statep->saa_port_list_mutex);
2425
2426 saa_portp = saa_statep->saa_port_list;
2427 while (saa_portp != NULL) {
2428
2429 if (saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid == ci_guid &&
2430 saa_portp->saa_pt_ibmf_reginfo.ir_port_num == port_num) {
2431
2432 mutex_enter(&saa_portp->saa_pt_mutex);
2433
2434 is_ready = (saa_portp->saa_pt_state
2435 == IBMF_SAA_PORT_STATE_READY) ? B_TRUE : B_FALSE;
2436
2437 /*
2438 * increment reference count to account for cpi and
2439 * informinfos. All 4 informinfo's sent are treated as
2440 * one port client reference
2441 */
2442 if (is_ready)
2443 saa_portp->saa_pt_reference_count += 2;
2444
2445 mutex_exit(&saa_portp->saa_pt_mutex);
2446
2447 if (is_ready)
2448 break; /* normally, only 1 port entry */
2449 }
2450 saa_portp = saa_portp->next;
2451 }
2452
2453 mutex_exit(&saa_statep->saa_port_list_mutex);
2454
2455 if (saa_portp != NULL && is_ready == B_TRUE) {
2456
2457 /* verify whether master sm lid changed */
2458
2459 /* first query the portinfo to see if the lid changed */
2460 ibt_status = ibt_query_hca_ports_byguid(ci_guid, port_num,
2461 &ibt_portinfop, &nports, &size);
2462
2463 if (ibt_status != IBT_SUCCESS) {
2464
2465 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
2466 ibmf_saa_impl_port_rereg_err, IBMF_TNF_ERROR, "",
2467 "ibmf_saa_impl_client_rereg: %s, ibmf_status ="
2468 " %d\n", tnf_string, msg,
2469 "ibt_query_hca_ports_byguid() failed",
2470 tnf_int, ibt_status, ibt_status);
2471
2472 goto bail;
2473 }
2474
2475 master_sm_lid = ibt_portinfop->p_sm_lid;
2476
2477 ibt_free_portinfo(ibt_portinfop, size);
2478
2479 /* check whether we need to subscribe for events */
2480 mutex_enter(&saa_portp->saa_pt_event_sub_mutex);
2481
2482 event_subs = (saa_portp->saa_pt_event_sub_client_list != NULL) ?
2483 B_TRUE : B_FALSE;
2484
2485 mutex_exit(&saa_portp->saa_pt_event_sub_mutex);
2486
2487 /* update the master smlid */
2488 mutex_enter(&saa_portp->saa_pt_mutex);
2489
2490 /* update the master sm lid value in ibmf_saa */
2491 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid =
2492 master_sm_lid;
2493
2494 /* if we're not subscribed for events, dec reference count */
2495 if (event_subs == B_FALSE)
2496 saa_portp->saa_pt_reference_count--;
2497
2498 mutex_exit(&saa_portp->saa_pt_mutex);
2499
2500 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
2501 ibmf_saa_impl_port_rereg, IBMF_TNF_TRACE, "",
2502 "ibmf_saa_impl_client_rereg: %s, master_sm_lid = 0x%x\n",
2503 tnf_string, msg,
2504 "port is up. Sending classportinfo request",
2505 tnf_opaque, master_sm_lid, master_sm_lid);
2506
2507 /* get the classportinfo again */
2508 ibmf_saa_impl_get_classportinfo(saa_portp);
2509
2510 /*
2511 * resubscribe to events if there are subscribers since SA may
2512 * have removed our subscription records when the port went down
2513 */
2514 if (event_subs == B_TRUE)
2515 ibmf_saa_subscribe_events(saa_portp, B_TRUE, B_FALSE);
2516 }
2517
2518 bail:
2519
2520 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_port_rereg_end,
2521 IBMF_TNF_TRACE, "", "ibmf_saa_impl_client_rereg() exit\n");
2522 }
2523 /*
2524 * ibmf_saa_impl_port_up:
2525 */
2526 static void
ibmf_saa_impl_port_up(ib_guid_t ci_guid,uint8_t port_num)2527 ibmf_saa_impl_port_up(ib_guid_t ci_guid, uint8_t port_num)
2528 {
2529 saa_port_t *saa_portp = NULL;
2530 int is_ready;
2531 ibt_hca_portinfo_t *ibt_portinfop;
2532 ib_lid_t master_sm_lid;
2533 uint_t nports, size;
2534 ibt_status_t ibt_status;
2535 boolean_t event_subs = B_FALSE;
2536
2537 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_port_up_start,
2538 IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_up: Handling port up"
2539 " guid %016" PRIx64 " port %d\n",
2540 tnf_opaque, hca_guid, ci_guid, tnf_uint, port, port_num);
2541
2542 /* Get classportinfo of corresponding entry */
2543 mutex_enter(&saa_statep->saa_port_list_mutex);
2544
2545 saa_portp = saa_statep->saa_port_list;
2546 while (saa_portp != NULL) {
2547
2548 if (saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid == ci_guid &&
2549 saa_portp->saa_pt_ibmf_reginfo.ir_port_num == port_num) {
2550
2551 mutex_enter(&saa_portp->saa_pt_mutex);
2552
2553 is_ready = (saa_portp->saa_pt_state
2554 == IBMF_SAA_PORT_STATE_READY) ? B_TRUE : B_FALSE;
2555
2556 /*
2557 * increment reference count to account for cpi and
2558 * informinfos. All 4 informinfo's sent are treated as
2559 * one port client reference
2560 */
2561 if (is_ready == B_TRUE)
2562 saa_portp->saa_pt_reference_count += 2;
2563
2564 mutex_exit(&saa_portp->saa_pt_mutex);
2565
2566 if (is_ready == B_TRUE)
2567 break; /* normally, only 1 port entry */
2568 }
2569 saa_portp = saa_portp->next;
2570 }
2571
2572 mutex_exit(&saa_statep->saa_port_list_mutex);
2573
2574 if (saa_portp != NULL && is_ready == B_TRUE) {
2575
2576 /* verify whether master sm lid changed */
2577
2578 /* first query the portinfo to see if the lid changed */
2579 ibt_status = ibt_query_hca_ports_byguid(ci_guid, port_num,
2580 &ibt_portinfop, &nports, &size);
2581
2582 if (ibt_status != IBT_SUCCESS) {
2583
2584 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
2585 ibmf_saa_impl_port_up_err, IBMF_TNF_ERROR, "",
2586 "ibmf_saa_impl_port_up: %s, ibmf_status ="
2587 " %d\n", tnf_string, msg,
2588 "ibt_query_hca_ports_byguid() failed",
2589 tnf_int, ibt_status, ibt_status);
2590
2591 goto bail;
2592 }
2593
2594 master_sm_lid = ibt_portinfop->p_sm_lid;
2595
2596 ibt_free_portinfo(ibt_portinfop, size);
2597
2598 /* check whether we need to subscribe for events */
2599 mutex_enter(&saa_portp->saa_pt_event_sub_mutex);
2600
2601 event_subs = (saa_portp->saa_pt_event_sub_client_list != NULL) ?
2602 B_TRUE : B_FALSE;
2603
2604 mutex_exit(&saa_portp->saa_pt_event_sub_mutex);
2605
2606 /* update the master smlid */
2607 mutex_enter(&saa_portp->saa_pt_mutex);
2608
2609 /* update the master sm lid value in ibmf_saa */
2610 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid =
2611 master_sm_lid;
2612
2613 /* if we're not subscribed for events, dec reference count */
2614 if (event_subs == B_FALSE)
2615 saa_portp->saa_pt_reference_count--;
2616
2617 mutex_exit(&saa_portp->saa_pt_mutex);
2618
2619 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
2620 ibmf_saa_impl_port_up, IBMF_TNF_TRACE, "",
2621 "ibmf_saa_impl_port_up: %s, master_sm_lid = 0x%x\n",
2622 tnf_string, msg,
2623 "port is up. Sending classportinfo request",
2624 tnf_opaque, master_sm_lid, master_sm_lid);
2625
2626 /* get the classportinfo again */
2627 ibmf_saa_impl_get_classportinfo(saa_portp);
2628
2629 /*
2630 * resubscribe to events if there are subscribers since SA may
2631 * have removed our subscription records when the port went down
2632 */
2633 if (event_subs == B_TRUE)
2634 ibmf_saa_subscribe_events(saa_portp, B_TRUE, B_FALSE);
2635 }
2636
2637 bail:
2638
2639 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_port_up_end,
2640 IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_up() exit\n");
2641 }
2642
2643 /*
2644 * ibmf_saa_impl_port_down:
2645 */
2646 static void
ibmf_saa_impl_port_down(ib_guid_t ci_guid,uint8_t port_num)2647 ibmf_saa_impl_port_down(ib_guid_t ci_guid, uint8_t port_num)
2648 {
2649
2650 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_port_down_start,
2651 IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_down: Handling port down"
2652 " guid %016" PRIx64 " port %d\n",
2653 tnf_opaque, hca_guid, ci_guid, tnf_uint, port, port_num);
2654
2655
2656 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_port_down_end,
2657 IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_down() exit\n");
2658 }
2659
2660 /*
2661 * ibmf_saa_impl_hca_detach:
2662 * find entry, unregister if there are no clients
2663 * have to unregister since ibmf needs to close the hca and will only do this if
2664 * no clients are registered
2665 */
2666 static void
ibmf_saa_impl_hca_detach(saa_port_t * saa_removed)2667 ibmf_saa_impl_hca_detach(saa_port_t *saa_removed)
2668 {
2669 saa_port_t *saa_portp;
2670 boolean_t must_unreg, must_unsub;
2671
2672 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_hca_detach_start,
2673 IBMF_TNF_TRACE, "", "ibmf_saa_impl_hca_detach: Detaching"
2674 " entry %016" PRIx64 "\n", tnf_opaque, entry, saa_removed);
2675
2676 /* find this entry */
2677 mutex_enter(&saa_statep->saa_port_list_mutex);
2678
2679 saa_portp = saa_statep->saa_port_list;
2680 while (saa_portp != NULL) {
2681
2682 if (saa_portp == saa_removed)
2683 break;
2684
2685 saa_portp = saa_portp->next;
2686 }
2687 mutex_exit(&saa_statep->saa_port_list_mutex);
2688
2689 ASSERT(saa_portp != NULL);
2690
2691 if (saa_portp == NULL) {
2692
2693 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1,
2694 ibmf_saa_impl_hca_detach, IBMF_TNF_TRACE, "",
2695 "ibmf_saa_impl_hca_detach: %s, entry %016"
2696 PRIx64 "\n",
2697 tnf_string, msg,
2698 "Port entry NOT found",
2699 tnf_opaque, entryp, saa_removed);
2700
2701 goto bail;
2702 }
2703
2704 /* if there are clients expecting Reports(), unsusbscribe */
2705 mutex_enter(&saa_portp->saa_pt_event_sub_mutex);
2706
2707 must_unsub = (saa_portp->saa_pt_event_sub_client_list != NULL) ?
2708 B_TRUE : B_FALSE;
2709
2710 mutex_exit(&saa_portp->saa_pt_event_sub_mutex);
2711
2712 /* fail if outstanding transactions */
2713 mutex_enter(&saa_portp->saa_pt_mutex);
2714
2715 if (saa_portp->saa_pt_num_outstanding_trans > 0) {
2716
2717 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L1,
2718 ibmf_saa_impl_fini_err, IBMF_TNF_TRACE, "",
2719 "ibmf_saa_impl_fini: %s, port = %016" PRIx64
2720 ", num transactions = %d\n",
2721 tnf_string, msg, "Detaching HCA."
2722 " Outstanding transactions on port.",
2723 tnf_opaque, port,
2724 saa_portp->saa_pt_port_guid,
2725 tnf_uint, outstanding_transactions,
2726 saa_portp->saa_pt_num_outstanding_trans);
2727
2728 mutex_exit(&saa_portp->saa_pt_mutex);
2729
2730 goto bail;
2731 }
2732
2733
2734 /*
2735 * increment reference count by one to account for unsubscribe requests
2736 * that are about to be sent. All four informinfo's are treated as one
2737 * port client reference. The count will be decremented by
2738 * subscribe_events() before the call returns.
2739 */
2740 if (must_unsub == B_TRUE)
2741 saa_portp->saa_pt_reference_count++;
2742
2743 mutex_exit(&saa_portp->saa_pt_mutex);
2744
2745 /*
2746 * try and unsubscribe from SA. Generate synchronous, unsequenced
2747 * unsubscribe requests.
2748 */
2749 if (must_unsub == B_TRUE)
2750 ibmf_saa_subscribe_events(saa_portp, B_FALSE, B_TRUE);
2751
2752 /* warning if registered clients */
2753 mutex_enter(&saa_portp->saa_pt_mutex);
2754
2755 if (saa_portp->saa_pt_reference_count > 0) {
2756
2757 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1,
2758 ibmf_saa_impl_hca_detach, IBMF_TNF_TRACE, "",
2759 "ibmf_saa_impl_hca_detach: %s, port %016"
2760 PRIx64 "\n",
2761 tnf_string, msg,
2762 "Detaching HCA for port with clients still"
2763 " registered", tnf_opaque, port,
2764 saa_portp->saa_pt_port_guid);
2765 }
2766
2767 /* synchronize on end of registration */
2768 while (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_REGISTERING) {
2769
2770 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L1,
2771 ibmf_saa_impl_hca_detach, IBMF_TNF_TRACE, "",
2772 "ibmf_saa_impl_hca_detach: %s\n",
2773 tnf_string, msg, "someone is registering. waiting"
2774 " for them to finish");
2775
2776 cv_wait(&saa_portp->saa_pt_ibmf_reg_cv,
2777 &saa_portp->saa_pt_mutex);
2778
2779 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L1,
2780 ibmf_saa_impl_hca_detach,
2781 IBMF_TNF_TRACE, "", "ibmf_saa_impl_hca_detach: %s\n",
2782 tnf_string, msg, "done waiting");
2783 }
2784
2785 /* unregister from ibmf */
2786 if (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_READY) {
2787 must_unreg = B_TRUE;
2788 } else
2789 must_unreg = B_FALSE;
2790
2791 ibmf_saa_impl_invalidate_port(saa_portp);
2792
2793 mutex_exit(&saa_portp->saa_pt_mutex);
2794
2795 if (must_unreg == B_TRUE) {
2796 if (ibmf_saa_impl_ibmf_unreg(saa_portp) != IBMF_SUCCESS) {
2797 mutex_enter(&saa_portp->saa_pt_mutex);
2798 mutex_enter(&saa_portp->saa_pt_kstat_mutex);
2799 (void) ibmf_saa_impl_init_kstats(saa_portp);
2800 mutex_exit(&saa_portp->saa_pt_kstat_mutex);
2801 saa_portp->saa_pt_state = IBMF_SAA_PORT_STATE_READY;
2802 if (must_unsub == B_TRUE)
2803 saa_portp->saa_pt_reference_count++;
2804 mutex_exit(&saa_portp->saa_pt_mutex);
2805
2806 if (must_unsub == B_TRUE) {
2807 ibmf_saa_subscribe_events(saa_portp, B_TRUE,
2808 B_FALSE);
2809 }
2810 }
2811 }
2812 bail:
2813 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_hca_detach_end,
2814 IBMF_TNF_TRACE, "", "ibmf_saa_impl_hca_detach() exit\n");
2815 }
2816
2817 /* ARGSUSED */
2818 void
ibmf_saa_async_cb(ibmf_handle_t ibmf_handle,ibmf_msg_t * msgp,void * args)2819 ibmf_saa_async_cb(ibmf_handle_t ibmf_handle, ibmf_msg_t *msgp, void *args)
2820 {
2821 saa_impl_trans_info_t *trans_info;
2822 int status;
2823 size_t length;
2824 void *result;
2825 saa_port_t *saa_portp;
2826 saa_client_data_t *client_data;
2827 int ibmf_status;
2828 boolean_t ignore_data;
2829 ibmf_retrans_t ibmf_retrans;
2830 boolean_t sa_is_redirected = B_FALSE;
2831
2832 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_async_cb_start,
2833 IBMF_TNF_TRACE, "", "ibmf_saa_async_cb() enter\n");
2834
2835 trans_info = (saa_impl_trans_info_t *)args;
2836
2837 client_data = trans_info->si_trans_client_data;
2838 saa_portp = trans_info->si_trans_port;
2839
2840 mutex_enter(&saa_portp->saa_pt_mutex);
2841 sa_is_redirected = saa_portp->saa_pt_redirect_active;
2842 mutex_exit(&saa_portp->saa_pt_mutex);
2843
2844 if ((msgp->im_msg_status == IBMF_TRANS_TIMEOUT) &&
2845 (sa_is_redirected == B_TRUE)) {
2846
2847 /*
2848 * We should retry the request using SM_LID and QP1 if we
2849 * have been using redirect up until now
2850 */
2851 ibmf_status = ibmf_saa_impl_revert_to_qp1(
2852 saa_portp, msgp, ibmf_saa_async_cb, args,
2853 trans_info->si_trans_transport_flags);
2854
2855 /*
2856 * If revert_to_qp1 returns success msg was resent.
2857 * Otherwise msg could not be resent. Continue normally
2858 */
2859 if (ibmf_status == IBMF_SUCCESS)
2860 goto bail;
2861
2862 } else if (msgp->im_msg_status == IBMF_TRANS_TIMEOUT) {
2863
2864
2865 ibmf_status = ibmf_saa_impl_new_smlid_retry(saa_portp, msgp,
2866 ibmf_saa_async_cb, args,
2867 trans_info->si_trans_transport_flags);
2868
2869 /*
2870 * if smlid_retry() returns success sm lid changed and msg
2871 * was resent. Otherwise, lid did not change or msg could not
2872 * be resent. Continue normally.
2873 */
2874 if (ibmf_status == IBMF_SUCCESS)
2875 goto bail;
2876
2877 /*
2878 * check whether we've received anything from the SA in a while.
2879 * If we have, this function will retry and return success. If
2880 * we haven't continue normally so that we return a timeout to
2881 * the client
2882 */
2883 ibmf_status = ibmf_saa_check_sa_and_retry(
2884 saa_portp, msgp, ibmf_saa_async_cb, args,
2885 trans_info->si_trans_send_time,
2886 trans_info->si_trans_transport_flags);
2887
2888 if (ibmf_status == IBMF_SUCCESS)
2889 goto bail;
2890 }
2891
2892 /*
2893 * If SA returned success but mad status is busy, retry a few times.
2894 * If SA returned success but mad status says redirect is required,
2895 * update the address info and retry the request to the new SA address
2896 */
2897 if (msgp->im_msg_status == IBMF_SUCCESS) {
2898
2899 ASSERT(msgp->im_msgbufs_recv.im_bufs_mad_hdr != NULL);
2900
2901 if ((b2h16(msgp->im_msgbufs_recv.im_bufs_mad_hdr->Status) ==
2902 MAD_STATUS_BUSY) &&
2903 (trans_info->si_trans_retry_busy_count <
2904 IBMF_SAA_MAX_BUSY_RETRY_COUNT)) {
2905
2906 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2,
2907 ibmf_saa_async_cb, IBMF_TNF_TRACE, "",
2908 "ibmf_saa_async_cb: %s, retry_count = %d\n",
2909 tnf_string, msg,
2910 "async response returned busy status",
2911 tnf_int, retry_count,
2912 trans_info->si_trans_retry_busy_count);
2913
2914 trans_info->si_trans_retry_busy_count++;
2915
2916 bcopy(&saa_portp->saa_pt_ibmf_retrans, &ibmf_retrans,
2917 sizeof (ibmf_retrans_t));
2918
2919 ibmf_status = ibmf_msg_transport(
2920 saa_portp->saa_pt_ibmf_handle,
2921 saa_portp->saa_pt_qp_handle, msgp, &ibmf_retrans,
2922 ibmf_saa_async_cb, args,
2923 trans_info->si_trans_transport_flags);
2924
2925 /*
2926 * if retry is successful, quit here since async_cb will
2927 * get called again; otherwise, let this function call
2928 * handle the cleanup
2929 */
2930 if (ibmf_status == IBMF_SUCCESS)
2931 goto bail;
2932 } else if (b2h16(msgp->im_msgbufs_recv.im_bufs_mad_hdr->Status)
2933 == MAD_STATUS_REDIRECT_REQUIRED) {
2934
2935 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L2,
2936 ibmf_saa_async_cb, IBMF_TNF_TRACE, "",
2937 "ibmf_saa_async_cb: "
2938 "async response returned redirect status\n");
2939
2940 /* update address info and copy it into msgp */
2941 ibmf_saa_impl_update_sa_address_info(saa_portp, msgp);
2942
2943 /* retry with new address info */
2944 bcopy(&saa_portp->saa_pt_ibmf_retrans, &ibmf_retrans,
2945 sizeof (ibmf_retrans_t));
2946
2947 ibmf_status = ibmf_msg_transport(
2948 saa_portp->saa_pt_ibmf_handle,
2949 saa_portp->saa_pt_qp_handle, msgp, &ibmf_retrans,
2950 ibmf_saa_async_cb, args,
2951 trans_info->si_trans_transport_flags);
2952
2953 /*
2954 * if retry is successful, quit here since async_cb will
2955 * get called again; otherwise, let this function call
2956 * handle the cleanup
2957 */
2958 if (ibmf_status == IBMF_SUCCESS)
2959 goto bail;
2960 }
2961 }
2962
2963 mutex_enter(&saa_portp->saa_pt_mutex);
2964
2965 ASSERT(saa_portp->saa_pt_num_outstanding_trans > 0);
2966 saa_portp->saa_pt_num_outstanding_trans--;
2967
2968 mutex_exit(&saa_portp->saa_pt_mutex);
2969
2970 if ((trans_info->si_trans_callback == NULL) &&
2971 (trans_info->si_trans_sub_callback == NULL))
2972 ignore_data = B_TRUE;
2973 else
2974 ignore_data = B_FALSE;
2975
2976 ibmf_saa_impl_prepare_response(ibmf_handle, msgp, ignore_data, &status,
2977 &result, &length, B_FALSE);
2978
2979 mutex_enter(&saa_portp->saa_pt_kstat_mutex);
2980
2981 IBMF_SAA_SUB32_KSTATS(saa_portp, outstanding_requests, 1);
2982
2983 if (status != IBMF_SUCCESS)
2984 IBMF_SAA_ADD32_KSTATS(saa_portp, failed_requests, 1);
2985
2986 if (status == IBMF_TRANS_TIMEOUT)
2987 IBMF_SAA_ADD32_KSTATS(saa_portp, requests_timedout, 1);
2988
2989 mutex_exit(&saa_portp->saa_pt_kstat_mutex);
2990
2991 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
2992 ibmf_saa_async_cb, IBMF_TNF_TRACE, "",
2993 "ibmf_saa_async_cb: %s\n", tnf_string, msg,
2994 "Calling ibmf_saa client's callback");
2995
2996 /*
2997 * there are three classes or trans_info users: ibmf_saa clients and
2998 * classportinfo requests; informinfo subscribe requests, and report
2999 * responses. For the first two, call the correct callback. For report
3000 * responses there's no need to notify anyone.
3001 */
3002 if (trans_info->si_trans_callback != NULL) {
3003 /* ibmf_saa client or classportinfo request */
3004 trans_info->si_trans_callback(trans_info->si_trans_callback_arg,
3005 length, result, status);
3006 } else if (trans_info->si_trans_sub_callback != NULL) {
3007 /* informinfo subscribe request */
3008 trans_info->si_trans_sub_callback(
3009 trans_info->si_trans_callback_arg, length, result, status,
3010 trans_info->si_trans_sub_producer_type);
3011 }
3012
3013 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
3014 ibmf_saa_async_cb, IBMF_TNF_TRACE, "",
3015 "ibmf_saa_async_cb: %s\n", tnf_string, msg,
3016 "Returned from callback");
3017
3018 if (client_data != NULL) {
3019 mutex_enter(&client_data->saa_client_mutex);
3020
3021 ASSERT(client_data->saa_client_num_pending_trans > 0);
3022 client_data->saa_client_num_pending_trans--;
3023
3024 if ((client_data->saa_client_num_pending_trans == 0) &&
3025 (client_data->saa_client_state == SAA_CLIENT_STATE_WAITING))
3026 cv_signal(&client_data->saa_client_state_cv);
3027
3028 mutex_exit(&client_data->saa_client_mutex);
3029 }
3030
3031 kmem_free(trans_info, sizeof (saa_impl_trans_info_t));
3032
3033 bail:
3034
3035 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_async_cb_end,
3036 IBMF_TNF_TRACE, "", "ibmf_saa_async_cb() exit\n");
3037 }
3038
3039 /*
3040 * ibmf_saa_check_sa_and_retry:
3041 *
3042 * If a particular transaction times out, we don't want to give up if we know
3043 * the SA is responding. Check the time since we last received a response. If
3044 * it's less than ibmf_saa_max_wait_time retry the request.
3045 *
3046 * msgp, ibmf_callback, ibmf_callback_arg, and transport flags should be the
3047 * same values passed to the original ibmf_msg_transport that timed out. The
3048 * ibmf_retrans parameter will be re-retrieved from the saa_portp structure.
3049 *
3050 * If max_wait_time seconds have passed, this function returns IBMF_TIMEOUT.
3051 * That way, callers can simply return the result of this function.
3052 *
3053 * Input Arguments
3054 * saa_portp pointer to saa_port structure
3055 * msgp ibmf message that timedout
3056 * ibmf_callback callback that should be called by msg_transport
3057 * ibmf_callback_arg args for ibmf_callback
3058 * transport_flags flags for ibmf_msg_transport
3059 *
3060 * Output Arguments
3061 * none
3062 *
3063 * Returns
3064 * IBMF_SUCCESS if we've recently received data from the SA and request was
3065 * resent.
3066 * IBMF_TRANS_TIMEOUT if no data has been received from the SA in max_wait_time
3067 * same values as ibmf_msg_transport() if data has been received but request
3068 * could not be resent.
3069 */
3070 static int
ibmf_saa_check_sa_and_retry(saa_port_t * saa_portp,ibmf_msg_t * msgp,ibmf_msg_cb_t ibmf_callback,void * ibmf_callback_arg,hrtime_t trans_send_time,int transport_flags)3071 ibmf_saa_check_sa_and_retry(saa_port_t *saa_portp, ibmf_msg_t *msgp,
3072 ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_arg,
3073 hrtime_t trans_send_time, int transport_flags)
3074 {
3075 hrtime_t curr_time, sa_uptime;
3076 ibmf_retrans_t ibmf_retrans;
3077 int ibmf_status;
3078
3079 do {
3080
3081 mutex_enter(&saa_portp->saa_pt_mutex);
3082
3083 sa_uptime = saa_portp->saa_pt_sa_uptime;
3084
3085 /* if nothing received from SA since we sent */
3086 curr_time = gethrtime();
3087
3088 /*
3089 * check if it's been a very long time since this
3090 * particular transaction was sent
3091 */
3092 if (((curr_time - trans_send_time) / 1000000000) >
3093 ibmf_saa_trans_wait_time) {
3094
3095 mutex_exit(&saa_portp->saa_pt_mutex);
3096
3097 IBMF_TRACE_5(IBMF_TNF_DEBUG, DPRINT_L1,
3098 ibmf_saa_check_sa_and_retry_err, IBMF_TNF_ERROR, "",
3099 "ibmf_saa_check_sa_and_retry: %s, msgp = "
3100 "%p sa_uptime = %" PRIu64 ", trans send time = %"
3101 PRIu64 ", curr_time = %" PRIu64 "\n",
3102 tnf_string, msg,
3103 "Nothing received for this transaction",
3104 tnf_opaque, msgp, msgp,
3105 tnf_long, sa_uptime, sa_uptime,
3106 tnf_long, trans_send_time, trans_send_time,
3107 tnf_long, curr_time, curr_time);
3108
3109 ibmf_status = IBMF_TRANS_TIMEOUT;
3110
3111 break;
3112 }
3113
3114 /*
3115 * check time since we received something,
3116 * and make sure that it hasn't been an extra long
3117 * time for this particular transaction
3118 */
3119 if (((curr_time - sa_uptime) / 1000000000) <
3120 ibmf_saa_max_wait_time) {
3121
3122 IBMF_TRACE_5(IBMF_TNF_DEBUG, DPRINT_L2,
3123 ibmf_saa_check_sa_and_retry, IBMF_TNF_TRACE, "",
3124 "ibmf_saa_check_sa_and_retry: %s, msgp = "
3125 "%p sa_uptime = %" PRIu64 " trans_send_time = %"
3126 PRIu64 " curr_time = %" PRIu64 "\n",
3127 tnf_string, msg, "Something received. Retrying",
3128 tnf_opaque, msgp, msgp,
3129 tnf_long, sa_uptime, sa_uptime,
3130 tnf_long, trans_send_time, trans_send_time,
3131 tnf_long, curr_time, curr_time);
3132
3133 /*
3134 * something received in WAIT_TIME_IN_SECS;
3135 * resend request
3136 */
3137
3138 /* new tid needed */
3139 msgp->im_msgbufs_send.im_bufs_mad_hdr->TransactionID =
3140 h2b64(saa_portp->saa_pt_current_tid++);
3141
3142 /*
3143 * We are going to retry the access to the SM but
3144 * Master SMLID could have changed due to a port change
3145 * event. So update the remote_lid of the message with
3146 * the SMLID from saa_portp for this port before the
3147 * retry.
3148 */
3149 msgp->im_local_addr.ia_remote_lid =
3150 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid;
3151
3152 bcopy(&saa_portp->saa_pt_ibmf_retrans,
3153 &ibmf_retrans, sizeof (ibmf_retrans_t));
3154
3155 mutex_exit(&saa_portp->saa_pt_mutex);
3156
3157 ibmf_status = ibmf_msg_transport(
3158 saa_portp->saa_pt_ibmf_handle,
3159 saa_portp->saa_pt_qp_handle, msgp,
3160 &ibmf_retrans, ibmf_callback, ibmf_callback_arg,
3161 transport_flags);
3162
3163 if (ibmf_status == IBMF_SUCCESS)
3164 goto bail;
3165
3166 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2,
3167 ibmf_saa_check_sa_and_retry, IBMF_TNF_TRACE, "",
3168 "ibmf_saa_check_sa_and_retry: %s, ibmf_status = "
3169 "%d\n", tnf_string, msg,
3170 "ibmf_msg_transport() failed",
3171 tnf_int, ibmf_status, ibmf_status);
3172 } else {
3173
3174 mutex_exit(&saa_portp->saa_pt_mutex);
3175
3176 IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L1,
3177 ibmf_saa_check_sa_and_retry_err, IBMF_TNF_ERROR, "",
3178 "ibmf_saa_check_sa_and_retry: %s, msgp = "
3179 "%p sa_uptime = %" PRIu64 " curr_time = %"
3180 PRIu64 "\n", tnf_string, msg,
3181 "Nothing received. Timing out",
3182 tnf_opaque, msgp, msgp,
3183 tnf_long, sa_uptime, sa_uptime,
3184 tnf_long, curr_time, curr_time);
3185
3186 ibmf_status = IBMF_TRANS_TIMEOUT;
3187
3188 break;
3189 }
3190 } while (ibmf_status == IBMF_TRANS_TIMEOUT);
3191
3192 bail:
3193 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
3194 ibmf_saa_check_sa_and_retry_end,
3195 IBMF_TNF_TRACE, "", "ibmf_saa_check_sa_and_retry() exiting"
3196 " ibmf_status = %d\n", tnf_int, result, ibmf_status);
3197
3198 return (ibmf_status);
3199 }
3200
3201
3202 /*
3203 * ibmf_saa_impl_prepare_response:
3204 */
3205 static void
ibmf_saa_impl_prepare_response(ibmf_handle_t ibmf_handle,ibmf_msg_t * msgp,boolean_t ignore_data,int * status,void ** result,size_t * length,boolean_t sleep_flag)3206 ibmf_saa_impl_prepare_response(ibmf_handle_t ibmf_handle,
3207 ibmf_msg_t *msgp, boolean_t ignore_data, int *status, void **result,
3208 size_t *length, boolean_t sleep_flag)
3209 {
3210 ibmf_msg_bufs_t *resp_buf;
3211 uint16_t attr_id;
3212 uint8_t method;
3213 boolean_t is_get_resp;
3214 uint16_t mad_status;
3215 uint16_t attr_offset;
3216 ib_sa_hdr_t *sa_hdr;
3217
3218 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
3219 ibmf_saa_impl_prepare_response_start,
3220 IBMF_TNF_TRACE, "", "ibmf_saa_impl_prepare_response() enter\n");
3221
3222 _NOTE(ASSUMING_PROTECTED(*msgp))
3223
3224 *result = NULL;
3225 *length = 0;
3226 sa_hdr = NULL;
3227
3228 resp_buf = &msgp->im_msgbufs_recv;
3229
3230 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*resp_buf))
3231
3232 if (msgp->im_msg_status != IBMF_SUCCESS) {
3233
3234 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2,
3235 ibmf_saa_impl_prepare_response, IBMF_TNF_TRACE, "",
3236 "ibmf_saa_impl_prepare_response: %s, msg_status = %d\n",
3237 tnf_string, msg, "Bad ibmf status",
3238 tnf_int, msg_status, msgp->im_msg_status);
3239
3240 *status = msgp->im_msg_status;
3241
3242 goto exit;
3243 }
3244
3245 if (resp_buf->im_bufs_mad_hdr == NULL) {
3246
3247 /*
3248 * this was an unsequenced transaction (from an unsubscribe for
3249 * following a CI_OFFLINE event)
3250 */
3251 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
3252 ibmf_saa_impl_prepare_response, IBMF_TNF_TRACE, "",
3253 "ibmf_saa_impl_prepare_response: %s\n",
3254 tnf_string, msg, "Unsequenced transaction callback");
3255
3256 goto exit;
3257 }
3258
3259 if ((mad_status = b2h16(resp_buf->im_bufs_mad_hdr->Status)) !=
3260 MAD_STATUS_NO_INVALID_FIELDS) {
3261
3262 /* convert mad packet status to IBMF status */
3263 switch (mad_status) {
3264
3265 case SA_STATUS_ERR_NO_RESOURCES:
3266 *status = IBMF_NO_RESOURCES;
3267 break;
3268 case SA_STATUS_ERR_REQ_INVALID:
3269 *status = IBMF_REQ_INVALID;
3270 break;
3271 case SA_STATUS_ERR_NO_RECORDS:
3272 *status = IBMF_NO_RECORDS;
3273 break;
3274 case SA_STATUS_ERR_TOO_MANY_RECORDS:
3275 *status = IBMF_TOO_MANY_RECORDS;
3276 break;
3277 case SA_STATUS_ERR_REQ_INVALID_GID:
3278 *status = IBMF_INVALID_GID;
3279 break;
3280 case SA_STATUS_ERR_REQ_INSUFFICIENT_COMPONENTS:
3281 *status = IBMF_INSUFF_COMPS;
3282 break;
3283 case MAD_STATUS_UNSUPP_METHOD:
3284 *status = IBMF_UNSUPP_METHOD;
3285 break;
3286 case MAD_STATUS_UNSUPP_METHOD_ATTR:
3287 *status = IBMF_UNSUPP_METHOD_ATTR;
3288 break;
3289 case MAD_STATUS_INVALID_FIELD:
3290 *status = IBMF_INVALID_FIELD;
3291 break;
3292 default:
3293 *status = IBMF_REQ_INVALID;
3294 break;
3295 }
3296
3297 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2,
3298 ibmf_saa_impl_prepare_response, IBMF_TNF_TRACE, "",
3299 "ibmf_saa_impl_prepare_response: %s, mad_status = %x\n",
3300 tnf_string, msg, "Bad MAD status",
3301 tnf_int, mad_status, mad_status);
3302
3303 goto exit;
3304 }
3305
3306 attr_id = b2h16(resp_buf->im_bufs_mad_hdr->AttributeID);
3307 method = resp_buf->im_bufs_mad_hdr->R_Method;
3308
3309 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
3310 ibmf_saa_impl_prepare_response, IBMF_TNF_TRACE, "",
3311 "ibmf_saa_impl_prepare_response: attr_id = 0x%x, method = "
3312 "0x%x\n",
3313 tnf_opaque, attr_id, attr_id,
3314 tnf_opaque, method, method);
3315
3316 /*
3317 * ignore any data from deleteresp since there's no way to know whether
3318 * real data was returned; also ignore data if this was a Report
3319 * response
3320 */
3321 if (method == SA_SUBN_ADM_DELETE_RESP) {
3322
3323 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
3324 ibmf_saa_impl_prepare_response, IBMF_TNF_TRACE, "",
3325 "impf_saa_impl_prepare_response: %s\n",
3326 tnf_string, msg,
3327 "DeleteResp or NoticeResp returned. "
3328 "Ignoring response data");
3329
3330 *status = IBMF_SUCCESS;
3331
3332 *length = 0;
3333 *result = NULL;
3334
3335 goto exit;
3336 }
3337
3338 if (attr_id == SA_MULTIPATHRECORD_ATTRID) {
3339
3340 /*
3341 * getmulti is only for requests; attribute should not
3342 * be returned from SA
3343 */
3344 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1,
3345 ibmf_saa_impl_prepare_response_err, IBMF_TNF_ERROR,
3346 "", "ibmf_saa_impl_prepare_response: %s\n",
3347 tnf_string, msg, "SA returned getmulti record");
3348
3349 *status = IBMF_REQ_INVALID;
3350
3351 goto exit;
3352 }
3353
3354 /* if we are supposed to ignore data, stop here */
3355 if (ignore_data == B_TRUE) {
3356
3357 *status = IBMF_SUCCESS;
3358
3359 goto exit;
3360 }
3361
3362 is_get_resp = resp_buf->im_bufs_mad_hdr->R_Method ==
3363 SA_SUBN_ADM_GET_RESP ? B_TRUE: B_FALSE;
3364
3365 /* unpack the sa header to get the attribute offset */
3366 *status = ibmf_saa_utils_unpack_sa_hdr(resp_buf->im_bufs_cl_hdr,
3367 resp_buf->im_bufs_cl_hdr_len, &sa_hdr, sleep_flag);
3368 if (*status != IBMF_SUCCESS) {
3369
3370 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1,
3371 ibmf_saa_impl_prepare_response_err,
3372 IBMF_TNF_TRACE, "", "ibmf_saa_impl_prepare_response: %s,"
3373 " ibmf_status = %d\n", tnf_string, msg,
3374 "Could not unpack sa hdr", tnf_int, ibmf_status, *status);
3375
3376 goto exit;
3377 }
3378
3379 attr_offset = sa_hdr->AttributeOffset;
3380
3381 /*
3382 * unpack data payload; if unpack function doesn't return success
3383 * (because it could not allocate memory) forward this status to waiting
3384 * client
3385 */
3386 *status = ibmf_saa_utils_unpack_payload(resp_buf->im_bufs_cl_data,
3387 resp_buf->im_bufs_cl_data_len, attr_id, result, length,
3388 attr_offset, is_get_resp, sleep_flag);
3389 if (*status == IBMF_SUCCESS) {
3390
3391 IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L3,
3392 ibmf_saa_impl_prepare_response,
3393 IBMF_TNF_TRACE, "",
3394 "ibmf_saa_impl_prepare_response: attr_id = "
3395 "0x%x, attr_offset = %d, packed_payload_len = %d, "
3396 "unpacked_payload_len = %d\n",
3397 tnf_opaque, attr_id, attr_id,
3398 tnf_opaque, attr_offset, attr_offset,
3399 tnf_opaque, packed_payload_len,
3400 resp_buf->im_bufs_cl_data_len,
3401 tnf_opaque, unpacked_payload_len, *length);
3402 } else {
3403
3404 IBMF_TRACE_5(IBMF_TNF_DEBUG, DPRINT_L1,
3405 ibmf_saa_impl_prepare_response_err,
3406 IBMF_TNF_TRACE, "", "ibmf_saa_impl_prepare_response: %s,"
3407 "attr_id = 0x%x, attr_offset = %d, packed_payload_len = %d,"
3408 "status = %d\n",
3409 tnf_string, msg, "Could not unpack payload",
3410 tnf_opaque, attr_id, attr_id,
3411 tnf_int, attr_offset, attr_offset,
3412 tnf_int, packed_payload_len,
3413 resp_buf->im_bufs_cl_data_len,
3414 tnf_int, status, *status);
3415 }
3416 exit:
3417 if (sa_hdr != NULL)
3418 kmem_free(sa_hdr, sizeof (ib_sa_hdr_t));
3419
3420 ibmf_saa_impl_free_msg(ibmf_handle, msgp);
3421
3422 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
3423 ibmf_saa_impl_prepare_response_end,
3424 IBMF_TNF_TRACE, "", "ibmf_saa_impl_prepare_response() exit,"
3425 " status = 0x%d\n", tnf_int, status, *status);
3426 }
3427
3428
3429 /*
3430 * ibmf_saa_impl_check_sa_support:
3431 * Checks the capability mask (returned from the SA classportinfo response) to
3432 * determine whether the sa supports the specified attribute ID.
3433 *
3434 * Input Arguments
3435 * cap_mask 16-bit capability mask returned in SA's classportinfo
3436 * attr_id attribute ID of current request
3437 *
3438 * Returns
3439 * IBMF_NOT_SUPPORTED if capability mask indicates SA does not support attribute
3440 * IBMF_SUCCESS otherwise
3441 */
3442 static int
ibmf_saa_impl_check_sa_support(uint16_t cap_mask,uint16_t attr_id)3443 ibmf_saa_impl_check_sa_support(uint16_t cap_mask, uint16_t attr_id)
3444 {
3445 boolean_t attr_supported = B_TRUE;
3446
3447 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
3448 ibmf_saa_impl_check_sa_support, IBMF_TNF_TRACE, "",
3449 "ibmf_saa_impl_check_sa_support: cap_mask = 0x%x, "
3450 "attr_id = 0x%x\n", tnf_opaque, cap_mask, cap_mask,
3451 tnf_opaque, attr_id, attr_id);
3452
3453 switch (attr_id) {
3454
3455 case SA_SWITCHINFORECORD_ATTRID:
3456 case SA_LINEARFDBRECORD_ATTRID:
3457 case SA_RANDOMFDBRECORD_ATTRID:
3458 case SA_MULTICASTFDBRECORD_ATTRID:
3459 case SA_SMINFORECORD_ATTRID:
3460 case SA_INFORMINFORECORD_ATTRID:
3461 case SA_LINKRECORD_ATTRID:
3462 case SA_GUIDINFORECORD_ATTRID:
3463 case SA_TRACERECORD_ATTRID:
3464 case SA_SERVICEASSNRECORD_ATTRID:
3465
3466 if ((cap_mask &
3467 SA_CAPMASK_OPT_RECORDS_SUPPORTED) == 0) {
3468
3469 IBMF_TRACE_3(IBMF_TNF_NODEBUG, DPRINT_L1,
3470 ibmf_saa_impl_check_sa_support,
3471 IBMF_TNF_ERROR, "",
3472 "ibmf_saa_impl_check_sa_support: %s, "
3473 "cap_mask = 0x%x\n", tnf_string, msg,
3474 "SA does not support optional records",
3475 tnf_opaque, cap_mask, cap_mask,
3476 tnf_opaque, attr_id, attr_id);
3477
3478 attr_supported = B_FALSE;
3479 }
3480 break;
3481
3482 case SA_MULTIPATHRECORD_ATTRID:
3483
3484 if ((cap_mask & SA_CAPMASK_MULTIPATH_SUPPORTED) == 0) {
3485
3486 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
3487 ibmf_saa_impl_check_sa_support,
3488 IBMF_TNF_ERROR, "",
3489 "ibmf_saa_impl_check_sa_support: %s, "
3490 "cap_mask = 0x%x\n", tnf_string, msg,
3491 "SA does not support multipath records",
3492 tnf_opaque, cap_mask, cap_mask);
3493
3494 attr_supported = B_FALSE;
3495 }
3496 break;
3497
3498 case SA_MCMEMBERRECORD_ATTRID:
3499
3500 if ((cap_mask & SA_CAPMASK_UD_MCAST_SUPPORTED) == 0) {
3501
3502 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
3503 ibmf_saa_impl_check_sa_support,
3504 IBMF_TNF_ERROR, "",
3505 "ibmf_saa_impl_check_sa_support: %s, "
3506 "cap_mask = 0x%x\n", tnf_string, msg,
3507 "SA does not support ud multicast",
3508 tnf_opaque, cap_mask, cap_mask);
3509
3510 attr_supported = B_FALSE;
3511 }
3512 break;
3513
3514 default:
3515 break;
3516 } /* switch */
3517
3518 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
3519 ibmf_saa_impl_check_sa_support_end, IBMF_TNF_TRACE, "",
3520 "ibmf_saa_impl_check_sa_support() exiting, attr_supported = %d\n",
3521 tnf_opaque, attr_supported, attr_supported);
3522
3523 if (attr_supported == B_FALSE)
3524 return (IBMF_UNSUPP_METHOD_ATTR);
3525 else
3526 return (IBMF_SUCCESS);
3527 }
3528
3529 /*
3530 * ibmf_saa_impl_get_attr_id_length:
3531 *
3532 * Returns the host size of the specified sa record. Returns 0 for unknown
3533 * attributes. multipath record size is a dynamic value given as a parameter
3534 * specified with the ibmf_sa_access() call.
3535 */
3536 static uint_t
ibmf_saa_impl_get_attr_id_length(uint16_t attr_id)3537 ibmf_saa_impl_get_attr_id_length(uint16_t attr_id)
3538 {
3539 uint_t attr_length;
3540
3541 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
3542 ibmf_saa_impl_get_attr_id_length_start,
3543 IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_attr_id_length() enter\n");
3544
3545 /* this function should not be used for multipath record */
3546 ASSERT(attr_id != SA_MULTIPATHRECORD_ATTRID);
3547
3548 switch (attr_id) {
3549 case SA_CLASSPORTINFO_ATTRID:
3550 attr_length = sizeof (ib_mad_classportinfo_t);
3551 break;
3552 case SA_NOTICE_ATTRID:
3553 attr_length = sizeof (ib_mad_notice_t);
3554 break;
3555 case SA_INFORMINFO_ATTRID:
3556 attr_length = sizeof (ib_mad_informinfo_t);
3557 break;
3558 case SA_NODERECORD_ATTRID:
3559 attr_length = sizeof (sa_node_record_t);
3560 break;
3561 case SA_PORTINFORECORD_ATTRID:
3562 attr_length = sizeof (sa_portinfo_record_t);
3563 break;
3564 case SA_SLTOVLRECORD_ATTRID:
3565 attr_length = sizeof (sa_SLtoVLmapping_record_t);
3566 break;
3567 case SA_SWITCHINFORECORD_ATTRID:
3568 attr_length = sizeof (sa_switchinfo_record_t);
3569 break;
3570 case SA_LINEARFDBRECORD_ATTRID:
3571 attr_length = sizeof (sa_linearft_record_t);
3572 break;
3573 case SA_RANDOMFDBRECORD_ATTRID:
3574 attr_length = sizeof (sa_randomft_record_t);
3575 break;
3576 case SA_MULTICASTFDBRECORD_ATTRID:
3577 attr_length = sizeof (sa_multicastft_record_t);
3578 break;
3579 case SA_SMINFORECORD_ATTRID:
3580 attr_length = sizeof (sa_sminfo_record_t);
3581 break;
3582 case SA_INFORMINFORECORD_ATTRID:
3583 attr_length = sizeof (sa_informinfo_record_t);
3584 break;
3585 case SA_LINKRECORD_ATTRID:
3586 attr_length = sizeof (sa_link_record_t);
3587 break;
3588 case SA_GUIDINFORECORD_ATTRID:
3589 attr_length = sizeof (sa_guidinfo_record_t);
3590 break;
3591 case SA_SERVICERECORD_ATTRID:
3592 attr_length = sizeof (sa_service_record_t);
3593 break;
3594 case SA_PARTITIONRECORD_ATTRID:
3595 attr_length = sizeof (sa_pkey_table_record_t);
3596 break;
3597 case SA_PATHRECORD_ATTRID:
3598 attr_length = sizeof (sa_path_record_t);
3599 break;
3600 case SA_VLARBRECORD_ATTRID:
3601 attr_length = sizeof (sa_VLarb_table_record_t);
3602 break;
3603 case SA_MCMEMBERRECORD_ATTRID:
3604 attr_length = sizeof (sa_mcmember_record_t);
3605 break;
3606 case SA_TRACERECORD_ATTRID:
3607 attr_length = sizeof (sa_trace_record_t);
3608 break;
3609 case SA_SERVICEASSNRECORD_ATTRID:
3610 attr_length = sizeof (sa_service_assn_record_t);
3611 break;
3612 default:
3613 /* should only get the above type of packets */
3614 attr_length = 0;
3615 break;
3616 }
3617
3618 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
3619 ibmf_saa_impl_get_attr_id_length_end,
3620 IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_attr_id_length():"
3621 " attr_id: 0x%x size %d\n",
3622 tnf_opaque, attr_id, attr_id, tnf_uint, attr_length, attr_length);
3623
3624 return (attr_length);
3625 }
3626
3627 /*
3628 * ibmf_saa_impl_free_msg:
3629 * Takes a completed message and free memory associated with the message,
3630 * including the individual fields of the im_msgbufs_send.
3631 * ibmf_free_msg, called at the end of this function, takes a pointer to the
3632 * message pointer so that it can set the message pointer to NULL. This
3633 * function takes just the message pointer so the msgp will not be NULL after
3634 * this function returns.
3635 *
3636 * Input Arguments
3637 * ibmf_hdl ibmf handle used in ibmf_msg_alloc
3638 * msgp pointer to ibmf_msg_t to free
3639 *
3640 * Returns
3641 * void
3642 */
3643 static void
ibmf_saa_impl_free_msg(ibmf_handle_t ibmf_hdl,ibmf_msg_t * msgp)3644 ibmf_saa_impl_free_msg(ibmf_handle_t ibmf_hdl, ibmf_msg_t *msgp)
3645 {
3646 int res;
3647
3648 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
3649 ibmf_saa_impl_free_msg_start,
3650 IBMF_TNF_TRACE, "", "ibmf_saa_impl_free_msg() enter: msg %p\n",
3651 tnf_opaque, msg, msgp);
3652
3653 ASSERT(msgp != NULL);
3654
3655 kmem_free(msgp->im_msgbufs_send.im_bufs_mad_hdr,
3656 sizeof (ib_mad_hdr_t));
3657
3658 kmem_free(msgp->im_msgbufs_send.im_bufs_cl_hdr,
3659 msgp->im_msgbufs_send.im_bufs_cl_hdr_len);
3660
3661 if (msgp->im_msgbufs_send.im_bufs_cl_data_len > 0)
3662 kmem_free(msgp->im_msgbufs_send.im_bufs_cl_data,
3663 msgp->im_msgbufs_send.im_bufs_cl_data_len);
3664
3665 res = ibmf_free_msg(ibmf_hdl, &msgp);
3666 ASSERT(res == IBMF_SUCCESS);
3667
3668 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
3669 ibmf_saa_impl_free_msg_end,
3670 IBMF_TNF_TRACE, "", "ibmf_saa_impl_free_msg() exit\n");
3671 }
3672
3673 /*
3674 * ibmf_saa_impl_get_port_guid:
3675 */
3676 static int
ibmf_saa_impl_get_port_guid(ibt_hca_portinfo_t * ibt_portinfop,ib_guid_t * guid_ret)3677 ibmf_saa_impl_get_port_guid(ibt_hca_portinfo_t *ibt_portinfop,
3678 ib_guid_t *guid_ret)
3679 {
3680 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
3681 ibmf_saa_impl_get_port_guid_start,
3682 IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_port_guid() enter\n");
3683
3684 if (ibt_portinfop->p_linkstate != IBT_PORT_ACTIVE) {
3685
3686 return (IBMF_BAD_PORT_STATE);
3687 }
3688
3689 if (ibt_portinfop->p_sgid_tbl_sz == 0) {
3690
3691 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L2,
3692 ibmf_saa_impl_get_port_guid_end, IBMF_TNF_TRACE, "",
3693 "ibmf_saa_impl_get_port_guid: %s\n", tnf_string, msg,
3694 "portinfo sgid table size is 0. Exiting.\n");
3695
3696 return (IBMF_TRANSPORT_FAILURE);
3697 }
3698
3699 *guid_ret = ibt_portinfop->p_sgid_tbl[0].gid_guid;
3700
3701 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3,
3702 ibmf_saa_impl_get_port_guid_end, IBMF_TNF_TRACE, "",
3703 "ibmf_saa_impl_get_port_guid: Returning port_guid %016" PRIx64 "\n",
3704 tnf_opaque, port_guid, *guid_ret);
3705
3706 return (IBMF_SUCCESS);
3707 }
3708
3709 /*
3710 * ibmf_saa_impl_set_transaction_params:
3711 */
3712 static void
ibmf_saa_impl_set_transaction_params(saa_port_t * saa_portp,ibt_hca_portinfo_t * portinfop)3713 ibmf_saa_impl_set_transaction_params(saa_port_t *saa_portp,
3714 ibt_hca_portinfo_t *portinfop)
3715 {
3716 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
3717 ibmf_saa_impl_set_transaction_params_start,
3718 IBMF_TNF_TRACE, "",
3719 "ibmf_saa_impl_set_transaction_params() enter\n");
3720
3721 _NOTE(ASSUMING_PROTECTED(*saa_portp))
3722
3723 saa_portp->saa_pt_ibmf_retrans.retrans_retries =
3724 ibmf_saa_retrans_retries;
3725 /*
3726 * For the first transaction (generally getting the
3727 * classportinfo) have ibmf pick our timeouts. It should be using the
3728 * default IB spec values.
3729 * Once we get the classportinfo we'll update the correct response time
3730 * value (rtv) and round-trip time (rttv). ibmf should always calculate
3731 * trans_to since it depends on the particular transaction's number of
3732 * packets.
3733 */
3734 saa_portp->saa_pt_ibmf_retrans.retrans_rtv = 0;
3735 saa_portp->saa_pt_ibmf_retrans.retrans_rttv = 0;
3736 saa_portp->saa_pt_ibmf_retrans.retrans_trans_to = 0;
3737
3738 /*
3739 * Assume that the SA supports all optional records. If it
3740 * does not, the request will get returned with ERR_NOT_SUPP. When
3741 * the classportinfo response comes back we will update the cap mask
3742 * to prevent unnecessary unsupported requests.
3743 */
3744 saa_portp->saa_pt_sa_cap_mask = 0xFFFF;
3745
3746 saa_portp->saa_pt_ibmf_msg_flags = 0;
3747 saa_portp->saa_pt_ibmf_addr_info.ia_remote_qno = 1;
3748 saa_portp->saa_pt_ibmf_addr_info.ia_p_key =
3749 IB_PKEY_DEFAULT_LIMITED;
3750 saa_portp->saa_pt_ibmf_addr_info.ia_q_key = IB_GSI_QKEY;
3751
3752 /*
3753 * fill out addr information for MADs that will be sent
3754 * to SA on this port
3755 */
3756 saa_portp->saa_pt_ibmf_addr_info.ia_local_lid = portinfop->p_base_lid;
3757 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid = portinfop->p_sm_lid;
3758 saa_portp->saa_pt_ibmf_addr_info.ia_service_level = portinfop->p_sm_sl;
3759
3760 /* place upper bound on subnet timeout in case of faulty SM */
3761 saa_portp->saa_pt_timeout = portinfop->p_subnet_timeout;
3762
3763 if (saa_portp->saa_pt_timeout > ibmf_saa_max_subnet_timeout)
3764 saa_portp->saa_pt_timeout = ibmf_saa_max_subnet_timeout;
3765
3766 IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L3,
3767 ibmf_saa_impl_set_transaction_params,
3768 IBMF_TNF_TRACE, "",
3769 "ibmf_saa_impl_set_transaction_params: local_lid = 0x%x, "
3770 "sm_lid = 0x%x, sm_sl = 0x%x, sn_timeout = 0x%x\n",
3771 tnf_opaque, local_lid, portinfop->p_base_lid,
3772 tnf_opaque, sm_lid, portinfop->p_sm_lid,
3773 tnf_opaque, sm_sl, portinfop->p_sm_sl,
3774 tnf_opaque, subnet_timeout, portinfop->p_subnet_timeout);
3775
3776 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
3777 ibmf_saa_impl_set_transaction_params_end,
3778 IBMF_TNF_TRACE, "",
3779 "ibmf_saa_impl_set_transaction_params() exit\n");
3780 }
3781
3782
3783 /*
3784 * ibmf_saa_impl_update_sa_address_info
3785 */
3786 static void
ibmf_saa_impl_update_sa_address_info(saa_port_t * saa_portp,ibmf_msg_t * msgp)3787 ibmf_saa_impl_update_sa_address_info(saa_port_t *saa_portp, ibmf_msg_t *msgp)
3788 {
3789 void *result;
3790 ib_sa_hdr_t *sa_hdr;
3791 int rv;
3792 size_t length;
3793 uint16_t attr_id;
3794 ib_mad_classportinfo_t *cpi;
3795 ibmf_global_addr_info_t *gaddrp = &saa_portp->saa_pt_ibmf_global_addr;
3796 ibt_hca_portinfo_t *ibt_pinfo;
3797 uint_t nports, size;
3798 ibt_status_t ibt_status;
3799
3800 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
3801 ibmf_saa_impl_update_sa_address_info,
3802 IBMF_TNF_TRACE, "",
3803 "ibmf_saa_impl_update_sa_address_info() enter\n");
3804
3805 /*
3806 * decode the respons of msgp as a classportinfo attribute
3807 */
3808 rv = ibmf_saa_utils_unpack_sa_hdr(msgp->im_msgbufs_recv.im_bufs_cl_hdr,
3809 msgp->im_msgbufs_recv.im_bufs_cl_hdr_len, &sa_hdr, KM_NOSLEEP);
3810 if (rv != IBMF_SUCCESS) {
3811
3812 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1,
3813 ibmf_saa_impl_update_sa_address_err,
3814 IBMF_TNF_TRACE, "", "ibmf_saa_impl_update_sa_address_info: "
3815 "%s, ibmf_status = %d\n", tnf_string, msg,
3816 "Could not unpack sa hdr", tnf_int, ibmf_status, rv);
3817
3818 return;
3819 }
3820
3821 attr_id = b2h16(msgp->im_msgbufs_recv.im_bufs_mad_hdr->AttributeID);
3822 if (attr_id != MAD_ATTR_ID_CLASSPORTINFO) {
3823 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1,
3824 ibmf_saa_impl_update_sa_address_info_err,
3825 IBMF_TNF_TRACE, "", "ibmf_saa_impl_update_sa_address_info: "
3826 "%s, attrID = %x\n", tnf_string, msg,
3827 "Wrong attribute ID", tnf_int, ibmf_status, attr_id);
3828
3829 kmem_free(sa_hdr, sizeof (ib_sa_hdr_t));
3830 return;
3831 }
3832 rv = ibmf_saa_utils_unpack_payload(
3833 msgp->im_msgbufs_recv.im_bufs_cl_data,
3834 msgp->im_msgbufs_recv.im_bufs_cl_data_len, attr_id, &result,
3835 &length, sa_hdr->AttributeOffset, B_TRUE, KM_NOSLEEP);
3836 if (rv != IBMF_SUCCESS) {
3837
3838 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1,
3839 ibmf_saa_impl_update_sa_address_err,
3840 IBMF_TNF_TRACE, "", "ibmf_saa_impl_update_sa_address_info: "
3841 "%s, ibmf_status = %d\n", tnf_string, msg,
3842 "Could not unpack payload", tnf_int, ibmf_status, rv);
3843
3844 kmem_free(sa_hdr, sizeof (ib_sa_hdr_t));
3845 return;
3846 }
3847
3848 kmem_free(sa_hdr, sizeof (ib_sa_hdr_t));
3849
3850 /*
3851 * Use the classportinfo contents to update the SA address info
3852 */
3853 cpi = (ib_mad_classportinfo_t *)result;
3854 mutex_enter(&saa_portp->saa_pt_mutex);
3855 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid = cpi->RedirectLID;
3856 saa_portp->saa_pt_ibmf_addr_info.ia_remote_qno = cpi->RedirectQP;
3857 saa_portp->saa_pt_ibmf_addr_info.ia_p_key = cpi->RedirectP_Key;
3858 saa_portp->saa_pt_ibmf_addr_info.ia_q_key = cpi->RedirectQ_Key;
3859 saa_portp->saa_pt_ibmf_addr_info.ia_service_level = cpi->RedirectSL;
3860
3861 saa_portp->saa_pt_redirect_active = B_TRUE;
3862
3863 if ((cpi->RedirectGID_hi != 0) || (cpi->RedirectGID_lo != 0)) {
3864
3865 mutex_exit(&saa_portp->saa_pt_mutex);
3866 ibt_status = ibt_query_hca_ports_byguid(
3867 saa_portp->saa_pt_node_guid, saa_portp->saa_pt_port_num,
3868 &ibt_pinfo, &nports, &size);
3869 if (ibt_status != IBT_SUCCESS) {
3870
3871 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1,
3872 ibmf_saa_impl_update_sa_address_err, IBMF_TNF_TRACE,
3873 "", "ibmf_saa_impl_update_sa_address_info: "
3874 "%s, ibt_status = %d\n", tnf_string, msg,
3875 "Could not query hca port",
3876 tnf_int, ibt_status, ibt_status);
3877
3878 kmem_free(result, length);
3879 return;
3880 }
3881
3882 mutex_enter(&saa_portp->saa_pt_mutex);
3883 /*
3884 * Fill in global address info parameters
3885 *
3886 * NOTE: The HopLimit value is not specified through the
3887 * contents of ClassPortInfo. It may be possible to find
3888 * out the proper value to use even for SA beeing redirected
3889 * to another subnet. But we do only support redirect within
3890 * our local subnet
3891 */
3892 gaddrp->ig_sender_gid.gid_prefix =
3893 ibt_pinfo->p_sgid_tbl[0].gid_prefix;
3894 gaddrp->ig_sender_gid.gid_guid = saa_portp->saa_pt_port_guid;
3895 gaddrp->ig_recver_gid.gid_prefix = cpi->RedirectGID_hi;
3896 gaddrp->ig_recver_gid.gid_guid = cpi->RedirectGID_lo;
3897 gaddrp->ig_flow_label = cpi->RedirectFL;
3898 gaddrp->ig_tclass = cpi->RedirectTC;
3899 gaddrp->ig_hop_limit = 0;
3900
3901 saa_portp->saa_pt_ibmf_msg_flags =
3902 IBMF_MSG_FLAGS_GLOBAL_ADDRESS;
3903
3904 mutex_exit(&saa_portp->saa_pt_mutex);
3905 ibt_free_portinfo(ibt_pinfo, size);
3906 } else {
3907 saa_portp->saa_pt_ibmf_msg_flags = 0;
3908 mutex_exit(&saa_portp->saa_pt_mutex);
3909 }
3910 kmem_free(result, length);
3911
3912 /*
3913 * Update the address info of msgp with the new address parameters
3914 */
3915 mutex_enter(&saa_portp->saa_pt_mutex);
3916 bcopy(&saa_portp->saa_pt_ibmf_addr_info, &msgp->im_local_addr,
3917 sizeof (ibmf_addr_info_t));
3918 if (saa_portp->saa_pt_ibmf_msg_flags & IBMF_MSG_FLAGS_GLOBAL_ADDRESS) {
3919
3920 msgp->im_msg_flags = IBMF_MSG_FLAGS_GLOBAL_ADDRESS;
3921
3922 bcopy(&saa_portp->saa_pt_ibmf_global_addr,
3923 &msgp->im_global_addr, sizeof (ibmf_global_addr_info_t));
3924 } else {
3925 msgp->im_msg_flags = 0;
3926 }
3927 mutex_exit(&saa_portp->saa_pt_mutex);
3928 }
3929
3930 /*
3931 * ibmf_saa_impl_ibmf_unreg:
3932 */
3933 static int
ibmf_saa_impl_ibmf_unreg(saa_port_t * saa_portp)3934 ibmf_saa_impl_ibmf_unreg(saa_port_t *saa_portp)
3935 {
3936 int ibmf_status;
3937
3938 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_ibmf_unreg_start,
3939 IBMF_TNF_TRACE, "", "ibmf_saa_impl_ibmf_unreg() enter\n");
3940
3941 /* teardown async cb */
3942 ibmf_status = ibmf_tear_down_async_cb(saa_portp->saa_pt_ibmf_handle,
3943 saa_portp->saa_pt_qp_handle, 0);
3944 if (ibmf_status != IBMF_SUCCESS) {
3945
3946 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1,
3947 ibmf_saa_impl_ibmf_unreg, IBMF_TNF_TRACE, "",
3948 "ibmf_saa_impl_ibmf_unreg: %s, ibmf_status = %d\n",
3949 tnf_string, msg, "Could not tear down async cb",
3950 tnf_int, ibmf_status, ibmf_status);
3951
3952 goto bail;
3953 }
3954
3955 /* free qp */
3956 ibmf_status = ibmf_free_qp(saa_portp->saa_pt_ibmf_handle,
3957 &saa_portp->saa_pt_qp_handle, 0);
3958
3959 if (ibmf_status != IBMF_SUCCESS) {
3960
3961 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1,
3962 ibmf_saa_impl_ibmf_unreg, IBMF_TNF_TRACE, "",
3963 "ibmf_saa_impl_ibmf_unreg: %s, ibmf_status = %d\n",
3964 tnf_string, msg, "Could not free queue pair",
3965 tnf_int, ibmf_status, ibmf_status);
3966
3967 (void) ibmf_saa_impl_setup_qp_async_cb(saa_portp, 1);
3968 goto bail;
3969 }
3970
3971 ibmf_status = ibmf_unregister(&saa_portp->saa_pt_ibmf_handle, 0);
3972
3973 if (ibmf_status != IBMF_SUCCESS) {
3974
3975 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1,
3976 ibmf_saa_impl_ibmf_unreg, IBMF_TNF_TRACE, "",
3977 "ibmf_saa_impl_ibmf_unreg: %s, ibmf_status = %d\n",
3978 tnf_string, msg, "ibmf_unregister() failed",
3979 tnf_int, ibmf_status, ibmf_status);
3980
3981 (void) ibmf_saa_impl_setup_qp_async_cb(saa_portp, 0);
3982 }
3983
3984 bail:
3985 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_ibmf_unreg_end,
3986 IBMF_TNF_TRACE, "", "ibmf_saa_impl_ibmf_unreg() exit\n");
3987
3988 return (ibmf_status);
3989 }
3990