xref: /illumos-gate/usr/src/uts/common/io/ib/ibtl/ibtl_cm.c (revision f67ca41a3fe371a8ac34045eb45b3c5449ee601c)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #pragma ident	"%Z%%M%	%I%	%E% SMI"
27 
28 #include <sys/ib/ibtl/impl/ibtl.h>
29 #include <sys/ib/ibtl/impl/ibtl_cm.h>
30 
31 /*
32  * ibtl_cm.c
33  *    These routines tie the Communication Manager into IBTL.
34  */
35 
36 /*
37  * Globals.
38  */
39 static char 		ibtf_cm[] = "ibtl_cm";
40 boolean_t		ibtl_fast_gid_cache_valid = B_FALSE;
41 
42 /*
43  * Function:
44  *	ibtl_cm_set_chan_private
45  * Input:
46  *	chan		Channel Handle.
47  *	cm_private	CM private data.
48  * Output:
49  *	none.
50  * Returns:
51  *	none.
52  * Description:
53  *	A helper function to store CM's Private data in the specified channel.
54  */
55 void
56 ibtl_cm_set_chan_private(ibt_channel_hdl_t chan, void *cm_private)
57 {
58 	IBTF_DPRINTF_L3(ibtf_cm, "ibtl_cm_set_chan_private(%p, %p)",
59 	    chan, cm_private);
60 
61 	mutex_enter(&chan->ch_cm_mutex);
62 	chan->ch_cm_private = cm_private;
63 	if (cm_private == NULL)
64 		cv_signal(&chan->ch_cm_cv);
65 	mutex_exit(&chan->ch_cm_mutex);
66 }
67 
68 
69 /*
70  * Function:
71  *	ibtl_cm_get_chan_private
72  * Input:
73  *	chan		Channel Handle.
74  * Output:
75  *	cm_private_p	The CM private data.
76  * Returns:
77  *	CM private data.
78  * Description:
79  *	A helper function to get CM's Private data for the specified channel.
80  */
81 void *
82 ibtl_cm_get_chan_private(ibt_channel_hdl_t chan)
83 {
84 	void *cm_private;
85 
86 	IBTF_DPRINTF_L3(ibtf_cm, "ibtl_cm_get_chan_private(%p)", chan);
87 	mutex_enter(&chan->ch_cm_mutex);
88 	cm_private = chan->ch_cm_private;
89 #ifndef __lock_lint
90 	/* IBCM will call the release function if cm_private is non-NULL */
91 	if (cm_private == NULL)
92 #endif
93 		mutex_exit(&chan->ch_cm_mutex);
94 	return (cm_private);
95 }
96 
97 void
98 ibtl_cm_release_chan_private(ibt_channel_hdl_t chan)
99 {
100 #ifndef __lock_lint
101 	mutex_exit(&chan->ch_cm_mutex);
102 #endif
103 }
104 
105 void
106 ibtl_cm_wait_chan_private(ibt_channel_hdl_t chan)
107 {
108 	mutex_enter(&chan->ch_cm_mutex);
109 	if (chan->ch_cm_private != NULL)
110 		cv_wait(&chan->ch_cm_cv, &chan->ch_cm_mutex);
111 	mutex_exit(&chan->ch_cm_mutex);
112 	delay(drv_usectohz(50000));
113 }
114 
115 
116 /*
117  * Function:
118  *	ibtl_cm_get_chan_type
119  * Input:
120  *	chan		Channel Handle.
121  * Output:
122  *	none.
123  * Returns:
124  *	Channel transport type.
125  * Description:
126  *	A helper function to get channel transport type.
127  */
128 ibt_tran_srv_t
129 ibtl_cm_get_chan_type(ibt_channel_hdl_t chan)
130 {
131 	IBTF_DPRINTF_L3(ibtf_cm, "ibtl_cm_get_chan_type(%p)", chan);
132 
133 	return (chan->ch_qp.qp_type);
134 }
135 
136 /*
137  * Function:
138  *	ibtl_cm_change_service_cnt
139  * Input:
140  *	ibt_hdl		Client's IBT Handle.
141  *	delta_num_sids	The change in the number of service ids
142  *			(positive for ibt_register_service() and
143  *			negative fo ibt_service_deregister()).
144  */
145 void
146 ibtl_cm_change_service_cnt(ibt_clnt_hdl_t ibt_hdl, int delta_num_sids)
147 {
148 	IBTF_DPRINTF_L3(ibtf_cm, "ibtl_cm_change_service_cnt(%p. %d)",
149 	    ibt_hdl, delta_num_sids);
150 
151 	mutex_enter(&ibtl_clnt_list_mutex);
152 	if ((delta_num_sids < 0) && (-delta_num_sids > ibt_hdl->clnt_srv_cnt)) {
153 		IBTF_DPRINTF_L2(ibtf_cm, "ibtl_cm_change_service_cnt: "
154 		    "ERROR: service registration counter underflow\n"
155 		    "current count = %d, requested delta = %d",
156 		    ibt_hdl->clnt_srv_cnt, delta_num_sids);
157 	}
158 	ibt_hdl->clnt_srv_cnt += delta_num_sids;
159 	mutex_exit(&ibtl_clnt_list_mutex);
160 }
161 
162 
163 /*
164  * Function:
165  *	ibtl_cm_get_hca_port
166  * Input:
167  *	gid		Source GID.
168  *	hca_guid	Optional source HCA GUID on which SGID is available.
169  *			Ignored if zero.
170  * Output:
171  *	hca_port	Pointer to ibtl_cm_hca_port_t struct.
172  * Returns:
173  *	IBT_SUCCESS.
174  * Description:
175  *	A helper function to get HCA node GUID, Base LID, SGID Index,
176  *	port number, LMC and MTU for the specified SGID.
177  *	Also filling default SGID, to be used in ibmf_sa_session_open.
178  */
179 ibt_status_t
180 ibtl_cm_get_hca_port(ib_gid_t gid, ib_guid_t hca_guid,
181     ibtl_cm_hca_port_t *hca_port)
182 {
183 	ibtl_hca_devinfo_t	*hca_devp;	/* HCA Dev Info */
184 	ibt_hca_portinfo_t	*portinfop;
185 	uint_t			ports, port;
186 	uint_t			i;
187 	ib_gid_t		*sgid;
188 	static ib_gid_t		fast_gid;	/* fast_gid_cache data */
189 	static uint8_t		fast_sgid_ix;
190 	static ibt_hca_portinfo_t *fast_portinfop;
191 	static ib_guid_t	fast_node_guid;
192 	static ib_guid_t	fast_port_guid;
193 
194 	IBTF_DPRINTF_L3(ibtf_cm, "ibtl_cm_get_hca_port(%llX:%llX, %llX)",
195 	    gid.gid_prefix, gid.gid_guid, hca_guid);
196 
197 	if ((gid.gid_prefix == 0) || (gid.gid_guid == 0)) {
198 		IBTF_DPRINTF_L2(ibtf_cm, "ibtl_cm_get_hca_port: "
199 		    "NULL SGID specified.");
200 		return (IBT_INVALID_PARAM);
201 	}
202 
203 	mutex_enter(&ibtl_clnt_list_mutex);
204 
205 	if ((ibtl_fast_gid_cache_valid == B_TRUE) &&
206 	    (gid.gid_guid == fast_gid.gid_guid) &&
207 	    (gid.gid_prefix == fast_gid.gid_prefix)) {
208 
209 		if ((hca_guid != 0) && (hca_guid != fast_node_guid)) {
210 			IBTF_DPRINTF_L3(ibtf_cm, "ibtl_cm_get_hca_port: "
211 			    "Mis-match hca_guid v/s sgid combination.");
212 			mutex_exit(&ibtl_clnt_list_mutex);
213 			return (IBT_INVALID_PARAM);
214 		}
215 
216 		portinfop = fast_portinfop;
217 		hca_port->hp_base_lid = portinfop->p_base_lid;
218 		hca_port->hp_port = portinfop->p_port_num;
219 		hca_port->hp_sgid_ix = fast_sgid_ix;
220 		hca_port->hp_lmc = portinfop->p_lmc;
221 		hca_port->hp_mtu = portinfop->p_mtu;
222 		hca_port->hp_hca_guid = fast_node_guid;
223 		hca_port->hp_port_guid = fast_port_guid;
224 
225 		mutex_exit(&ibtl_clnt_list_mutex);
226 
227 		return (IBT_SUCCESS);
228 	}
229 
230 	/* If HCA GUID is specified, then lookup in that device only. */
231 	if (hca_guid) {
232 		hca_devp = ibtl_get_hcadevinfo(hca_guid);
233 	} else {
234 		hca_devp = ibtl_hca_list;
235 	}
236 
237 	while (hca_devp != NULL) {
238 
239 		ports = hca_devp->hd_hca_attr->hca_nports;
240 		portinfop = hca_devp->hd_portinfop;
241 
242 		for (port = 0; port < ports; port++, portinfop++) {
243 			if (portinfop->p_linkstate != IBT_PORT_ACTIVE)
244 				continue;
245 			sgid = &portinfop->p_sgid_tbl[0];
246 			for (i = 0; i < portinfop->p_sgid_tbl_sz; i++, sgid++) {
247 				if ((gid.gid_guid != sgid->gid_guid) ||
248 				    (gid.gid_prefix != sgid->gid_prefix))
249 					continue;
250 
251 				/*
252 				 * Found the matching GID.
253 				 */
254 				ibtl_fast_gid_cache_valid = B_TRUE;
255 				fast_gid = gid;
256 				fast_portinfop = portinfop;
257 				fast_node_guid = hca_port->hp_hca_guid =
258 				    hca_devp->hd_hca_attr->hca_node_guid;
259 				fast_sgid_ix = hca_port->hp_sgid_ix = i;
260 				fast_port_guid =
261 				    portinfop->p_sgid_tbl[0].gid_guid;
262 				hca_port->hp_port_guid = fast_port_guid;
263 				hca_port->hp_base_lid = portinfop->p_base_lid;
264 				hca_port->hp_port = portinfop->p_port_num;
265 				hca_port->hp_lmc = portinfop->p_lmc;
266 				hca_port->hp_mtu = portinfop->p_mtu;
267 
268 				mutex_exit(&ibtl_clnt_list_mutex);
269 
270 				return (IBT_SUCCESS);
271 			}
272 		}
273 
274 		/* Asked to look in the specified HCA device only?. */
275 		if (hca_guid)
276 			break;
277 
278 		/* Get next in the list */
279 		hca_devp = hca_devp->hd_hca_dev_link;
280 	}
281 
282 	mutex_exit(&ibtl_clnt_list_mutex);
283 
284 	/* If we are here, then we failed to get a match, so return error. */
285 	return (IBT_INVALID_PARAM);
286 }
287 
288 
289 static ibt_status_t
290 ibtl_cm_get_cnt(ibt_path_attr_t *attr, ibt_path_flags_t flags,
291     ibtl_cm_port_list_t *plistp, uint_t *count)
292 {
293 	ibtl_hca_devinfo_t	*hdevp;
294 	ibt_hca_portinfo_t	*pinfop;
295 	ib_guid_t		hca_guid, tmp_hca_guid = 0;
296 	ib_gid_t		gid;
297 	uint_t			pcount = 0, tmp_pcount = 0;
298 	uint_t			cnt = *count;
299 	ibt_status_t		retval = IBT_SUCCESS;
300 	uint_t			i, j;
301 
302 	*count = 0;
303 
304 	/* If HCA GUID is specified, then lookup in that device only. */
305 	if (attr->pa_hca_guid) {
306 		hdevp = ibtl_get_hcadevinfo(attr->pa_hca_guid);
307 	} else {
308 		hdevp = ibtl_hca_list;
309 	}
310 
311 	while (hdevp != NULL) {
312 		hca_guid = hdevp->hd_hca_attr->hca_node_guid;
313 
314 		if ((flags & IBT_PATH_APM) &&
315 		    (!(hdevp->hd_hca_attr->hca_flags &
316 		    IBT_HCA_AUTO_PATH_MIG))) {
317 
318 			IBTF_DPRINTF_L2(ibtf_cm, "ibtl_cm_get_cnt: "
319 			    "HCA (%llX) - APM NOT SUPPORTED ", hca_guid);
320 
321 			retval = IBT_APM_NOT_SUPPORTED;
322 
323 			if (attr->pa_hca_guid)
324 				break;
325 			hdevp = hdevp->hd_hca_dev_link;
326 			continue;
327 		}
328 
329 		for (i = 0; i < hdevp->hd_hca_attr->hca_nports; i++) {
330 
331 			if ((attr->pa_hca_port_num) &&
332 			    (attr->pa_hca_port_num != (i + 1))) {
333 				IBTF_DPRINTF_L3(ibtf_cm, "ibtl_cm_get_cnt: "
334 				    "Asked only on Port# %d, so skip this "
335 				    "port(%d)", attr->pa_hca_port_num, (i + 1));
336 				continue;
337 			}
338 			pinfop = hdevp->hd_portinfop + i;
339 
340 			if (pinfop->p_linkstate != IBT_PORT_ACTIVE) {
341 				retval = IBT_HCA_PORT_NOT_ACTIVE;
342 				continue;
343 			}
344 			if (attr->pa_mtu.r_mtu) {
345 				if ((attr->pa_mtu.r_selector == IBT_GT) &&
346 				    (attr->pa_mtu.r_mtu >= pinfop->p_mtu))
347 					continue;
348 				else if ((attr->pa_mtu.r_selector == IBT_EQU) &&
349 				    (attr->pa_mtu.r_mtu > pinfop->p_mtu))
350 					continue;
351 			}
352 
353 			for (j = 0; j < pinfop->p_sgid_tbl_sz; j++) {
354 				gid = pinfop->p_sgid_tbl[j];
355 				if (gid.gid_prefix && gid.gid_guid) {
356 					pcount++;
357 					if (plistp) {
358 						plistp->p_hca_guid = hca_guid;
359 						plistp->p_mtu = pinfop->p_mtu;
360 						plistp->p_base_lid =
361 						    pinfop->p_base_lid;
362 						plistp->p_port_num =
363 						    pinfop->p_port_num;
364 						plistp->p_sgid_ix = j;
365 						plistp->p_sgid = gid;
366 						plistp->p_count = cnt;
367 						if (hdevp->hd_multism)
368 							plistp->p_multi |=
369 							    IBTL_CM_MULTI_SM;
370 
371 						IBTF_DPRINTF_L3(ibtf_cm,
372 						    "ibtl_cm_get_cnt: HCA"
373 						    "(%llX,%d) SGID(%llX:%llX)",
374 						    plistp->p_hca_guid,
375 						    plistp->p_port_num,
376 						    plistp->p_sgid.gid_prefix,
377 						    plistp->p_sgid.gid_guid);
378 
379 						plistp++;
380 					}
381 				}
382 			}
383 		}
384 		/* Asked to look in the specified HCA device only?. */
385 		if (attr->pa_hca_guid)
386 			break;
387 
388 		if (flags & IBT_PATH_APM) {
389 			if (pcount == 2) {
390 				attr->pa_hca_guid = hca_guid;
391 				break;
392 			} else if (pcount == 1) {
393 				if (hdevp->hd_hca_dev_link) {
394 					tmp_hca_guid = hca_guid;
395 					tmp_pcount = pcount;
396 					pcount = 0;
397 				} else if (tmp_hca_guid) {
398 					attr->pa_hca_guid = tmp_hca_guid;
399 				} else {
400 					attr->pa_hca_guid = hca_guid;
401 				}
402 			} else if ((pcount == 0) && (tmp_hca_guid)) {
403 				attr->pa_hca_guid = tmp_hca_guid;
404 				pcount = tmp_pcount;
405 			}
406 		}
407 		hdevp = hdevp->hd_hca_dev_link;
408 	}
409 
410 	*count = pcount;
411 
412 	if (pcount) {
413 		retval = IBT_SUCCESS;
414 	} else {
415 		IBTF_DPRINTF_L2(ibtf_cm, "ibtl_cm_get_cnt: "
416 		    "Appropriate Source Points NOT found");
417 		if (retval == IBT_SUCCESS)
418 			retval = IBT_NO_HCAS_AVAILABLE;
419 	}
420 
421 	return (retval);
422 }
423 
424 
425 ibt_status_t
426 ibtl_cm_get_active_plist(ibt_path_attr_t *attr, ibt_path_flags_t flags,
427     ibtl_cm_port_list_t **port_list_p)
428 {
429 	ibtl_cm_port_list_t	*p_listp, tmp;
430 	uint_t			i, j;
431 	uint_t			count, rcount;
432 	boolean_t		multi_hca = B_FALSE;
433 	ibt_status_t		retval = IBT_SUCCESS;
434 
435 	IBTF_DPRINTF_L3(ibtf_cm, "ibtl_cm_get_active_plist(%p, %X)",
436 	    attr, flags);
437 
438 get_plist_start:
439 	*port_list_p = NULL;
440 
441 	/* Get "number of active src points" so that we can allocate memory. */
442 	mutex_enter(&ibtl_clnt_list_mutex);
443 	retval = ibtl_cm_get_cnt(attr, flags, NULL, &count);
444 	mutex_exit(&ibtl_clnt_list_mutex);
445 
446 	IBTF_DPRINTF_L3(ibtf_cm, "ibtl_cm_get_active_plist: Found %d SrcPoint",
447 	    count);
448 	if (retval != IBT_SUCCESS)
449 		return (retval);
450 
451 	/* Allocate Memory to hold Src Point information. */
452 	p_listp = kmem_zalloc(count * sizeof (ibtl_cm_port_list_t), KM_SLEEP);
453 
454 	/*
455 	 * Verify that the count we got previously is still valid, as we had
456 	 * dropped mutex to allocate memory. If not, restart the process.
457 	 */
458 	mutex_enter(&ibtl_clnt_list_mutex);
459 	retval = ibtl_cm_get_cnt(attr, flags, NULL, &rcount);
460 	if (retval != IBT_SUCCESS) {
461 		mutex_exit(&ibtl_clnt_list_mutex);
462 		kmem_free(p_listp, count * sizeof (ibtl_cm_port_list_t));
463 		return (retval);
464 	} else if (rcount != count) {
465 		mutex_exit(&ibtl_clnt_list_mutex);
466 		kmem_free(p_listp, count * sizeof (ibtl_cm_port_list_t));
467 		goto get_plist_start;
468 	}
469 
470 	*port_list_p = p_listp;
471 	/*
472 	 * Src count hasn't changed, still holding the lock fill-in the
473 	 * required source point information.
474 	 */
475 	retval = ibtl_cm_get_cnt(attr, flags, p_listp, &rcount);
476 	mutex_exit(&ibtl_clnt_list_mutex);
477 	if (retval != IBT_SUCCESS) {
478 		kmem_free(p_listp, count * sizeof (ibtl_cm_port_list_t));
479 		*port_list_p = NULL;
480 		return (retval);
481 	}
482 
483 	p_listp = *port_list_p;
484 
485 	_NOTE(NO_COMPETING_THREADS_NOW)
486 
487 	for (i = 0; i < count - 1; i++) {
488 		for (j = 0; j < count - 1 - i; j++) {
489 			if (p_listp[j].p_hca_guid != p_listp[j+1].p_hca_guid) {
490 				multi_hca = B_TRUE;
491 				break;
492 			}
493 		}
494 		if (multi_hca == B_TRUE)
495 			break;
496 	}
497 
498 	if (multi_hca == B_TRUE)
499 		for (i = 0; i < count; i++)
500 			p_listp[i].p_multi |= IBTL_CM_MULTI_HCA;
501 
502 	/*
503 	 * Sort (bubble sort) the list based on MTU quality (higher on top).
504 	 * Sorting is only performed, if IBT_PATH_AVAIL is set.
505 	 */
506 	if (((attr->pa_mtu.r_selector == IBT_GT) || (flags & IBT_PATH_AVAIL)) &&
507 	    (!(flags & IBT_PATH_APM))) {
508 		for (i = 0; i < count - 1; i++) {
509 			for (j = 0; j < count - 1 - i; j++) {
510 				if (p_listp[j].p_mtu < p_listp[j+1].p_mtu) {
511 					tmp = p_listp[j];
512 					p_listp[j] = p_listp[j+1];
513 					p_listp[j+1] = tmp;
514 				}
515 			}
516 		}
517 	}
518 
519 	if ((p_listp->p_multi & IBTL_CM_MULTI_HCA) &&
520 	    (flags & IBT_PATH_AVAIL) && (!(flags & IBT_PATH_APM))) {
521 		/* Avoid having same HCA next to each other in the list. */
522 		for (i = 0; i < count - 1; i++) {
523 			for (j = 0; j < (count - 1 - i); j++) {
524 				if ((p_listp[j].p_hca_guid ==
525 				    p_listp[j+1].p_hca_guid) &&
526 				    (j+2 < count)) {
527 					tmp = p_listp[j+1];
528 					p_listp[j+1] = p_listp[j+2];
529 					p_listp[j+2] = tmp;
530 				}
531 			}
532 		}
533 	}
534 
535 	/*
536 	 * If SGID is specified, then make sure that SGID info is first
537 	 * in the array.
538 	 */
539 	if (attr->pa_sgid.gid_guid && (p_listp->p_count > 1) &&
540 	    (p_listp[0].p_sgid.gid_guid != attr->pa_sgid.gid_guid)) {
541 		for (i = 1; i < count; i++) {
542 			if (p_listp[i].p_sgid.gid_guid ==
543 			    attr->pa_sgid.gid_guid) {
544 				tmp = p_listp[i];
545 				p_listp[i] = p_listp[0];
546 				p_listp[0] = tmp;
547 			}
548 		}
549 	}
550 
551 #ifndef lint
552 	_NOTE(COMPETING_THREADS_NOW)
553 #endif
554 
555 	IBTF_DPRINTF_L3(ibtf_cm, "ibtl_cm_get_active_plist: "
556 	    "Returned <%d> entries @0x%p", count, *port_list_p);
557 
558 	return (retval);
559 }
560 
561 
562 void
563 ibtl_cm_free_active_plist(ibtl_cm_port_list_t *plist)
564 {
565 	int count;
566 
567 	IBTF_DPRINTF_L3(ibtf_cm, "ibtl_cm_free_active_plist(%p)", plist);
568 
569 	if (plist != NULL) {
570 		_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*plist))
571 		count = plist->p_count;
572 		_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*plist))
573 
574 		kmem_free(plist, count * sizeof (ibtl_cm_port_list_t));
575 	}
576 }
577 
578 /*
579  * Function:
580  *	ibtl_cm_get_1st_full_pkey_ix
581  * Input:
582  *	hca_guid	HCA GUID.
583  *	port		Port Number.
584  * Output:
585  *	None.
586  * Returns:
587  *	P_Key Index of the first full member available from the P_Key table
588  *	of the specified HCA<->Port.
589  * Description:
590  *	A helper function to get P_Key Index of the first full member P_Key
591  *	available on the specified HCA and Port combination.
592  */
593 uint16_t
594 ibtl_cm_get_1st_full_pkey_ix(ib_guid_t hca_guid, uint8_t port)
595 {
596 	ibtl_hca_devinfo_t	*hca_devp;	/* HCA Dev Info */
597 	uint16_t		pkey_ix = 0;
598 
599 	IBTF_DPRINTF_L3(ibtf_cm, "ibtl_cm_get_1st_full_pkey_ix(%llX, %d)",
600 	    hca_guid, port);
601 
602 	mutex_enter(&ibtl_clnt_list_mutex);
603 	hca_devp = ibtl_get_hcadevinfo(hca_guid);
604 
605 	if ((hca_devp != NULL) && (port <= hca_devp->hd_hca_attr->hca_nports) &&
606 	    (port != 0)) {
607 		pkey_ix = hca_devp->hd_portinfop[port - 1].p_def_pkey_ix;
608 	} else {
609 		IBTF_DPRINTF_L2(ibtf_cm, "ibtl_cm_get_1st_full_pkey_ix: "
610 		    "Invalid HCA (%llX), Port (%d) specified.", hca_guid, port);
611 	}
612 	mutex_exit(&ibtl_clnt_list_mutex);
613 
614 	return (pkey_ix);
615 }
616 
617 
618 ibt_status_t
619 ibtl_cm_get_local_comp_gids(ib_guid_t hca_guid, ib_gid_t gid, ib_gid_t **gids_p,
620     uint_t *num_gids_p)
621 {
622 	ibtl_hca_devinfo_t	*hdevp;	/* HCA Dev Info */
623 	ibt_hca_portinfo_t	*pinfop;
624 	ib_gid_t		sgid;
625 	ib_gid_t		*gidp = NULL;
626 	int			i, j, k;
627 	int			count = 0;
628 	int			gid_specified;
629 
630 	IBTF_DPRINTF_L3(ibtf_cm, "ibtl_cm_get_local_comp_gids(%llX, %llX:%llX)",
631 	    hca_guid, gid.gid_prefix, gid.gid_guid);
632 
633 	mutex_enter(&ibtl_clnt_list_mutex);
634 	hdevp = ibtl_get_hcadevinfo(hca_guid);
635 
636 	if (hdevp == NULL) {
637 		IBTF_DPRINTF_L3(ibtf_cm, "ibtl_cm_get_local_comp_gids: ",
638 		    "NO HCA (%llX) availble", hca_guid);
639 		mutex_exit(&ibtl_clnt_list_mutex);
640 		return (IBT_NO_HCAS_AVAILABLE);
641 	}
642 
643 	if (gid.gid_prefix && gid.gid_guid)
644 		gid_specified = 1;
645 	else
646 		gid_specified = 0;
647 
648 	for (i = 0; i < hdevp->hd_hca_attr->hca_nports; i++) {
649 		pinfop = hdevp->hd_portinfop + i;
650 
651 		if (pinfop->p_linkstate != IBT_PORT_ACTIVE)
652 			continue;
653 
654 		for (j = 0; j < pinfop->p_sgid_tbl_sz; j++) {
655 			sgid = pinfop->p_sgid_tbl[j];
656 			if (sgid.gid_prefix && sgid.gid_guid) {
657 				if (gid_specified &&
658 				    ((gid.gid_prefix == sgid.gid_prefix) &&
659 				    (gid.gid_guid == sgid.gid_guid))) {
660 					/*
661 					 * Don't return the input specified
662 					 * GID
663 					 */
664 					continue;
665 				}
666 				count++;
667 			}
668 		}
669 	}
670 
671 	if (count == 0) {
672 		IBTF_DPRINTF_L2(ibtf_cm, "ibtl_cm_get_local_comp_gids: "
673 		    "Companion GIDs not available");
674 		mutex_exit(&ibtl_clnt_list_mutex);
675 		return (IBT_GIDS_NOT_FOUND);
676 	}
677 
678 	gidp = kmem_zalloc(count * sizeof (ib_gid_t), KM_SLEEP);
679 	*num_gids_p = count;
680 	*gids_p = gidp;
681 	k = 0;
682 
683 	for (i = 0; i < hdevp->hd_hca_attr->hca_nports; i++) {
684 		pinfop = hdevp->hd_portinfop + i;
685 
686 		if (pinfop->p_linkstate != IBT_PORT_ACTIVE)
687 			continue;
688 
689 		for (j = 0; j < pinfop->p_sgid_tbl_sz; j++) {
690 			sgid = pinfop->p_sgid_tbl[j];
691 			if (sgid.gid_prefix && sgid.gid_guid) {
692 				if (gid_specified &&
693 				    ((gid.gid_prefix == sgid.gid_prefix) &&
694 				    (gid.gid_guid == sgid.gid_guid)))
695 					continue;
696 
697 				gidp[k].gid_prefix = sgid.gid_prefix;
698 				gidp[k].gid_guid = sgid.gid_guid;
699 
700 				IBTF_DPRINTF_L3(ibtf_cm,
701 				    "ibtl_cm_get_local_comp_gids: GID[%d]="
702 				    "%llX:%llX", k, gidp[k].gid_prefix,
703 				    gidp[k].gid_guid);
704 				k++;
705 				if (k == count)
706 					break;
707 			}
708 		}
709 		if (k == count)
710 			break;
711 	}
712 	mutex_exit(&ibtl_clnt_list_mutex);
713 
714 	return (IBT_SUCCESS);
715 }
716 
717 
718 int
719 ibtl_cm_is_multi_sm(ib_guid_t hca_guid)
720 {
721 	ibtl_hca_devinfo_t	*hdevp;	/* HCA Dev Info */
722 	uint_t			multi_sm;
723 
724 	mutex_enter(&ibtl_clnt_list_mutex);
725 	hdevp = ibtl_get_hcadevinfo(hca_guid);
726 	if (hdevp == NULL) {
727 		IBTF_DPRINTF_L2(ibtf_cm, "ibtl_cm_is_multi_sm: NO HCA (%llX) "
728 		    "availble", hca_guid);
729 		mutex_exit(&ibtl_clnt_list_mutex);
730 		return (-1);
731 	}
732 	multi_sm = hdevp->hd_multism;
733 	mutex_exit(&ibtl_clnt_list_mutex);
734 
735 	IBTF_DPRINTF_L3(ibtf_cm, "ibtl_cm_is_multi_sm(%llX): %d", hca_guid,
736 	    multi_sm);
737 
738 	return (multi_sm);
739 }
740