xref: /illumos-gate/usr/src/uts/common/io/ib/clients/eoib/enx_q.c (revision 6a634c9dca3093f3922e4b7ab826d7bdf17bf78e)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
24  */
25 
26 #include <sys/types.h>
27 #include <sys/kmem.h>
28 #include <sys/conf.h>
29 #include <sys/ddi.h>
30 #include <sys/sunddi.h>
31 #include <sys/ksynch.h>
32 
33 #include <sys/ib/clients/eoib/enx_impl.h>
34 
35 /*
36  * Acquire an SWQE
37  */
38 
39 /*ARGSUSED*/
40 eibnx_wqe_t *
eibnx_acquire_swqe(eibnx_thr_info_t * info,int flag)41 eibnx_acquire_swqe(eibnx_thr_info_t *info, int flag)
42 {
43 	eibnx_wqe_t *wqe = NULL;
44 	eibnx_tx_t *snd_p = &info->ti_snd;
45 	int i;
46 
47 	for (i = 0; i < ENX_NUM_SWQE; i++) {
48 		wqe = &(snd_p->tx_wqe[i]);
49 
50 		mutex_enter(&wqe->qe_lock);
51 		if ((wqe->qe_flags & ENX_QEFL_INUSE) == 0) {
52 			wqe->qe_flags |= ENX_QEFL_INUSE;
53 			mutex_exit(&wqe->qe_lock);
54 			break;
55 		}
56 		mutex_exit(&wqe->qe_lock);
57 	}
58 
59 	/*
60 	 * We probably have enough swqe entries for doing our solicitations.
61 	 * If we find it not enough in practice, we need to implement some
62 	 * sort of dynamic allocation.
63 	 */
64 	if (i == ENX_NUM_SWQE)
65 		wqe = NULL;
66 
67 	return (wqe);
68 }
69 
70 /*
71  * Return a SWQE from completion. We may have to release
72  * it or keep it.
73  */
74 void
eibnx_return_swqe(eibnx_wqe_t * wqe)75 eibnx_return_swqe(eibnx_wqe_t *wqe)
76 {
77 	ASSERT(wqe->qe_type == ENX_QETYP_SWQE);
78 
79 	mutex_enter(&wqe->qe_lock);
80 
81 	/*
82 	 * This send wqe is from the completion queue.  We need to
83 	 * clear the 'posted' flag first.
84 	 */
85 	ASSERT((wqe->qe_flags & ENX_QEFL_POSTED) == ENX_QEFL_POSTED);
86 	wqe->qe_flags &= (~ENX_QEFL_POSTED);
87 
88 	/*
89 	 * See if we need to release this send wqe back to the pool
90 	 * on completion. We may not need to do so if, for example,
91 	 * this were a swqe acquired specifically for a particular gw.
92 	 */
93 	if (wqe->qe_flags & ENX_QEFL_RELONCOMP) {
94 		wqe->qe_sgl.ds_len = wqe->qe_bufsz;
95 		wqe->qe_flags &= (~ENX_QEFL_INUSE);
96 
97 		wqe->qe_flags &= (~ENX_QEFL_RELONCOMP);
98 	}
99 
100 	mutex_exit(&wqe->qe_lock);
101 }
102 
103 /*
104  * Return a RWQE from completion. We probably have to repost it.
105  */
106 void
eibnx_return_rwqe(eibnx_thr_info_t * info,eibnx_wqe_t * wqe)107 eibnx_return_rwqe(eibnx_thr_info_t *info, eibnx_wqe_t *wqe)
108 {
109 	ibt_status_t ret;
110 
111 	ASSERT(wqe->qe_type == ENX_QETYP_RWQE);
112 
113 	mutex_enter(&wqe->qe_lock);
114 
115 	/*
116 	 * We should never need to free an rwqe on completion.
117 	 */
118 	ASSERT((wqe->qe_flags & ENX_QEFL_RELONCOMP) == 0);
119 
120 	/*
121 	 * An rwqe is always in-use and posted, so we only need to make
122 	 * sure the ds_len is adjusted back to the value it's supposed
123 	 * to have.
124 	 */
125 	wqe->qe_sgl.ds_len = wqe->qe_bufsz;
126 
127 	/*
128 	 * Repost the recv wqe
129 	 */
130 	ret = ibt_post_recv(info->ti_chan, &(wqe->qe_wr.recv), 1, NULL);
131 	if (ret != IBT_SUCCESS) {
132 		ENX_DPRINTF_WARN("ibt_post_recv(chan_hdl=0x%llx) failed, "
133 		    "ret=%d", info->ti_chan, ret);
134 	}
135 
136 	mutex_exit(&wqe->qe_lock);
137 }
138 
139 /*
140  * Release an SWQE that was acquired earlier.
141  */
142 void
eibnx_release_swqe(eibnx_wqe_t * wqe)143 eibnx_release_swqe(eibnx_wqe_t *wqe)
144 {
145 	ASSERT(wqe->qe_type == ENX_QETYP_SWQE);
146 
147 	mutex_enter(&wqe->qe_lock);
148 
149 	/*
150 	 * Make sure this swqe is in use. Since this routine may also be
151 	 * called when we're trying to cleanup the eoib nodes, we
152 	 * should clear all flag bits.
153 	 */
154 	ASSERT((wqe->qe_flags & ENX_QEFL_INUSE) == ENX_QEFL_INUSE);
155 	wqe->qe_flags = 0;
156 
157 	mutex_exit(&wqe->qe_lock);
158 }
159 
160 /*
161  * Insert the passed child to the head of the queue
162  */
163 void
eibnx_enqueue_child(eibnx_thr_info_t * info,eibnx_gw_info_t * gwi,char * node_name,dev_info_t * dip)164 eibnx_enqueue_child(eibnx_thr_info_t *info, eibnx_gw_info_t *gwi,
165     char *node_name, dev_info_t *dip)
166 {
167 	eibnx_child_t *ch;
168 	eibnx_child_t *new_ch;
169 
170 	new_ch = kmem_zalloc(sizeof (eibnx_child_t), KM_SLEEP);
171 	new_ch->ch_dip = dip;
172 	new_ch->ch_node_name = node_name;
173 	new_ch->ch_gwi = gwi;
174 
175 	mutex_enter(&info->ti_child_lock);
176 
177 	/*
178 	 * Search existing children to see if we already have this
179 	 * child.  If so, simply update its dip and node_name
180 	 */
181 	for (ch = info->ti_child; ch; ch = ch->ch_next) {
182 		if (ch->ch_gwi->gw_portid == gwi->gw_portid) {
183 			ch->ch_dip = dip;
184 			if (ch->ch_node_name) {
185 				kmem_free(ch->ch_node_name, MAXNAMELEN);
186 			}
187 			ch->ch_node_name = node_name;
188 			kmem_free(new_ch, sizeof (eibnx_child_t));
189 			return;
190 		}
191 	}
192 
193 	/*
194 	 * If not, add the new child to the list of children
195 	 */
196 	new_ch->ch_next = info->ti_child;
197 	info->ti_child = new_ch;
198 
199 	mutex_exit(&info->ti_child_lock);
200 }
201 
202 int
eibnx_update_child(eibnx_thr_info_t * info,eibnx_gw_info_t * gwi,dev_info_t * dip)203 eibnx_update_child(eibnx_thr_info_t *info, eibnx_gw_info_t *gwi,
204     dev_info_t *dip)
205 {
206 	eibnx_child_t *ch;
207 
208 	mutex_enter(&info->ti_child_lock);
209 	for (ch = info->ti_child; ch; ch = ch->ch_next) {
210 		if (ch->ch_gwi->gw_portid == gwi->gw_portid) {
211 			if (ch->ch_dip != dip) {
212 				ENX_DPRINTF_DEBUG("updating child dip for "
213 				    "gw portid 0x%x to 0x%llx",
214 				    gwi->gw_portid, dip);
215 				ch->ch_dip = dip;
216 			}
217 			mutex_exit(&info->ti_child_lock);
218 
219 			return (ENX_E_SUCCESS);
220 		}
221 	}
222 	mutex_exit(&info->ti_child_lock);
223 
224 	return (ENX_E_FAILURE);
225 }
226 
227 dev_info_t *
eibnx_find_child_dip_by_inst(eibnx_thr_info_t * info,int inst)228 eibnx_find_child_dip_by_inst(eibnx_thr_info_t *info, int inst)
229 {
230 	eibnx_child_t *ch;
231 	dev_info_t *dip = NULL;
232 
233 	mutex_enter(&info->ti_child_lock);
234 	for (ch = info->ti_child; ch != NULL; ch = ch->ch_next) {
235 		dip = ch->ch_dip;
236 		if (ddi_get_instance(dip) == inst)
237 			break;
238 	}
239 	mutex_exit(&info->ti_child_lock);
240 
241 	return (dip);
242 }
243 
244 dev_info_t *
eibnx_find_child_dip_by_gw(eibnx_thr_info_t * info,uint16_t gw_portid)245 eibnx_find_child_dip_by_gw(eibnx_thr_info_t *info, uint16_t gw_portid)
246 {
247 	eibnx_child_t *ch;
248 	dev_info_t *dip = NULL;
249 
250 	mutex_enter(&info->ti_child_lock);
251 	for (ch = info->ti_child; ch != NULL; ch = ch->ch_next) {
252 		dip = ch->ch_dip;
253 		if (ch->ch_gwi->gw_portid == gw_portid)
254 			break;
255 	}
256 	mutex_exit(&info->ti_child_lock);
257 
258 	return (dip);
259 }
260 
261 /*
262  * See if the passed gateway is already found in our list.  Note
263  * that we assume that the gateway port id uniquely identifies each
264  * gateway.
265  */
266 eibnx_gw_info_t *
eibnx_find_gw_in_gwlist(eibnx_thr_info_t * info,eibnx_gw_info_t * gwi)267 eibnx_find_gw_in_gwlist(eibnx_thr_info_t *info, eibnx_gw_info_t *gwi)
268 {
269 	eibnx_gw_info_t *lgw = NULL;
270 
271 	mutex_enter(&info->ti_gw_lock);
272 	for (lgw = info->ti_gw; lgw; lgw = lgw->gw_next) {
273 		if (lgw->gw_portid == gwi->gw_portid)
274 			break;
275 	}
276 	mutex_exit(&info->ti_gw_lock);
277 
278 	return (lgw);
279 }
280 
281 /*
282  * Add a newly discovered gateway to the gateway list.  Since we'll
283  * need to send unicast solicitations to this gateway soon, we'll
284  * also grab a swqe entry, and initialize basic gw adress parameters
285  * such as the gid, qpn, qkey and pkey of the GW.  When we eventually
286  * get to sending the unicast to this gateway for the first time,
287  * we'll discover the path to this gateway using these parameters
288  * and modify the ud destination handle appropriately.
289  */
290 eibnx_gw_info_t *
eibnx_add_gw_to_gwlist(eibnx_thr_info_t * info,eibnx_gw_info_t * gwi,ibt_wc_t * wc,uint8_t * recv_buf)291 eibnx_add_gw_to_gwlist(eibnx_thr_info_t *info, eibnx_gw_info_t *gwi,
292     ibt_wc_t *wc, uint8_t *recv_buf)
293 {
294 	eibnx_gw_info_t *new_gwi;
295 	eibnx_wqe_t *wqe;
296 	ib_grh_t *grh;
297 	ib_gid_t sgid;
298 	clock_t timeout_usecs;
299 
300 	/*
301 	 * For now, we'll simply do KM_NOSLEEP allocation, since this code
302 	 * is called from within rx processing
303 	 */
304 	new_gwi = kmem_zalloc(sizeof (eibnx_gw_info_t), KM_NOSLEEP);
305 	if (new_gwi == NULL) {
306 		ENX_DPRINTF_WARN("no memory, gw port_id 0x%x "
307 		    "will be ignored by hca_guid=0x%llx, port=0x%x",
308 		    gwi->gw_portid, info->ti_hca_guid,
309 		    info->ti_pi->p_port_num);
310 		return (NULL);
311 	}
312 
313 	/*
314 	 * We also need to acquire a send wqe to do unicast solicitations
315 	 * to this gateway later on. We should've enough pre-allocated swqes
316 	 * to do this without sleeping.
317 	 */
318 	if ((wqe = eibnx_acquire_swqe(info, KM_NOSLEEP)) == NULL) {
319 		ENX_DPRINTF_WARN("no swqe available, gw port_id 0x%x "
320 		    "will be ignored by hca_guid=0x%llx, port=0x%x",
321 		    gwi->gw_portid, info->ti_hca_guid,
322 		    info->ti_pi->p_port_num);
323 		kmem_free(new_gwi, sizeof (eibnx_gw_info_t));
324 		return (NULL);
325 	}
326 
327 	/*
328 	 * Initialize gw state and wqe information.
329 	 */
330 	new_gwi->gw_next = NULL;
331 	new_gwi->gw_swqe = wqe;
332 	new_gwi->gw_state = gwi->gw_state;
333 
334 	/*
335 	 * Set up gateway advertisement monitoring parameters. Since we
336 	 * always need to check against a timeout value of 2.5 * gw_adv_period,
337 	 * we'll keep this pre-calculated value as well.
338 	 */
339 	mutex_init(&new_gwi->gw_adv_lock, NULL, MUTEX_DRIVER, NULL);
340 	new_gwi->gw_adv_flag = gwi->gw_adv_flag;
341 	new_gwi->gw_adv_last_lbolt = ddi_get_lbolt64();
342 	timeout_usecs = gwi->gw_adv_period * 1000;
343 	timeout_usecs = ((timeout_usecs << 2) + timeout_usecs) >> 1;
344 	new_gwi->gw_adv_timeout_ticks = drv_usectohz(timeout_usecs);
345 
346 	/*
347 	 * Initialize gateway address information. Note that if the message has
348 	 * a GRH, we'll use the subnet prefix, otherwise we'll assume that the
349 	 * gateway is in the same subnet as ourselves.
350 	 */
351 	new_gwi->gw_addr.ga_vect = NULL;
352 	if (wc->wc_flags & IBT_WC_GRH_PRESENT) {
353 		grh = (ib_grh_t *)(uintptr_t)recv_buf;
354 		new_gwi->gw_addr.ga_gid.gid_prefix =
355 		    ntohll(grh->SGID.gid_prefix);
356 	} else {
357 		sgid = info->ti_pi->p_sgid_tbl[0];
358 		new_gwi->gw_addr.ga_gid.gid_prefix =
359 		    sgid.gid_prefix;
360 	}
361 	new_gwi->gw_addr.ga_gid.gid_guid = gwi->gw_guid;
362 	new_gwi->gw_addr.ga_qpn = gwi->gw_ctrl_qpn;
363 	new_gwi->gw_addr.ga_qkey = EIB_FIP_QKEY;
364 	new_gwi->gw_addr.ga_pkey = EIB_ADMIN_PKEY;
365 
366 	/*
367 	 * Initialize gateway parameters received via the advertisement
368 	 */
369 	new_gwi->gw_system_guid = gwi->gw_system_guid;
370 	new_gwi->gw_guid = gwi->gw_guid;
371 	new_gwi->gw_adv_period = gwi->gw_adv_period;
372 	new_gwi->gw_ka_period = gwi->gw_ka_period;
373 	new_gwi->gw_vnic_ka_period = gwi->gw_vnic_ka_period;
374 	new_gwi->gw_ctrl_qpn = gwi->gw_ctrl_qpn;
375 	new_gwi->gw_lid = gwi->gw_lid;
376 	new_gwi->gw_portid = gwi->gw_portid;
377 	new_gwi->gw_num_net_vnics = gwi->gw_num_net_vnics;
378 	new_gwi->gw_is_host_adm_vnics = gwi->gw_is_host_adm_vnics;
379 	new_gwi->gw_sl = gwi->gw_sl;
380 	new_gwi->gw_n_rss_qpn = gwi->gw_n_rss_qpn;
381 	new_gwi->gw_flag_ucast_advt = gwi->gw_flag_ucast_advt;
382 	new_gwi->gw_flag_available = gwi->gw_flag_available;
383 	bcopy(gwi->gw_system_name, new_gwi->gw_system_name,
384 	    sizeof (new_gwi->gw_system_name));
385 	bcopy(gwi->gw_port_name, new_gwi->gw_port_name,
386 	    sizeof (new_gwi->gw_port_name));
387 	bcopy(gwi->gw_vendor_id, new_gwi->gw_vendor_id,
388 	    sizeof (new_gwi->gw_vendor_id));
389 
390 	/*
391 	 * Queue up the new gwi and return it
392 	 */
393 	mutex_enter(&info->ti_gw_lock);
394 	new_gwi->gw_next = info->ti_gw;
395 	info->ti_gw = new_gwi;
396 	mutex_exit(&info->ti_gw_lock);
397 
398 	return (new_gwi);
399 }
400 
401 /*
402  * Update old data for the gateway in our list with the new data.
403  */
404 void
eibnx_replace_gw_in_gwlist(eibnx_thr_info_t * info,eibnx_gw_info_t * orig_gwi,eibnx_gw_info_t * new_gwi,ibt_wc_t * wc,uint8_t * recv_buf,boolean_t * gwi_changed)405 eibnx_replace_gw_in_gwlist(eibnx_thr_info_t *info, eibnx_gw_info_t *orig_gwi,
406     eibnx_gw_info_t *new_gwi, ibt_wc_t *wc, uint8_t *recv_buf,
407     boolean_t *gwi_changed)
408 {
409 	ib_sn_prefix_t new_gw_sn_prefix;
410 	ib_grh_t *grh;
411 	ib_gid_t sgid;
412 	boolean_t changed = B_FALSE;
413 	boolean_t gw_addr_changed = B_TRUE;
414 
415 	/*
416 	 * We'll update all info received in the new advertisement in
417 	 * the original gwi and also move the gw_state to that of the state
418 	 * in the new gwi.
419 	 */
420 	mutex_enter(&info->ti_gw_lock);
421 
422 	orig_gwi->gw_state = new_gwi->gw_state;
423 
424 	/*
425 	 * The guids shouldn't really change for the "same" gateway
426 	 */
427 	if (new_gwi->gw_system_guid != orig_gwi->gw_system_guid) {
428 		ENX_DPRINTF_WARN("gateway system guid changed for the "
429 		    "*same* gateway from 0x%llx to 0x%llx",
430 		    orig_gwi->gw_system_guid, new_gwi->gw_system_guid);
431 
432 		orig_gwi->gw_system_guid = new_gwi->gw_system_guid;
433 		changed = B_TRUE;
434 	}
435 	if (new_gwi->gw_guid != orig_gwi->gw_guid) {
436 		ENX_DPRINTF_WARN("gateway guid changed for the "
437 		    "*same* gateway from 0x%llx to 0x%llx",
438 		    orig_gwi->gw_guid, new_gwi->gw_guid);
439 
440 		orig_gwi->gw_guid = new_gwi->gw_guid;
441 		changed = B_TRUE;
442 		gw_addr_changed = B_TRUE;
443 	}
444 
445 	if (new_gwi->gw_adv_period != orig_gwi->gw_adv_period) {
446 		ENX_DPRINTF_DEBUG("gateway adv period changed "
447 		    "from 0x%lx to 0x%lx", orig_gwi->gw_adv_period,
448 		    new_gwi->gw_adv_period);
449 
450 		orig_gwi->gw_adv_period = new_gwi->gw_adv_period;
451 		changed = B_TRUE;
452 	}
453 	if (new_gwi->gw_ka_period != orig_gwi->gw_ka_period) {
454 		ENX_DPRINTF_DEBUG("gateway ka period changed "
455 		    "from 0x%lx to 0x%lx", orig_gwi->gw_ka_period,
456 		    new_gwi->gw_ka_period);
457 
458 		orig_gwi->gw_ka_period = new_gwi->gw_ka_period;
459 		changed = B_TRUE;
460 	}
461 	if (new_gwi->gw_vnic_ka_period != orig_gwi->gw_vnic_ka_period) {
462 		ENX_DPRINTF_DEBUG("vnic ka period changed "
463 		    "from 0x%lx to 0x%lx", orig_gwi->gw_vnic_ka_period,
464 		    new_gwi->gw_vnic_ka_period);
465 
466 		orig_gwi->gw_vnic_ka_period = new_gwi->gw_vnic_ka_period;
467 		changed = B_TRUE;
468 	}
469 	if (new_gwi->gw_ctrl_qpn != orig_gwi->gw_ctrl_qpn) {
470 		ENX_DPRINTF_DEBUG("gateway control qpn changed "
471 		    "from 0x%lx to 0x%lx", orig_gwi->gw_ctrl_qpn,
472 		    new_gwi->gw_ctrl_qpn);
473 
474 		orig_gwi->gw_ctrl_qpn = new_gwi->gw_ctrl_qpn;
475 		changed = B_TRUE;
476 	}
477 	if (new_gwi->gw_lid != orig_gwi->gw_lid) {
478 		ENX_DPRINTF_DEBUG("gateway lid changed from 0x%x to 0x%x",
479 		    orig_gwi->gw_lid, new_gwi->gw_lid);
480 
481 		orig_gwi->gw_lid = new_gwi->gw_lid;
482 		changed = B_TRUE;
483 		gw_addr_changed = B_TRUE;
484 	}
485 
486 	/*
487 	 * The identity of the gateway is currently defined by its portid,
488 	 * so this cannot be different or eibnx_find_gw_in_gwlist() wouldn't
489 	 * have thought it's the same.  For now though, we'll treat it
490 	 * like any other parameter, and flag it if we find this different.
491 	 */
492 	if (new_gwi->gw_portid != orig_gwi->gw_portid) {
493 		ENX_DPRINTF_WARN("gateway portid changed for the *same* "
494 		    "gateway from 0x%x to 0x%x", orig_gwi->gw_portid,
495 		    new_gwi->gw_portid);
496 
497 		orig_gwi->gw_portid = new_gwi->gw_portid;
498 		changed = B_TRUE;
499 	}
500 
501 	if (new_gwi->gw_is_host_adm_vnics != orig_gwi->gw_is_host_adm_vnics) {
502 		ENX_DPRINTF_DEBUG("host adm vnics changed from 0x%x to 0x%x",
503 		    orig_gwi->gw_is_host_adm_vnics,
504 		    new_gwi->gw_is_host_adm_vnics);
505 
506 		orig_gwi->gw_is_host_adm_vnics = new_gwi->gw_is_host_adm_vnics;
507 		changed = B_TRUE;
508 	}
509 	if (new_gwi->gw_sl != orig_gwi->gw_sl) {
510 		ENX_DPRINTF_DEBUG("gateway sl changed from 0x%x to 0x%x",
511 		    orig_gwi->gw_sl, new_gwi->gw_sl);
512 
513 		orig_gwi->gw_sl = new_gwi->gw_sl;
514 		changed = B_TRUE;
515 	}
516 	if (new_gwi->gw_n_rss_qpn != orig_gwi->gw_n_rss_qpn) {
517 		ENX_DPRINTF_DEBUG("gateway n_rss_qpn changed from 0x%x to 0x%x",
518 		    orig_gwi->gw_n_rss_qpn, new_gwi->gw_n_rss_qpn);
519 
520 		orig_gwi->gw_n_rss_qpn = new_gwi->gw_n_rss_qpn;
521 		changed = B_TRUE;
522 	}
523 
524 	/*
525 	 * The gw_flag_ucast_advt and gw_flag_available are expected to
526 	 * change over time (and even gw_num_net_vnics could change, but
527 	 * it's of no use to us presently), and we shouldn't trigger any
528 	 * flag for these
529 	 */
530 	orig_gwi->gw_flag_ucast_advt = new_gwi->gw_flag_ucast_advt;
531 	orig_gwi->gw_flag_available = new_gwi->gw_flag_available;
532 	orig_gwi->gw_num_net_vnics = new_gwi->gw_num_net_vnics;
533 
534 	if (strncmp((const char *)new_gwi->gw_system_name,
535 	    (const char *)orig_gwi->gw_system_name, EIB_GW_SYSNAME_LEN) != 0) {
536 		ENX_DPRINTF_DEBUG("gateway system name changed from %s to %s",
537 		    orig_gwi->gw_system_name, new_gwi->gw_system_name);
538 
539 		bcopy(new_gwi->gw_system_name, orig_gwi->gw_system_name,
540 		    EIB_GW_SYSNAME_LEN);
541 		changed = B_TRUE;
542 	}
543 	if (strncmp((const char *)new_gwi->gw_port_name,
544 	    (const char *)orig_gwi->gw_port_name, EIB_GW_PORTNAME_LEN) != 0) {
545 		ENX_DPRINTF_DEBUG("gateway port name changed from %s to %s",
546 		    orig_gwi->gw_port_name, new_gwi->gw_port_name);
547 
548 		bcopy(new_gwi->gw_port_name, orig_gwi->gw_port_name,
549 		    EIB_GW_PORTNAME_LEN);
550 		changed = B_TRUE;
551 	}
552 	if (strncmp((const char *)new_gwi->gw_vendor_id,
553 	    (const char *)orig_gwi->gw_vendor_id, EIB_GW_VENDOR_LEN) != 0) {
554 		ENX_DPRINTF_DEBUG("vendor id changed from %s to %s",
555 		    orig_gwi->gw_vendor_id, new_gwi->gw_vendor_id);
556 
557 		bcopy(new_gwi->gw_vendor_id, orig_gwi->gw_vendor_id,
558 		    EIB_GW_VENDOR_LEN);
559 		changed = B_TRUE;
560 	}
561 
562 	/*
563 	 * See if the subnet prefix for the gateway has changed
564 	 */
565 	if (wc->wc_flags & IBT_WC_GRH_PRESENT) {
566 		grh = (ib_grh_t *)(uintptr_t)recv_buf;
567 		new_gw_sn_prefix = ntohll(grh->SGID.gid_prefix);
568 	} else {
569 		sgid = info->ti_pi->p_sgid_tbl[0];
570 		new_gw_sn_prefix = sgid.gid_prefix;
571 	}
572 	if (new_gw_sn_prefix != orig_gwi->gw_addr.ga_gid.gid_prefix) {
573 		ENX_DPRINTF_WARN("subnet prefix changed from 0x%llx to 0x%llx",
574 		    orig_gwi->gw_addr.ga_gid.gid_prefix, new_gw_sn_prefix);
575 
576 		changed = B_TRUE;
577 		gw_addr_changed = B_TRUE;
578 	}
579 
580 	/*
581 	 * If the gateway address has changed in any way, clear the current
582 	 * address vector and update the gateway guid and gateway qpn. The
583 	 * address vector will be created the next time a unicast solicit
584 	 * is attempted for this gateway.
585 	 */
586 	if (gw_addr_changed) {
587 		if (orig_gwi->gw_addr.ga_vect != NULL) {
588 			kmem_free(orig_gwi->gw_addr.ga_vect,
589 			    sizeof (ibt_adds_vect_t));
590 			orig_gwi->gw_addr.ga_vect = NULL;
591 		}
592 		orig_gwi->gw_addr.ga_gid.gid_prefix = new_gw_sn_prefix;
593 		orig_gwi->gw_addr.ga_gid.gid_guid = new_gwi->gw_guid;
594 		orig_gwi->gw_addr.ga_qpn = new_gwi->gw_ctrl_qpn;
595 		orig_gwi->gw_addr.ga_qkey = EIB_FIP_QKEY;
596 		orig_gwi->gw_addr.ga_pkey = EIB_ADMIN_PKEY;
597 	}
598 
599 	mutex_exit(&info->ti_gw_lock);
600 
601 	if (gwi_changed) {
602 		*gwi_changed = changed;
603 	}
604 }
605 
606 /*
607  * Queue up a node for EoIB instantiation and wake up the thread
608  * that creates eoib nodes.
609  */
610 void
eibnx_queue_for_creation(eibnx_thr_info_t * info,eibnx_gw_info_t * gwi)611 eibnx_queue_for_creation(eibnx_thr_info_t *info, eibnx_gw_info_t *gwi)
612 {
613 	eibnx_t *ss = enx_global_ss;
614 	eibnx_nodeq_t *new_node;
615 
616 	/*
617 	 * For now, we'll simply do KM_NOSLEEP allocation, since this
618 	 * code is called from within rx processing
619 	 */
620 	new_node = kmem_zalloc(sizeof (eibnx_nodeq_t), KM_NOSLEEP);
621 	if (new_node == NULL) {
622 		ENX_DPRINTF_WARN("no memory, eoib node will not be "
623 		    "created for hca_guid=0x%llx, hca_port=0x%x, "
624 		    "gw_port_id=0x%x", info->ti_hca_guid,
625 		    info->ti_pi->p_port_num, gwi->gw_portid);
626 		return;
627 	}
628 	new_node->nc_info = info;
629 	new_node->nc_gwi = gwi;
630 
631 	/*
632 	 * If the eoib node creation thread is dying (or dead), don't
633 	 * queue up any more requests for creation
634 	 */
635 	mutex_enter(&ss->nx_nodeq_lock);
636 	if (ss->nx_nodeq_thr_die) {
637 		kmem_free(new_node, sizeof (eibnx_nodeq_t));
638 	} else {
639 		new_node->nc_next = ss->nx_nodeq;
640 		ss->nx_nodeq = new_node;
641 		cv_signal(&ss->nx_nodeq_cv);
642 	}
643 	mutex_exit(&ss->nx_nodeq_lock);
644 }
645