xref: /illumos-gate/usr/src/uts/common/io/ib/clients/ibd/ibd.c (revision b31b5de1357c915fe7dab4d9646d9d84f9fe69bc)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 /*
28  * An implementation of the IPoIB standard based on PSARC 2001/289.
29  */
30 
31 #include <sys/types.h>
32 #include <sys/conf.h>
33 #include <sys/ddi.h>
34 #include <sys/sunddi.h>
35 #include <sys/modctl.h>
36 #include <sys/stropts.h>
37 #include <sys/stream.h>
38 #include <sys/strsun.h>
39 #include <sys/strsubr.h>
40 #include <sys/dlpi.h>
41 #include <sys/mac_provider.h>
42 
43 #include <sys/pattr.h>		/* for HCK_FULLCKSUM */
44 #include <sys/sysmacros.h>	/* for offsetof */
45 #include <sys/disp.h>		/* for async thread pri */
46 #include <sys/atomic.h>		/* for atomic_add*() */
47 #include <sys/ethernet.h>	/* for ETHERTYPE_IP */
48 #include <netinet/in.h>		/* for netinet/ip.h below */
49 #include <netinet/ip.h>		/* for struct ip */
50 #include <netinet/udp.h>	/* for struct udphdr */
51 #include <inet/common.h>	/* for inet/ip.h below */
52 #include <inet/ip.h>		/* for ipha_t */
53 #include <inet/ip_if.h>		/* for IP6_DL_SAP */
54 #include <inet/ip6.h>		/* for ip6_t */
55 #include <inet/tcp.h>		/* for tcph_t */
56 #include <netinet/icmp6.h>	/* for icmp6_t */
57 #include <sys/callb.h>
58 #include <sys/modhash.h>
59 
60 #include <sys/ib/clients/ibd/ibd.h>
61 #include <sys/ib/mgt/sm_attr.h>	/* for SM_INIT_TYPE_* */
62 #include <sys/note.h>
63 #include <sys/multidata.h>
64 
65 #include <sys/ib/mgt/ibmf/ibmf.h>	/* for ibd_get_portspeed */
66 
67 /*
68  * Per-interface tunables
69  *
70  * ibd_tx_copy_thresh
71  *     This sets the threshold at which ibd will attempt to do a bcopy of the
72  *     outgoing data into a pre-mapped buffer. The IPoIB driver's send behavior
73  *     is restricted by various parameters, so setting of this value must be
74  *     made after careful considerations only.  For instance, IB HCAs currently
75  *     impose a relatively small limit (when compared to ethernet NICs) on the
76  *     length of the SGL for transmit. On the other hand, the ip stack could
77  *     send down mp chains that are quite long when LSO is enabled.
78  *
79  * ibd_num_swqe
80  *     Number of "send WQE" elements that will be allocated and used by ibd.
81  *     When tuning this parameter, the size of pre-allocated, pre-mapped copy
82  *     buffer in each of these send wqes must be taken into account. This
83  *     copy buffer size is determined by the value of IBD_TX_BUF_SZ (this is
84  *     currently set to the same value of ibd_tx_copy_thresh, but may be
85  *     changed independently if needed).
86  *
87  * ibd_num_rwqe
88  *     Number of "receive WQE" elements that will be allocated and used by
89  *     ibd. This parameter is limited by the maximum channel size of the HCA.
90  *     Each buffer in the receive wqe will be of MTU size.
91  *
92  * ibd_num_lso_bufs
93  *     Number of "larger-than-MTU" copy buffers to use for cases when the
94  *     outgoing mblk chain is too fragmented to be used with ibt_map_mem_iov()
95  *     and too large to be used with regular MTU-sized copy buffers. It is
96  *     not recommended to tune this variable without understanding the
97  *     application environment and/or memory resources. The size of each of
98  *     these lso buffers is determined by the value of IBD_LSO_BUFSZ.
99  *
100  * ibd_num_ah
101  *     Number of AH cache entries to allocate
102  *
103  * ibd_hash_size
104  *     Hash table size for the active AH list
105  *
106  * ibd_separate_cqs
107  * ibd_txcomp_poll
108  *     These boolean variables (1 or 0) may be used to tune the behavior of
109  *     ibd in managing the send and receive completion queues and in deciding
110  *     whether or not transmit completions should be polled or interrupt
111  *     driven (when the completion queues are separate). If both the completion
112  *     queues are interrupt driven, it may not be possible for the handlers to
113  *     be invoked concurrently, depending on how the interrupts are tied on
114  *     the PCI intr line.  Note that some combination of these two parameters
115  *     may not be meaningful (and therefore not allowed).
116  *
117  * ibd_tx_softintr
118  * ibd_rx_softintr
119  *     The softintr mechanism allows ibd to avoid event queue overflows if
120  *     the receive/completion handlers are to be expensive. These are enabled
121  *     by default.
122  *
123  * ibd_log_sz
124  *     This specifies the size of the ibd log buffer in bytes. The buffer is
125  *     allocated and logging is enabled only when IBD_LOGGING is defined.
126  *
127  */
128 uint_t ibd_tx_copy_thresh = 0x1000;
129 uint_t ibd_num_swqe = 4000;
130 uint_t ibd_num_rwqe = 4000;
131 uint_t ibd_num_lso_bufs = 0x400;
132 uint_t ibd_num_ah = 64;
133 uint_t ibd_hash_size = 32;
134 uint_t ibd_separate_cqs = 1;
135 uint_t ibd_txcomp_poll = 0;
136 uint_t ibd_rx_softintr = 1;
137 uint_t ibd_tx_softintr = 1;
138 #ifdef IBD_LOGGING
139 uint_t ibd_log_sz = 0x20000;
140 #endif
141 
142 #define	IBD_TX_COPY_THRESH		ibd_tx_copy_thresh
143 #define	IBD_TX_BUF_SZ			ibd_tx_copy_thresh
144 #define	IBD_NUM_SWQE			ibd_num_swqe
145 #define	IBD_NUM_RWQE			ibd_num_rwqe
146 #define	IBD_NUM_LSO_BUFS		ibd_num_lso_bufs
147 #define	IBD_NUM_AH			ibd_num_ah
148 #define	IBD_HASH_SIZE			ibd_hash_size
149 #ifdef IBD_LOGGING
150 #define	IBD_LOG_SZ			ibd_log_sz
151 #endif
152 
153 /*
154  * Receive CQ moderation parameters: NOT tunables
155  */
156 static uint_t ibd_rxcomp_count = 4;
157 static uint_t ibd_rxcomp_usec = 10;
158 
159 /*
160  * Thresholds
161  *
162  * When waiting for resources (swqes or lso buffers) to become available,
163  * the first two thresholds below determine how long to wait before informing
164  * the network layer to start sending packets again. The IBD_TX_POLL_THRESH
165  * determines how low the available swqes should go before we start polling
166  * the completion queue.
167  */
168 #define	IBD_FREE_LSOS_THRESH		8
169 #define	IBD_FREE_SWQES_THRESH		20
170 #define	IBD_TX_POLL_THRESH		80
171 
172 /*
173  * When doing multiple-send-wr or multiple-recv-wr posts, this value
174  * determines how many to do at a time (in a single ibt_post_send/recv).
175  */
176 #define	IBD_MAX_POST_MULTIPLE		4
177 
178 /*
179  * Maximum length for returning chained mps back to crossbow
180  */
181 #define	IBD_MAX_RX_MP_LEN		16
182 
183 /*
184  * LSO parameters
185  */
186 #define	IBD_LSO_MAXLEN			65536
187 #define	IBD_LSO_BUFSZ			8192
188 #define	IBD_PROP_LSO_POLICY		"lso-policy"
189 
190 /*
191  * Completion queue polling control
192  */
193 #define	IBD_RX_CQ_POLLING		0x1
194 #define	IBD_TX_CQ_POLLING		0x2
195 #define	IBD_REDO_RX_CQ_POLLING		0x4
196 #define	IBD_REDO_TX_CQ_POLLING		0x8
197 
198 /*
199  * Flag bits for resources to reap
200  */
201 #define	IBD_RSRC_SWQE			0x1
202 #define	IBD_RSRC_LSOBUF			0x2
203 
204 /*
205  * Async operation types
206  */
207 #define	IBD_ASYNC_GETAH			1
208 #define	IBD_ASYNC_JOIN			2
209 #define	IBD_ASYNC_LEAVE			3
210 #define	IBD_ASYNC_PROMON		4
211 #define	IBD_ASYNC_PROMOFF		5
212 #define	IBD_ASYNC_REAP			6
213 #define	IBD_ASYNC_TRAP			7
214 #define	IBD_ASYNC_SCHED			8
215 #define	IBD_ASYNC_LINK			9
216 #define	IBD_ASYNC_EXIT			10
217 
218 /*
219  * Async operation states
220  */
221 #define	IBD_OP_NOTSTARTED		0
222 #define	IBD_OP_ONGOING			1
223 #define	IBD_OP_COMPLETED		2
224 #define	IBD_OP_ERRORED			3
225 #define	IBD_OP_ROUTERED			4
226 
227 /*
228  * Miscellaneous constants
229  */
230 #define	IBD_SEND			0
231 #define	IBD_RECV			1
232 #define	IB_MGID_IPV4_LOWGRP_MASK	0xFFFFFFFF
233 #ifdef IBD_LOGGING
234 #define	IBD_DMAX_LINE			100
235 #endif
236 
237 /*
238  * Enumerations for link states
239  */
240 typedef enum {
241 	IBD_LINK_DOWN,
242 	IBD_LINK_UP,
243 	IBD_LINK_UP_ABSENT
244 } ibd_link_op_t;
245 
246 /*
247  * Driver State Pointer
248  */
249 void *ibd_list;
250 
251 /*
252  * Logging
253  */
254 #ifdef IBD_LOGGING
255 kmutex_t ibd_lbuf_lock;
256 uint8_t *ibd_lbuf;
257 uint32_t ibd_lbuf_ndx;
258 #endif
259 
260 /*
261  * Required system entry points
262  */
263 static int ibd_attach(dev_info_t *dip, ddi_attach_cmd_t cmd);
264 static int ibd_detach(dev_info_t *dip, ddi_detach_cmd_t cmd);
265 
266 /*
267  * Required driver entry points for GLDv3
268  */
269 static int ibd_m_stat(void *, uint_t, uint64_t *);
270 static int ibd_m_start(void *);
271 static void ibd_m_stop(void *);
272 static int ibd_m_promisc(void *, boolean_t);
273 static int ibd_m_multicst(void *, boolean_t, const uint8_t *);
274 static int ibd_m_unicst(void *, const uint8_t *);
275 static mblk_t *ibd_m_tx(void *, mblk_t *);
276 static boolean_t ibd_m_getcapab(void *, mac_capab_t, void *);
277 
278 /*
279  * Private driver entry points for GLDv3
280  */
281 
282 /*
283  * Initialization
284  */
285 static int ibd_state_init(ibd_state_t *, dev_info_t *);
286 static int ibd_drv_init(ibd_state_t *);
287 static int ibd_init_txlist(ibd_state_t *);
288 static int ibd_init_rxlist(ibd_state_t *);
289 static int ibd_acache_init(ibd_state_t *);
290 #ifdef IBD_LOGGING
291 static void ibd_log_init(void);
292 #endif
293 
294 /*
295  * Termination/cleanup
296  */
297 static void ibd_state_fini(ibd_state_t *);
298 static void ibd_drv_fini(ibd_state_t *);
299 static void ibd_fini_txlist(ibd_state_t *);
300 static void ibd_fini_rxlist(ibd_state_t *);
301 static void ibd_tx_cleanup(ibd_state_t *, ibd_swqe_t *);
302 static void ibd_acache_fini(ibd_state_t *);
303 #ifdef IBD_LOGGING
304 static void ibd_log_fini(void);
305 #endif
306 
307 /*
308  * Allocation/acquire/map routines
309  */
310 static int ibd_alloc_swqe(ibd_state_t *, ibd_swqe_t **, int, ibt_lkey_t);
311 static int ibd_alloc_rwqe(ibd_state_t *, ibd_rwqe_t **);
312 static int ibd_alloc_tx_copybufs(ibd_state_t *);
313 static int ibd_alloc_tx_lsobufs(ibd_state_t *);
314 static int ibd_acquire_swqe(ibd_state_t *, ibd_swqe_t **);
315 static int ibd_acquire_lsobufs(ibd_state_t *, uint_t, ibt_wr_ds_t *,
316     uint32_t *);
317 
318 /*
319  * Free/release/unmap routines
320  */
321 static void ibd_free_swqe(ibd_state_t *, ibd_swqe_t *);
322 static void ibd_free_rwqe(ibd_state_t *, ibd_rwqe_t *);
323 static void ibd_delete_rwqe(ibd_state_t *, ibd_rwqe_t *);
324 static void ibd_free_tx_copybufs(ibd_state_t *);
325 static void ibd_free_tx_lsobufs(ibd_state_t *);
326 static void ibd_release_swqe(ibd_state_t *, ibd_swqe_t *);
327 static void ibd_release_lsobufs(ibd_state_t *, ibt_wr_ds_t *, uint32_t);
328 static void ibd_free_lsohdr(ibd_swqe_t *, mblk_t *);
329 static void ibd_unmap_mem(ibd_state_t *, ibd_swqe_t *);
330 
331 /*
332  * Handlers/callback routines
333  */
334 static uint_t ibd_intr(char *);
335 static uint_t ibd_tx_recycle(char *);
336 static void ibd_rcq_handler(ibt_cq_hdl_t, void *);
337 static void ibd_scq_handler(ibt_cq_hdl_t, void *);
338 static void ibd_poll_compq(ibd_state_t *, ibt_cq_hdl_t);
339 static uint_t ibd_drain_cq(ibd_state_t *, ibt_cq_hdl_t, ibt_wc_t *, uint_t);
340 static void ibd_freemsg_cb(char *);
341 static void ibd_async_handler(void *, ibt_hca_hdl_t, ibt_async_code_t,
342     ibt_async_event_t *);
343 static void ibd_snet_notices_handler(void *, ib_gid_t,
344     ibt_subnet_event_code_t, ibt_subnet_event_t *);
345 
346 /*
347  * Send/receive routines
348  */
349 static boolean_t ibd_send(ibd_state_t *, mblk_t *);
350 static void ibd_post_send(ibd_state_t *, ibd_swqe_t *);
351 static int ibd_post_rwqe(ibd_state_t *, ibd_rwqe_t *, boolean_t);
352 static void ibd_process_rx(ibd_state_t *, ibd_rwqe_t *, ibt_wc_t *);
353 static void ibd_flush_rx(ibd_state_t *, mblk_t *);
354 
355 /*
356  * Threads
357  */
358 static void ibd_async_work(ibd_state_t *);
359 
360 /*
361  * Async tasks
362  */
363 static void ibd_async_acache(ibd_state_t *, ipoib_mac_t *);
364 static void ibd_async_multicast(ibd_state_t *, ib_gid_t, int);
365 static void ibd_async_setprom(ibd_state_t *);
366 static void ibd_async_unsetprom(ibd_state_t *);
367 static void ibd_async_reap_group(ibd_state_t *, ibd_mce_t *, ib_gid_t, uint8_t);
368 static void ibd_async_trap(ibd_state_t *, ibd_req_t *);
369 static void ibd_async_txsched(ibd_state_t *);
370 static void ibd_async_link(ibd_state_t *, ibd_req_t *);
371 
372 /*
373  * Async task helpers
374  */
375 static ibd_mce_t *ibd_async_mcache(ibd_state_t *, ipoib_mac_t *, boolean_t *);
376 static ibd_mce_t *ibd_join_group(ibd_state_t *, ib_gid_t, uint8_t);
377 static ibd_mce_t *ibd_mcache_find(ib_gid_t, struct list *);
378 static boolean_t ibd_get_allroutergroup(ibd_state_t *,
379     ipoib_mac_t *, ipoib_mac_t *);
380 static void ibd_leave_group(ibd_state_t *, ib_gid_t, uint8_t);
381 static void ibd_reacquire_group(ibd_state_t *, ibd_mce_t *);
382 static ibt_status_t ibd_iba_join(ibd_state_t *, ib_gid_t, ibd_mce_t *);
383 static ibt_status_t ibd_find_bgroup(ibd_state_t *);
384 static void ibd_n2h_gid(ipoib_mac_t *, ib_gid_t *);
385 static void ibd_h2n_mac(ipoib_mac_t *, ib_qpn_t, ib_sn_prefix_t, ib_guid_t);
386 static uint64_t ibd_get_portspeed(ibd_state_t *);
387 static int ibd_get_portpkey(ibd_state_t *, ib_guid_t *);
388 static boolean_t ibd_async_safe(ibd_state_t *);
389 static void ibd_async_done(ibd_state_t *);
390 static ibd_ace_t *ibd_acache_find(ibd_state_t *, ipoib_mac_t *, boolean_t, int);
391 static ibd_ace_t *ibd_acache_lookup(ibd_state_t *, ipoib_mac_t *, int *, int);
392 static ibd_ace_t *ibd_acache_get_unref(ibd_state_t *);
393 static boolean_t ibd_acache_recycle(ibd_state_t *, ipoib_mac_t *, boolean_t);
394 static void ibd_link_mod(ibd_state_t *, ibt_async_code_t);
395 
396 /*
397  * Miscellaneous helpers
398  */
399 static int ibd_sched_poll(ibd_state_t *, int, int);
400 static void ibd_queue_work_slot(ibd_state_t *, ibd_req_t *, int);
401 static int ibd_resume_transmission(ibd_state_t *);
402 static int ibd_setup_lso(ibd_swqe_t *, mblk_t *, uint32_t, ibt_ud_dest_hdl_t);
403 static int ibd_prepare_sgl(ibd_state_t *, mblk_t *, ibd_swqe_t *, uint_t);
404 static void *list_get_head(list_t *);
405 static int ibd_hash_key_cmp(mod_hash_key_t, mod_hash_key_t);
406 static uint_t ibd_hash_by_id(void *, mod_hash_key_t);
407 static void ibd_print_warn(ibd_state_t *, char *, ...);
408 #ifdef IBD_LOGGING
409 static void ibd_log(const char *, ...);
410 #endif
411 
412 DDI_DEFINE_STREAM_OPS(ibd_dev_ops, nulldev, nulldev, ibd_attach, ibd_detach,
413     nodev, NULL, D_MP, NULL, ddi_quiesce_not_needed);
414 
415 /* Module Driver Info */
416 static struct modldrv ibd_modldrv = {
417 	&mod_driverops,			/* This one is a driver */
418 	"InfiniBand GLDv3 Driver",	/* short description */
419 	&ibd_dev_ops			/* driver specific ops */
420 };
421 
422 /* Module Linkage */
423 static struct modlinkage ibd_modlinkage = {
424 	MODREV_1, (void *)&ibd_modldrv, NULL
425 };
426 
427 /*
428  * Module (static) info passed to IBTL during ibt_attach
429  */
430 static struct ibt_clnt_modinfo_s ibd_clnt_modinfo = {
431 	IBTI_V_CURR,
432 	IBT_NETWORK,
433 	ibd_async_handler,
434 	NULL,
435 	"IPIB"
436 };
437 
438 /*
439  * GLDv3 entry points
440  */
441 #define	IBD_M_CALLBACK_FLAGS	(MC_GETCAPAB)
442 static mac_callbacks_t ib_m_callbacks = {
443 	IBD_M_CALLBACK_FLAGS,
444 	ibd_m_stat,
445 	ibd_m_start,
446 	ibd_m_stop,
447 	ibd_m_promisc,
448 	ibd_m_multicst,
449 	ibd_m_unicst,
450 	ibd_m_tx,
451 	NULL,
452 	ibd_m_getcapab
453 };
454 
455 /*
456  * Fill/clear <scope> and <p_key> in multicast/broadcast address
457  */
458 #define	IBD_FILL_SCOPE_PKEY(maddr, scope, pkey)		\
459 {							\
460 	*(uint32_t *)((char *)(maddr) + 4) |=		\
461 	    htonl((uint32_t)(scope) << 16);		\
462 	*(uint32_t *)((char *)(maddr) + 8) |=		\
463 	    htonl((uint32_t)(pkey) << 16);		\
464 }
465 
466 #define	IBD_CLEAR_SCOPE_PKEY(maddr)			\
467 {							\
468 	*(uint32_t *)((char *)(maddr) + 4) &=		\
469 	    htonl(~((uint32_t)0xF << 16));		\
470 	*(uint32_t *)((char *)(maddr) + 8) &=		\
471 	    htonl(~((uint32_t)0xFFFF << 16));		\
472 }
473 
474 /*
475  * Rudimentary debugging support
476  */
477 #ifdef DEBUG
478 int ibd_debuglevel = 100;
479 static void
480 debug_print(int l, char *fmt, ...)
481 {
482 	va_list ap;
483 
484 	if (l < ibd_debuglevel)
485 		return;
486 	va_start(ap, fmt);
487 	vcmn_err(CE_CONT, fmt, ap);
488 	va_end(ap);
489 }
490 #define	DPRINT		debug_print
491 #else
492 #define	DPRINT
493 #endif
494 
495 /*
496  * Common routine to print warning messages; adds in hca guid, port number
497  * and pkey to be able to identify the IBA interface.
498  */
499 static void
500 ibd_print_warn(ibd_state_t *state, char *fmt, ...)
501 {
502 	ib_guid_t hca_guid;
503 	char ibd_print_buf[256];
504 	int len;
505 	va_list ap;
506 
507 	hca_guid = ddi_prop_get_int64(DDI_DEV_T_ANY, state->id_dip,
508 	    0, "hca-guid", 0);
509 	len = snprintf(ibd_print_buf, sizeof (ibd_print_buf),
510 	    "%s%d: HCA GUID %016llx port %d PKEY %02x ",
511 	    ddi_driver_name(state->id_dip), ddi_get_instance(state->id_dip),
512 	    (u_longlong_t)hca_guid, state->id_port, state->id_pkey);
513 	va_start(ap, fmt);
514 	(void) vsnprintf(ibd_print_buf + len, sizeof (ibd_print_buf) - len,
515 	    fmt, ap);
516 	cmn_err(CE_NOTE, "!%s", ibd_print_buf);
517 	va_end(ap);
518 }
519 
520 /*
521  * Warlock directives
522  */
523 
524 /*
525  * id_lso_lock
526  *
527  * state->id_lso->bkt_nfree may be accessed without a lock to
528  * determine the threshold at which we have to ask the nw layer
529  * to resume transmission (see ibd_resume_transmission()).
530  */
531 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_lso_lock,
532     ibd_state_t::id_lso))
533 _NOTE(DATA_READABLE_WITHOUT_LOCK(ibd_state_t::id_lso))
534 _NOTE(DATA_READABLE_WITHOUT_LOCK(ibd_lsobkt_t::bkt_nfree))
535 
536 /*
537  * id_cq_poll_lock
538  */
539 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_cq_poll_lock,
540     ibd_state_t::id_cq_poll_busy))
541 
542 /*
543  * id_txpost_lock
544  */
545 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_txpost_lock,
546     ibd_state_t::id_tx_head))
547 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_txpost_lock,
548     ibd_state_t::id_tx_busy))
549 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_txpost_lock,
550     ibd_state_t::id_tx_tailp))
551 
552 /*
553  * id_rxpost_lock
554  */
555 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_rxpost_lock,
556     ibd_state_t::id_rx_head))
557 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_rxpost_lock,
558     ibd_state_t::id_rx_busy))
559 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_rxpost_lock,
560     ibd_state_t::id_rx_tailp))
561 
562 /*
563  * id_acache_req_lock
564  */
565 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_acache_req_lock,
566     ibd_state_t::id_acache_req_cv))
567 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_acache_req_lock,
568     ibd_state_t::id_req_list))
569 
570 /*
571  * id_ac_mutex
572  *
573  * This mutex is actually supposed to protect id_ah_op as well,
574  * but this path of the code isn't clean (see update of id_ah_op
575  * in ibd_async_acache(), immediately after the call to
576  * ibd_async_mcache()). For now, we'll skip this check by
577  * declaring that id_ah_op is protected by some internal scheme
578  * that warlock isn't aware of.
579  */
580 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_ac_mutex,
581     ibd_state_t::id_ah_active))
582 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_ac_mutex,
583     ibd_state_t::id_ah_free))
584 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_ac_mutex,
585     ibd_state_t::id_ah_addr))
586 _NOTE(SCHEME_PROTECTS_DATA("ac mutex should protect this",
587     ibd_state_t::id_ah_op))
588 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_ac_mutex,
589     ibd_state_t::id_ah_error))
590 _NOTE(DATA_READABLE_WITHOUT_LOCK(ibd_state_t::id_ah_error))
591 
592 /*
593  * id_mc_mutex
594  */
595 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_mc_mutex,
596     ibd_state_t::id_mc_full))
597 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_mc_mutex,
598     ibd_state_t::id_mc_non))
599 
600 /*
601  * id_trap_lock
602  */
603 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_trap_lock,
604     ibd_state_t::id_trap_cv))
605 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_trap_lock,
606     ibd_state_t::id_trap_stop))
607 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_trap_lock,
608     ibd_state_t::id_trap_inprog))
609 
610 /*
611  * id_prom_op
612  */
613 _NOTE(SCHEME_PROTECTS_DATA("only by async thread",
614     ibd_state_t::id_prom_op))
615 
616 /*
617  * id_sched_lock
618  */
619 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_sched_lock,
620     ibd_state_t::id_sched_needed))
621 
622 /*
623  * id_link_mutex
624  */
625 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_link_mutex,
626     ibd_state_t::id_link_state))
627 _NOTE(DATA_READABLE_WITHOUT_LOCK(ibd_state_t::id_link_state))
628 _NOTE(SCHEME_PROTECTS_DATA("only async thr and drv init",
629     ibd_state_t::id_link_speed))
630 
631 /*
632  * id_tx_list.dl_mutex
633  */
634 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_tx_list.dl_mutex,
635     ibd_state_t::id_tx_list.dl_head))
636 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_tx_list.dl_mutex,
637     ibd_state_t::id_tx_list.dl_tail))
638 _NOTE(SCHEME_PROTECTS_DATA("atomic or dl mutex or single thr",
639     ibd_state_t::id_tx_list.dl_pending_sends))
640 _NOTE(SCHEME_PROTECTS_DATA("atomic or dl mutex or single thr",
641     ibd_state_t::id_tx_list.dl_cnt))
642 
643 /*
644  * id_rx_list.dl_mutex
645  */
646 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_rx_list.dl_mutex,
647     ibd_state_t::id_rx_list.dl_head))
648 _NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_rx_list.dl_mutex,
649     ibd_state_t::id_rx_list.dl_tail))
650 _NOTE(SCHEME_PROTECTS_DATA("atomic or dl mutex or single thr",
651     ibd_state_t::id_rx_list.dl_bufs_outstanding))
652 _NOTE(SCHEME_PROTECTS_DATA("atomic or dl mutex or single thr",
653     ibd_state_t::id_rx_list.dl_cnt))
654 
655 
656 /*
657  * Items protected by atomic updates
658  */
659 _NOTE(SCHEME_PROTECTS_DATA("atomic update only",
660     ibd_state_s::id_brd_rcv
661     ibd_state_s::id_brd_xmt
662     ibd_state_s::id_multi_rcv
663     ibd_state_s::id_multi_xmt
664     ibd_state_s::id_num_intrs
665     ibd_state_s::id_rcv_bytes
666     ibd_state_s::id_rcv_pkt
667     ibd_state_s::id_tx_short
668     ibd_state_s::id_xmt_bytes
669     ibd_state_s::id_xmt_pkt))
670 
671 /*
672  * Non-mutex protection schemes for data elements. Almost all of
673  * these are non-shared items.
674  */
675 _NOTE(SCHEME_PROTECTS_DATA("unshared or single-threaded",
676     callb_cpr
677     ib_gid_s
678     ib_header_info
679     ibd_acache_rq
680     ibd_acache_s::ac_mce
681     ibd_mcache::mc_fullreap
682     ibd_mcache::mc_jstate
683     ibd_mcache::mc_req
684     ibd_rwqe_s
685     ibd_swqe_s
686     ibd_wqe_s
687     ibt_wr_ds_s::ds_va
688     ibt_wr_lso_s
689     ipoib_mac::ipoib_qpn
690     mac_capab_lso_s
691     msgb::b_next
692     msgb::b_rptr
693     msgb::b_wptr))
694 
695 int
696 _init()
697 {
698 	int status;
699 
700 	/*
701 	 * Sanity check some parameter settings. Tx completion polling
702 	 * only makes sense with separate CQs for Tx and Rx.
703 	 */
704 	if ((ibd_txcomp_poll == 1) && (ibd_separate_cqs == 0)) {
705 		cmn_err(CE_NOTE, "!ibd: %s",
706 		    "Setting ibd_txcomp_poll = 0 for combined CQ");
707 		ibd_txcomp_poll = 0;
708 	}
709 
710 	status = ddi_soft_state_init(&ibd_list, sizeof (ibd_state_t), 0);
711 	if (status != 0) {
712 		DPRINT(10, "_init:failed in ddi_soft_state_init()");
713 		return (status);
714 	}
715 
716 	mac_init_ops(&ibd_dev_ops, "ibd");
717 	status = mod_install(&ibd_modlinkage);
718 	if (status != 0) {
719 		DPRINT(10, "_init:failed in mod_install()");
720 		ddi_soft_state_fini(&ibd_list);
721 		mac_fini_ops(&ibd_dev_ops);
722 		return (status);
723 	}
724 
725 #ifdef IBD_LOGGING
726 	ibd_log_init();
727 #endif
728 	return (0);
729 }
730 
731 int
732 _info(struct modinfo *modinfop)
733 {
734 	return (mod_info(&ibd_modlinkage, modinfop));
735 }
736 
737 int
738 _fini()
739 {
740 	int status;
741 
742 	status = mod_remove(&ibd_modlinkage);
743 	if (status != 0)
744 		return (status);
745 
746 	mac_fini_ops(&ibd_dev_ops);
747 	ddi_soft_state_fini(&ibd_list);
748 #ifdef IBD_LOGGING
749 	ibd_log_fini();
750 #endif
751 	return (0);
752 }
753 
754 /*
755  * Convert the GID part of the mac address from network byte order
756  * to host order.
757  */
758 static void
759 ibd_n2h_gid(ipoib_mac_t *mac, ib_gid_t *dgid)
760 {
761 	ib_sn_prefix_t nbopref;
762 	ib_guid_t nboguid;
763 
764 	bcopy(mac->ipoib_gidpref, &nbopref, sizeof (ib_sn_prefix_t));
765 	bcopy(mac->ipoib_gidsuff, &nboguid, sizeof (ib_guid_t));
766 	dgid->gid_prefix = b2h64(nbopref);
767 	dgid->gid_guid = b2h64(nboguid);
768 }
769 
770 /*
771  * Create the IPoIB address in network byte order from host order inputs.
772  */
773 static void
774 ibd_h2n_mac(ipoib_mac_t *mac, ib_qpn_t qpn, ib_sn_prefix_t prefix,
775     ib_guid_t guid)
776 {
777 	ib_sn_prefix_t nbopref;
778 	ib_guid_t nboguid;
779 
780 	mac->ipoib_qpn = htonl(qpn);
781 	nbopref = h2b64(prefix);
782 	nboguid = h2b64(guid);
783 	bcopy(&nbopref, mac->ipoib_gidpref, sizeof (ib_sn_prefix_t));
784 	bcopy(&nboguid, mac->ipoib_gidsuff, sizeof (ib_guid_t));
785 }
786 
787 /*
788  * Send to the appropriate all-routers group when the IBA multicast group
789  * does not exist, based on whether the target group is v4 or v6.
790  */
791 static boolean_t
792 ibd_get_allroutergroup(ibd_state_t *state, ipoib_mac_t *mcmac,
793     ipoib_mac_t *rmac)
794 {
795 	boolean_t retval = B_TRUE;
796 	uint32_t adjscope = state->id_scope << 16;
797 	uint32_t topword;
798 
799 	/*
800 	 * Copy the first 4 bytes in without assuming any alignment of
801 	 * input mac address; this will have IPoIB signature, flags and
802 	 * scope bits.
803 	 */
804 	bcopy(mcmac->ipoib_gidpref, &topword, sizeof (uint32_t));
805 	topword = ntohl(topword);
806 
807 	/*
808 	 * Generate proper address for IPv4/v6, adding in the Pkey properly.
809 	 */
810 	if ((topword == (IB_MCGID_IPV4_PREFIX | adjscope)) ||
811 	    (topword == (IB_MCGID_IPV6_PREFIX | adjscope)))
812 		ibd_h2n_mac(rmac, IB_MC_QPN, (((uint64_t)topword << 32) |
813 		    ((uint32_t)(state->id_pkey << 16))),
814 		    (INADDR_ALLRTRS_GROUP - INADDR_UNSPEC_GROUP));
815 	else
816 		/*
817 		 * Does not have proper bits in the mgid address.
818 		 */
819 		retval = B_FALSE;
820 
821 	return (retval);
822 }
823 
824 /*
825  * Padding for nd6 Neighbor Solicitation and Advertisement needs to be at
826  * front of optional src/tgt link layer address. Right now Solaris inserts
827  * padding by default at the end. The routine which is doing is nce_xmit()
828  * in ip_ndp.c. It copies the nd_lla_addr after the nd_opt_hdr_t. So when
829  * the packet comes down from IP layer to the IBD driver, it is in the
830  * following format: [IPoIB_PTXHDR_T][INET6 packet][ICMP6][OPT_ND_HDR_T]
831  * This size is 2 bytes followed by [22 bytes of ipoib_machdr]. As a result
832  * machdr is not 4 byte aligned and had 2 bytes of padding at the end.
833  *
834  * The send routine at IBD driver changes this packet as follows:
835  * [IPoIB_HDR_T][INET6 packet][ICMP6][OPT_ND_HDR_T + 2 bytes of padding]
836  * followed by [22 bytes of ipoib_machdr] resulting in machdr 4 byte
837  * aligned.
838  *
839  * At the receiving side again ibd_process_rx takes the above packet and
840  * removes the two bytes of front padding and inserts it at the end. This
841  * is since the IP layer does not understand padding at the front.
842  */
843 #define	IBD_PAD_NSNA(ip6h, len, type) {					\
844 	uchar_t 	*nd_lla_ptr;					\
845 	icmp6_t 	*icmp6;						\
846 	nd_opt_hdr_t	*opt;						\
847 	int 		i;						\
848 									\
849 	icmp6 = (icmp6_t *)&ip6h[1];					\
850 	len -= sizeof (nd_neighbor_advert_t);				\
851 	if (((icmp6->icmp6_type == ND_NEIGHBOR_SOLICIT) ||		\
852 	    (icmp6->icmp6_type == ND_NEIGHBOR_ADVERT)) &&		\
853 	    (len != 0)) {						\
854 		opt = (nd_opt_hdr_t *)((uint8_t *)ip6h			\
855 		    + IPV6_HDR_LEN + sizeof (nd_neighbor_advert_t));	\
856 		ASSERT(opt != NULL);					\
857 		nd_lla_ptr = (uchar_t *)&opt[1];			\
858 		if (type == IBD_SEND) {					\
859 			for (i = IPOIB_ADDRL; i > 0; i--)		\
860 				*(nd_lla_ptr + i + 1) =			\
861 				    *(nd_lla_ptr + i - 1);		\
862 		} else {						\
863 			for (i = 0; i < IPOIB_ADDRL; i++)		\
864 				*(nd_lla_ptr + i) =			\
865 				    *(nd_lla_ptr + i + 2);		\
866 		}							\
867 		*(nd_lla_ptr + i) = 0;					\
868 		*(nd_lla_ptr + i + 1) = 0;				\
869 	}								\
870 }
871 
872 /*
873  * Address handle entries maintained by the driver are kept in the
874  * free and active lists. Each entry starts out in the free list;
875  * it migrates to the active list when primed using ibt_get_paths()
876  * and ibt_modify_ud_dest() for transmission to a specific destination.
877  * In the active list, the entry has a reference count indicating the
878  * number of ongoing/uncompleted transmits that reference it. The
879  * entry is left in the active list even after the reference count
880  * goes to 0, since successive transmits can find it there and do
881  * not need to set up another entry (ie the path information is
882  * cached using the active list). Entries on the active list are
883  * also hashed using the destination link address as a key for faster
884  * lookups during transmits.
885  *
886  * For any destination address (unicast or multicast, whatever the
887  * join states), there will be at most one entry in the active list.
888  * Entries with a 0 reference count on the active list can be reused
889  * for a transmit to a new destination, if the free list is empty.
890  *
891  * The AH free list insertion/deletion is protected with the id_ac_mutex,
892  * since the async thread and Tx callback handlers insert/delete. The
893  * active list does not need a lock (all operations are done by the
894  * async thread) but updates to the reference count are atomically
895  * done (increments done by Tx path, decrements by the Tx callback handler).
896  */
897 #define	IBD_ACACHE_INSERT_FREE(state, ce) \
898 	list_insert_head(&state->id_ah_free, ce)
899 #define	IBD_ACACHE_GET_FREE(state) \
900 	list_get_head(&state->id_ah_free)
901 #define	IBD_ACACHE_INSERT_ACTIVE(state, ce) {			\
902 	int _ret_;						\
903 	list_insert_head(&state->id_ah_active, ce);		\
904 	_ret_ = mod_hash_insert(state->id_ah_active_hash,	\
905 	    (mod_hash_key_t)&ce->ac_mac, (mod_hash_val_t)ce);	\
906 	ASSERT(_ret_ == 0);					\
907 }
908 #define	IBD_ACACHE_PULLOUT_ACTIVE(state, ce) {			\
909 	list_remove(&state->id_ah_active, ce);			\
910 	(void) mod_hash_remove(state->id_ah_active_hash,	\
911 	    (mod_hash_key_t)&ce->ac_mac, (mod_hash_val_t)ce);	\
912 }
913 #define	IBD_ACACHE_GET_ACTIVE(state) \
914 	list_get_head(&state->id_ah_active)
915 
916 /*
917  * Membership states for different mcg's are tracked by two lists:
918  * the "non" list is used for promiscuous mode, when all mcg traffic
919  * needs to be inspected. This type of membership is never used for
920  * transmission, so there can not be an AH in the active list
921  * corresponding to a member in this list. This list does not need
922  * any protection, since all operations are performed by the async
923  * thread.
924  *
925  * "Full" and "SendOnly" membership is tracked using a single list,
926  * the "full" list. This is because this single list can then be
927  * searched during transmit to a multicast group (if an AH for the
928  * mcg is not found in the active list), since at least one type
929  * of membership must be present before initiating the transmit.
930  * This list is also emptied during driver detach, since sendonly
931  * membership acquired during transmit is dropped at detach time
932  * alongwith ipv4 broadcast full membership. Insert/deletes to
933  * this list are done only by the async thread, but it is also
934  * searched in program context (see multicast disable case), thus
935  * the id_mc_mutex protects the list. The driver detach path also
936  * deconstructs the "full" list, but it ensures that the async
937  * thread will not be accessing the list (by blocking out mcg
938  * trap handling and making sure no more Tx reaping will happen).
939  *
940  * Currently, an IBA attach is done in the SendOnly case too,
941  * although this is not required.
942  */
943 #define	IBD_MCACHE_INSERT_FULL(state, mce) \
944 	list_insert_head(&state->id_mc_full, mce)
945 #define	IBD_MCACHE_INSERT_NON(state, mce) \
946 	list_insert_head(&state->id_mc_non, mce)
947 #define	IBD_MCACHE_FIND_FULL(state, mgid) \
948 	ibd_mcache_find(mgid, &state->id_mc_full)
949 #define	IBD_MCACHE_FIND_NON(state, mgid) \
950 	ibd_mcache_find(mgid, &state->id_mc_non)
951 #define	IBD_MCACHE_PULLOUT_FULL(state, mce) \
952 	list_remove(&state->id_mc_full, mce)
953 #define	IBD_MCACHE_PULLOUT_NON(state, mce) \
954 	list_remove(&state->id_mc_non, mce)
955 
956 /*
957  * AH and MCE active list manipulation:
958  *
959  * Multicast disable requests and MCG delete traps are two cases
960  * where the active AH entry for the mcg (if any unreferenced one exists)
961  * will be moved to the free list (to force the next Tx to the mcg to
962  * join the MCG in SendOnly mode). Port up handling will also move AHs
963  * from active to free list.
964  *
965  * In the case when some transmits are still pending on an entry
966  * for an mcg, but a multicast disable has already been issued on the
967  * mcg, there are some options to consider to preserve the join state
968  * to ensure the emitted packet is properly routed on the IBA fabric.
969  * For the AH, we can
970  * 1. take out of active list at multicast disable time.
971  * 2. take out of active list only when last pending Tx completes.
972  * For the MCE, we can
973  * 3. take out of active list at multicast disable time.
974  * 4. take out of active list only when last pending Tx completes.
975  * 5. move from active list to stale list at multicast disable time.
976  * We choose to use 2,4. We use option 4 so that if a multicast enable
977  * is tried before the pending Tx completes, the enable code finds the
978  * mce in the active list and just has to make sure it will not be reaped
979  * (ie the mcg leave done) when the pending Tx does complete. Alternatively,
980  * a stale list (#5) that would be checked in the enable code would need
981  * to be implemented. Option 2 is used, because otherwise, a Tx attempt
982  * after the multicast disable would try to put an AH in the active list,
983  * and associate the mce it finds in the active list to this new AH,
984  * whereas the mce is already associated with the previous AH (taken off
985  * the active list), and will be removed once the pending Tx's complete
986  * (unless a reference count on mce's is implemented). One implication of
987  * using 2,4 is that new Tx's posted before the pending Tx's complete will
988  * grab new references on the AH, further delaying the leave.
989  *
990  * In the case of mcg delete (or create) trap when the port is sendonly
991  * joined, the AH and MCE handling is different: the AH and MCE has to be
992  * immediately taken off the active lists (forcing a join and path lookup
993  * at the next Tx is the only guaranteed means of ensuring a proper Tx
994  * to an mcg as it is repeatedly created and deleted and goes thru
995  * reincarnations).
996  *
997  * When a port is already sendonly joined, and a multicast enable is
998  * attempted, the same mce structure is promoted; this ensures only a
999  * single mce on the active list tracks the most powerful join state.
1000  *
1001  * In the case of port up event handling, the MCE for sendonly membership
1002  * is freed up, and the ACE is put into the free list as soon as possible
1003  * (depending on whether posted Tx's have completed). For fullmembership
1004  * MCE's though, the ACE is similarly handled; but the MCE is kept around
1005  * (a re-JOIN is attempted) only if the DLPI leave has not already been
1006  * done; else the mce is deconstructed (mc_fullreap case).
1007  *
1008  * MCG creation and deletion trap handling:
1009  *
1010  * These traps are unreliable (meaning sometimes the trap might never
1011  * be delivered to the subscribed nodes) and may arrive out-of-order
1012  * since they use UD transport. An alternative to relying on these
1013  * unreliable traps is to poll for mcg presence every so often, but
1014  * instead of doing that, we try to be as conservative as possible
1015  * while handling the traps, and hope that the traps do arrive at
1016  * the subscribed nodes soon. Note that if a node is fullmember
1017  * joined to an mcg, it can not possibly receive a mcg create/delete
1018  * trap for that mcg (by fullmember definition); if it does, it is
1019  * an old trap from a previous incarnation of the mcg.
1020  *
1021  * Whenever a trap is received, the driver cleans up its sendonly
1022  * membership to the group; we choose to do a sendonly leave even
1023  * on a creation trap to handle the case of a prior deletion of the mcg
1024  * having gone unnoticed. Consider an example scenario:
1025  * T1: MCG M is deleted, and fires off deletion trap D1.
1026  * T2: MCG M is recreated, fires off creation trap C1, which is lost.
1027  * T3: Node N tries to transmit to M, joining in sendonly mode.
1028  * T4: MCG M is deleted, and fires off deletion trap D2.
1029  * T5: N receives a deletion trap, but can not distinguish D1 from D2.
1030  *     If the trap is D2, then a LEAVE is not required, since the mcg
1031  *     is already deleted; but if it is D1, a LEAVE is required. A safe
1032  *     approach is to always LEAVE, but the SM may be confused if it
1033  *     receives a LEAVE without a prior JOIN.
1034  *
1035  * Management of the non-membership to an mcg is similar to the above,
1036  * except that if the interface is in promiscuous mode, it is required
1037  * to attempt to re-join the mcg after receiving a trap. Unfortunately,
1038  * if the re-join attempt fails (in which case a warning message needs
1039  * to be printed), it is not clear whether it failed due to the mcg not
1040  * existing, or some fabric/hca issues, due to the delayed nature of
1041  * trap delivery. Querying the SA to establish presence/absence of the
1042  * mcg is also racy at best. Thus, the driver just prints a warning
1043  * message when it can not rejoin after receiving a create trap, although
1044  * this might be (on rare occassions) a mis-warning if the create trap is
1045  * received after the mcg was deleted.
1046  */
1047 
1048 /*
1049  * Implementation of atomic "recycle" bits and reference count
1050  * on address handles. This utilizes the fact that max reference
1051  * count on any handle is limited by number of send wqes, thus
1052  * high bits in the ac_ref field can be used as the recycle bits,
1053  * and only the low bits hold the number of pending Tx requests.
1054  * This atomic AH reference counting allows the Tx completion
1055  * handler not to acquire the id_ac_mutex to process every completion,
1056  * thus reducing lock contention problems between completion and
1057  * the Tx path.
1058  */
1059 #define	CYCLEVAL		0x80000
1060 #define	CLEAR_REFCYCLE(ace)	(ace)->ac_ref = 0
1061 #define	CYCLE_SET(ace)		(((ace)->ac_ref & CYCLEVAL) == CYCLEVAL)
1062 #define	GET_REF(ace)		((ace)->ac_ref)
1063 #define	GET_REF_CYCLE(ace) (				\
1064 	/*						\
1065 	 * Make sure "cycle" bit is set.		\
1066 	 */						\
1067 	ASSERT(CYCLE_SET(ace)),				\
1068 	((ace)->ac_ref & ~(CYCLEVAL))			\
1069 )
1070 #define	INC_REF(ace, num) {				\
1071 	atomic_add_32(&(ace)->ac_ref, num);		\
1072 }
1073 #define	SET_CYCLE_IF_REF(ace) (				\
1074 	CYCLE_SET(ace) ? B_TRUE :			\
1075 	    atomic_add_32_nv(&ace->ac_ref, CYCLEVAL) ==	\
1076 		CYCLEVAL ?				\
1077 		/*					\
1078 		 * Clear the "cycle" bit we just set;	\
1079 		 * ref count known to be 0 from above.	\
1080 		 */					\
1081 		CLEAR_REFCYCLE(ace), B_FALSE :		\
1082 		/*					\
1083 		 * We set "cycle" bit; let caller know.	\
1084 		 */					\
1085 		B_TRUE					\
1086 )
1087 #define	DEC_REF_DO_CYCLE(ace) (				\
1088 	atomic_add_32_nv(&ace->ac_ref, -1) ==		\
1089 	    CYCLEVAL ?					\
1090 		/*					\
1091 		 * Ref count known to be 0 from above.	\
1092 		 */					\
1093 		B_TRUE :				\
1094 		B_FALSE					\
1095 )
1096 
1097 static void *
1098 list_get_head(list_t *list)
1099 {
1100 	list_node_t *lhead = list_head(list);
1101 
1102 	if (lhead != NULL)
1103 		list_remove(list, lhead);
1104 	return (lhead);
1105 }
1106 
1107 /*
1108  * This is always guaranteed to be able to queue the work.
1109  */
1110 static void
1111 ibd_queue_work_slot(ibd_state_t *state, ibd_req_t *ptr, int op)
1112 {
1113 	/* Initialize request */
1114 	DPRINT(1, "ibd_queue_work_slot : op: %d \n", op);
1115 	ptr->rq_op = op;
1116 
1117 	/*
1118 	 * Queue provided slot onto request pool.
1119 	 */
1120 	mutex_enter(&state->id_acache_req_lock);
1121 	list_insert_tail(&state->id_req_list, ptr);
1122 
1123 	/* Go, fetch, async thread */
1124 	cv_signal(&state->id_acache_req_cv);
1125 	mutex_exit(&state->id_acache_req_lock);
1126 }
1127 
1128 /*
1129  * Main body of the per interface async thread.
1130  */
1131 static void
1132 ibd_async_work(ibd_state_t *state)
1133 {
1134 	ibd_req_t *ptr;
1135 	callb_cpr_t cprinfo;
1136 
1137 	mutex_enter(&state->id_acache_req_lock);
1138 	CALLB_CPR_INIT(&cprinfo, &state->id_acache_req_lock,
1139 	    callb_generic_cpr, "ibd_async_work");
1140 
1141 	for (;;) {
1142 		ptr = list_get_head(&state->id_req_list);
1143 		if (ptr != NULL) {
1144 			mutex_exit(&state->id_acache_req_lock);
1145 
1146 			/*
1147 			 * Once we have done the operation, there is no
1148 			 * guarantee the request slot is going to be valid,
1149 			 * it might be freed up (as in IBD_ASYNC_LEAVE, REAP,
1150 			 * TRAP).
1151 			 *
1152 			 * Perform the request.
1153 			 */
1154 			switch (ptr->rq_op) {
1155 				case IBD_ASYNC_GETAH:
1156 					ibd_async_acache(state, &ptr->rq_mac);
1157 					break;
1158 				case IBD_ASYNC_JOIN:
1159 				case IBD_ASYNC_LEAVE:
1160 					ibd_async_multicast(state,
1161 					    ptr->rq_gid, ptr->rq_op);
1162 					break;
1163 				case IBD_ASYNC_PROMON:
1164 					ibd_async_setprom(state);
1165 					break;
1166 				case IBD_ASYNC_PROMOFF:
1167 					ibd_async_unsetprom(state);
1168 					break;
1169 				case IBD_ASYNC_REAP:
1170 					ibd_async_reap_group(state,
1171 					    ptr->rq_ptr, ptr->rq_gid,
1172 					    IB_MC_JSTATE_FULL);
1173 					/*
1174 					 * the req buf contains in mce
1175 					 * structure, so we do not need
1176 					 * to free it here.
1177 					 */
1178 					ptr = NULL;
1179 					break;
1180 				case IBD_ASYNC_TRAP:
1181 					ibd_async_trap(state, ptr);
1182 					break;
1183 				case IBD_ASYNC_SCHED:
1184 					ibd_async_txsched(state);
1185 					break;
1186 				case IBD_ASYNC_LINK:
1187 					ibd_async_link(state, ptr);
1188 					break;
1189 				case IBD_ASYNC_EXIT:
1190 					mutex_enter(&state->id_acache_req_lock);
1191 #ifndef __lock_lint
1192 					CALLB_CPR_EXIT(&cprinfo);
1193 #else
1194 					mutex_exit(&state->id_acache_req_lock);
1195 #endif
1196 					return;
1197 			}
1198 			if (ptr != NULL)
1199 				kmem_cache_free(state->id_req_kmc, ptr);
1200 
1201 			mutex_enter(&state->id_acache_req_lock);
1202 		} else {
1203 #ifndef __lock_lint
1204 			/*
1205 			 * Nothing to do: wait till new request arrives.
1206 			 */
1207 			CALLB_CPR_SAFE_BEGIN(&cprinfo);
1208 			cv_wait(&state->id_acache_req_cv,
1209 			    &state->id_acache_req_lock);
1210 			CALLB_CPR_SAFE_END(&cprinfo,
1211 			    &state->id_acache_req_lock);
1212 #endif
1213 		}
1214 	}
1215 
1216 	/*NOTREACHED*/
1217 	_NOTE(NOT_REACHED)
1218 }
1219 
1220 /*
1221  * Return when it is safe to queue requests to the async daemon; primarily
1222  * for subnet trap and async event handling. Disallow requests before the
1223  * daemon is created, and when interface deinitilization starts.
1224  */
1225 static boolean_t
1226 ibd_async_safe(ibd_state_t *state)
1227 {
1228 	mutex_enter(&state->id_trap_lock);
1229 	if (state->id_trap_stop) {
1230 		mutex_exit(&state->id_trap_lock);
1231 		return (B_FALSE);
1232 	}
1233 	state->id_trap_inprog++;
1234 	mutex_exit(&state->id_trap_lock);
1235 	return (B_TRUE);
1236 }
1237 
1238 /*
1239  * Wake up ibd_drv_fini() if the detach code is waiting for pending subnet
1240  * trap or event handling to complete to kill the async thread and deconstruct
1241  * the mcg/ace list.
1242  */
1243 static void
1244 ibd_async_done(ibd_state_t *state)
1245 {
1246 	mutex_enter(&state->id_trap_lock);
1247 	if (--state->id_trap_inprog == 0)
1248 		cv_signal(&state->id_trap_cv);
1249 	mutex_exit(&state->id_trap_lock);
1250 }
1251 
1252 /*
1253  * Hash functions:
1254  * ibd_hash_by_id: Returns the qpn as the hash entry into bucket.
1255  * ibd_hash_key_cmp: Compares two keys, return 0 on success or else 1.
1256  * These operate on mac addresses input into ibd_send, but there is no
1257  * guarantee on the alignment of the ipoib_mac_t structure.
1258  */
1259 /*ARGSUSED*/
1260 static uint_t
1261 ibd_hash_by_id(void *hash_data, mod_hash_key_t key)
1262 {
1263 	ulong_t ptraddr = (ulong_t)key;
1264 	uint_t hval;
1265 
1266 	/*
1267 	 * If the input address is 4 byte aligned, we can just dereference
1268 	 * it. This is most common, since IP will send in a 4 byte aligned
1269 	 * IP header, which implies the 24 byte IPoIB psuedo header will be
1270 	 * 4 byte aligned too.
1271 	 */
1272 	if ((ptraddr & 3) == 0)
1273 		return ((uint_t)((ipoib_mac_t *)key)->ipoib_qpn);
1274 
1275 	bcopy(&(((ipoib_mac_t *)key)->ipoib_qpn), &hval, sizeof (uint_t));
1276 	return (hval);
1277 }
1278 
1279 static int
1280 ibd_hash_key_cmp(mod_hash_key_t key1, mod_hash_key_t key2)
1281 {
1282 	if (bcmp((char *)key1, (char *)key2, IPOIB_ADDRL) == 0)
1283 		return (0);
1284 	else
1285 		return (1);
1286 }
1287 
1288 /*
1289  * Initialize all the per interface caches and lists; AH cache,
1290  * MCG list etc.
1291  */
1292 static int
1293 ibd_acache_init(ibd_state_t *state)
1294 {
1295 	ibd_ace_t *ce;
1296 	int i;
1297 
1298 	mutex_init(&state->id_acache_req_lock, NULL, MUTEX_DRIVER, NULL);
1299 	cv_init(&state->id_acache_req_cv, NULL, CV_DEFAULT, NULL);
1300 
1301 	mutex_init(&state->id_ac_mutex, NULL, MUTEX_DRIVER, NULL);
1302 	mutex_init(&state->id_mc_mutex, NULL, MUTEX_DRIVER, NULL);
1303 	list_create(&state->id_ah_free, sizeof (ibd_ace_t),
1304 	    offsetof(ibd_ace_t, ac_list));
1305 	list_create(&state->id_ah_active, sizeof (ibd_ace_t),
1306 	    offsetof(ibd_ace_t, ac_list));
1307 	state->id_ah_active_hash = mod_hash_create_extended("IBD AH hash",
1308 	    IBD_HASH_SIZE, mod_hash_null_keydtor, mod_hash_null_valdtor,
1309 	    ibd_hash_by_id, NULL, ibd_hash_key_cmp, KM_SLEEP);
1310 	list_create(&state->id_mc_full, sizeof (ibd_mce_t),
1311 	    offsetof(ibd_mce_t, mc_list));
1312 	list_create(&state->id_mc_non, sizeof (ibd_mce_t),
1313 	    offsetof(ibd_mce_t, mc_list));
1314 	list_create(&state->id_req_list, sizeof (ibd_req_t),
1315 	    offsetof(ibd_req_t, rq_list));
1316 
1317 	state->id_ac_list = ce = (ibd_ace_t *)kmem_zalloc(sizeof (ibd_ace_t) *
1318 	    IBD_NUM_AH, KM_SLEEP);
1319 	for (i = 0; i < IBD_NUM_AH; i++, ce++) {
1320 		if (ibt_alloc_ud_dest(state->id_hca_hdl, IBT_UD_DEST_NO_FLAGS,
1321 		    state->id_pd_hdl, &ce->ac_dest) != IBT_SUCCESS) {
1322 			ibd_acache_fini(state);
1323 			return (DDI_FAILURE);
1324 		} else {
1325 			CLEAR_REFCYCLE(ce);
1326 			ce->ac_mce = NULL;
1327 			IBD_ACACHE_INSERT_FREE(state, ce);
1328 		}
1329 	}
1330 	return (DDI_SUCCESS);
1331 }
1332 
1333 static void
1334 ibd_acache_fini(ibd_state_t *state)
1335 {
1336 	ibd_ace_t *ptr;
1337 
1338 	mutex_enter(&state->id_ac_mutex);
1339 
1340 	while ((ptr = IBD_ACACHE_GET_ACTIVE(state)) != NULL) {
1341 		ASSERT(GET_REF(ptr) == 0);
1342 		(void) ibt_free_ud_dest(ptr->ac_dest);
1343 	}
1344 
1345 	while ((ptr = IBD_ACACHE_GET_FREE(state)) != NULL) {
1346 		ASSERT(GET_REF(ptr) == 0);
1347 		(void) ibt_free_ud_dest(ptr->ac_dest);
1348 	}
1349 
1350 	list_destroy(&state->id_ah_free);
1351 	list_destroy(&state->id_ah_active);
1352 	list_destroy(&state->id_mc_full);
1353 	list_destroy(&state->id_mc_non);
1354 	list_destroy(&state->id_req_list);
1355 	kmem_free(state->id_ac_list, sizeof (ibd_ace_t) * IBD_NUM_AH);
1356 	mutex_exit(&state->id_ac_mutex);
1357 	mutex_destroy(&state->id_ac_mutex);
1358 	mutex_destroy(&state->id_mc_mutex);
1359 	mutex_destroy(&state->id_acache_req_lock);
1360 	cv_destroy(&state->id_acache_req_cv);
1361 }
1362 
1363 /*
1364  * Search AH active hash list for a cached path to input destination.
1365  * If we are "just looking", hold == F. When we are in the Tx path,
1366  * we set hold == T to grab a reference on the AH so that it can not
1367  * be recycled to a new destination while the Tx request is posted.
1368  */
1369 static ibd_ace_t *
1370 ibd_acache_find(ibd_state_t *state, ipoib_mac_t *mac, boolean_t hold, int num)
1371 {
1372 	ibd_ace_t *ptr;
1373 
1374 	ASSERT(mutex_owned(&state->id_ac_mutex));
1375 
1376 	/*
1377 	 * Do hash search.
1378 	 */
1379 	if (mod_hash_find(state->id_ah_active_hash,
1380 	    (mod_hash_key_t)mac, (mod_hash_val_t)&ptr) == 0) {
1381 		if (hold)
1382 			INC_REF(ptr, num);
1383 		return (ptr);
1384 	}
1385 	return (NULL);
1386 }
1387 
1388 /*
1389  * This is called by the tx side; if an initialized AH is found in
1390  * the active list, it is locked down and can be used; if no entry
1391  * is found, an async request is queued to do path resolution.
1392  */
1393 static ibd_ace_t *
1394 ibd_acache_lookup(ibd_state_t *state, ipoib_mac_t *mac, int *err, int numwqe)
1395 {
1396 	ibd_ace_t *ptr;
1397 	ibd_req_t *req;
1398 
1399 	/*
1400 	 * Only attempt to print when we can; in the mdt pattr case, the
1401 	 * address is not aligned properly.
1402 	 */
1403 	if (((ulong_t)mac & 3) == 0) {
1404 		DPRINT(4,
1405 		    "ibd_acache_lookup : lookup for %08X:%08X:%08X:%08X:%08X",
1406 		    htonl(mac->ipoib_qpn), htonl(mac->ipoib_gidpref[0]),
1407 		    htonl(mac->ipoib_gidpref[1]), htonl(mac->ipoib_gidsuff[0]),
1408 		    htonl(mac->ipoib_gidsuff[1]));
1409 	}
1410 
1411 	mutex_enter(&state->id_ac_mutex);
1412 
1413 	if ((ptr = ibd_acache_find(state, mac, B_TRUE, numwqe)) != NULL) {
1414 		mutex_exit(&state->id_ac_mutex);
1415 		return (ptr);
1416 	}
1417 
1418 	/*
1419 	 * Implementation of a single outstanding async request; if
1420 	 * the operation is not started yet, queue a request and move
1421 	 * to ongoing state. Remember in id_ah_addr for which address
1422 	 * we are queueing the request, in case we need to flag an error;
1423 	 * Any further requests, for the same or different address, until
1424 	 * the operation completes, is sent back to GLDv3 to be retried.
1425 	 * The async thread will update id_ah_op with an error indication
1426 	 * or will set it to indicate the next look up can start; either
1427 	 * way, it will mac_tx_update() so that all blocked requests come
1428 	 * back here.
1429 	 */
1430 	*err = EAGAIN;
1431 	if (state->id_ah_op == IBD_OP_NOTSTARTED) {
1432 		req = kmem_cache_alloc(state->id_req_kmc, KM_NOSLEEP);
1433 		if (req != NULL) {
1434 			/*
1435 			 * We did not even find the entry; queue a request
1436 			 * for it.
1437 			 */
1438 			bcopy(mac, &(req->rq_mac), IPOIB_ADDRL);
1439 			ibd_queue_work_slot(state, req, IBD_ASYNC_GETAH);
1440 			state->id_ah_op = IBD_OP_ONGOING;
1441 			bcopy(mac, &state->id_ah_addr, IPOIB_ADDRL);
1442 		}
1443 	} else if ((state->id_ah_op != IBD_OP_ONGOING) &&
1444 	    (bcmp(&state->id_ah_addr, mac, IPOIB_ADDRL) == 0)) {
1445 		/*
1446 		 * Check the status of the pathrecord lookup request
1447 		 * we had queued before.
1448 		 */
1449 		if (state->id_ah_op == IBD_OP_ERRORED) {
1450 			*err = EFAULT;
1451 			state->id_ah_error++;
1452 		} else {
1453 			/*
1454 			 * IBD_OP_ROUTERED case: We need to send to the
1455 			 * all-router MCG. If we can find the AH for
1456 			 * the mcg, the Tx will be attempted. If we
1457 			 * do not find the AH, we return NORESOURCES
1458 			 * to retry.
1459 			 */
1460 			ipoib_mac_t routermac;
1461 
1462 			(void) ibd_get_allroutergroup(state, mac, &routermac);
1463 			ptr = ibd_acache_find(state, &routermac, B_TRUE,
1464 			    numwqe);
1465 		}
1466 		state->id_ah_op = IBD_OP_NOTSTARTED;
1467 	} else if ((state->id_ah_op != IBD_OP_ONGOING) &&
1468 	    (bcmp(&state->id_ah_addr, mac, IPOIB_ADDRL) != 0)) {
1469 		/*
1470 		 * This case can happen when we get a higher band
1471 		 * packet. The easiest way is to reset the state machine
1472 		 * to accommodate the higher priority packet.
1473 		 */
1474 		state->id_ah_op = IBD_OP_NOTSTARTED;
1475 	}
1476 	mutex_exit(&state->id_ac_mutex);
1477 
1478 	return (ptr);
1479 }
1480 
1481 /*
1482  * Grab a not-currently-in-use AH/PathRecord from the active
1483  * list to recycle to a new destination. Only the async thread
1484  * executes this code.
1485  */
1486 static ibd_ace_t *
1487 ibd_acache_get_unref(ibd_state_t *state)
1488 {
1489 	ibd_ace_t *ptr = list_head(&state->id_ah_active);
1490 
1491 	ASSERT(mutex_owned(&state->id_ac_mutex));
1492 
1493 	/*
1494 	 * Do plain linear search.
1495 	 */
1496 	while (ptr != NULL) {
1497 		/*
1498 		 * Note that it is possible that the "cycle" bit
1499 		 * is set on the AH w/o any reference count. The
1500 		 * mcg must have been deleted, and the tx cleanup
1501 		 * just decremented the reference count to 0, but
1502 		 * hasn't gotten around to grabbing the id_ac_mutex
1503 		 * to move the AH into the free list.
1504 		 */
1505 		if (GET_REF(ptr) == 0) {
1506 			IBD_ACACHE_PULLOUT_ACTIVE(state, ptr);
1507 			break;
1508 		}
1509 		ptr = list_next(&state->id_ah_active, ptr);
1510 	}
1511 	return (ptr);
1512 }
1513 
1514 /*
1515  * Invoked to clean up AH from active list in case of multicast
1516  * disable and to handle sendonly memberships during mcg traps.
1517  * And for port up processing for multicast and unicast AHs.
1518  * Normally, the AH is taken off the active list, and put into
1519  * the free list to be recycled for a new destination. In case
1520  * Tx requests on the AH have not completed yet, the AH is marked
1521  * for reaping (which will put the AH on the free list) once the Tx's
1522  * complete; in this case, depending on the "force" input, we take
1523  * out the AH from the active list right now, or leave it also for
1524  * the reap operation. Returns TRUE if the AH is taken off the active
1525  * list (and either put into the free list right now, or arranged for
1526  * later), FALSE otherwise.
1527  */
1528 static boolean_t
1529 ibd_acache_recycle(ibd_state_t *state, ipoib_mac_t *acmac, boolean_t force)
1530 {
1531 	ibd_ace_t *acactive;
1532 	boolean_t ret = B_TRUE;
1533 
1534 	ASSERT(mutex_owned(&state->id_ac_mutex));
1535 
1536 	if ((acactive = ibd_acache_find(state, acmac, B_FALSE, 0)) != NULL) {
1537 
1538 		/*
1539 		 * Note that the AH might already have the cycle bit set
1540 		 * on it; this might happen if sequences of multicast
1541 		 * enables and disables are coming so fast, that posted
1542 		 * Tx's to the mcg have not completed yet, and the cycle
1543 		 * bit is set successively by each multicast disable.
1544 		 */
1545 		if (SET_CYCLE_IF_REF(acactive)) {
1546 			if (!force) {
1547 				/*
1548 				 * The ace is kept on the active list, further
1549 				 * Tx's can still grab a reference on it; the
1550 				 * ace is reaped when all pending Tx's
1551 				 * referencing the AH complete.
1552 				 */
1553 				ret = B_FALSE;
1554 			} else {
1555 				/*
1556 				 * In the mcg trap case, we always pull the
1557 				 * AH from the active list. And also the port
1558 				 * up multi/unicast case.
1559 				 */
1560 				IBD_ACACHE_PULLOUT_ACTIVE(state, acactive);
1561 				acactive->ac_mce = NULL;
1562 			}
1563 		} else {
1564 			/*
1565 			 * Determined the ref count is 0, thus reclaim
1566 			 * immediately after pulling out the ace from
1567 			 * the active list.
1568 			 */
1569 			IBD_ACACHE_PULLOUT_ACTIVE(state, acactive);
1570 			acactive->ac_mce = NULL;
1571 			IBD_ACACHE_INSERT_FREE(state, acactive);
1572 		}
1573 
1574 	}
1575 	return (ret);
1576 }
1577 
1578 /*
1579  * Helper function for async path record lookup. If we are trying to
1580  * Tx to a MCG, check our membership, possibly trying to join the
1581  * group if required. If that fails, try to send the packet to the
1582  * all router group (indicated by the redirect output), pointing
1583  * the input mac address to the router mcg address.
1584  */
1585 static ibd_mce_t *
1586 ibd_async_mcache(ibd_state_t *state, ipoib_mac_t *mac, boolean_t *redirect)
1587 {
1588 	ib_gid_t mgid;
1589 	ibd_mce_t *mce;
1590 	ipoib_mac_t routermac;
1591 
1592 	*redirect = B_FALSE;
1593 	ibd_n2h_gid(mac, &mgid);
1594 
1595 	/*
1596 	 * Check the FullMember+SendOnlyNonMember list.
1597 	 * Since we are the only one who manipulates the
1598 	 * id_mc_full list, no locks are needed.
1599 	 */
1600 	mce = IBD_MCACHE_FIND_FULL(state, mgid);
1601 	if (mce != NULL) {
1602 		DPRINT(4, "ibd_async_mcache : already joined to group");
1603 		return (mce);
1604 	}
1605 
1606 	/*
1607 	 * Not found; try to join(SendOnlyNonMember) and attach.
1608 	 */
1609 	DPRINT(4, "ibd_async_mcache : not joined to group");
1610 	if ((mce = ibd_join_group(state, mgid, IB_MC_JSTATE_SEND_ONLY_NON)) !=
1611 	    NULL) {
1612 		DPRINT(4, "ibd_async_mcache : nonmem joined to group");
1613 		return (mce);
1614 	}
1615 
1616 	/*
1617 	 * MCGroup not present; try to join the all-router group. If
1618 	 * any of the following steps succeed, we will be redirecting
1619 	 * to the all router group.
1620 	 */
1621 	DPRINT(4, "ibd_async_mcache : nonmem join failed");
1622 	if (!ibd_get_allroutergroup(state, mac, &routermac))
1623 		return (NULL);
1624 	*redirect = B_TRUE;
1625 	ibd_n2h_gid(&routermac, &mgid);
1626 	bcopy(&routermac, mac, IPOIB_ADDRL);
1627 	DPRINT(4, "ibd_async_mcache : router mgid : %016llx:%016llx\n",
1628 	    mgid.gid_prefix, mgid.gid_guid);
1629 
1630 	/*
1631 	 * Are we already joined to the router group?
1632 	 */
1633 	if ((mce = IBD_MCACHE_FIND_FULL(state, mgid)) != NULL) {
1634 		DPRINT(4, "ibd_async_mcache : using already joined router"
1635 		    "group\n");
1636 		return (mce);
1637 	}
1638 
1639 	/*
1640 	 * Can we join(SendOnlyNonMember) the router group?
1641 	 */
1642 	DPRINT(4, "ibd_async_mcache : attempting join to router grp");
1643 	if ((mce = ibd_join_group(state, mgid, IB_MC_JSTATE_SEND_ONLY_NON)) !=
1644 	    NULL) {
1645 		DPRINT(4, "ibd_async_mcache : joined to router grp");
1646 		return (mce);
1647 	}
1648 
1649 	return (NULL);
1650 }
1651 
1652 /*
1653  * Async path record lookup code.
1654  */
1655 static void
1656 ibd_async_acache(ibd_state_t *state, ipoib_mac_t *mac)
1657 {
1658 	ibd_ace_t *ce;
1659 	ibd_mce_t *mce = NULL;
1660 	ibt_path_attr_t path_attr;
1661 	ibt_path_info_t path_info;
1662 	ib_gid_t destgid;
1663 	int ret = IBD_OP_NOTSTARTED;
1664 
1665 	DPRINT(4, "ibd_async_acache :  %08X:%08X:%08X:%08X:%08X",
1666 	    htonl(mac->ipoib_qpn), htonl(mac->ipoib_gidpref[0]),
1667 	    htonl(mac->ipoib_gidpref[1]), htonl(mac->ipoib_gidsuff[0]),
1668 	    htonl(mac->ipoib_gidsuff[1]));
1669 
1670 	/*
1671 	 * Check whether we are trying to transmit to a MCG.
1672 	 * In that case, we need to make sure we are a member of
1673 	 * the MCG.
1674 	 */
1675 	if (mac->ipoib_qpn == htonl(IB_MC_QPN)) {
1676 		boolean_t redirected;
1677 
1678 		/*
1679 		 * If we can not find or join the group or even
1680 		 * redirect, error out.
1681 		 */
1682 		if ((mce = ibd_async_mcache(state, mac, &redirected)) ==
1683 		    NULL) {
1684 			state->id_ah_op = IBD_OP_ERRORED;
1685 			return;
1686 		}
1687 
1688 		/*
1689 		 * If we got redirected, we need to determine whether
1690 		 * the AH for the new mcg is in the cache already, and
1691 		 * not pull it in then; otherwise proceed to get the
1692 		 * path for the new mcg. There is no guarantee that
1693 		 * if the AH is currently in the cache, it will still be
1694 		 * there when we look in ibd_acache_lookup(), but that's
1695 		 * okay, we will come back here.
1696 		 */
1697 		if (redirected) {
1698 			ret = IBD_OP_ROUTERED;
1699 			DPRINT(4, "ibd_async_acache :  redirected to "
1700 			    "%08X:%08X:%08X:%08X:%08X",
1701 			    htonl(mac->ipoib_qpn), htonl(mac->ipoib_gidpref[0]),
1702 			    htonl(mac->ipoib_gidpref[1]),
1703 			    htonl(mac->ipoib_gidsuff[0]),
1704 			    htonl(mac->ipoib_gidsuff[1]));
1705 
1706 			mutex_enter(&state->id_ac_mutex);
1707 			if (ibd_acache_find(state, mac, B_FALSE, 0) != NULL) {
1708 				state->id_ah_op = IBD_OP_ROUTERED;
1709 				mutex_exit(&state->id_ac_mutex);
1710 				DPRINT(4, "ibd_async_acache : router AH found");
1711 				return;
1712 			}
1713 			mutex_exit(&state->id_ac_mutex);
1714 		}
1715 	}
1716 
1717 	/*
1718 	 * Get an AH from the free list.
1719 	 */
1720 	mutex_enter(&state->id_ac_mutex);
1721 	if ((ce = IBD_ACACHE_GET_FREE(state)) == NULL) {
1722 		/*
1723 		 * No free ones; try to grab an unreferenced active
1724 		 * one. Maybe we need to make the active list LRU,
1725 		 * but that will create more work for Tx callbacks.
1726 		 * Is there a way of not having to pull out the
1727 		 * entry from the active list, but just indicate it
1728 		 * is being recycled? Yes, but that creates one more
1729 		 * check in the fast lookup path.
1730 		 */
1731 		if ((ce = ibd_acache_get_unref(state)) == NULL) {
1732 			/*
1733 			 * Pretty serious shortage now.
1734 			 */
1735 			state->id_ah_op = IBD_OP_NOTSTARTED;
1736 			mutex_exit(&state->id_ac_mutex);
1737 			DPRINT(10, "ibd_async_acache : failed to find AH "
1738 			    "slot\n");
1739 			return;
1740 		}
1741 		/*
1742 		 * We could check whether ac_mce points to a SendOnly
1743 		 * member and drop that membership now. Or do it lazily
1744 		 * at detach time.
1745 		 */
1746 		ce->ac_mce = NULL;
1747 	}
1748 	mutex_exit(&state->id_ac_mutex);
1749 	ASSERT(ce->ac_mce == NULL);
1750 
1751 	/*
1752 	 * Update the entry.
1753 	 */
1754 	bcopy((char *)mac, &ce->ac_mac, IPOIB_ADDRL);
1755 
1756 	bzero(&path_info, sizeof (path_info));
1757 	bzero(&path_attr, sizeof (ibt_path_attr_t));
1758 	path_attr.pa_sgid = state->id_sgid;
1759 	path_attr.pa_num_dgids = 1;
1760 	ibd_n2h_gid(&ce->ac_mac, &destgid);
1761 	path_attr.pa_dgids = &destgid;
1762 	path_attr.pa_sl = state->id_mcinfo->mc_adds_vect.av_srvl;
1763 	if (ibt_get_paths(state->id_ibt_hdl, IBT_PATH_NO_FLAGS,
1764 	    &path_attr, 1, &path_info, NULL) != IBT_SUCCESS) {
1765 		DPRINT(10, "ibd_async_acache : failed in ibt_get_paths");
1766 		goto error;
1767 	}
1768 	if (ibt_modify_ud_dest(ce->ac_dest, state->id_mcinfo->mc_qkey,
1769 	    ntohl(ce->ac_mac.ipoib_qpn),
1770 	    &path_info.pi_prim_cep_path.cep_adds_vect) != IBT_SUCCESS) {
1771 		DPRINT(10, "ibd_async_acache : failed in ibt_modify_ud_dest");
1772 		goto error;
1773 	}
1774 
1775 	/*
1776 	 * mce is set whenever an AH is being associated with a
1777 	 * MCG; this will come in handy when we leave the MCG. The
1778 	 * lock protects Tx fastpath from scanning the active list.
1779 	 */
1780 	if (mce != NULL)
1781 		ce->ac_mce = mce;
1782 	mutex_enter(&state->id_ac_mutex);
1783 	IBD_ACACHE_INSERT_ACTIVE(state, ce);
1784 	state->id_ah_op = ret;
1785 	mutex_exit(&state->id_ac_mutex);
1786 	return;
1787 error:
1788 	/*
1789 	 * We might want to drop SendOnly membership here if we
1790 	 * joined above. The lock protects Tx callbacks inserting
1791 	 * into the free list.
1792 	 */
1793 	mutex_enter(&state->id_ac_mutex);
1794 	state->id_ah_op = IBD_OP_ERRORED;
1795 	IBD_ACACHE_INSERT_FREE(state, ce);
1796 	mutex_exit(&state->id_ac_mutex);
1797 }
1798 
1799 /*
1800  * While restoring port's presence on the subnet on a port up, it is possible
1801  * that the port goes down again.
1802  */
1803 static void
1804 ibd_async_link(ibd_state_t *state, ibd_req_t *req)
1805 {
1806 	ibd_link_op_t opcode = (ibd_link_op_t)req->rq_ptr;
1807 	link_state_t lstate = (opcode == IBD_LINK_DOWN) ? LINK_STATE_DOWN :
1808 	    LINK_STATE_UP;
1809 	ibd_mce_t *mce, *pmce;
1810 	ibd_ace_t *ace, *pace;
1811 
1812 	DPRINT(10, "ibd_async_link(): %d", opcode);
1813 
1814 	/*
1815 	 * On a link up, revalidate the link speed/width. No point doing
1816 	 * this on a link down, since we will be unable to do SA operations,
1817 	 * defaulting to the lowest speed. Also notice that we update our
1818 	 * notion of speed before calling mac_link_update(), which will do
1819 	 * neccesary higher level notifications for speed changes.
1820 	 */
1821 	if ((opcode == IBD_LINK_UP_ABSENT) || (opcode == IBD_LINK_UP)) {
1822 		_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*state))
1823 		state->id_link_speed = ibd_get_portspeed(state);
1824 		_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*state))
1825 	}
1826 
1827 	/*
1828 	 * Do all the work required to establish our presence on
1829 	 * the subnet.
1830 	 */
1831 	if (opcode == IBD_LINK_UP_ABSENT) {
1832 		/*
1833 		 * If in promiscuous mode ...
1834 		 */
1835 		if (state->id_prom_op == IBD_OP_COMPLETED) {
1836 			/*
1837 			 * Drop all nonmembership.
1838 			 */
1839 			ibd_async_unsetprom(state);
1840 
1841 			/*
1842 			 * Then, try to regain nonmembership to all mcg's.
1843 			 */
1844 			ibd_async_setprom(state);
1845 
1846 		}
1847 
1848 		/*
1849 		 * Drop all sendonly membership (which also gets rid of the
1850 		 * AHs); try to reacquire all full membership.
1851 		 */
1852 		mce = list_head(&state->id_mc_full);
1853 		while ((pmce = mce) != NULL) {
1854 			mce = list_next(&state->id_mc_full, mce);
1855 			if (pmce->mc_jstate == IB_MC_JSTATE_SEND_ONLY_NON)
1856 				ibd_leave_group(state,
1857 				    pmce->mc_info.mc_adds_vect.av_dgid,
1858 				    IB_MC_JSTATE_SEND_ONLY_NON);
1859 			else
1860 				ibd_reacquire_group(state, pmce);
1861 		}
1862 
1863 		/*
1864 		 * Recycle all active AHs to free list (and if there are
1865 		 * pending posts, make sure they will go into the free list
1866 		 * once the Tx's complete). Grab the lock to prevent
1867 		 * concurrent Tx's as well as Tx cleanups.
1868 		 */
1869 		mutex_enter(&state->id_ac_mutex);
1870 		ace = list_head(&state->id_ah_active);
1871 		while ((pace = ace) != NULL) {
1872 			boolean_t cycled;
1873 
1874 			ace = list_next(&state->id_ah_active, ace);
1875 			mce = pace->ac_mce;
1876 			cycled = ibd_acache_recycle(state, &pace->ac_mac,
1877 			    B_TRUE);
1878 			/*
1879 			 * If this is for an mcg, it must be for a fullmember,
1880 			 * since we got rid of send-only members above when
1881 			 * processing the mce list.
1882 			 */
1883 			ASSERT(cycled && ((mce == NULL) || (mce->mc_jstate ==
1884 			    IB_MC_JSTATE_FULL)));
1885 
1886 			/*
1887 			 * Check if the fullmember mce needs to be torn down,
1888 			 * ie whether the DLPI disable has already been done.
1889 			 * If so, do some of the work of tx_cleanup, namely
1890 			 * causing leave (which will fail), detach and
1891 			 * mce-freeing. tx_cleanup will put the AH into free
1892 			 * list. The reason to duplicate some of this
1893 			 * tx_cleanup work is because we want to delete the
1894 			 * AH right now instead of waiting for tx_cleanup, to
1895 			 * force subsequent Tx's to reacquire an AH.
1896 			 */
1897 			if ((mce != NULL) && (mce->mc_fullreap))
1898 				ibd_async_reap_group(state, mce,
1899 				    mce->mc_info.mc_adds_vect.av_dgid,
1900 				    mce->mc_jstate);
1901 		}
1902 		mutex_exit(&state->id_ac_mutex);
1903 	}
1904 
1905 	/*
1906 	 * mac handle is guaranteed to exist since driver does ibt_close_hca()
1907 	 * (which stops further events from being delivered) before
1908 	 * mac_unregister(). At this point, it is guaranteed that mac_register
1909 	 * has already been done.
1910 	 */
1911 	mutex_enter(&state->id_link_mutex);
1912 	state->id_link_state = lstate;
1913 	mac_link_update(state->id_mh, lstate);
1914 	mutex_exit(&state->id_link_mutex);
1915 
1916 	ibd_async_done(state);
1917 }
1918 
1919 /*
1920  * When the link is notified up, we need to do a few things, based
1921  * on the port's current p_init_type_reply claiming a reinit has been
1922  * done or not. The reinit steps are:
1923  * 1. If in InitTypeReply, NoLoadReply == PreserveContentReply == 0, verify
1924  *    the old Pkey and GID0 are correct.
1925  * 2. Register for mcg traps (already done by ibmf).
1926  * 3. If PreservePresenceReply indicates the SM has restored port's presence
1927  *    in subnet, nothing more to do. Else go to next steps (on async daemon).
1928  * 4. Give up all sendonly memberships.
1929  * 5. Acquire all full memberships.
1930  * 6. In promiscuous mode, acquire all non memberships.
1931  * 7. Recycle all AHs to free list.
1932  */
1933 static void
1934 ibd_link_mod(ibd_state_t *state, ibt_async_code_t code)
1935 {
1936 	ibt_hca_portinfo_t *port_infop;
1937 	ibt_status_t ibt_status;
1938 	uint_t psize, port_infosz;
1939 	ibd_link_op_t opcode;
1940 	ibd_req_t *req;
1941 
1942 	/*
1943 	 * Do not send a request to the async daemon if it has not
1944 	 * yet been created or is being destroyed. If the async
1945 	 * daemon has not yet been created, we still need to track
1946 	 * last known state of the link. If this code races with the
1947 	 * detach path, then we are assured that the detach path has
1948 	 * not yet done the ibt_close_hca (which waits for all async
1949 	 * events to complete). If the code races with the attach path,
1950 	 * we need to validate the pkey/gid (in the link_up case) if
1951 	 * the initialization path has already set these up and created
1952 	 * IBTF resources based on the values.
1953 	 */
1954 	mutex_enter(&state->id_link_mutex);
1955 
1956 	/*
1957 	 * If the init code in ibd_drv_init hasn't yet set up the
1958 	 * pkey/gid, nothing to do; that code will set the link state.
1959 	 */
1960 	if (state->id_link_state == LINK_STATE_UNKNOWN) {
1961 		mutex_exit(&state->id_link_mutex);
1962 		return;
1963 	}
1964 
1965 	if (code == IBT_EVENT_PORT_UP) {
1966 		uint8_t itreply;
1967 		boolean_t badup = B_FALSE;
1968 
1969 		ibt_status = ibt_query_hca_ports(state->id_hca_hdl,
1970 		    state->id_port, &port_infop, &psize, &port_infosz);
1971 		if ((ibt_status != IBT_SUCCESS) || (psize != 1)) {
1972 			mutex_exit(&state->id_link_mutex);
1973 			DPRINT(10, "ibd_link_up : failed in"
1974 			    " ibt_query_port()\n");
1975 			return;
1976 		}
1977 
1978 		/*
1979 		 * If the link already went down by the time the handler gets
1980 		 * here, give up; we can not even validate pkey/gid since those
1981 		 * are not valid.
1982 		 */
1983 		if (port_infop->p_linkstate != IBT_PORT_ACTIVE)
1984 			badup = B_TRUE;
1985 
1986 		itreply = port_infop->p_init_type_reply;
1987 
1988 		/*
1989 		 * In InitTypeReply, check if NoLoadReply ==
1990 		 * PreserveContentReply == 0, in which case, verify Pkey/GID0.
1991 		 */
1992 		if (((itreply & SM_INIT_TYPE_REPLY_NO_LOAD_REPLY) == 0) &&
1993 		    ((itreply & SM_INIT_TYPE_PRESERVE_CONTENT_REPLY) == 0) &&
1994 		    (!badup)) {
1995 			/*
1996 			 * Check that the subnet part of GID0 has not changed.
1997 			 */
1998 			if (bcmp(port_infop->p_sgid_tbl, &state->id_sgid,
1999 			    sizeof (ib_gid_t)) != 0)
2000 				badup = B_TRUE;
2001 
2002 			/*
2003 			 * Check that Pkey/index mapping is still valid.
2004 			 */
2005 			if ((port_infop->p_pkey_tbl_sz <= state->id_pkix) ||
2006 			    (port_infop->p_pkey_tbl[state->id_pkix] !=
2007 			    state->id_pkey))
2008 				badup = B_TRUE;
2009 		}
2010 
2011 		/*
2012 		 * In InitTypeReply, if PreservePresenceReply indicates the SM
2013 		 * has ensured that the port's presence in mcg, traps etc is
2014 		 * intact, nothing more to do.
2015 		 */
2016 		opcode = IBD_LINK_UP_ABSENT;
2017 		if ((itreply & SM_INIT_TYPE_PRESERVE_PRESENCE_REPLY) ==
2018 		    SM_INIT_TYPE_PRESERVE_PRESENCE_REPLY)
2019 			opcode = IBD_LINK_UP;
2020 
2021 		if (badup)
2022 			code = IBT_ERROR_PORT_DOWN;
2023 		ibt_free_portinfo(port_infop, port_infosz);
2024 	}
2025 
2026 	if (!ibd_async_safe(state)) {
2027 		state->id_link_state = ((code == IBT_EVENT_PORT_UP) ?
2028 		    LINK_STATE_UP : LINK_STATE_DOWN);
2029 		mutex_exit(&state->id_link_mutex);
2030 		return;
2031 	}
2032 	mutex_exit(&state->id_link_mutex);
2033 
2034 	if (code == IBT_ERROR_PORT_DOWN)
2035 		opcode = IBD_LINK_DOWN;
2036 
2037 	req = kmem_cache_alloc(state->id_req_kmc, KM_SLEEP);
2038 	req->rq_ptr = (void *)opcode;
2039 	ibd_queue_work_slot(state, req, IBD_ASYNC_LINK);
2040 }
2041 
2042 /*
2043  * For the port up/down events, IBTL guarantees there will not be concurrent
2044  * invocations of the handler. IBTL might coalesce link transition events,
2045  * and not invoke the handler for _each_ up/down transition, but it will
2046  * invoke the handler with last known state
2047  */
2048 static void
2049 ibd_async_handler(void *clnt_private, ibt_hca_hdl_t hca_hdl,
2050     ibt_async_code_t code, ibt_async_event_t *event)
2051 {
2052 	ibd_state_t *state = (ibd_state_t *)clnt_private;
2053 
2054 	switch (code) {
2055 	case IBT_ERROR_CATASTROPHIC_CHAN:
2056 		ibd_print_warn(state, "catastrophic channel error");
2057 		break;
2058 	case IBT_ERROR_CQ:
2059 		ibd_print_warn(state, "completion queue error");
2060 		break;
2061 	case IBT_ERROR_PORT_DOWN:
2062 	case IBT_EVENT_PORT_UP:
2063 		/*
2064 		 * Events will be delivered to all instances that have
2065 		 * done ibt_open_hca() but not yet done ibt_close_hca().
2066 		 * Only need to do work for our port; IBTF will deliver
2067 		 * events for other ports on the hca we have ibt_open_hca'ed
2068 		 * too. Note that ibd_drv_init() initializes id_port before
2069 		 * doing ibt_open_hca().
2070 		 */
2071 		ASSERT(state->id_hca_hdl == hca_hdl);
2072 		if (state->id_port != event->ev_port)
2073 			break;
2074 
2075 		ibd_link_mod(state, code);
2076 		break;
2077 
2078 	case IBT_HCA_ATTACH_EVENT:
2079 	case IBT_HCA_DETACH_EVENT:
2080 		/*
2081 		 * When a new card is plugged to the system, attach_event is
2082 		 * invoked. Additionally, a cfgadm needs to be run to make the
2083 		 * card known to the system, and an ifconfig needs to be run to
2084 		 * plumb up any ibd interfaces on the card. In the case of card
2085 		 * unplug, a cfgadm is run that will trigger any RCM scripts to
2086 		 * unplumb the ibd interfaces on the card; when the card is
2087 		 * actually unplugged, the detach_event is invoked;
2088 		 * additionally, if any ibd instances are still active on the
2089 		 * card (eg there were no associated RCM scripts), driver's
2090 		 * detach routine is invoked.
2091 		 */
2092 		break;
2093 	default:
2094 		break;
2095 	}
2096 }
2097 
2098 /*
2099  * Attach device to the IO framework.
2100  */
2101 static int
2102 ibd_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
2103 {
2104 	mac_register_t *macp;
2105 	ibd_state_t *state;
2106 	int instance;
2107 	int err;
2108 
2109 	switch (cmd) {
2110 		case DDI_ATTACH:
2111 			break;
2112 		case DDI_RESUME:
2113 			/* This driver does not support resume */
2114 		default:
2115 			return (DDI_FAILURE);
2116 	}
2117 
2118 	/*
2119 	 * Allocate soft device data structure
2120 	 */
2121 	instance = ddi_get_instance(dip);
2122 	if (ddi_soft_state_zalloc(ibd_list, instance) == DDI_FAILURE)
2123 		return (DDI_FAILURE);
2124 	state = ddi_get_soft_state(ibd_list, instance);
2125 
2126 	/* pre ibt_attach() soft state initialization */
2127 	if (ibd_state_init(state, dip) != DDI_SUCCESS) {
2128 		DPRINT(10, "ibd_attach : failed in ibd_state_init()");
2129 		goto attach_fail_state_init;
2130 	}
2131 
2132 	/* alloc rx soft intr */
2133 	if ((ibd_rx_softintr == 1) &&
2134 	    ddi_add_softintr(dip, DDI_SOFTINT_LOW, &state->id_rx,
2135 	    NULL, NULL, ibd_intr, (caddr_t)state) != DDI_SUCCESS) {
2136 		DPRINT(10, "ibd_attach : failed in ddi_add_softintr()");
2137 		goto attach_fail_ddi_add_rx_softintr;
2138 	}
2139 
2140 	/* alloc tx soft intr */
2141 	if ((ibd_tx_softintr == 1) &&
2142 	    ddi_add_softintr(dip, DDI_SOFTINT_LOW, &state->id_tx,
2143 	    NULL, NULL, ibd_tx_recycle, (caddr_t)state) != DDI_SUCCESS) {
2144 		DPRINT(10, "ibd_attach : failed in ddi_add_softintr()");
2145 		goto attach_fail_ddi_add_tx_softintr;
2146 	}
2147 
2148 	/* "attach" to IBTL */
2149 	if (ibt_attach(&ibd_clnt_modinfo, dip, state,
2150 	    &state->id_ibt_hdl) != IBT_SUCCESS) {
2151 		DPRINT(10, "ibd_attach : failed in ibt_attach()");
2152 		goto attach_fail_ibt_attach;
2153 	}
2154 
2155 	/* Finish initializing this driver */
2156 	if (ibd_drv_init(state) != DDI_SUCCESS) {
2157 		DPRINT(10, "ibd_attach : failed in ibd_drv_init()\n");
2158 		goto attach_fail_drv_init;
2159 	}
2160 
2161 	/*
2162 	 * Initialize pointers to device specific functions which will be
2163 	 * used by the generic layer.
2164 	 */
2165 	if ((macp = mac_alloc(MAC_VERSION)) == NULL) {
2166 		DPRINT(10, "ibd_attach : failed in mac_alloc()");
2167 		goto attach_fail_drv_init;
2168 	}
2169 
2170 	macp->m_type_ident = MAC_PLUGIN_IDENT_IB;
2171 	macp->m_driver = state;
2172 	macp->m_dip = state->id_dip;
2173 	macp->m_src_addr = (uint8_t *)&state->id_macaddr;
2174 	macp->m_callbacks = &ib_m_callbacks;
2175 	macp->m_min_sdu = 0;
2176 	macp->m_max_sdu = state->id_mtu - IPOIB_HDRSIZE;
2177 
2178 	/*
2179 	 *  Register ourselves with the GLDv3 interface
2180 	 */
2181 	err = mac_register(macp, &state->id_mh);
2182 	mac_free(macp);
2183 	if (err != 0) {
2184 		DPRINT(10, "ibd_attach : failed in mac_register()");
2185 		goto attach_fail_mac_register;
2186 	}
2187 
2188 	/*
2189 	 * Setup the handler we will use for regular DLPI stuff. Its important
2190 	 * to setup the recv handler after registering with gldv3.
2191 	 */
2192 	ibt_set_cq_handler(state->id_rcq_hdl, ibd_rcq_handler, state);
2193 	if (ibt_enable_cq_notify(state->id_rcq_hdl, IBT_NEXT_COMPLETION) !=
2194 	    IBT_SUCCESS) {
2195 		DPRINT(10, "ibd_attach : failed in ibt_enable_cq_notify()\n");
2196 		goto attach_fail_setup_handler;
2197 	}
2198 
2199 	/*
2200 	 * Setup the subnet notices handler after we initialize the a/mcaches
2201 	 * and start the async thread, both of which are required for the
2202 	 * trap handler to function properly. Enable the trap handler to
2203 	 * queue requests to the async thread after the mac_register, because
2204 	 * the async daemon invokes mac_tx_update(), which must be done after
2205 	 * mac_register().
2206 	 */
2207 	ibt_register_subnet_notices(state->id_ibt_hdl,
2208 	    ibd_snet_notices_handler, state);
2209 	mutex_enter(&state->id_trap_lock);
2210 	state->id_trap_stop = B_FALSE;
2211 	mutex_exit(&state->id_trap_lock);
2212 
2213 	/*
2214 	 * Indicate link status to GLDv3 and higher layers. By default,
2215 	 * we assume we are in up state (which must have been true at
2216 	 * least at the time the broadcast mcg's were probed); if there
2217 	 * were any up/down transitions till the time we come here, the
2218 	 * async handler will have updated last known state, which we
2219 	 * use to tell GLDv3. The async handler will not send any
2220 	 * notifications to GLDv3 till we reach here in the initialization
2221 	 * sequence.
2222 	 */
2223 	mac_link_update(state->id_mh, state->id_link_state);
2224 
2225 	return (DDI_SUCCESS);
2226 
2227 	/* Attach failure points, cleanup */
2228 attach_fail_setup_handler:
2229 	(void) mac_unregister(state->id_mh);
2230 
2231 attach_fail_mac_register:
2232 	ibd_drv_fini(state);
2233 
2234 attach_fail_drv_init:
2235 	if (ibt_detach(state->id_ibt_hdl) != IBT_SUCCESS)
2236 		ibd_print_warn(state, "failed to free IB resources");
2237 
2238 attach_fail_ibt_attach:
2239 	if (ibd_tx_softintr == 1)
2240 		ddi_remove_softintr(state->id_tx);
2241 
2242 attach_fail_ddi_add_tx_softintr:
2243 	if (ibd_rx_softintr == 1)
2244 		ddi_remove_softintr(state->id_rx);
2245 
2246 attach_fail_ddi_add_rx_softintr:
2247 	ibd_state_fini(state);
2248 
2249 attach_fail_state_init:
2250 	ddi_soft_state_free(ibd_list, instance);
2251 
2252 	return (DDI_FAILURE);
2253 }
2254 
2255 /*
2256  * Detach device from the IO framework.
2257  */
2258 static int
2259 ibd_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
2260 {
2261 	ibd_state_t *state;
2262 	int status;
2263 	int instance;
2264 
2265 	switch (cmd) {
2266 		case DDI_DETACH:
2267 			break;
2268 		case DDI_SUSPEND:
2269 		default:
2270 			return (DDI_FAILURE);
2271 	}
2272 
2273 	instance = ddi_get_instance(dip);
2274 	state = ddi_get_soft_state(ibd_list, instance);
2275 
2276 	/*
2277 	 * First, stop receive interrupts; this stops the
2278 	 * driver from handing up buffers to higher layers.
2279 	 * Wait for receive buffers to be returned; give up
2280 	 * after 5 seconds.
2281 	 */
2282 	ibt_set_cq_handler(state->id_rcq_hdl, 0, 0);
2283 	status = 50;
2284 	while (state->id_rx_list.dl_bufs_outstanding > 0) {
2285 		delay(drv_usectohz(100000));
2286 		if (--status == 0) {
2287 			DPRINT(2, "ibd_detach : reclaiming failed");
2288 			goto failed;
2289 		}
2290 	}
2291 
2292 	if (mac_unregister(state->id_mh) != DDI_SUCCESS) {
2293 		DPRINT(10, "ibd_detach : failed in mac_unregister()");
2294 		goto failed;
2295 	}
2296 
2297 	if (ibd_rx_softintr == 1)
2298 		ddi_remove_softintr(state->id_rx);
2299 
2300 	if (ibd_tx_softintr == 1)
2301 		ddi_remove_softintr(state->id_tx);
2302 
2303 	ibd_drv_fini(state);
2304 
2305 	if (ibt_detach(state->id_ibt_hdl) != IBT_SUCCESS)
2306 		ibd_print_warn(state, "failed to free all IB resources at "
2307 		    "driver detach time");
2308 
2309 	ibd_state_fini(state);
2310 	ddi_soft_state_free(ibd_list, instance);
2311 	return (DDI_SUCCESS);
2312 
2313 failed:
2314 	/*
2315 	 * Reap all the Tx/Rx completions that were posted since we
2316 	 * turned off the notification. Turn on notifications. There
2317 	 * is a race in that we do not reap completions that come in
2318 	 * after the poll and before notifications get turned on. That
2319 	 * is okay, the next rx/tx packet will trigger a completion
2320 	 * that will reap any missed completions.
2321 	 */
2322 	ibd_poll_compq(state, state->id_rcq_hdl);
2323 	ibt_set_cq_handler(state->id_rcq_hdl, ibd_rcq_handler, state);
2324 	return (DDI_FAILURE);
2325 }
2326 
2327 /*
2328  * Pre ibt_attach() driver initialization
2329  */
2330 static int
2331 ibd_state_init(ibd_state_t *state, dev_info_t *dip)
2332 {
2333 	char buf[64];
2334 
2335 	mutex_init(&state->id_link_mutex, NULL, MUTEX_DRIVER, NULL);
2336 	state->id_link_state = LINK_STATE_UNKNOWN;
2337 
2338 	mutex_init(&state->id_trap_lock, NULL, MUTEX_DRIVER, NULL);
2339 	cv_init(&state->id_trap_cv, NULL, CV_DEFAULT, NULL);
2340 	state->id_trap_stop = B_TRUE;
2341 	state->id_trap_inprog = 0;
2342 
2343 	mutex_init(&state->id_cq_poll_lock, NULL, MUTEX_DRIVER, NULL);
2344 	state->id_dip = dip;
2345 
2346 	mutex_init(&state->id_sched_lock, NULL, MUTEX_DRIVER, NULL);
2347 
2348 	state->id_tx_list.dl_head = NULL;
2349 	state->id_tx_list.dl_tail = NULL;
2350 	state->id_tx_list.dl_pending_sends = B_FALSE;
2351 	state->id_tx_list.dl_cnt = 0;
2352 	mutex_init(&state->id_tx_list.dl_mutex, NULL, MUTEX_DRIVER, NULL);
2353 	mutex_init(&state->id_txpost_lock, NULL, MUTEX_DRIVER, NULL);
2354 	state->id_tx_busy = 0;
2355 
2356 	state->id_rx_list.dl_head = NULL;
2357 	state->id_rx_list.dl_tail = NULL;
2358 	state->id_rx_list.dl_bufs_outstanding = 0;
2359 	state->id_rx_list.dl_cnt = 0;
2360 	mutex_init(&state->id_rx_list.dl_mutex, NULL, MUTEX_DRIVER, NULL);
2361 	mutex_init(&state->id_rxpost_lock, NULL, MUTEX_DRIVER, NULL);
2362 
2363 	(void) sprintf(buf, "ibd_req%d", ddi_get_instance(dip));
2364 	state->id_req_kmc = kmem_cache_create(buf, sizeof (ibd_req_t),
2365 	    0, NULL, NULL, NULL, NULL, NULL, 0);
2366 
2367 #ifdef IBD_LOGGING
2368 	mutex_init(&ibd_lbuf_lock, NULL, MUTEX_DRIVER, NULL);
2369 #endif
2370 
2371 	return (DDI_SUCCESS);
2372 }
2373 
2374 /*
2375  * Post ibt_detach() driver deconstruction
2376  */
2377 static void
2378 ibd_state_fini(ibd_state_t *state)
2379 {
2380 	kmem_cache_destroy(state->id_req_kmc);
2381 
2382 	mutex_destroy(&state->id_rxpost_lock);
2383 	mutex_destroy(&state->id_rx_list.dl_mutex);
2384 
2385 	mutex_destroy(&state->id_txpost_lock);
2386 	mutex_destroy(&state->id_tx_list.dl_mutex);
2387 
2388 	mutex_destroy(&state->id_sched_lock);
2389 	mutex_destroy(&state->id_cq_poll_lock);
2390 
2391 	cv_destroy(&state->id_trap_cv);
2392 	mutex_destroy(&state->id_trap_lock);
2393 	mutex_destroy(&state->id_link_mutex);
2394 
2395 #ifdef IBD_LOGGING
2396 	mutex_destroy(&ibd_lbuf_lock);
2397 #endif
2398 }
2399 
2400 /*
2401  * Fetch IBA parameters for the network device from IB nexus.
2402  */
2403 static int
2404 ibd_get_portpkey(ibd_state_t *state, ib_guid_t *hca_guid)
2405 {
2406 	/*
2407 	 * Get the IBA Pkey ... allow only fullmembers, per IPoIB spec.
2408 	 * Note that the default partition is also allowed.
2409 	 */
2410 	state->id_pkey = ddi_prop_get_int(DDI_DEV_T_ANY, state->id_dip,
2411 	    0, "port-pkey", IB_PKEY_INVALID_LIMITED);
2412 	if (state->id_pkey <= IB_PKEY_INVALID_FULL) {
2413 		DPRINT(10, "ibd_get_portpkey : ERROR: IBport device has wrong"
2414 		    "partition\n");
2415 		return (DDI_FAILURE);
2416 	}
2417 
2418 	/*
2419 	 * ... the IBA port ...
2420 	 */
2421 	state->id_port = ddi_prop_get_int(DDI_DEV_T_ANY, state->id_dip,
2422 	    0, "port-number", 0);
2423 	if (state->id_port == 0) {
2424 		DPRINT(10, "ibd_get_portpkey : ERROR: invalid port number\n");
2425 		return (DDI_FAILURE);
2426 	}
2427 
2428 	/*
2429 	 * ... and HCA GUID.
2430 	 */
2431 	*hca_guid = ddi_prop_get_int64(DDI_DEV_T_ANY, state->id_dip,
2432 	    0, "hca-guid", 0);
2433 	if (*hca_guid == 0) {
2434 		DPRINT(10, "ibd_get_portpkey : ERROR: IBport hca has wrong "
2435 		    "guid\n");
2436 		return (DDI_FAILURE);
2437 	}
2438 
2439 	return (DDI_SUCCESS);
2440 }
2441 
2442 /*
2443  * Fetch link speed from SA for snmp ifspeed reporting.
2444  */
2445 static uint64_t
2446 ibd_get_portspeed(ibd_state_t *state)
2447 {
2448 	int			ret;
2449 	ibt_path_info_t		path;
2450 	ibt_path_attr_t		path_attr;
2451 	uint8_t			num_paths;
2452 	uint64_t		ifspeed;
2453 
2454 	/*
2455 	 * Due to serdes 8b10b encoding on the wire, 2.5 Gbps on wire
2456 	 * translates to 2 Gbps data rate. Thus, 1X single data rate is
2457 	 * 2000000000. Start with that as default.
2458 	 */
2459 	ifspeed = 2000000000;
2460 
2461 	bzero(&path_attr, sizeof (path_attr));
2462 
2463 	/*
2464 	 * Get the port speed from Loopback path information.
2465 	 */
2466 	path_attr.pa_dgids = &state->id_sgid;
2467 	path_attr.pa_num_dgids = 1;
2468 	path_attr.pa_sgid = state->id_sgid;
2469 
2470 	if (ibt_get_paths(state->id_ibt_hdl, IBT_PATH_NO_FLAGS,
2471 	    &path_attr, 1, &path, &num_paths) != IBT_SUCCESS)
2472 		goto earlydone;
2473 
2474 	if (num_paths < 1)
2475 		goto earlydone;
2476 
2477 	/*
2478 	 * In case SA does not return an expected value, report the default
2479 	 * speed as 1X.
2480 	 */
2481 	ret = 1;
2482 	switch (path.pi_prim_cep_path.cep_adds_vect.av_srate) {
2483 		case IBT_SRATE_2:	/*  1X SDR i.e 2.5 Gbps */
2484 			ret = 1;
2485 			break;
2486 		case IBT_SRATE_10:	/*  4X SDR or 1X QDR i.e 10 Gbps */
2487 			ret = 4;
2488 			break;
2489 		case IBT_SRATE_30:	/* 12X SDR i.e 30 Gbps */
2490 			ret = 12;
2491 			break;
2492 		case IBT_SRATE_5:	/*  1X DDR i.e  5 Gbps */
2493 			ret = 2;
2494 			break;
2495 		case IBT_SRATE_20:	/*  4X DDR or 8X SDR i.e 20 Gbps */
2496 			ret = 8;
2497 			break;
2498 		case IBT_SRATE_40:	/*  8X DDR or 4X QDR i.e 40 Gbps */
2499 			ret = 16;
2500 			break;
2501 		case IBT_SRATE_60:	/* 12X DDR i.e 60 Gbps */
2502 			ret = 24;
2503 			break;
2504 		case IBT_SRATE_80:	/*  8X QDR i.e 80 Gbps */
2505 			ret = 32;
2506 			break;
2507 		case IBT_SRATE_120:	/* 12X QDR i.e 120 Gbps */
2508 			ret = 48;
2509 			break;
2510 	}
2511 
2512 	ifspeed *= ret;
2513 
2514 earlydone:
2515 	return (ifspeed);
2516 }
2517 
2518 /*
2519  * Search input mcg list (id_mc_full or id_mc_non) for an entry
2520  * representing the input mcg mgid.
2521  */
2522 static ibd_mce_t *
2523 ibd_mcache_find(ib_gid_t mgid, struct list *mlist)
2524 {
2525 	ibd_mce_t *ptr = list_head(mlist);
2526 
2527 	/*
2528 	 * Do plain linear search.
2529 	 */
2530 	while (ptr != NULL) {
2531 		if (bcmp(&mgid, &ptr->mc_info.mc_adds_vect.av_dgid,
2532 		    sizeof (ib_gid_t)) == 0)
2533 			return (ptr);
2534 		ptr = list_next(mlist, ptr);
2535 	}
2536 	return (NULL);
2537 }
2538 
2539 /*
2540  * Execute IBA JOIN.
2541  */
2542 static ibt_status_t
2543 ibd_iba_join(ibd_state_t *state, ib_gid_t mgid, ibd_mce_t *mce)
2544 {
2545 	ibt_mcg_attr_t mcg_attr;
2546 
2547 	bzero(&mcg_attr, sizeof (ibt_mcg_attr_t));
2548 	mcg_attr.mc_qkey = state->id_mcinfo->mc_qkey;
2549 	mcg_attr.mc_mgid = mgid;
2550 	mcg_attr.mc_join_state = mce->mc_jstate;
2551 	mcg_attr.mc_scope = state->id_scope;
2552 	mcg_attr.mc_pkey = state->id_pkey;
2553 	mcg_attr.mc_flow = state->id_mcinfo->mc_adds_vect.av_flow;
2554 	mcg_attr.mc_sl = state->id_mcinfo->mc_adds_vect.av_srvl;
2555 	mcg_attr.mc_tclass = state->id_mcinfo->mc_adds_vect.av_tclass;
2556 	return (ibt_join_mcg(state->id_sgid, &mcg_attr, &mce->mc_info,
2557 	    NULL, NULL));
2558 }
2559 
2560 /*
2561  * This code JOINs the port in the proper way (depending on the join
2562  * state) so that IBA fabric will forward mcg packets to/from the port.
2563  * It also attaches the QPN to the mcg so it can receive those mcg
2564  * packets. This code makes sure not to attach the mcg to the QP if
2565  * that has been previously done due to the mcg being joined with a
2566  * different join state, even though this is not required by SWG_0216,
2567  * refid 3610.
2568  */
2569 static ibd_mce_t *
2570 ibd_join_group(ibd_state_t *state, ib_gid_t mgid, uint8_t jstate)
2571 {
2572 	ibt_status_t ibt_status;
2573 	ibd_mce_t *mce, *tmce, *omce = NULL;
2574 	boolean_t do_attach = B_TRUE;
2575 
2576 	DPRINT(2, "ibd_join_group : join_group state %d : %016llx:%016llx\n",
2577 	    jstate, mgid.gid_prefix, mgid.gid_guid);
2578 
2579 	/*
2580 	 * For enable_multicast Full member joins, we need to do some
2581 	 * extra work. If there is already an mce on the list that
2582 	 * indicates full membership, that means the membership has
2583 	 * not yet been dropped (since the disable_multicast was issued)
2584 	 * because there are pending Tx's to the mcg; in that case, just
2585 	 * mark the mce not to be reaped when the Tx completion queues
2586 	 * an async reap operation.
2587 	 *
2588 	 * If there is already an mce on the list indicating sendonly
2589 	 * membership, try to promote to full membership. Be careful
2590 	 * not to deallocate the old mce, since there might be an AH
2591 	 * pointing to it; instead, update the old mce with new data
2592 	 * that tracks the full membership.
2593 	 */
2594 	if ((jstate == IB_MC_JSTATE_FULL) && ((omce =
2595 	    IBD_MCACHE_FIND_FULL(state, mgid)) != NULL)) {
2596 		if (omce->mc_jstate == IB_MC_JSTATE_FULL) {
2597 			ASSERT(omce->mc_fullreap);
2598 			omce->mc_fullreap = B_FALSE;
2599 			return (omce);
2600 		} else {
2601 			ASSERT(omce->mc_jstate == IB_MC_JSTATE_SEND_ONLY_NON);
2602 		}
2603 	}
2604 
2605 	/*
2606 	 * Allocate the ibd_mce_t to track this JOIN.
2607 	 */
2608 	mce = kmem_zalloc(sizeof (ibd_mce_t), KM_SLEEP);
2609 	mce->mc_fullreap = B_FALSE;
2610 	mce->mc_jstate = jstate;
2611 
2612 	if ((ibt_status = ibd_iba_join(state, mgid, mce)) != IBT_SUCCESS) {
2613 		DPRINT(10, "ibd_join_group : failed ibt_join_mcg() %d",
2614 		    ibt_status);
2615 		kmem_free(mce, sizeof (ibd_mce_t));
2616 		return (NULL);
2617 	}
2618 
2619 	/*
2620 	 * Is an IBA attach required? Not if the interface is already joined
2621 	 * to the mcg in a different appropriate join state.
2622 	 */
2623 	if (jstate == IB_MC_JSTATE_NON) {
2624 		tmce = IBD_MCACHE_FIND_FULL(state, mgid);
2625 		if ((tmce != NULL) && (tmce->mc_jstate == IB_MC_JSTATE_FULL))
2626 			do_attach = B_FALSE;
2627 	} else if (jstate == IB_MC_JSTATE_FULL) {
2628 		if (IBD_MCACHE_FIND_NON(state, mgid) != NULL)
2629 			do_attach = B_FALSE;
2630 	} else {	/* jstate == IB_MC_JSTATE_SEND_ONLY_NON */
2631 		do_attach = B_FALSE;
2632 	}
2633 
2634 	if (do_attach) {
2635 		/*
2636 		 * Do the IBA attach.
2637 		 */
2638 		DPRINT(10, "ibd_join_group: ibt_attach_mcg \n");
2639 		if ((ibt_status = ibt_attach_mcg(state->id_chnl_hdl,
2640 		    &mce->mc_info)) != IBT_SUCCESS) {
2641 			DPRINT(10, "ibd_join_group : failed qp attachment "
2642 			    "%d\n", ibt_status);
2643 			/*
2644 			 * NOTE that we should probably preserve the join info
2645 			 * in the list and later try to leave again at detach
2646 			 * time.
2647 			 */
2648 			(void) ibt_leave_mcg(state->id_sgid, mgid,
2649 			    state->id_sgid, jstate);
2650 			kmem_free(mce, sizeof (ibd_mce_t));
2651 			return (NULL);
2652 		}
2653 	}
2654 
2655 	/*
2656 	 * Insert the ibd_mce_t in the proper list.
2657 	 */
2658 	if (jstate == IB_MC_JSTATE_NON) {
2659 		IBD_MCACHE_INSERT_NON(state, mce);
2660 	} else {
2661 		/*
2662 		 * Set up the mc_req fields used for reaping the
2663 		 * mcg in case of delayed tx completion (see
2664 		 * ibd_tx_cleanup()). Also done for sendonly join in
2665 		 * case we are promoted to fullmembership later and
2666 		 * keep using the same mce.
2667 		 */
2668 		mce->mc_req.rq_gid = mgid;
2669 		mce->mc_req.rq_ptr = mce;
2670 		/*
2671 		 * Check whether this is the case of trying to join
2672 		 * full member, and we were already joined send only.
2673 		 * We try to drop our SendOnly membership, but it is
2674 		 * possible that the mcg does not exist anymore (and
2675 		 * the subnet trap never reached us), so the leave
2676 		 * operation might fail.
2677 		 */
2678 		if (omce != NULL) {
2679 			(void) ibt_leave_mcg(state->id_sgid, mgid,
2680 			    state->id_sgid, IB_MC_JSTATE_SEND_ONLY_NON);
2681 			omce->mc_jstate = IB_MC_JSTATE_FULL;
2682 			bcopy(&mce->mc_info, &omce->mc_info,
2683 			    sizeof (ibt_mcg_info_t));
2684 			kmem_free(mce, sizeof (ibd_mce_t));
2685 			return (omce);
2686 		}
2687 		mutex_enter(&state->id_mc_mutex);
2688 		IBD_MCACHE_INSERT_FULL(state, mce);
2689 		mutex_exit(&state->id_mc_mutex);
2690 	}
2691 
2692 	return (mce);
2693 }
2694 
2695 /*
2696  * Called during port up event handling to attempt to reacquire full
2697  * membership to an mcg. Stripped down version of ibd_join_group().
2698  * Note that it is possible that the mcg might have gone away, and
2699  * gets recreated at this point.
2700  */
2701 static void
2702 ibd_reacquire_group(ibd_state_t *state, ibd_mce_t *mce)
2703 {
2704 	ib_gid_t mgid;
2705 
2706 	/*
2707 	 * If the mc_fullreap flag is set, or this join fails, a subsequent
2708 	 * reap/leave is going to try to leave the group. We could prevent
2709 	 * that by adding a boolean flag into ibd_mce_t, if required.
2710 	 */
2711 	if (mce->mc_fullreap)
2712 		return;
2713 
2714 	mgid = mce->mc_info.mc_adds_vect.av_dgid;
2715 
2716 	DPRINT(2, "ibd_reacquire_group : %016llx:%016llx\n", mgid.gid_prefix,
2717 	    mgid.gid_guid);
2718 
2719 	if (ibd_iba_join(state, mgid, mce) != IBT_SUCCESS)
2720 		ibd_print_warn(state, "Failure on port up to rejoin "
2721 		    "multicast gid %016llx:%016llx",
2722 		    (u_longlong_t)mgid.gid_prefix,
2723 		    (u_longlong_t)mgid.gid_guid);
2724 }
2725 
2726 /*
2727  * This code handles delayed Tx completion cleanups for mcg's to which
2728  * disable_multicast has been issued, regular mcg related cleanups during
2729  * disable_multicast, disable_promiscous and mcg traps, as well as
2730  * cleanups during driver detach time. Depending on the join state,
2731  * it deletes the mce from the appropriate list and issues the IBA
2732  * leave/detach; except in the disable_multicast case when the mce
2733  * is left on the active list for a subsequent Tx completion cleanup.
2734  */
2735 static void
2736 ibd_async_reap_group(ibd_state_t *state, ibd_mce_t *mce, ib_gid_t mgid,
2737     uint8_t jstate)
2738 {
2739 	ibd_mce_t *tmce;
2740 	boolean_t do_detach = B_TRUE;
2741 
2742 	/*
2743 	 * Before detaching, we must check whether the other list
2744 	 * contains the mcg; if we detach blindly, the consumer
2745 	 * who set up the other list will also stop receiving
2746 	 * traffic.
2747 	 */
2748 	if (jstate == IB_MC_JSTATE_FULL) {
2749 		/*
2750 		 * The following check is only relevant while coming
2751 		 * from the Tx completion path in the reap case.
2752 		 */
2753 		if (!mce->mc_fullreap)
2754 			return;
2755 		mutex_enter(&state->id_mc_mutex);
2756 		IBD_MCACHE_PULLOUT_FULL(state, mce);
2757 		mutex_exit(&state->id_mc_mutex);
2758 		if (IBD_MCACHE_FIND_NON(state, mgid) != NULL)
2759 			do_detach = B_FALSE;
2760 	} else if (jstate == IB_MC_JSTATE_NON) {
2761 		IBD_MCACHE_PULLOUT_NON(state, mce);
2762 		tmce = IBD_MCACHE_FIND_FULL(state, mgid);
2763 		if ((tmce != NULL) && (tmce->mc_jstate == IB_MC_JSTATE_FULL))
2764 			do_detach = B_FALSE;
2765 	} else {	/* jstate == IB_MC_JSTATE_SEND_ONLY_NON */
2766 		mutex_enter(&state->id_mc_mutex);
2767 		IBD_MCACHE_PULLOUT_FULL(state, mce);
2768 		mutex_exit(&state->id_mc_mutex);
2769 		do_detach = B_FALSE;
2770 	}
2771 
2772 	/*
2773 	 * If we are reacting to a mcg trap and leaving our sendonly or
2774 	 * non membership, the mcg is possibly already gone, so attempting
2775 	 * to leave might fail. On the other hand, we must try to leave
2776 	 * anyway, since this might be a trap from long ago, and we could
2777 	 * have potentially sendonly joined to a recent incarnation of
2778 	 * the mcg and are about to loose track of this information.
2779 	 */
2780 	if (do_detach) {
2781 		DPRINT(2, "ibd_async_reap_group : ibt_detach_mcg : "
2782 		    "%016llx:%016llx\n", mgid.gid_prefix, mgid.gid_guid);
2783 		(void) ibt_detach_mcg(state->id_chnl_hdl, &mce->mc_info);
2784 	}
2785 
2786 	(void) ibt_leave_mcg(state->id_sgid, mgid, state->id_sgid, jstate);
2787 	kmem_free(mce, sizeof (ibd_mce_t));
2788 }
2789 
2790 /*
2791  * Async code executed due to multicast and promiscuous disable requests
2792  * and mcg trap handling; also executed during driver detach. Mostly, a
2793  * leave and detach is done; except for the fullmember case when Tx
2794  * requests are pending, whence arrangements are made for subsequent
2795  * cleanup on Tx completion.
2796  */
2797 static void
2798 ibd_leave_group(ibd_state_t *state, ib_gid_t mgid, uint8_t jstate)
2799 {
2800 	ipoib_mac_t mcmac;
2801 	boolean_t recycled;
2802 	ibd_mce_t *mce;
2803 
2804 	DPRINT(2, "ibd_leave_group : leave_group state %d : %016llx:%016llx\n",
2805 	    jstate, mgid.gid_prefix, mgid.gid_guid);
2806 
2807 	if (jstate == IB_MC_JSTATE_NON) {
2808 		recycled = B_TRUE;
2809 		mce = IBD_MCACHE_FIND_NON(state, mgid);
2810 		/*
2811 		 * In case we are handling a mcg trap, we might not find
2812 		 * the mcg in the non list.
2813 		 */
2814 		if (mce == NULL)
2815 			return;
2816 	} else {
2817 		mce = IBD_MCACHE_FIND_FULL(state, mgid);
2818 
2819 		/*
2820 		 * In case we are handling a mcg trap, make sure the trap
2821 		 * is not arriving late; if we have an mce that indicates
2822 		 * that we are already a fullmember, that would be a clear
2823 		 * indication that the trap arrived late (ie, is for a
2824 		 * previous incarnation of the mcg).
2825 		 */
2826 		if (jstate == IB_MC_JSTATE_SEND_ONLY_NON) {
2827 			if ((mce == NULL) || (mce->mc_jstate ==
2828 			    IB_MC_JSTATE_FULL))
2829 				return;
2830 		} else {
2831 			ASSERT(jstate == IB_MC_JSTATE_FULL);
2832 
2833 			/*
2834 			 * If join group failed, mce will be NULL here.
2835 			 * This is because in GLDv3 driver, set multicast
2836 			 *  will always return success.
2837 			 */
2838 			if (mce == NULL)
2839 				return;
2840 
2841 			mce->mc_fullreap = B_TRUE;
2842 		}
2843 
2844 		/*
2845 		 * If no pending Tx's remain that reference the AH
2846 		 * for the mcg, recycle it from active to free list.
2847 		 * Else in the IB_MC_JSTATE_FULL case, just mark the AH,
2848 		 * so the last completing Tx will cause an async reap
2849 		 * operation to be invoked, at which time we will drop our
2850 		 * membership to the mcg so that the pending Tx's complete
2851 		 * successfully. Refer to comments on "AH and MCE active
2852 		 * list manipulation" at top of this file. The lock protects
2853 		 * against Tx fast path and Tx cleanup code.
2854 		 */
2855 		mutex_enter(&state->id_ac_mutex);
2856 		ibd_h2n_mac(&mcmac, IB_MC_QPN, mgid.gid_prefix, mgid.gid_guid);
2857 		recycled = ibd_acache_recycle(state, &mcmac, (jstate ==
2858 		    IB_MC_JSTATE_SEND_ONLY_NON));
2859 		mutex_exit(&state->id_ac_mutex);
2860 	}
2861 
2862 	if (recycled) {
2863 		DPRINT(2, "ibd_leave_group : leave_group reaping : "
2864 		    "%016llx:%016llx\n", mgid.gid_prefix, mgid.gid_guid);
2865 		ibd_async_reap_group(state, mce, mgid, jstate);
2866 	}
2867 }
2868 
2869 /*
2870  * Find the broadcast address as defined by IPoIB; implicitly
2871  * determines the IBA scope, mtu, tclass etc of the link the
2872  * interface is going to be a member of.
2873  */
2874 static ibt_status_t
2875 ibd_find_bgroup(ibd_state_t *state)
2876 {
2877 	ibt_mcg_attr_t mcg_attr;
2878 	uint_t numg;
2879 	uchar_t scopes[] = { IB_MC_SCOPE_SUBNET_LOCAL,
2880 	    IB_MC_SCOPE_SITE_LOCAL, IB_MC_SCOPE_ORG_LOCAL,
2881 	    IB_MC_SCOPE_GLOBAL };
2882 	int i, mcgmtu;
2883 	boolean_t found = B_FALSE;
2884 
2885 	bzero(&mcg_attr, sizeof (ibt_mcg_attr_t));
2886 	mcg_attr.mc_pkey = state->id_pkey;
2887 	state->id_mgid.gid_guid = IB_MGID_IPV4_LOWGRP_MASK;
2888 
2889 	for (i = 0; i < sizeof (scopes)/sizeof (scopes[0]); i++) {
2890 		state->id_scope = mcg_attr.mc_scope = scopes[i];
2891 
2892 		/*
2893 		 * Look for the IPoIB broadcast group.
2894 		 */
2895 		state->id_mgid.gid_prefix =
2896 		    (((uint64_t)IB_MCGID_IPV4_PREFIX << 32) |
2897 		    ((uint64_t)state->id_scope << 48) |
2898 		    ((uint32_t)(state->id_pkey << 16)));
2899 		mcg_attr.mc_mgid = state->id_mgid;
2900 		if (ibt_query_mcg(state->id_sgid, &mcg_attr, 1,
2901 		    &state->id_mcinfo, &numg) == IBT_SUCCESS) {
2902 			found = B_TRUE;
2903 			break;
2904 		}
2905 
2906 	}
2907 
2908 	if (!found) {
2909 		ibd_print_warn(state, "IPoIB broadcast group absent");
2910 		return (IBT_FAILURE);
2911 	}
2912 
2913 	/*
2914 	 * Assert that the mcg mtu <= id_mtu. Fill in updated id_mtu.
2915 	 */
2916 	mcgmtu = (128 << state->id_mcinfo->mc_mtu);
2917 	if (state->id_mtu < mcgmtu) {
2918 		ibd_print_warn(state, "IPoIB broadcast group MTU %d "
2919 		    "greater than port's maximum MTU %d", mcgmtu,
2920 		    state->id_mtu);
2921 		return (IBT_FAILURE);
2922 	}
2923 	state->id_mtu = mcgmtu;
2924 
2925 	return (IBT_SUCCESS);
2926 }
2927 
2928 /*
2929  * Post ibt_attach() initialization.
2930  */
2931 static int
2932 ibd_drv_init(ibd_state_t *state)
2933 {
2934 	kthread_t *kht;
2935 	ibt_ud_chan_alloc_args_t ud_alloc_attr;
2936 	ibt_ud_chan_query_attr_t ud_chan_attr;
2937 	ibt_hca_portinfo_t *port_infop;
2938 	ibt_hca_attr_t hca_attrs;
2939 	ibt_status_t ibt_status;
2940 	ibt_cq_attr_t cq_attr;
2941 	ib_guid_t hca_guid;
2942 	uint32_t real_size;
2943 	uint32_t *ptr;
2944 	char pathname[OBP_MAXPATHLEN];
2945 	uint_t psize, port_infosz;
2946 
2947 	/*
2948 	 * Initialize id_port before ibt_open_hca because of
2949 	 * ordering requirements in port up/down handling.
2950 	 */
2951 	if (ibd_get_portpkey(state, &hca_guid) != DDI_SUCCESS)
2952 		return (DDI_FAILURE);
2953 
2954 	if (ibt_open_hca(state->id_ibt_hdl, hca_guid,
2955 	    &state->id_hca_hdl) != IBT_SUCCESS) {
2956 		DPRINT(10, "ibd_drv_init : failed in ibt_open_hca()\n");
2957 		return (DDI_FAILURE);
2958 	}
2959 
2960 	mutex_enter(&state->id_link_mutex);
2961 	ibt_status = ibt_query_hca_ports(state->id_hca_hdl,
2962 	    state->id_port, &port_infop, &psize,
2963 	    &port_infosz);
2964 	if ((ibt_status != IBT_SUCCESS) || (psize != 1)) {
2965 		mutex_exit(&state->id_link_mutex);
2966 		DPRINT(10, "ibd_drv_init : failed in ibt_query_port()\n");
2967 		(void) ibt_close_hca(state->id_hca_hdl);
2968 		return (DDI_FAILURE);
2969 	}
2970 
2971 	/*
2972 	 * If the link already went down by the time we get here, give up;
2973 	 * we can not even get the gid since that is not valid. We would
2974 	 * fail in ibd_find_bgroup() anyway.
2975 	 */
2976 	if (port_infop->p_linkstate != IBT_PORT_ACTIVE) {
2977 		mutex_exit(&state->id_link_mutex);
2978 		ibt_free_portinfo(port_infop, port_infosz);
2979 		(void) ibt_close_hca(state->id_hca_hdl);
2980 		ibd_print_warn(state, "Port is not active");
2981 		return (DDI_FAILURE);
2982 	}
2983 
2984 	/*
2985 	 * This verifies the Pkey ibnexus handed us is still valid.
2986 	 * This is also the point from which the pkey table for the
2987 	 * port must hold the exact pkey value at the exact index
2988 	 * across port up/downs.
2989 	 */
2990 	if (ibt_pkey2index(state->id_hca_hdl, state->id_port,
2991 	    state->id_pkey, &state->id_pkix) != IBT_SUCCESS) {
2992 		mutex_exit(&state->id_link_mutex);
2993 		ibt_free_portinfo(port_infop, port_infosz);
2994 		DPRINT(10, "ibd_drv_init : failed in ibt_pkey2index()\n");
2995 		(void) ibt_close_hca(state->id_hca_hdl);
2996 		return (DDI_FAILURE);
2997 	}
2998 
2999 	state->id_mtu = (128 << port_infop->p_mtu);
3000 	state->id_sgid = *port_infop->p_sgid_tbl;
3001 	state->id_link_state = LINK_STATE_UP;
3002 	mutex_exit(&state->id_link_mutex);
3003 
3004 	ibt_free_portinfo(port_infop, port_infosz);
3005 
3006 	state->id_link_speed = ibd_get_portspeed(state);
3007 
3008 	/*
3009 	 * Read drv conf and record what the policy is on enabling LSO
3010 	 */
3011 	if (ddi_prop_get_int(DDI_DEV_T_ANY, state->id_dip,
3012 	    DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, IBD_PROP_LSO_POLICY, 1)) {
3013 		state->id_lso_policy = B_TRUE;
3014 	} else {
3015 		state->id_lso_policy = B_FALSE;
3016 	}
3017 
3018 	ibt_status = ibt_query_hca(state->id_hca_hdl, &hca_attrs);
3019 	ASSERT(ibt_status == IBT_SUCCESS);
3020 
3021 	if (ibd_find_bgroup(state) != IBT_SUCCESS) {
3022 		DPRINT(10, "ibd_drv_init : failed in ibd_find_bgroup\n");
3023 		goto drv_init_fail_find_bgroup;
3024 	}
3025 
3026 	if (ibt_alloc_pd(state->id_hca_hdl, IBT_PD_NO_FLAGS,
3027 	    &state->id_pd_hdl) != IBT_SUCCESS) {
3028 		DPRINT(10, "ibd_drv_init : failed in ibt_alloc_pd()\n");
3029 		goto drv_init_fail_alloc_pd;
3030 	}
3031 
3032 	/* Initialize the parallel ARP cache and AHs */
3033 	if (ibd_acache_init(state) != DDI_SUCCESS) {
3034 		DPRINT(10, "ibd_drv_init : failed in ibd_acache_init()\n");
3035 		goto drv_init_fail_acache;
3036 	}
3037 
3038 	if ((hca_attrs.hca_flags2 & IBT_HCA2_RES_LKEY) == IBT_HCA2_RES_LKEY) {
3039 		state->id_hca_res_lkey_capab = 1;
3040 		state->id_res_lkey = hca_attrs.hca_reserved_lkey;
3041 	}
3042 
3043 	/*
3044 	 * Check various tunable limits.
3045 	 */
3046 
3047 	/*
3048 	 * See if extended sgl size information is provided by the hca; if yes,
3049 	 * use the correct one and set the maximum sqseg value.
3050 	 */
3051 	if (hca_attrs.hca_flags & IBT_HCA_WQE_SIZE_INFO)
3052 		state->id_max_sqseg = hca_attrs.hca_ud_send_sgl_sz;
3053 	else
3054 		state->id_max_sqseg = hca_attrs.hca_max_sgl;
3055 
3056 	/*
3057 	 * Set LSO capability and maximum length
3058 	 */
3059 	if (hca_attrs.hca_max_lso_size > 0) {
3060 		state->id_lso_capable = B_TRUE;
3061 		if (hca_attrs.hca_max_lso_size > IBD_LSO_MAXLEN)
3062 			state->id_lso_maxlen = IBD_LSO_MAXLEN;
3063 		else
3064 			state->id_lso_maxlen = hca_attrs.hca_max_lso_size;
3065 	} else {
3066 		state->id_lso_capable = B_FALSE;
3067 		state->id_lso_maxlen = 0;
3068 	}
3069 
3070 
3071 	/*
3072 	 * Check #r/s wqes against max channel size.
3073 	 */
3074 	if (hca_attrs.hca_max_chan_sz < IBD_NUM_RWQE)
3075 		state->id_num_rwqe = hca_attrs.hca_max_chan_sz;
3076 	else
3077 		state->id_num_rwqe = IBD_NUM_RWQE;
3078 
3079 	if (hca_attrs.hca_max_chan_sz < IBD_NUM_SWQE)
3080 		state->id_num_swqe = hca_attrs.hca_max_chan_sz;
3081 	else
3082 		state->id_num_swqe = IBD_NUM_SWQE;
3083 
3084 	/*
3085 	 * Check the hardware checksum capability. Currently we only consider
3086 	 * full checksum offload.
3087 	 */
3088 	if ((hca_attrs.hca_flags & IBT_HCA_CKSUM_FULL) == IBT_HCA_CKSUM_FULL) {
3089 		state->id_hwcksum_capab = IBT_HCA_CKSUM_FULL;
3090 	}
3091 
3092 	/*
3093 	 * Allocate Rx/combined CQ:
3094 	 * Theoretically, there is no point in having more than #rwqe
3095 	 * plus #swqe cqe's, except that the CQ will be signalled for
3096 	 * overflow when the last wqe completes, if none of the previous
3097 	 * cqe's have been polled. Thus, we allocate just a few less wqe's
3098 	 * to make sure such overflow does not occur.
3099 	 */
3100 	cq_attr.cq_sched = NULL;
3101 	cq_attr.cq_flags = IBT_CQ_NO_FLAGS;
3102 
3103 	if (ibd_separate_cqs == 1) {
3104 		/*
3105 		 * Allocate Receive CQ.
3106 		 */
3107 		if (hca_attrs.hca_max_cq_sz >= (state->id_num_rwqe + 1)) {
3108 			cq_attr.cq_size = state->id_num_rwqe + 1;
3109 		} else {
3110 			cq_attr.cq_size = hca_attrs.hca_max_cq_sz;
3111 			state->id_num_rwqe = cq_attr.cq_size - 1;
3112 		}
3113 
3114 		if (ibt_alloc_cq(state->id_hca_hdl, &cq_attr,
3115 		    &state->id_rcq_hdl, &real_size) != IBT_SUCCESS) {
3116 			DPRINT(10, "ibd_drv_init : failed in ibt_alloc_cq()\n");
3117 			goto drv_init_fail_alloc_rcq;
3118 		}
3119 
3120 		if (ibt_modify_cq(state->id_rcq_hdl,
3121 		    ibd_rxcomp_count, ibd_rxcomp_usec, 0) != IBT_SUCCESS) {
3122 			DPRINT(10, "ibd_drv_init: Receive CQ interrupt "
3123 			    "moderation failed\n");
3124 		}
3125 
3126 		state->id_rxwcs_size = state->id_num_rwqe + 1;
3127 		state->id_rxwcs = kmem_alloc(sizeof (ibt_wc_t) *
3128 		    state->id_rxwcs_size, KM_SLEEP);
3129 
3130 		/*
3131 		 * Allocate Send CQ.
3132 		 */
3133 		if (hca_attrs.hca_max_cq_sz >= (state->id_num_swqe + 1)) {
3134 			cq_attr.cq_size = state->id_num_swqe + 1;
3135 		} else {
3136 			cq_attr.cq_size = hca_attrs.hca_max_cq_sz;
3137 			state->id_num_swqe = cq_attr.cq_size - 1;
3138 		}
3139 
3140 		if (ibt_alloc_cq(state->id_hca_hdl, &cq_attr,
3141 		    &state->id_scq_hdl, &real_size) != IBT_SUCCESS) {
3142 			DPRINT(10, "ibd_drv_init : failed in ibt_alloc_cq()\n");
3143 			goto drv_init_fail_alloc_scq;
3144 		}
3145 		if (ibt_modify_cq(state->id_scq_hdl,
3146 		    10, 300, 0) != IBT_SUCCESS) {
3147 			DPRINT(10, "ibd_drv_init: Send CQ interrupt "
3148 			    "moderation failed\n");
3149 		}
3150 
3151 		state->id_txwcs_size = state->id_num_swqe + 1;
3152 		state->id_txwcs = kmem_alloc(sizeof (ibt_wc_t) *
3153 		    state->id_txwcs_size, KM_SLEEP);
3154 	} else {
3155 		/*
3156 		 * Allocate combined Send/Receive CQ.
3157 		 */
3158 		if (hca_attrs.hca_max_cq_sz >= (state->id_num_rwqe +
3159 		    state->id_num_swqe + 1)) {
3160 			cq_attr.cq_size = state->id_num_rwqe +
3161 			    state->id_num_swqe + 1;
3162 		} else {
3163 			cq_attr.cq_size = hca_attrs.hca_max_cq_sz;
3164 			state->id_num_rwqe = ((cq_attr.cq_size - 1) *
3165 			    state->id_num_rwqe) / (state->id_num_rwqe +
3166 			    state->id_num_swqe);
3167 			state->id_num_swqe = cq_attr.cq_size - 1 -
3168 			    state->id_num_rwqe;
3169 		}
3170 
3171 		state->id_rxwcs_size = cq_attr.cq_size;
3172 		state->id_txwcs_size = state->id_rxwcs_size;
3173 
3174 		if (ibt_alloc_cq(state->id_hca_hdl, &cq_attr,
3175 		    &state->id_rcq_hdl, &real_size) != IBT_SUCCESS) {
3176 			DPRINT(10, "ibd_drv_init : failed in ibt_alloc_cq()\n");
3177 			goto drv_init_fail_alloc_rcq;
3178 		}
3179 		state->id_scq_hdl = state->id_rcq_hdl;
3180 		state->id_rxwcs = kmem_alloc(sizeof (ibt_wc_t) *
3181 		    state->id_rxwcs_size, KM_SLEEP);
3182 		state->id_txwcs = state->id_rxwcs;
3183 	}
3184 
3185 	/*
3186 	 * Print message in case we could not allocate as many wqe's
3187 	 * as was requested. Note that in the combined CQ case, we will
3188 	 * get the following message.
3189 	 */
3190 	if (state->id_num_rwqe != IBD_NUM_RWQE)
3191 		ibd_print_warn(state, "Setting #rwqe = %d instead of default "
3192 		    "%d", state->id_num_rwqe, IBD_NUM_RWQE);
3193 	if (state->id_num_swqe != IBD_NUM_SWQE)
3194 		ibd_print_warn(state, "Setting #swqe = %d instead of default "
3195 		    "%d", state->id_num_swqe, IBD_NUM_SWQE);
3196 
3197 	ud_alloc_attr.ud_flags  = IBT_WR_SIGNALED;
3198 	if (state->id_hca_res_lkey_capab)
3199 		ud_alloc_attr.ud_flags |= IBT_FAST_REG_RES_LKEY;
3200 	if (state->id_lso_policy && state->id_lso_capable)
3201 		ud_alloc_attr.ud_flags |= IBT_USES_LSO;
3202 
3203 	ud_alloc_attr.ud_hca_port_num	= state->id_port;
3204 	ud_alloc_attr.ud_sizes.cs_sq_sgl = state->id_max_sqseg;
3205 	ud_alloc_attr.ud_sizes.cs_rq_sgl = IBD_MAX_RQSEG;
3206 	ud_alloc_attr.ud_sizes.cs_sq	= state->id_num_swqe;
3207 	ud_alloc_attr.ud_sizes.cs_rq	= state->id_num_rwqe;
3208 	ud_alloc_attr.ud_qkey		= state->id_mcinfo->mc_qkey;
3209 	ud_alloc_attr.ud_scq		= state->id_scq_hdl;
3210 	ud_alloc_attr.ud_rcq		= state->id_rcq_hdl;
3211 	ud_alloc_attr.ud_pd		= state->id_pd_hdl;
3212 	ud_alloc_attr.ud_pkey_ix	= state->id_pkix;
3213 	ud_alloc_attr.ud_clone_chan	= NULL;
3214 
3215 	if (ibt_alloc_ud_channel(state->id_hca_hdl, IBT_ACHAN_NO_FLAGS,
3216 	    &ud_alloc_attr, &state->id_chnl_hdl, NULL) != IBT_SUCCESS) {
3217 		DPRINT(10, "ibd_drv_init : failed in ibt_alloc_ud_channel()"
3218 		    "\n");
3219 		goto drv_init_fail_alloc_chan;
3220 	}
3221 
3222 	if (ibt_query_ud_channel(state->id_chnl_hdl, &ud_chan_attr) !=
3223 	    DDI_SUCCESS) {
3224 		DPRINT(10, "ibd_drv_init : failed in ibt_query_ud_channel()");
3225 		goto drv_init_fail_query_chan;
3226 	}
3227 
3228 	state->id_qpnum = ud_chan_attr.ud_qpn;
3229 	/* state->id_max_sqseg = ud_chan_attr.ud_chan_sizes.cs_sq_sgl; */
3230 
3231 	if (state->id_max_sqseg > IBD_MAX_SQSEG) {
3232 		state->id_max_sqseg = IBD_MAX_SQSEG;
3233 	} else if (state->id_max_sqseg < IBD_MAX_SQSEG) {
3234 		ibd_print_warn(state, "Set #sgl = %d instead of default %d",
3235 		    state->id_max_sqseg, IBD_MAX_SQSEG);
3236 	}
3237 
3238 	/* Initialize the Transmit buffer list */
3239 	if (ibd_init_txlist(state) != DDI_SUCCESS) {
3240 		DPRINT(10, "ibd_drv_init : failed in ibd_init_txlist()\n");
3241 		goto drv_init_fail_txlist_init;
3242 	}
3243 
3244 	if ((ibd_separate_cqs == 1) && (ibd_txcomp_poll == 0)) {
3245 		/*
3246 		 * Setup the handler we will use for regular DLPI stuff
3247 		 */
3248 		ibt_set_cq_handler(state->id_scq_hdl, ibd_scq_handler, state);
3249 		if (ibt_enable_cq_notify(state->id_scq_hdl,
3250 		    IBT_NEXT_COMPLETION) != IBT_SUCCESS) {
3251 			DPRINT(10, "ibd_drv_init : failed in"
3252 			    " ibt_enable_cq_notify()\n");
3253 			goto drv_init_fail_cq_notify;
3254 		}
3255 	}
3256 
3257 	/* Initialize the Receive buffer list */
3258 	if (ibd_init_rxlist(state) != DDI_SUCCESS) {
3259 		DPRINT(10, "ibd_drv_init : failed in ibd_init_rxlist()\n");
3260 		goto drv_init_fail_rxlist_init;
3261 	}
3262 
3263 	/* Join to IPoIB broadcast group as required by IPoIB */
3264 	if (ibd_join_group(state, state->id_mgid, IB_MC_JSTATE_FULL) == NULL) {
3265 		DPRINT(10, "ibd_drv_init : failed in ibd_join_group\n");
3266 		goto drv_init_fail_join_group;
3267 	}
3268 
3269 	/*
3270 	 * Create the async thread; thread_create never fails.
3271 	 */
3272 	kht = thread_create(NULL, 0, ibd_async_work, state, 0, &p0,
3273 	    TS_RUN, minclsyspri);
3274 
3275 	state->id_async_thrid = kht->t_did;
3276 
3277 	/*
3278 	 * The local mac address is now known. Create the IPoIB
3279 	 * address.
3280 	 */
3281 	ibd_h2n_mac(&state->id_macaddr, state->id_qpnum,
3282 	    state->id_sgid.gid_prefix, state->id_sgid.gid_guid);
3283 	/*
3284 	 * Similarly, program in the broadcast mac address.
3285 	 */
3286 	ibd_h2n_mac(&state->id_bcaddr, IB_QPN_MASK, state->id_mgid.gid_prefix,
3287 	    state->id_mgid.gid_guid);
3288 
3289 	ptr = (uint32_t *)&state->id_macaddr;
3290 	DPRINT(10, "ibd_drv_init : INFO: MAC %08X:%08X:%08X:%08X:%08X\n",
3291 	    *ptr, *(ptr+1), *(ptr+2), *(ptr+3), *(ptr+4));
3292 	ptr = (uint32_t *)&state->id_bcaddr;
3293 	DPRINT(10, "ibd_drv_init : INFO: BCMAC %08X:%08X:%08X:%08X:%08X\n",
3294 	    *ptr, *(ptr+1), *(ptr+2), *(ptr+3), *(ptr+4));
3295 	DPRINT(10, "ibd_drv_init : INFO: Pkey 0x%x, Mgid %016llx%016llx\n",
3296 	    state->id_pkey, state->id_mgid.gid_prefix,
3297 	    state->id_mgid.gid_guid);
3298 	DPRINT(10, "ibd_drv_init : INFO: GID %016llx%016llx\n",
3299 	    state->id_sgid.gid_prefix, state->id_sgid.gid_guid);
3300 	DPRINT(10, "ibd_drv_init : INFO: PKEY %04x\n", state->id_pkey);
3301 	DPRINT(10, "ibd_drv_init : INFO: MTU %d\n", state->id_mtu);
3302 	(void) ddi_pathname(state->id_dip, pathname);
3303 	DPRINT(10, "ibd_drv_init : INFO: Pathname %s\n", pathname);
3304 
3305 	return (DDI_SUCCESS);
3306 
3307 drv_init_fail_join_group:
3308 	ibd_fini_rxlist(state);
3309 
3310 drv_init_fail_rxlist_init:
3311 drv_init_fail_cq_notify:
3312 	ibd_fini_txlist(state);
3313 
3314 drv_init_fail_txlist_init:
3315 drv_init_fail_query_chan:
3316 	if (ibt_free_channel(state->id_chnl_hdl) != IBT_SUCCESS)
3317 		DPRINT(10, "ibd_drv_init : failed in ibt_free_channel()");
3318 
3319 drv_init_fail_alloc_chan:
3320 	if ((ibd_separate_cqs == 1) && (ibt_free_cq(state->id_scq_hdl) !=
3321 	    IBT_SUCCESS))
3322 		DPRINT(10, "ibd_drv_init : Tx ibt_free_cq()");
3323 
3324 	if (ibd_separate_cqs == 1)
3325 		kmem_free(state->id_txwcs, sizeof (ibt_wc_t) *
3326 		    state->id_txwcs_size);
3327 
3328 drv_init_fail_alloc_scq:
3329 	if (ibt_free_cq(state->id_rcq_hdl) != IBT_SUCCESS)
3330 		DPRINT(10, "ibd_drv_init : Rx ibt_free_cq()");
3331 	kmem_free(state->id_rxwcs, sizeof (ibt_wc_t) * state->id_rxwcs_size);
3332 
3333 drv_init_fail_alloc_rcq:
3334 	ibd_acache_fini(state);
3335 drv_init_fail_acache:
3336 	if (ibt_free_pd(state->id_hca_hdl, state->id_pd_hdl) != IBT_SUCCESS)
3337 		DPRINT(10, "ibd_drv_init : failed in ibt_free_pd()");
3338 
3339 drv_init_fail_alloc_pd:
3340 	ibt_free_mcg_info(state->id_mcinfo, 1);
3341 drv_init_fail_find_bgroup:
3342 	if (ibt_close_hca(state->id_hca_hdl) != IBT_SUCCESS)
3343 		DPRINT(10, "ibd_drv_init : failed in ibt_close_hca()");
3344 
3345 	return (DDI_FAILURE);
3346 }
3347 
3348 
3349 static int
3350 ibd_alloc_tx_copybufs(ibd_state_t *state)
3351 {
3352 	ibt_mr_attr_t mem_attr;
3353 
3354 	/*
3355 	 * Allocate one big chunk for all regular tx copy bufs
3356 	 */
3357 	state->id_tx_buf_sz = state->id_mtu;
3358 	if (state->id_lso_policy && state->id_lso_capable &&
3359 	    (IBD_TX_BUF_SZ > state->id_mtu)) {
3360 		state->id_tx_buf_sz = IBD_TX_BUF_SZ;
3361 	}
3362 
3363 	state->id_tx_bufs = kmem_zalloc(state->id_num_swqe *
3364 	    state->id_tx_buf_sz, KM_SLEEP);
3365 
3366 	/*
3367 	 * Do one memory registration on the entire txbuf area
3368 	 */
3369 	mem_attr.mr_vaddr = (uint64_t)(uintptr_t)state->id_tx_bufs;
3370 	mem_attr.mr_len = state->id_num_swqe * state->id_tx_buf_sz;
3371 	mem_attr.mr_as = NULL;
3372 	mem_attr.mr_flags = IBT_MR_SLEEP;
3373 	if (ibt_register_mr(state->id_hca_hdl, state->id_pd_hdl, &mem_attr,
3374 	    &state->id_tx_mr_hdl, &state->id_tx_mr_desc) != IBT_SUCCESS) {
3375 		DPRINT(10, "ibd_alloc_tx_copybufs: ibt_register_mr failed");
3376 		kmem_free(state->id_tx_bufs,
3377 		    state->id_num_swqe * state->id_tx_buf_sz);
3378 		state->id_tx_bufs = NULL;
3379 		return (DDI_FAILURE);
3380 	}
3381 
3382 	return (DDI_SUCCESS);
3383 }
3384 
3385 static int
3386 ibd_alloc_tx_lsobufs(ibd_state_t *state)
3387 {
3388 	ibt_mr_attr_t mem_attr;
3389 	ibd_lsobuf_t *buflist;
3390 	ibd_lsobuf_t *lbufp;
3391 	ibd_lsobuf_t *tail;
3392 	ibd_lsobkt_t *bktp;
3393 	uint8_t *membase;
3394 	uint8_t *memp;
3395 	uint_t memsz;
3396 	int i;
3397 
3398 	/*
3399 	 * Allocate the lso bucket
3400 	 */
3401 	bktp = kmem_zalloc(sizeof (ibd_lsobkt_t), KM_SLEEP);
3402 
3403 	/*
3404 	 * Allocate the entire lso memory and register it
3405 	 */
3406 	memsz = IBD_NUM_LSO_BUFS * IBD_LSO_BUFSZ;
3407 	membase = kmem_zalloc(memsz, KM_SLEEP);
3408 
3409 	mem_attr.mr_vaddr = (uint64_t)(uintptr_t)membase;
3410 	mem_attr.mr_len = memsz;
3411 	mem_attr.mr_as = NULL;
3412 	mem_attr.mr_flags = IBT_MR_SLEEP;
3413 	if (ibt_register_mr(state->id_hca_hdl, state->id_pd_hdl,
3414 	    &mem_attr, &bktp->bkt_mr_hdl, &bktp->bkt_mr_desc) != IBT_SUCCESS) {
3415 		DPRINT(10, "ibd_alloc_tx_lsobufs: ibt_register_mr failed");
3416 		kmem_free(membase, memsz);
3417 		kmem_free(bktp, sizeof (ibd_lsobkt_t));
3418 		return (DDI_FAILURE);
3419 	}
3420 
3421 	/*
3422 	 * Now allocate the buflist.  Note that the elements in the buflist and
3423 	 * the buffers in the lso memory have a permanent 1-1 relation, so we
3424 	 * can always derive the address of a buflist entry from the address of
3425 	 * an lso buffer.
3426 	 */
3427 	buflist = kmem_zalloc(IBD_NUM_LSO_BUFS * sizeof (ibd_lsobuf_t),
3428 	    KM_SLEEP);
3429 
3430 	/*
3431 	 * Set up the lso buf chain
3432 	 */
3433 	memp = membase;
3434 	lbufp = buflist;
3435 	for (i = 0; i < IBD_NUM_LSO_BUFS; i++) {
3436 		lbufp->lb_isfree = 1;
3437 		lbufp->lb_buf = memp;
3438 		lbufp->lb_next = lbufp + 1;
3439 
3440 		tail = lbufp;
3441 
3442 		memp += IBD_LSO_BUFSZ;
3443 		lbufp++;
3444 	}
3445 	tail->lb_next = NULL;
3446 
3447 	/*
3448 	 * Set up the LSO buffer information in ibd state
3449 	 */
3450 	bktp->bkt_bufl = buflist;
3451 	bktp->bkt_free_head = buflist;
3452 	bktp->bkt_mem = membase;
3453 	bktp->bkt_nelem = IBD_NUM_LSO_BUFS;
3454 	bktp->bkt_nfree = bktp->bkt_nelem;
3455 
3456 	state->id_lso = bktp;
3457 
3458 	return (DDI_SUCCESS);
3459 }
3460 
3461 /*
3462  * Statically allocate Tx buffer list(s).
3463  */
3464 static int
3465 ibd_init_txlist(ibd_state_t *state)
3466 {
3467 	ibd_swqe_t *swqe;
3468 	ibt_lkey_t lkey;
3469 	int i;
3470 
3471 	if (ibd_alloc_tx_copybufs(state) != DDI_SUCCESS)
3472 		return (DDI_FAILURE);
3473 
3474 	if (state->id_lso_policy && state->id_lso_capable) {
3475 		if (ibd_alloc_tx_lsobufs(state) != DDI_SUCCESS)
3476 			state->id_lso_policy = B_FALSE;
3477 	}
3478 
3479 	/*
3480 	 * Allocate and setup the swqe list
3481 	 */
3482 	lkey = state->id_tx_mr_desc.md_lkey;
3483 	for (i = 0; i < state->id_num_swqe; i++) {
3484 		if (ibd_alloc_swqe(state, &swqe, i, lkey) != DDI_SUCCESS) {
3485 			DPRINT(10, "ibd_init_txlist: ibd_alloc_swqe failed");
3486 			ibd_fini_txlist(state);
3487 			return (DDI_FAILURE);
3488 		}
3489 
3490 		/* add to list */
3491 		state->id_tx_list.dl_cnt++;
3492 		if (state->id_tx_list.dl_head == NULL) {
3493 			swqe->swqe_prev = NULL;
3494 			swqe->swqe_next = NULL;
3495 			state->id_tx_list.dl_head = SWQE_TO_WQE(swqe);
3496 			state->id_tx_list.dl_tail = SWQE_TO_WQE(swqe);
3497 		} else {
3498 			swqe->swqe_prev = state->id_tx_list.dl_tail;
3499 			swqe->swqe_next = NULL;
3500 			state->id_tx_list.dl_tail->w_next = SWQE_TO_WQE(swqe);
3501 			state->id_tx_list.dl_tail = SWQE_TO_WQE(swqe);
3502 		}
3503 	}
3504 
3505 	return (DDI_SUCCESS);
3506 }
3507 
3508 static int
3509 ibd_acquire_lsobufs(ibd_state_t *state, uint_t req_sz, ibt_wr_ds_t *sgl_p,
3510     uint32_t *nds_p)
3511 {
3512 	ibd_lsobkt_t *bktp;
3513 	ibd_lsobuf_t *lbufp;
3514 	ibd_lsobuf_t *nextp;
3515 	ibt_lkey_t lso_lkey;
3516 	uint_t frag_sz;
3517 	uint_t num_needed;
3518 	int i;
3519 
3520 	ASSERT(sgl_p != NULL);
3521 	ASSERT(nds_p != NULL);
3522 	ASSERT(req_sz != 0);
3523 
3524 	/*
3525 	 * Determine how many bufs we'd need for the size requested
3526 	 */
3527 	num_needed = req_sz / IBD_LSO_BUFSZ;
3528 	if ((frag_sz = req_sz % IBD_LSO_BUFSZ) != 0)
3529 		num_needed++;
3530 
3531 	mutex_enter(&state->id_lso_lock);
3532 
3533 	/*
3534 	 * If we don't have enough lso bufs, return failure
3535 	 */
3536 	ASSERT(state->id_lso != NULL);
3537 	bktp = state->id_lso;
3538 	if (bktp->bkt_nfree < num_needed) {
3539 		mutex_exit(&state->id_lso_lock);
3540 		return (-1);
3541 	}
3542 
3543 	/*
3544 	 * Pick the first 'num_needed' bufs from the free list
3545 	 */
3546 	lso_lkey = bktp->bkt_mr_desc.md_lkey;
3547 	lbufp = bktp->bkt_free_head;
3548 	for (i = 0; i < num_needed; i++) {
3549 		ASSERT(lbufp->lb_isfree != 0);
3550 		ASSERT(lbufp->lb_buf != NULL);
3551 
3552 		nextp = lbufp->lb_next;
3553 
3554 		sgl_p[i].ds_va = (ib_vaddr_t)(uintptr_t)lbufp->lb_buf;
3555 		sgl_p[i].ds_key = lso_lkey;
3556 		sgl_p[i].ds_len = IBD_LSO_BUFSZ;
3557 
3558 		lbufp->lb_isfree = 0;
3559 		lbufp->lb_next = NULL;
3560 
3561 		lbufp = nextp;
3562 	}
3563 	bktp->bkt_free_head = lbufp;
3564 
3565 	/*
3566 	 * If the requested size is not a multiple of IBD_LSO_BUFSZ, we need
3567 	 * to adjust the last sgl entry's length. Since we know we need atleast
3568 	 * one, the i-1 use below is ok.
3569 	 */
3570 	if (frag_sz) {
3571 		sgl_p[i-1].ds_len = frag_sz;
3572 	}
3573 
3574 	/*
3575 	 * Update nfree count and return
3576 	 */
3577 	bktp->bkt_nfree -= num_needed;
3578 
3579 	mutex_exit(&state->id_lso_lock);
3580 
3581 	*nds_p = num_needed;
3582 
3583 	return (0);
3584 }
3585 
3586 static void
3587 ibd_release_lsobufs(ibd_state_t *state, ibt_wr_ds_t *sgl_p, uint32_t nds)
3588 {
3589 	ibd_lsobkt_t *bktp;
3590 	ibd_lsobuf_t *lbufp;
3591 	uint8_t *lso_mem_end;
3592 	uint_t ndx;
3593 	int i;
3594 
3595 	mutex_enter(&state->id_lso_lock);
3596 
3597 	bktp = state->id_lso;
3598 	ASSERT(bktp != NULL);
3599 
3600 	lso_mem_end = bktp->bkt_mem + bktp->bkt_nelem * IBD_LSO_BUFSZ;
3601 	for (i = 0; i < nds; i++) {
3602 		uint8_t *va;
3603 
3604 		va = (uint8_t *)(uintptr_t)sgl_p[i].ds_va;
3605 		ASSERT(va >= bktp->bkt_mem && va < lso_mem_end);
3606 
3607 		/*
3608 		 * Figure out the buflist element this sgl buffer corresponds
3609 		 * to and put it back at the head
3610 		 */
3611 		ndx = (va - bktp->bkt_mem) / IBD_LSO_BUFSZ;
3612 		lbufp = bktp->bkt_bufl + ndx;
3613 
3614 		ASSERT(lbufp->lb_isfree == 0);
3615 		ASSERT(lbufp->lb_buf == va);
3616 
3617 		lbufp->lb_isfree = 1;
3618 		lbufp->lb_next = bktp->bkt_free_head;
3619 		bktp->bkt_free_head = lbufp;
3620 	}
3621 	bktp->bkt_nfree += nds;
3622 
3623 	mutex_exit(&state->id_lso_lock);
3624 }
3625 
3626 static void
3627 ibd_free_tx_copybufs(ibd_state_t *state)
3628 {
3629 	/*
3630 	 * Unregister txbuf mr
3631 	 */
3632 	if (ibt_deregister_mr(state->id_hca_hdl,
3633 	    state->id_tx_mr_hdl) != IBT_SUCCESS) {
3634 		DPRINT(10, "ibd_free_tx_copybufs: ibt_deregister_mr failed");
3635 	}
3636 	state->id_tx_mr_hdl = NULL;
3637 
3638 	/*
3639 	 * Free txbuf memory
3640 	 */
3641 	kmem_free(state->id_tx_bufs, state->id_num_swqe * state->id_tx_buf_sz);
3642 	state->id_tx_bufs = NULL;
3643 }
3644 
3645 static void
3646 ibd_free_tx_lsobufs(ibd_state_t *state)
3647 {
3648 	ibd_lsobkt_t *bktp;
3649 
3650 	mutex_enter(&state->id_lso_lock);
3651 
3652 	if ((bktp = state->id_lso) == NULL) {
3653 		mutex_exit(&state->id_lso_lock);
3654 		return;
3655 	}
3656 
3657 	/*
3658 	 * First, free the buflist
3659 	 */
3660 	ASSERT(bktp->bkt_bufl != NULL);
3661 	kmem_free(bktp->bkt_bufl, bktp->bkt_nelem * sizeof (ibd_lsobuf_t));
3662 
3663 	/*
3664 	 * Unregister the LSO memory and free it
3665 	 */
3666 	ASSERT(bktp->bkt_mr_hdl != NULL);
3667 	if (ibt_deregister_mr(state->id_hca_hdl,
3668 	    bktp->bkt_mr_hdl) != IBT_SUCCESS) {
3669 		DPRINT(10,
3670 		    "ibd_free_lsobufs: ibt_deregister_mr failed");
3671 	}
3672 	ASSERT(bktp->bkt_mem);
3673 	kmem_free(bktp->bkt_mem, bktp->bkt_nelem * IBD_LSO_BUFSZ);
3674 
3675 	/*
3676 	 * Finally free the bucket
3677 	 */
3678 	kmem_free(bktp, sizeof (ibd_lsobkt_t));
3679 	state->id_lso = NULL;
3680 
3681 	mutex_exit(&state->id_lso_lock);
3682 }
3683 
3684 /*
3685  * Free the statically allocated Tx buffer list.
3686  */
3687 static void
3688 ibd_fini_txlist(ibd_state_t *state)
3689 {
3690 	ibd_swqe_t *node;
3691 
3692 	/*
3693 	 * Free the allocated swqes
3694 	 */
3695 	mutex_enter(&state->id_tx_list.dl_mutex);
3696 	while (state->id_tx_list.dl_head != NULL) {
3697 		node = WQE_TO_SWQE(state->id_tx_list.dl_head);
3698 		state->id_tx_list.dl_head = node->swqe_next;
3699 		state->id_tx_list.dl_cnt--;
3700 		ASSERT(state->id_tx_list.dl_cnt >= 0);
3701 		ibd_free_swqe(state, node);
3702 	}
3703 	mutex_exit(&state->id_tx_list.dl_mutex);
3704 
3705 	ibd_free_tx_lsobufs(state);
3706 	ibd_free_tx_copybufs(state);
3707 }
3708 
3709 /*
3710  * Allocate a single send wqe and register it so it is almost
3711  * ready to be posted to the hardware.
3712  */
3713 static int
3714 ibd_alloc_swqe(ibd_state_t *state, ibd_swqe_t **wqe, int ndx, ibt_lkey_t lkey)
3715 {
3716 	ibd_swqe_t *swqe;
3717 
3718 	swqe = kmem_zalloc(sizeof (ibd_swqe_t), KM_SLEEP);
3719 	*wqe = swqe;
3720 
3721 	swqe->swqe_type = IBD_WQE_SEND;
3722 	swqe->swqe_next = NULL;
3723 	swqe->swqe_prev = NULL;
3724 	swqe->swqe_im_mblk = NULL;
3725 
3726 	swqe->swqe_copybuf.ic_sgl.ds_va = (ib_vaddr_t)(uintptr_t)
3727 	    (state->id_tx_bufs + ndx * state->id_tx_buf_sz);
3728 	swqe->swqe_copybuf.ic_sgl.ds_key = lkey;
3729 	swqe->swqe_copybuf.ic_sgl.ds_len = 0; /* set in send */
3730 
3731 	swqe->w_swr.wr_id = (ibt_wrid_t)(uintptr_t)swqe;
3732 	swqe->w_swr.wr_flags = IBT_WR_SEND_SIGNAL;
3733 	swqe->w_swr.wr_trans = IBT_UD_SRV;
3734 
3735 	/* These are set in send */
3736 	swqe->w_swr.wr_nds = 0;
3737 	swqe->w_swr.wr_sgl = NULL;
3738 	swqe->w_swr.wr_opcode = IBT_WRC_SEND;
3739 
3740 	return (DDI_SUCCESS);
3741 }
3742 
3743 /*
3744  * Free an allocated send wqe.
3745  */
3746 /*ARGSUSED*/
3747 static void
3748 ibd_free_swqe(ibd_state_t *state, ibd_swqe_t *swqe)
3749 {
3750 	kmem_free(swqe, sizeof (ibd_swqe_t));
3751 }
3752 
3753 /*
3754  * Post a rwqe to the hardware and add it to the Rx list. The
3755  * "recycle" parameter indicates whether an old rwqe is being
3756  * recycled, or this is a new one.
3757  */
3758 static int
3759 ibd_post_rwqe(ibd_state_t *state, ibd_rwqe_t *rwqe, boolean_t recycle)
3760 {
3761 	ibt_status_t ibt_status;
3762 
3763 	if (recycle == B_FALSE) {
3764 		mutex_enter(&state->id_rx_list.dl_mutex);
3765 		if (state->id_rx_list.dl_head == NULL) {
3766 			rwqe->rwqe_prev = NULL;
3767 			rwqe->rwqe_next = NULL;
3768 			state->id_rx_list.dl_head = RWQE_TO_WQE(rwqe);
3769 			state->id_rx_list.dl_tail = RWQE_TO_WQE(rwqe);
3770 		} else {
3771 			rwqe->rwqe_prev = state->id_rx_list.dl_tail;
3772 			rwqe->rwqe_next = NULL;
3773 			state->id_rx_list.dl_tail->w_next = RWQE_TO_WQE(rwqe);
3774 			state->id_rx_list.dl_tail = RWQE_TO_WQE(rwqe);
3775 		}
3776 		mutex_exit(&state->id_rx_list.dl_mutex);
3777 	}
3778 
3779 	mutex_enter(&state->id_rxpost_lock);
3780 	if (state->id_rx_busy) {
3781 		rwqe->w_post_link = NULL;
3782 		if (state->id_rx_head)
3783 			*(state->id_rx_tailp) = (ibd_wqe_t *)rwqe;
3784 		else
3785 			state->id_rx_head = rwqe;
3786 		state->id_rx_tailp = &(rwqe->w_post_link);
3787 	} else {
3788 		state->id_rx_busy = 1;
3789 		do {
3790 			mutex_exit(&state->id_rxpost_lock);
3791 
3792 			/*
3793 			 * Here we should add dl_cnt before post recv, because
3794 			 * we would have to make sure dl_cnt is updated before
3795 			 * the corresponding ibd_process_rx() is called.
3796 			 */
3797 			atomic_add_32(&state->id_rx_list.dl_cnt, 1);
3798 
3799 			ibt_status = ibt_post_recv(state->id_chnl_hdl,
3800 			    &rwqe->w_rwr, 1, NULL);
3801 			if (ibt_status != IBT_SUCCESS) {
3802 				(void) atomic_add_32_nv(
3803 				    &state->id_rx_list.dl_cnt, -1);
3804 				ibd_print_warn(state, "ibd_post_rwqe: "
3805 				    "posting failed, ret=%d", ibt_status);
3806 				return (DDI_FAILURE);
3807 			}
3808 
3809 			mutex_enter(&state->id_rxpost_lock);
3810 			rwqe = state->id_rx_head;
3811 			if (rwqe) {
3812 				state->id_rx_head =
3813 				    (ibd_rwqe_t *)(rwqe->w_post_link);
3814 			}
3815 		} while (rwqe);
3816 		state->id_rx_busy = 0;
3817 	}
3818 	mutex_exit(&state->id_rxpost_lock);
3819 
3820 	return (DDI_SUCCESS);
3821 }
3822 
3823 /*
3824  * Allocate the statically allocated Rx buffer list.
3825  */
3826 static int
3827 ibd_init_rxlist(ibd_state_t *state)
3828 {
3829 	ibd_rwqe_t *rwqe;
3830 	int i;
3831 
3832 	for (i = 0; i < state->id_num_rwqe; i++) {
3833 		if (ibd_alloc_rwqe(state, &rwqe) != DDI_SUCCESS) {
3834 			ibd_fini_rxlist(state);
3835 			return (DDI_FAILURE);
3836 		}
3837 
3838 		if (ibd_post_rwqe(state, rwqe, B_FALSE) == DDI_FAILURE) {
3839 			ibd_free_rwqe(state, rwqe);
3840 			ibd_fini_rxlist(state);
3841 			return (DDI_FAILURE);
3842 		}
3843 	}
3844 
3845 	return (DDI_SUCCESS);
3846 }
3847 
3848 /*
3849  * Free the statically allocated Rx buffer list.
3850  *
3851  */
3852 static void
3853 ibd_fini_rxlist(ibd_state_t *state)
3854 {
3855 	ibd_rwqe_t *node;
3856 
3857 	mutex_enter(&state->id_rx_list.dl_mutex);
3858 	while (state->id_rx_list.dl_head != NULL) {
3859 		node = WQE_TO_RWQE(state->id_rx_list.dl_head);
3860 		state->id_rx_list.dl_head = state->id_rx_list.dl_head->w_next;
3861 		state->id_rx_list.dl_cnt--;
3862 		ASSERT(state->id_rx_list.dl_cnt >= 0);
3863 
3864 		ibd_free_rwqe(state, node);
3865 	}
3866 	mutex_exit(&state->id_rx_list.dl_mutex);
3867 }
3868 
3869 /*
3870  * Allocate a single recv wqe and register it so it is almost
3871  * ready to be posted to the hardware.
3872  */
3873 static int
3874 ibd_alloc_rwqe(ibd_state_t *state, ibd_rwqe_t **wqe)
3875 {
3876 	ibt_mr_attr_t mem_attr;
3877 	ibd_rwqe_t *rwqe;
3878 
3879 	if ((rwqe = kmem_zalloc(sizeof (ibd_rwqe_t), KM_NOSLEEP)) == NULL) {
3880 		DPRINT(10, "ibd_alloc_rwqe: failed in kmem_alloc");
3881 		return (DDI_FAILURE);
3882 	}
3883 	*wqe = rwqe;
3884 	rwqe->rwqe_type = IBD_WQE_RECV;
3885 	rwqe->w_state = state;
3886 	rwqe->rwqe_next = NULL;
3887 	rwqe->rwqe_prev = NULL;
3888 	rwqe->w_freeing_wqe = B_FALSE;
3889 	rwqe->w_freemsg_cb.free_func = ibd_freemsg_cb;
3890 	rwqe->w_freemsg_cb.free_arg = (char *)rwqe;
3891 
3892 	rwqe->rwqe_copybuf.ic_bufaddr = kmem_alloc(state->id_mtu +
3893 	    IPOIB_GRH_SIZE, KM_NOSLEEP);
3894 	if (rwqe->rwqe_copybuf.ic_bufaddr == NULL) {
3895 		DPRINT(10, "ibd_alloc_rwqe: failed in kmem_alloc");
3896 		kmem_free(rwqe, sizeof (ibd_rwqe_t));
3897 		return (DDI_FAILURE);
3898 	}
3899 
3900 	if ((rwqe->rwqe_im_mblk = desballoc(rwqe->rwqe_copybuf.ic_bufaddr,
3901 	    state->id_mtu + IPOIB_GRH_SIZE, 0, &rwqe->w_freemsg_cb)) ==
3902 	    NULL) {
3903 		DPRINT(10, "ibd_alloc_rwqe : failed in desballoc()");
3904 		kmem_free(rwqe->rwqe_copybuf.ic_bufaddr,
3905 		    state->id_mtu + IPOIB_GRH_SIZE);
3906 		rwqe->rwqe_copybuf.ic_bufaddr = NULL;
3907 		kmem_free(rwqe, sizeof (ibd_rwqe_t));
3908 		return (DDI_FAILURE);
3909 	}
3910 
3911 	mem_attr.mr_vaddr = (uint64_t)(uintptr_t)rwqe->rwqe_copybuf.ic_bufaddr;
3912 	mem_attr.mr_len = state->id_mtu + IPOIB_GRH_SIZE;
3913 	mem_attr.mr_as = NULL;
3914 	mem_attr.mr_flags = IBT_MR_NOSLEEP | IBT_MR_ENABLE_LOCAL_WRITE;
3915 	if (ibt_register_mr(state->id_hca_hdl, state->id_pd_hdl, &mem_attr,
3916 	    &rwqe->rwqe_copybuf.ic_mr_hdl, &rwqe->rwqe_copybuf.ic_mr_desc) !=
3917 	    IBT_SUCCESS) {
3918 		DPRINT(10, "ibd_alloc_rwqe : failed in ibt_register_mem()");
3919 		rwqe->w_freeing_wqe = B_TRUE;
3920 		freemsg(rwqe->rwqe_im_mblk);
3921 		kmem_free(rwqe->rwqe_copybuf.ic_bufaddr,
3922 		    state->id_mtu + IPOIB_GRH_SIZE);
3923 		rwqe->rwqe_copybuf.ic_bufaddr = NULL;
3924 		kmem_free(rwqe, sizeof (ibd_rwqe_t));
3925 		return (DDI_FAILURE);
3926 	}
3927 
3928 	rwqe->rwqe_copybuf.ic_sgl.ds_va =
3929 	    (ib_vaddr_t)(uintptr_t)rwqe->rwqe_copybuf.ic_bufaddr;
3930 	rwqe->rwqe_copybuf.ic_sgl.ds_key =
3931 	    rwqe->rwqe_copybuf.ic_mr_desc.md_lkey;
3932 	rwqe->rwqe_copybuf.ic_sgl.ds_len = state->id_mtu + IPOIB_GRH_SIZE;
3933 	rwqe->w_rwr.wr_id = (ibt_wrid_t)(uintptr_t)rwqe;
3934 	rwqe->w_rwr.wr_nds = 1;
3935 	rwqe->w_rwr.wr_sgl = &rwqe->rwqe_copybuf.ic_sgl;
3936 
3937 	return (DDI_SUCCESS);
3938 }
3939 
3940 /*
3941  * Free an allocated recv wqe.
3942  */
3943 static void
3944 ibd_free_rwqe(ibd_state_t *state, ibd_rwqe_t *rwqe)
3945 {
3946 	if (ibt_deregister_mr(state->id_hca_hdl,
3947 	    rwqe->rwqe_copybuf.ic_mr_hdl) != IBT_SUCCESS) {
3948 		DPRINT(10, "ibd_free_rwqe: failed in ibt_deregister_mr()");
3949 		return;
3950 	}
3951 
3952 	/*
3953 	 * Indicate to the callback function that this rwqe/mblk
3954 	 * should not be recycled. The freemsg() will invoke
3955 	 * ibd_freemsg_cb().
3956 	 */
3957 	if (rwqe->rwqe_im_mblk != NULL) {
3958 		rwqe->w_freeing_wqe = B_TRUE;
3959 		freemsg(rwqe->rwqe_im_mblk);
3960 	}
3961 	kmem_free(rwqe->rwqe_copybuf.ic_bufaddr,
3962 	    state->id_mtu + IPOIB_GRH_SIZE);
3963 	rwqe->rwqe_copybuf.ic_bufaddr = NULL;
3964 	kmem_free(rwqe, sizeof (ibd_rwqe_t));
3965 }
3966 
3967 /*
3968  * Delete the rwqe being freed from the rx list.
3969  */
3970 static void
3971 ibd_delete_rwqe(ibd_state_t *state, ibd_rwqe_t *rwqe)
3972 {
3973 	mutex_enter(&state->id_rx_list.dl_mutex);
3974 	if (state->id_rx_list.dl_head == RWQE_TO_WQE(rwqe))
3975 		state->id_rx_list.dl_head = rwqe->rwqe_next;
3976 	else
3977 		rwqe->rwqe_prev->w_next = rwqe->rwqe_next;
3978 	if (state->id_rx_list.dl_tail == RWQE_TO_WQE(rwqe))
3979 		state->id_rx_list.dl_tail = rwqe->rwqe_prev;
3980 	else
3981 		rwqe->rwqe_next->w_prev = rwqe->rwqe_prev;
3982 	mutex_exit(&state->id_rx_list.dl_mutex);
3983 }
3984 
3985 /*
3986  * Pre ibt_detach() deconstruction.
3987  */
3988 static void
3989 ibd_drv_fini(ibd_state_t *state)
3990 {
3991 	ib_gid_t mgid;
3992 	ibd_mce_t *mce;
3993 	ibt_status_t status;
3994 	uint8_t jstate;
3995 
3996 	/*
3997 	 * Desubscribe from trap notices; we will be tearing down
3998 	 * the mcg lists soon. Make sure the trap handler does nothing
3999 	 * even if it is invoked (ie till we invoke ibt_detach()).
4000 	 */
4001 	ibt_register_subnet_notices(state->id_ibt_hdl, NULL, NULL);
4002 	mutex_enter(&state->id_trap_lock);
4003 	state->id_trap_stop = B_TRUE;
4004 	while (state->id_trap_inprog > 0)
4005 		cv_wait(&state->id_trap_cv, &state->id_trap_lock);
4006 	mutex_exit(&state->id_trap_lock);
4007 
4008 	/*
4009 	 * Flushing the channel ensures that all pending WQE's
4010 	 * are marked with flush_error and handed to the CQ. It
4011 	 * does not guarantee the invocation of the CQ handler.
4012 	 * This call is guaranteed to return successfully for UD QPNs.
4013 	 */
4014 	status = ibt_flush_channel(state->id_chnl_hdl);
4015 	ASSERT(status == IBT_SUCCESS);
4016 
4017 	/*
4018 	 * We possibly need a loop here to wait for all the Tx
4019 	 * callbacks to happen. The Tx handlers will retrieve
4020 	 * held resources like AH ac_ref count, registered memory
4021 	 * and possibly IBD_ASYNC_REAP requests. Rx interrupts were already
4022 	 * turned off (in ibd_detach()); turn off Tx interrupts and
4023 	 * poll. By the time the polling returns an empty indicator,
4024 	 * we are sure we have seen all pending Tx callbacks. Note
4025 	 * that after the ibt_set_cq_handler() returns, the old handler
4026 	 * is guaranteed not to be invoked anymore.
4027 	 */
4028 	if (ibd_separate_cqs == 1)
4029 		ibt_set_cq_handler(state->id_scq_hdl, 0, 0);
4030 	ibd_poll_compq(state, state->id_scq_hdl);
4031 
4032 	/*
4033 	 * No more async requests will be posted since the device has been
4034 	 * unregistered; completion handlers have been turned off, so Tx
4035 	 * handler will not cause any more IBD_ASYNC_REAP requests. Queue a
4036 	 * request for the async thread to exit, which will be serviced
4037 	 * after any pending ones. This can take a while, specially if the
4038 	 * SM is unreachable, since IBMF will slowly timeout each SM request
4039 	 * issued by the async thread. Reap the thread before continuing on,
4040 	 * we do not want it to be lingering in modunloaded code.
4041 	 */
4042 	ibd_queue_work_slot(state, &state->id_ah_req, IBD_ASYNC_EXIT);
4043 	thread_join(state->id_async_thrid);
4044 
4045 	/*
4046 	 * We can not be in promiscuous mode anymore, upper layers
4047 	 * would have made a request to disable it (if ever set previously)
4048 	 * before the detach is allowed to progress to this point; and the
4049 	 * aysnc thread would have processed that request by now. Thus the
4050 	 * nonmember list is guaranteed empty at this point.
4051 	 */
4052 	ASSERT(state->id_prom_op != IBD_OP_COMPLETED);
4053 
4054 	/*
4055 	 * Drop all residual full/non membership. This includes full
4056 	 * membership to the broadcast group, and any nonmembership
4057 	 * acquired during transmits. We do this after the Tx completion
4058 	 * handlers are done, since those might result in some late
4059 	 * leaves; this also eliminates a potential race with that
4060 	 * path wrt the mc full list insert/delete. Trap handling
4061 	 * has also been suppressed at this point. Thus, no locks
4062 	 * are required while traversing the mc full list.
4063 	 */
4064 	DPRINT(2, "ibd_drv_fini : clear full cache entries");
4065 	mce = list_head(&state->id_mc_full);
4066 	while (mce != NULL) {
4067 		mgid = mce->mc_info.mc_adds_vect.av_dgid;
4068 		jstate = mce->mc_jstate;
4069 		mce = list_next(&state->id_mc_full, mce);
4070 		ibd_leave_group(state, mgid, jstate);
4071 	}
4072 
4073 	ibt_free_mcg_info(state->id_mcinfo, 1);
4074 
4075 	/*
4076 	 * Kill the channel now; guaranteed to return successfully
4077 	 * for UD QPNs.
4078 	 */
4079 	status = ibt_free_channel(state->id_chnl_hdl);
4080 	ASSERT(status == IBT_SUCCESS);
4081 
4082 	/*
4083 	 * Kill the CQ; all completion handlers are guaranteed to
4084 	 * have terminated by the time this returns. Since we killed
4085 	 * the QPN above, we can not receive the IBT_CQ_BUSY error.
4086 	 */
4087 	status = ibt_free_cq(state->id_rcq_hdl);
4088 	ASSERT(status == IBT_SUCCESS);
4089 	kmem_free(state->id_rxwcs, sizeof (ibt_wc_t) * state->id_rxwcs_size);
4090 
4091 	if (ibd_separate_cqs == 1) {
4092 		status = ibt_free_cq(state->id_scq_hdl);
4093 		ASSERT(status == IBT_SUCCESS);
4094 		kmem_free(state->id_txwcs, sizeof (ibt_wc_t) *
4095 		    state->id_txwcs_size);
4096 	}
4097 
4098 	/*
4099 	 * Since these following will act on the Rx/Tx list, which
4100 	 * is also looked at by the Rx/Tx handlers, keep them around
4101 	 * till all handlers are guaranteed to have completed.
4102 	 */
4103 	ibd_fini_rxlist(state);
4104 	ibd_fini_txlist(state);
4105 
4106 	/*
4107 	 * Clean up the active AH hash list.
4108 	 */
4109 	mod_hash_destroy_hash(state->id_ah_active_hash);
4110 
4111 	/*
4112 	 * Free parallel ARP cache and AHs; we are sure all of these
4113 	 * resources have been released by the Tx completion handler.
4114 	 */
4115 	ibd_acache_fini(state);
4116 
4117 	/*
4118 	 * We freed the QPN, all the MRs and AHs. This step should not
4119 	 * fail; print a warning message if it does fail, due to a bug
4120 	 * in the driver.
4121 	 */
4122 	if (ibt_free_pd(state->id_hca_hdl, state->id_pd_hdl) != IBT_SUCCESS)
4123 		ibd_print_warn(state, "failed to free protection domain");
4124 
4125 	if (ibt_close_hca(state->id_hca_hdl) != IBT_SUCCESS)
4126 		ibd_print_warn(state, "failed to close HCA device");
4127 }
4128 
4129 /*
4130  * IBA Rx/Tx completion queue handler. Guaranteed to be single
4131  * threaded and nonreentrant for this CQ. When using combined CQ,
4132  * this handles Tx and Rx completions. With separate CQs, this handles
4133  * only Rx completions.
4134  */
4135 /* ARGSUSED */
4136 static void
4137 ibd_rcq_handler(ibt_cq_hdl_t cq_hdl, void *arg)
4138 {
4139 	ibd_state_t *state = (ibd_state_t *)arg;
4140 
4141 	atomic_add_64(&state->id_num_intrs, 1);
4142 
4143 	if (ibd_rx_softintr == 1)
4144 		ddi_trigger_softintr(state->id_rx);
4145 	else
4146 		(void) ibd_intr((char *)state);
4147 }
4148 
4149 /*
4150  * Separate CQ handler for Tx completions, when the Tx CQ is in
4151  * interrupt driven mode.
4152  */
4153 /* ARGSUSED */
4154 static void
4155 ibd_scq_handler(ibt_cq_hdl_t cq_hdl, void *arg)
4156 {
4157 	ibd_state_t *state = (ibd_state_t *)arg;
4158 
4159 	atomic_add_64(&state->id_num_intrs, 1);
4160 
4161 	if (ibd_tx_softintr == 1)
4162 		ddi_trigger_softintr(state->id_tx);
4163 	else
4164 		(void) ibd_tx_recycle((char *)state);
4165 }
4166 
4167 /*
4168  * Multicast group create/delete trap handler. These will be delivered
4169  * on a kernel thread (handling can thus block) and can be invoked
4170  * concurrently. The handler can be invoked anytime after it is
4171  * registered and before ibt_detach().
4172  */
4173 /* ARGSUSED */
4174 static void
4175 ibd_snet_notices_handler(void *arg, ib_gid_t gid, ibt_subnet_event_code_t code,
4176     ibt_subnet_event_t *event)
4177 {
4178 	ibd_state_t *state = (ibd_state_t *)arg;
4179 	ibd_req_t *req;
4180 
4181 	/*
4182 	 * The trap handler will get invoked once for every event for
4183 	 * evert port. The input "gid" is the GID0 of the port the
4184 	 * trap came in on; we just need to act on traps that came
4185 	 * to our port, meaning the port on which the ipoib interface
4186 	 * resides. Since ipoib uses GID0 of the port, we just match
4187 	 * the gids to check whether we need to handle the trap.
4188 	 */
4189 	if (bcmp(&gid, &state->id_sgid, sizeof (ib_gid_t)) != 0)
4190 		return;
4191 
4192 	DPRINT(10, "ibd_notices_handler : %d\n", code);
4193 
4194 	switch (code) {
4195 		case IBT_SM_EVENT_UNAVAILABLE:
4196 			/*
4197 			 * If we are in promiscuous mode or have
4198 			 * sendnonmembers, we need to print a warning
4199 			 * message right now. Else, just store the
4200 			 * information, print when we enter promiscuous
4201 			 * mode or attempt nonmember send. We might
4202 			 * also want to stop caching sendnonmember.
4203 			 */
4204 			ibd_print_warn(state, "IBA multicast support "
4205 			    "degraded due to unavailability of multicast "
4206 			    "traps");
4207 			break;
4208 		case IBT_SM_EVENT_AVAILABLE:
4209 			/*
4210 			 * If we printed a warning message above or
4211 			 * while trying to nonmember send or get into
4212 			 * promiscuous mode, print an okay message.
4213 			 */
4214 			ibd_print_warn(state, "IBA multicast support "
4215 			    "restored due to availability of multicast "
4216 			    "traps");
4217 			break;
4218 		case IBT_SM_EVENT_MCG_CREATED:
4219 		case IBT_SM_EVENT_MCG_DELETED:
4220 			/*
4221 			 * Common processing of creation/deletion traps.
4222 			 * First check if the instance is being
4223 			 * [de]initialized; back off then, without doing
4224 			 * anything more, since we are not sure if the
4225 			 * async thread is around, or whether we might
4226 			 * be racing with the detach code in ibd_drv_fini()
4227 			 * that scans the mcg list.
4228 			 */
4229 			if (!ibd_async_safe(state))
4230 				return;
4231 
4232 			req = kmem_cache_alloc(state->id_req_kmc, KM_SLEEP);
4233 			req->rq_gid = event->sm_notice_gid;
4234 			req->rq_ptr = (void *)code;
4235 			ibd_queue_work_slot(state, req, IBD_ASYNC_TRAP);
4236 			break;
4237 	}
4238 }
4239 
4240 static void
4241 ibd_async_trap(ibd_state_t *state, ibd_req_t *req)
4242 {
4243 	ib_gid_t mgid = req->rq_gid;
4244 	ibt_subnet_event_code_t code = (ibt_subnet_event_code_t)req->rq_ptr;
4245 
4246 	DPRINT(10, "ibd_async_trap : %d\n", code);
4247 
4248 	/*
4249 	 * Atomically search the nonmember and sendonlymember lists and
4250 	 * delete.
4251 	 */
4252 	ibd_leave_group(state, mgid, IB_MC_JSTATE_SEND_ONLY_NON);
4253 
4254 	if (state->id_prom_op == IBD_OP_COMPLETED) {
4255 		ibd_leave_group(state, mgid, IB_MC_JSTATE_NON);
4256 
4257 		/*
4258 		 * If in promiscuous mode, try to join/attach to the new
4259 		 * mcg. Given the unreliable out-of-order mode of trap
4260 		 * delivery, we can never be sure whether it is a problem
4261 		 * if the join fails. Thus, we warn the admin of a failure
4262 		 * if this was a creation trap. Note that the trap might
4263 		 * actually be reporting a long past event, and the mcg
4264 		 * might already have been deleted, thus we might be warning
4265 		 * in vain.
4266 		 */
4267 		if ((ibd_join_group(state, mgid, IB_MC_JSTATE_NON) ==
4268 		    NULL) && (code == IBT_SM_EVENT_MCG_CREATED))
4269 			ibd_print_warn(state, "IBA promiscuous mode missed "
4270 			    "new multicast gid %016llx:%016llx",
4271 			    (u_longlong_t)mgid.gid_prefix,
4272 			    (u_longlong_t)mgid.gid_guid);
4273 	}
4274 
4275 	/*
4276 	 * Free the request slot allocated by the subnet event thread.
4277 	 */
4278 	ibd_async_done(state);
4279 }
4280 
4281 /*
4282  * GLDv3 entry point to get capabilities.
4283  */
4284 static boolean_t
4285 ibd_m_getcapab(void *arg, mac_capab_t cap, void *cap_data)
4286 {
4287 	ibd_state_t *state = arg;
4288 
4289 	switch (cap) {
4290 	case MAC_CAPAB_HCKSUM: {
4291 		uint32_t *txflags = cap_data;
4292 
4293 		/*
4294 		 * We either do full checksum or not do it at all
4295 		 */
4296 		if (state->id_hwcksum_capab & IBT_HCA_CKSUM_FULL)
4297 			*txflags = HCK_FULLCKSUM | HCKSUM_INET_FULL_V4;
4298 		else
4299 			return (B_FALSE);
4300 		break;
4301 	}
4302 
4303 	case MAC_CAPAB_LSO: {
4304 		mac_capab_lso_t *cap_lso = cap_data;
4305 
4306 		/*
4307 		 * In addition to the capability and policy, since LSO
4308 		 * relies on hw checksum, we'll not enable LSO if we
4309 		 * don't have hw checksum.  Of course, if the HCA doesn't
4310 		 * provide the reserved lkey capability, enabling LSO will
4311 		 * actually affect performance adversely, so we'll disable
4312 		 * LSO even for that case.
4313 		 */
4314 		if (!state->id_lso_policy || !state->id_lso_capable)
4315 			return (B_FALSE);
4316 
4317 		if ((state->id_hwcksum_capab & IBT_HCA_CKSUM_FULL) == 0)
4318 			return (B_FALSE);
4319 
4320 		if (state->id_hca_res_lkey_capab == 0) {
4321 			ibd_print_warn(state, "no reserved-lkey capability, "
4322 			    "disabling LSO");
4323 			return (B_FALSE);
4324 		}
4325 
4326 		cap_lso->lso_flags = LSO_TX_BASIC_TCP_IPV4;
4327 		cap_lso->lso_basic_tcp_ipv4.lso_max = state->id_lso_maxlen - 1;
4328 		break;
4329 	}
4330 
4331 	default:
4332 		return (B_FALSE);
4333 	}
4334 
4335 	return (B_TRUE);
4336 }
4337 
4338 /*
4339  * GLDv3 entry point to start hardware.
4340  */
4341 /*ARGSUSED*/
4342 static int
4343 ibd_m_start(void *arg)
4344 {
4345 	return (0);
4346 }
4347 
4348 /*
4349  * GLDv3 entry point to stop hardware from receiving packets.
4350  */
4351 /*ARGSUSED*/
4352 static void
4353 ibd_m_stop(void *arg)
4354 {
4355 }
4356 
4357 /*
4358  * GLDv3 entry point to modify device's mac address. We do not
4359  * allow address modifications.
4360  */
4361 static int
4362 ibd_m_unicst(void *arg, const uint8_t *macaddr)
4363 {
4364 	ibd_state_t *state;
4365 
4366 	state = (ibd_state_t *)arg;
4367 	if (bcmp(macaddr, &state->id_macaddr, IPOIB_ADDRL) == 0)
4368 		return (0);
4369 	else
4370 		return (EINVAL);
4371 }
4372 
4373 /*
4374  * The blocking part of the IBA join/leave operations are done out
4375  * of here on the async thread.
4376  */
4377 static void
4378 ibd_async_multicast(ibd_state_t *state, ib_gid_t mgid, int op)
4379 {
4380 	DPRINT(3, "ibd_async_multicast : async_setmc op %d :"
4381 	    "%016llx:%016llx\n", op, mgid.gid_prefix, mgid.gid_guid);
4382 
4383 	if (op == IBD_ASYNC_JOIN) {
4384 
4385 		if (ibd_join_group(state, mgid, IB_MC_JSTATE_FULL) == NULL) {
4386 			ibd_print_warn(state, "Joint multicast group failed :"
4387 			"%016llx:%016llx", mgid.gid_prefix, mgid.gid_guid);
4388 		}
4389 	} else {
4390 		/*
4391 		 * Here, we must search for the proper mcg_info and
4392 		 * use that to leave the group.
4393 		 */
4394 		ibd_leave_group(state, mgid, IB_MC_JSTATE_FULL);
4395 	}
4396 }
4397 
4398 /*
4399  * GLDv3 entry point for multicast enable/disable requests.
4400  * This function queues the operation to the async thread and
4401  * return success for a valid multicast address.
4402  */
4403 static int
4404 ibd_m_multicst(void *arg, boolean_t add, const uint8_t *mcmac)
4405 {
4406 	ibd_state_t *state = (ibd_state_t *)arg;
4407 	ipoib_mac_t maddr, *mcast;
4408 	ib_gid_t mgid;
4409 	ibd_req_t *req;
4410 
4411 	/*
4412 	 * The incoming multicast address might not be aligned properly
4413 	 * on a 4 byte boundary to be considered an ipoib_mac_t. We force
4414 	 * it to look like one though, to get the offsets of the mc gid,
4415 	 * since we know we are not going to dereference any values with
4416 	 * the ipoib_mac_t pointer.
4417 	 */
4418 	bcopy(mcmac, &maddr, sizeof (ipoib_mac_t));
4419 	mcast = &maddr;
4420 
4421 	/*
4422 	 * Check validity of MCG address. We could additionally check
4423 	 * that a enable/disable is not being issued on the "broadcast"
4424 	 * mcg, but since this operation is only invokable by priviledged
4425 	 * programs anyway, we allow the flexibility to those dlpi apps.
4426 	 * Note that we do not validate the "scope" of the IBA mcg.
4427 	 */
4428 	if ((ntohl(mcast->ipoib_qpn) & IB_QPN_MASK) != IB_MC_QPN)
4429 		return (EINVAL);
4430 
4431 	/*
4432 	 * fill in multicast pkey and scope
4433 	 */
4434 	IBD_FILL_SCOPE_PKEY(mcast, state->id_scope, state->id_pkey);
4435 
4436 	/*
4437 	 * If someone is trying to JOIN/LEAVE the broadcast group, we do
4438 	 * nothing (ie we stay JOINed to the broadcast group done in
4439 	 * ibd_drv_init()), to mimic ethernet behavior. IPv4 specifically
4440 	 * requires to be joined to broadcast groups at all times.
4441 	 * ibd_join_group() has an ASSERT(omce->mc_fullreap) that also
4442 	 * depends on this.
4443 	 */
4444 	if (bcmp(mcast, &state->id_bcaddr, IPOIB_ADDRL) == 0)
4445 		return (0);
4446 
4447 	ibd_n2h_gid(mcast, &mgid);
4448 	req = kmem_cache_alloc(state->id_req_kmc, KM_NOSLEEP);
4449 	if (req == NULL)
4450 		return (ENOMEM);
4451 
4452 	req->rq_gid = mgid;
4453 
4454 	if (add) {
4455 		DPRINT(1, "ibd_m_multicst : %016llx:%016llx\n",
4456 		    mgid.gid_prefix, mgid.gid_guid);
4457 		ibd_queue_work_slot(state, req, IBD_ASYNC_JOIN);
4458 	} else {
4459 		DPRINT(1, "ibd_m_multicst : unset_multicast : "
4460 		    "%016llx:%016llx", mgid.gid_prefix, mgid.gid_guid);
4461 		ibd_queue_work_slot(state, req, IBD_ASYNC_LEAVE);
4462 	}
4463 	return (0);
4464 }
4465 
4466 /*
4467  * The blocking part of the IBA promiscuous operations are done
4468  * out of here on the async thread. The dlpireq parameter indicates
4469  * whether this invocation is due to a dlpi request or due to
4470  * a port up/down event.
4471  */
4472 static void
4473 ibd_async_unsetprom(ibd_state_t *state)
4474 {
4475 	ibd_mce_t *mce = list_head(&state->id_mc_non);
4476 	ib_gid_t mgid;
4477 
4478 	DPRINT(2, "ibd_async_unsetprom : async_unset_promisc");
4479 
4480 	while (mce != NULL) {
4481 		mgid = mce->mc_info.mc_adds_vect.av_dgid;
4482 		mce = list_next(&state->id_mc_non, mce);
4483 		ibd_leave_group(state, mgid, IB_MC_JSTATE_NON);
4484 	}
4485 	state->id_prom_op = IBD_OP_NOTSTARTED;
4486 }
4487 
4488 /*
4489  * The blocking part of the IBA promiscuous operations are done
4490  * out of here on the async thread. The dlpireq parameter indicates
4491  * whether this invocation is due to a dlpi request or due to
4492  * a port up/down event.
4493  */
4494 static void
4495 ibd_async_setprom(ibd_state_t *state)
4496 {
4497 	ibt_mcg_attr_t mcg_attr;
4498 	ibt_mcg_info_t *mcg_info;
4499 	ib_gid_t mgid;
4500 	uint_t numg;
4501 	int i, ret = IBD_OP_COMPLETED;
4502 
4503 	DPRINT(2, "ibd_async_setprom : async_set_promisc");
4504 
4505 	/*
4506 	 * Obtain all active MC groups on the IB fabric with
4507 	 * specified criteria (scope + Pkey + Qkey + mtu).
4508 	 */
4509 	bzero(&mcg_attr, sizeof (mcg_attr));
4510 	mcg_attr.mc_pkey = state->id_pkey;
4511 	mcg_attr.mc_scope = state->id_scope;
4512 	mcg_attr.mc_qkey = state->id_mcinfo->mc_qkey;
4513 	mcg_attr.mc_mtu_req.r_mtu = state->id_mcinfo->mc_mtu;
4514 	mcg_attr.mc_mtu_req.r_selector = IBT_EQU;
4515 	if (ibt_query_mcg(state->id_sgid, &mcg_attr, 0, &mcg_info, &numg) !=
4516 	    IBT_SUCCESS) {
4517 		ibd_print_warn(state, "Could not get list of IBA multicast "
4518 		    "groups");
4519 		ret = IBD_OP_ERRORED;
4520 		goto done;
4521 	}
4522 
4523 	/*
4524 	 * Iterate over the returned mcg's and join as NonMember
4525 	 * to the IP mcg's.
4526 	 */
4527 	for (i = 0; i < numg; i++) {
4528 		/*
4529 		 * Do a NonMember JOIN on the MC group.
4530 		 */
4531 		mgid = mcg_info[i].mc_adds_vect.av_dgid;
4532 		if (ibd_join_group(state, mgid, IB_MC_JSTATE_NON) == NULL)
4533 			ibd_print_warn(state, "IBA promiscuous mode missed "
4534 			    "multicast gid %016llx:%016llx",
4535 			    (u_longlong_t)mgid.gid_prefix,
4536 			    (u_longlong_t)mgid.gid_guid);
4537 	}
4538 
4539 	ibt_free_mcg_info(mcg_info, numg);
4540 	DPRINT(4, "ibd_async_setprom : async_set_promisc completes");
4541 done:
4542 	state->id_prom_op = ret;
4543 }
4544 
4545 /*
4546  * GLDv3 entry point for multicast promiscuous enable/disable requests.
4547  * GLDv3 assumes phys state receives more packets than multi state,
4548  * which is not true for IPoIB. Thus, treat the multi and phys
4549  * promiscuous states the same way to work with GLDv3's assumption.
4550  */
4551 static int
4552 ibd_m_promisc(void *arg, boolean_t on)
4553 {
4554 	ibd_state_t *state = (ibd_state_t *)arg;
4555 	ibd_req_t *req;
4556 
4557 	req = kmem_cache_alloc(state->id_req_kmc, KM_NOSLEEP);
4558 	if (req == NULL)
4559 		return (ENOMEM);
4560 	if (on) {
4561 		DPRINT(1, "ibd_m_promisc : set_promisc : %d", on);
4562 		ibd_queue_work_slot(state, req, IBD_ASYNC_PROMON);
4563 	} else {
4564 		DPRINT(1, "ibd_m_promisc : unset_promisc");
4565 		ibd_queue_work_slot(state, req, IBD_ASYNC_PROMOFF);
4566 	}
4567 
4568 	return (0);
4569 }
4570 
4571 /*
4572  * GLDv3 entry point for gathering statistics.
4573  */
4574 static int
4575 ibd_m_stat(void *arg, uint_t stat, uint64_t *val)
4576 {
4577 	ibd_state_t *state = (ibd_state_t *)arg;
4578 
4579 	switch (stat) {
4580 	case MAC_STAT_IFSPEED:
4581 		*val = state->id_link_speed;
4582 		break;
4583 	case MAC_STAT_MULTIRCV:
4584 		*val = state->id_multi_rcv;
4585 		break;
4586 	case MAC_STAT_BRDCSTRCV:
4587 		*val = state->id_brd_rcv;
4588 		break;
4589 	case MAC_STAT_MULTIXMT:
4590 		*val = state->id_multi_xmt;
4591 		break;
4592 	case MAC_STAT_BRDCSTXMT:
4593 		*val = state->id_brd_xmt;
4594 		break;
4595 	case MAC_STAT_RBYTES:
4596 		*val = state->id_rcv_bytes;
4597 		break;
4598 	case MAC_STAT_IPACKETS:
4599 		*val = state->id_rcv_pkt;
4600 		break;
4601 	case MAC_STAT_OBYTES:
4602 		*val = state->id_xmt_bytes;
4603 		break;
4604 	case MAC_STAT_OPACKETS:
4605 		*val = state->id_xmt_pkt;
4606 		break;
4607 	case MAC_STAT_OERRORS:
4608 		*val = state->id_ah_error;	/* failed AH translation */
4609 		break;
4610 	case MAC_STAT_IERRORS:
4611 		*val = 0;
4612 		break;
4613 	case MAC_STAT_NOXMTBUF:
4614 		*val = state->id_tx_short;
4615 		break;
4616 	case MAC_STAT_NORCVBUF:
4617 	default:
4618 		return (ENOTSUP);
4619 	}
4620 
4621 	return (0);
4622 }
4623 
4624 static void
4625 ibd_async_txsched(ibd_state_t *state)
4626 {
4627 	ibd_req_t *req;
4628 	int ret;
4629 
4630 	if (ibd_txcomp_poll)
4631 		ibd_poll_compq(state, state->id_scq_hdl);
4632 
4633 	ret = ibd_resume_transmission(state);
4634 	if (ret && ibd_txcomp_poll) {
4635 		if (req = kmem_cache_alloc(state->id_req_kmc, KM_NOSLEEP))
4636 			ibd_queue_work_slot(state, req, IBD_ASYNC_SCHED);
4637 		else {
4638 			ibd_print_warn(state, "ibd_async_txsched: "
4639 			    "no memory, can't schedule work slot");
4640 		}
4641 	}
4642 }
4643 
4644 static int
4645 ibd_resume_transmission(ibd_state_t *state)
4646 {
4647 	int flag;
4648 	int met_thresh = 0;
4649 	int ret = -1;
4650 
4651 	mutex_enter(&state->id_sched_lock);
4652 	if (state->id_sched_needed & IBD_RSRC_SWQE) {
4653 		met_thresh = (state->id_tx_list.dl_cnt >
4654 		    IBD_FREE_SWQES_THRESH);
4655 		flag = IBD_RSRC_SWQE;
4656 	} else if (state->id_sched_needed & IBD_RSRC_LSOBUF) {
4657 		ASSERT(state->id_lso != NULL);
4658 		met_thresh = (state->id_lso->bkt_nfree >
4659 		    IBD_FREE_LSOS_THRESH);
4660 		flag = IBD_RSRC_LSOBUF;
4661 	}
4662 	if (met_thresh) {
4663 		state->id_sched_needed &= ~flag;
4664 		ret = 0;
4665 	}
4666 	mutex_exit(&state->id_sched_lock);
4667 
4668 	if (ret == 0)
4669 		mac_tx_update(state->id_mh);
4670 
4671 	return (ret);
4672 }
4673 
4674 /*
4675  * Release the send wqe back into free list.
4676  */
4677 static void
4678 ibd_release_swqe(ibd_state_t *state, ibd_swqe_t *swqe)
4679 {
4680 	/*
4681 	 * Add back on Tx list for reuse.
4682 	 */
4683 	swqe->swqe_next = NULL;
4684 	mutex_enter(&state->id_tx_list.dl_mutex);
4685 	if (state->id_tx_list.dl_pending_sends) {
4686 		state->id_tx_list.dl_pending_sends = B_FALSE;
4687 	}
4688 	if (state->id_tx_list.dl_head == NULL) {
4689 		state->id_tx_list.dl_head = SWQE_TO_WQE(swqe);
4690 	} else {
4691 		state->id_tx_list.dl_tail->w_next = SWQE_TO_WQE(swqe);
4692 	}
4693 	state->id_tx_list.dl_tail = SWQE_TO_WQE(swqe);
4694 	state->id_tx_list.dl_cnt++;
4695 	mutex_exit(&state->id_tx_list.dl_mutex);
4696 }
4697 
4698 /*
4699  * Acquire a send wqe from free list.
4700  * Returns error number and send wqe pointer.
4701  */
4702 static int
4703 ibd_acquire_swqe(ibd_state_t *state, ibd_swqe_t **swqe)
4704 {
4705 	int rc = 0;
4706 	ibd_swqe_t *wqe;
4707 
4708 	/*
4709 	 * Check and reclaim some of the completed Tx requests.
4710 	 * If someone else is already in this code and pulling Tx
4711 	 * completions, no need to poll, since the current lock holder
4712 	 * will do the work anyway. Normally, we poll for completions
4713 	 * every few Tx attempts, but if we are short on Tx descriptors,
4714 	 * we always try to poll.
4715 	 */
4716 	if ((ibd_txcomp_poll == 1) &&
4717 	    (state->id_tx_list.dl_cnt < IBD_TX_POLL_THRESH)) {
4718 		ibd_poll_compq(state, state->id_scq_hdl);
4719 	}
4720 
4721 	/*
4722 	 * Grab required transmit wqes.
4723 	 */
4724 	mutex_enter(&state->id_tx_list.dl_mutex);
4725 	wqe = WQE_TO_SWQE(state->id_tx_list.dl_head);
4726 	if (wqe != NULL) {
4727 		state->id_tx_list.dl_cnt -= 1;
4728 		state->id_tx_list.dl_head = wqe->swqe_next;
4729 		if (state->id_tx_list.dl_tail == SWQE_TO_WQE(wqe))
4730 			state->id_tx_list.dl_tail = NULL;
4731 	} else {
4732 		/*
4733 		 * If we did not find the number we were looking for, flag
4734 		 * no resource. Adjust list appropriately in either case.
4735 		 */
4736 		rc = ENOENT;
4737 		state->id_tx_list.dl_pending_sends = B_TRUE;
4738 		DPRINT(5, "ibd_acquire_swqe: out of Tx wqe");
4739 		atomic_add_64(&state->id_tx_short, 1);
4740 	}
4741 	mutex_exit(&state->id_tx_list.dl_mutex);
4742 	*swqe = wqe;
4743 
4744 	return (rc);
4745 }
4746 
4747 static int
4748 ibd_setup_lso(ibd_swqe_t *node, mblk_t *mp, uint32_t mss,
4749     ibt_ud_dest_hdl_t ud_dest)
4750 {
4751 	mblk_t	*nmp;
4752 	int iph_len, tcph_len;
4753 	ibt_wr_lso_t *lso;
4754 	uintptr_t ip_start, tcp_start;
4755 	uint8_t *dst;
4756 	uint_t pending, mblen;
4757 
4758 	/*
4759 	 * The code in ibd_send would've set 'wr.ud.udwr_dest' by default;
4760 	 * we need to adjust it here for lso.
4761 	 */
4762 	lso = &(node->w_swr.wr.ud_lso);
4763 	lso->lso_ud_dest = ud_dest;
4764 	lso->lso_mss = mss;
4765 
4766 	/*
4767 	 * Calculate the LSO header size and set it in the UD LSO structure.
4768 	 * Note that the only assumption we make is that each of the IPoIB,
4769 	 * IP and TCP headers will be contained in a single mblk fragment;
4770 	 * together, the headers may span multiple mblk fragments.
4771 	 */
4772 	nmp = mp;
4773 	ip_start = (uintptr_t)(nmp->b_rptr) + IPOIB_HDRSIZE;
4774 	if (ip_start >= (uintptr_t)(nmp->b_wptr)) {
4775 		ip_start = (uintptr_t)nmp->b_cont->b_rptr
4776 		    + (ip_start - (uintptr_t)(nmp->b_wptr));
4777 		nmp = nmp->b_cont;
4778 
4779 	}
4780 	iph_len = IPH_HDR_LENGTH((ipha_t *)ip_start);
4781 
4782 	tcp_start = ip_start + iph_len;
4783 	if (tcp_start >= (uintptr_t)(nmp->b_wptr)) {
4784 		tcp_start = (uintptr_t)nmp->b_cont->b_rptr
4785 		    + (tcp_start - (uintptr_t)(nmp->b_wptr));
4786 		nmp = nmp->b_cont;
4787 	}
4788 	tcph_len = TCP_HDR_LENGTH((tcph_t *)tcp_start);
4789 	lso->lso_hdr_sz = IPOIB_HDRSIZE + iph_len + tcph_len;
4790 
4791 	/*
4792 	 * If the lso header fits entirely within a single mblk fragment,
4793 	 * we'll avoid an additional copy of the lso header here and just
4794 	 * pass the b_rptr of the mblk directly.
4795 	 *
4796 	 * If this isn't true, we'd have to allocate for it explicitly.
4797 	 */
4798 	if (lso->lso_hdr_sz <= MBLKL(mp)) {
4799 		lso->lso_hdr = mp->b_rptr;
4800 	} else {
4801 		/* On work completion, remember to free this allocated hdr */
4802 		lso->lso_hdr = kmem_zalloc(lso->lso_hdr_sz, KM_NOSLEEP);
4803 		if (lso->lso_hdr == NULL) {
4804 			DPRINT(10, "ibd_setup_lso: couldn't allocate lso hdr, "
4805 			    "sz = %d", lso->lso_hdr_sz);
4806 			lso->lso_hdr_sz = 0;
4807 			lso->lso_mss = 0;
4808 			return (-1);
4809 		}
4810 	}
4811 
4812 	/*
4813 	 * Copy in the lso header only if we need to
4814 	 */
4815 	if (lso->lso_hdr != mp->b_rptr) {
4816 		dst = lso->lso_hdr;
4817 		pending = lso->lso_hdr_sz;
4818 
4819 		for (nmp = mp; nmp && pending; nmp = nmp->b_cont) {
4820 			mblen = MBLKL(nmp);
4821 			if (pending > mblen) {
4822 				bcopy(nmp->b_rptr, dst, mblen);
4823 				dst += mblen;
4824 				pending -= mblen;
4825 			} else {
4826 				bcopy(nmp->b_rptr, dst, pending);
4827 				break;
4828 			}
4829 		}
4830 	}
4831 
4832 	return (0);
4833 }
4834 
4835 static void
4836 ibd_free_lsohdr(ibd_swqe_t *node, mblk_t *mp)
4837 {
4838 	ibt_wr_lso_t *lso;
4839 
4840 	if ((!node) || (!mp))
4841 		return;
4842 
4843 	/*
4844 	 * Free any header space that we might've allocated if we
4845 	 * did an LSO
4846 	 */
4847 	if (node->w_swr.wr_opcode == IBT_WRC_SEND_LSO) {
4848 		lso = &(node->w_swr.wr.ud_lso);
4849 		if ((lso->lso_hdr) && (lso->lso_hdr != mp->b_rptr)) {
4850 			kmem_free(lso->lso_hdr, lso->lso_hdr_sz);
4851 			lso->lso_hdr = NULL;
4852 			lso->lso_hdr_sz = 0;
4853 		}
4854 	}
4855 }
4856 
4857 static void
4858 ibd_post_send(ibd_state_t *state, ibd_swqe_t *node)
4859 {
4860 	uint_t		i;
4861 	uint_t		num_posted;
4862 	uint_t		n_wrs;
4863 	ibt_status_t	ibt_status;
4864 	ibt_send_wr_t	wrs[IBD_MAX_POST_MULTIPLE];
4865 	ibd_swqe_t	*elem;
4866 	ibd_swqe_t	*nodes[IBD_MAX_POST_MULTIPLE];
4867 
4868 	node->swqe_next = NULL;
4869 
4870 	mutex_enter(&state->id_txpost_lock);
4871 
4872 	/*
4873 	 * Enqueue the new node in chain of wqes to send
4874 	 */
4875 	if (state->id_tx_head) {
4876 		*(state->id_tx_tailp) = (ibd_wqe_t *)node;
4877 	} else {
4878 		state->id_tx_head = node;
4879 	}
4880 	state->id_tx_tailp = &(node->swqe_next);
4881 
4882 	/*
4883 	 * If someone else is helping out with the sends,
4884 	 * just go back
4885 	 */
4886 	if (state->id_tx_busy) {
4887 		mutex_exit(&state->id_txpost_lock);
4888 		return;
4889 	}
4890 
4891 	/*
4892 	 * Otherwise, mark the flag to indicate that we'll be
4893 	 * doing the dispatch of what's there in the wqe chain
4894 	 */
4895 	state->id_tx_busy = 1;
4896 
4897 	while (state->id_tx_head) {
4898 		/*
4899 		 * Collect pending requests, IBD_MAX_POST_MULTIPLE wrs
4900 		 * at a time if possible, and keep posting them.
4901 		 */
4902 		for (n_wrs = 0, elem = state->id_tx_head;
4903 		    (elem) && (n_wrs < IBD_MAX_POST_MULTIPLE);
4904 		    elem = WQE_TO_SWQE(elem->swqe_next), n_wrs++) {
4905 
4906 			nodes[n_wrs] = elem;
4907 			wrs[n_wrs] = elem->w_swr;
4908 		}
4909 		state->id_tx_head = elem;
4910 
4911 		/*
4912 		 * Release the txpost lock before posting the
4913 		 * send request to the hca; if the posting fails
4914 		 * for some reason, we'll never receive completion
4915 		 * intimation, so we'll need to cleanup.
4916 		 */
4917 		mutex_exit(&state->id_txpost_lock);
4918 
4919 		ASSERT(n_wrs != 0);
4920 
4921 		/*
4922 		 * If posting fails for some reason, we'll never receive
4923 		 * completion intimation, so we'll need to cleanup. But
4924 		 * we need to make sure we don't clean up nodes whose
4925 		 * wrs have been successfully posted. We assume that the
4926 		 * hca driver returns on the first failure to post and
4927 		 * therefore the first 'num_posted' entries don't need
4928 		 * cleanup here.
4929 		 */
4930 		num_posted = 0;
4931 		ibt_status = ibt_post_send(state->id_chnl_hdl,
4932 		    wrs, n_wrs, &num_posted);
4933 		if (ibt_status != IBT_SUCCESS) {
4934 
4935 			ibd_print_warn(state, "ibd_post_send: "
4936 			    "posting multiple wrs failed: "
4937 			    "requested=%d, done=%d, ret=%d",
4938 			    n_wrs, num_posted, ibt_status);
4939 
4940 			for (i = num_posted; i < n_wrs; i++)
4941 				ibd_tx_cleanup(state, nodes[i]);
4942 		}
4943 
4944 		/*
4945 		 * Grab the mutex before we go and check the tx Q again
4946 		 */
4947 		mutex_enter(&state->id_txpost_lock);
4948 	}
4949 
4950 	state->id_tx_busy = 0;
4951 	mutex_exit(&state->id_txpost_lock);
4952 }
4953 
4954 static int
4955 ibd_prepare_sgl(ibd_state_t *state, mblk_t *mp, ibd_swqe_t *node,
4956     uint_t lsohdr_sz)
4957 {
4958 	ibt_wr_ds_t *sgl;
4959 	ibt_status_t ibt_status;
4960 	mblk_t *nmp;
4961 	mblk_t *data_mp;
4962 	uchar_t *bufp;
4963 	size_t blksize;
4964 	size_t skip;
4965 	size_t avail;
4966 	uint_t pktsize;
4967 	uint_t frag_len;
4968 	uint_t pending_hdr;
4969 	uint_t hiwm;
4970 	int nmblks;
4971 	int i;
4972 
4973 	/*
4974 	 * Let's skip ahead to the data if this is LSO
4975 	 */
4976 	data_mp = mp;
4977 	pending_hdr = 0;
4978 	if (lsohdr_sz) {
4979 		pending_hdr = lsohdr_sz;
4980 		for (nmp = mp; nmp; nmp = nmp->b_cont) {
4981 			frag_len = nmp->b_wptr - nmp->b_rptr;
4982 			if (frag_len > pending_hdr)
4983 				break;
4984 			pending_hdr -= frag_len;
4985 		}
4986 		data_mp = nmp;	/* start of data past lso header */
4987 		ASSERT(data_mp != NULL);
4988 	}
4989 
4990 	/*
4991 	 * Calculate the size of message data and number of msg blocks
4992 	 */
4993 	pktsize = 0;
4994 	for (nmblks = 0, nmp = data_mp; nmp != NULL;
4995 	    nmp = nmp->b_cont, nmblks++) {
4996 		pktsize += MBLKL(nmp);
4997 	}
4998 	pktsize -= pending_hdr;
4999 
5000 	/*
5001 	 * Translating the virtual address regions into physical regions
5002 	 * for using the Reserved LKey feature results in a wr sgl that
5003 	 * is a little longer. Since failing ibt_map_mem_iov() is costly,
5004 	 * we'll fix a high-water mark (65%) for when we should stop.
5005 	 */
5006 	hiwm = (state->id_max_sqseg * 65) / 100;
5007 
5008 	/*
5009 	 * We only do ibt_map_mem_iov() if the pktsize is above the
5010 	 * "copy-threshold", and if the number of mp fragments is less than
5011 	 * the maximum acceptable.
5012 	 */
5013 	if ((state->id_hca_res_lkey_capab) &&
5014 	    (pktsize > IBD_TX_COPY_THRESH) &&
5015 	    (nmblks < hiwm)) {
5016 		ibt_iov_t iov_arr[IBD_MAX_SQSEG];
5017 		ibt_iov_attr_t iov_attr;
5018 
5019 		iov_attr.iov_as = NULL;
5020 		iov_attr.iov = iov_arr;
5021 		iov_attr.iov_buf = NULL;
5022 		iov_attr.iov_list_len = nmblks;
5023 		iov_attr.iov_wr_nds = state->id_max_sqseg;
5024 		iov_attr.iov_lso_hdr_sz = lsohdr_sz;
5025 		iov_attr.iov_flags = IBT_IOV_SLEEP;
5026 
5027 		for (nmp = data_mp, i = 0; i < nmblks; i++, nmp = nmp->b_cont) {
5028 			iov_arr[i].iov_addr = (caddr_t)(void *)nmp->b_rptr;
5029 			iov_arr[i].iov_len = MBLKL(nmp);
5030 			if (i == 0) {
5031 				iov_arr[i].iov_addr += pending_hdr;
5032 				iov_arr[i].iov_len -= pending_hdr;
5033 			}
5034 		}
5035 
5036 		node->w_buftype = IBD_WQE_MAPPED;
5037 		node->w_swr.wr_sgl = node->w_sgl;
5038 
5039 		ibt_status = ibt_map_mem_iov(state->id_hca_hdl, &iov_attr,
5040 		    (ibt_all_wr_t *)&node->w_swr, &node->w_mi_hdl);
5041 		if (ibt_status != IBT_SUCCESS) {
5042 			ibd_print_warn(state, "ibd_send: ibt_map_mem_iov "
5043 			    "failed, nmblks=%d, ret=%d\n", nmblks, ibt_status);
5044 			goto ibd_copy_path;
5045 		}
5046 
5047 		return (0);
5048 	}
5049 
5050 ibd_copy_path:
5051 	if (pktsize <= state->id_tx_buf_sz) {
5052 		node->swqe_copybuf.ic_sgl.ds_len = pktsize;
5053 		node->w_swr.wr_nds = 1;
5054 		node->w_swr.wr_sgl = &node->swqe_copybuf.ic_sgl;
5055 		node->w_buftype = IBD_WQE_TXBUF;
5056 
5057 		/*
5058 		 * Even though this is the copy path for transfers less than
5059 		 * id_tx_buf_sz, it could still be an LSO packet.  If so, it
5060 		 * is possible the first data mblk fragment (data_mp) still
5061 		 * contains part of the LSO header that we need to skip.
5062 		 */
5063 		bufp = (uchar_t *)(uintptr_t)node->w_swr.wr_sgl->ds_va;
5064 		for (nmp = data_mp; nmp != NULL; nmp = nmp->b_cont) {
5065 			blksize = MBLKL(nmp) - pending_hdr;
5066 			bcopy(nmp->b_rptr + pending_hdr, bufp, blksize);
5067 			bufp += blksize;
5068 			pending_hdr = 0;
5069 		}
5070 
5071 		return (0);
5072 	}
5073 
5074 	/*
5075 	 * Copy path for transfers greater than id_tx_buf_sz
5076 	 */
5077 	node->w_swr.wr_sgl = node->w_sgl;
5078 	if (ibd_acquire_lsobufs(state, pktsize,
5079 	    node->w_swr.wr_sgl, &(node->w_swr.wr_nds)) != 0) {
5080 		DPRINT(10, "ibd_prepare_sgl: lso bufs acquire failed");
5081 		return (-1);
5082 	}
5083 	node->w_buftype = IBD_WQE_LSOBUF;
5084 
5085 	/*
5086 	 * Copy the larger-than-id_tx_buf_sz packet into a set of
5087 	 * fixed-sized, pre-mapped LSO buffers. Note that we might
5088 	 * need to skip part of the LSO header in the first fragment
5089 	 * as before.
5090 	 */
5091 	nmp = data_mp;
5092 	skip = pending_hdr;
5093 	for (i = 0; i < node->w_swr.wr_nds; i++) {
5094 		sgl = node->w_swr.wr_sgl + i;
5095 		bufp = (uchar_t *)(uintptr_t)sgl->ds_va;
5096 		avail = IBD_LSO_BUFSZ;
5097 		while (nmp && avail) {
5098 			blksize = MBLKL(nmp) - skip;
5099 			if (blksize > avail) {
5100 				bcopy(nmp->b_rptr + skip, bufp, avail);
5101 				skip += avail;
5102 				avail = 0;
5103 			} else {
5104 				bcopy(nmp->b_rptr + skip, bufp, blksize);
5105 				skip = 0;
5106 				avail -= blksize;
5107 				bufp += blksize;
5108 				nmp = nmp->b_cont;
5109 			}
5110 		}
5111 	}
5112 
5113 	return (0);
5114 }
5115 
5116 /*
5117  * Schedule a completion queue polling to reap the resource we're
5118  * short on.  If we implement the change to reap tx completions
5119  * in a separate thread, we'll need to wake up that thread here.
5120  */
5121 static int
5122 ibd_sched_poll(ibd_state_t *state, int resource_type, int q_flag)
5123 {
5124 	ibd_req_t *req;
5125 
5126 	mutex_enter(&state->id_sched_lock);
5127 	state->id_sched_needed |= resource_type;
5128 	mutex_exit(&state->id_sched_lock);
5129 
5130 	/*
5131 	 * If we are asked to queue a work entry, we need to do it
5132 	 */
5133 	if (q_flag) {
5134 		req = kmem_cache_alloc(state->id_req_kmc, KM_NOSLEEP);
5135 		if (req == NULL)
5136 			return (-1);
5137 
5138 		ibd_queue_work_slot(state, req, IBD_ASYNC_SCHED);
5139 	}
5140 
5141 	return (0);
5142 }
5143 
5144 /*
5145  * The passed in packet has this format:
5146  * IPOIB_ADDRL b dest addr :: 2b sap :: 2b 0's :: data
5147  */
5148 static boolean_t
5149 ibd_send(ibd_state_t *state, mblk_t *mp)
5150 {
5151 	ibd_ace_t *ace;
5152 	ibd_swqe_t *node;
5153 	ipoib_mac_t *dest;
5154 	ib_header_info_t *ipibp;
5155 	ip6_t *ip6h;
5156 	uint_t pktsize;
5157 	uint32_t mss;
5158 	uint32_t hckflags;
5159 	uint32_t lsoflags = 0;
5160 	uint_t lsohdr_sz = 0;
5161 	int ret, len;
5162 	boolean_t dofree = B_FALSE;
5163 	boolean_t rc;
5164 
5165 	node = NULL;
5166 	if (ibd_acquire_swqe(state, &node) != 0) {
5167 		/*
5168 		 * If we don't have an swqe available, schedule a transmit
5169 		 * completion queue cleanup and hold off on sending more
5170 		 * more packets until we have some free swqes
5171 		 */
5172 		if (ibd_sched_poll(state, IBD_RSRC_SWQE, ibd_txcomp_poll) == 0)
5173 			return (B_FALSE);
5174 
5175 		/*
5176 		 * If a poll cannot be scheduled, we have no choice but
5177 		 * to drop this packet
5178 		 */
5179 		ibd_print_warn(state, "ibd_send: no swqe, pkt drop");
5180 		return (B_TRUE);
5181 	}
5182 
5183 	/*
5184 	 * Initialize the commonly used fields in swqe to NULL to protect
5185 	 * against ibd_tx_cleanup accidentally misinterpreting these on a
5186 	 * failure.
5187 	 */
5188 	node->swqe_im_mblk = NULL;
5189 	node->w_swr.wr_nds = 0;
5190 	node->w_swr.wr_sgl = NULL;
5191 	node->w_swr.wr_opcode = IBT_WRC_SEND;
5192 
5193 	/*
5194 	 * Obtain an address handle for the destination.
5195 	 */
5196 	ipibp = (ib_header_info_t *)mp->b_rptr;
5197 	dest = (ipoib_mac_t *)&ipibp->ib_dst;
5198 	if ((ntohl(dest->ipoib_qpn) & IB_QPN_MASK) == IB_MC_QPN)
5199 		IBD_FILL_SCOPE_PKEY(dest, state->id_scope, state->id_pkey);
5200 
5201 	pktsize = msgsize(mp);
5202 
5203 	atomic_add_64(&state->id_xmt_bytes, pktsize);
5204 	atomic_inc_64(&state->id_xmt_pkt);
5205 	if (bcmp(&ipibp->ib_dst, &state->id_bcaddr, IPOIB_ADDRL) == 0)
5206 		atomic_inc_64(&state->id_brd_xmt);
5207 	else if ((ntohl(ipibp->ib_dst.ipoib_qpn) & IB_QPN_MASK) == IB_MC_QPN)
5208 		atomic_inc_64(&state->id_multi_xmt);
5209 
5210 	if ((ace = ibd_acache_lookup(state, dest, &ret, 1)) != NULL) {
5211 		node->w_ahandle = ace;
5212 		node->w_swr.wr.ud.udwr_dest = ace->ac_dest;
5213 	} else {
5214 		DPRINT(5,
5215 		    "ibd_send: acache lookup %s for %08X:%08X:%08X:%08X:%08X",
5216 		    ((ret == EFAULT) ? "failed" : "queued"),
5217 		    htonl(dest->ipoib_qpn), htonl(dest->ipoib_gidpref[0]),
5218 		    htonl(dest->ipoib_gidpref[1]),
5219 		    htonl(dest->ipoib_gidsuff[0]),
5220 		    htonl(dest->ipoib_gidsuff[1]));
5221 		node->w_ahandle = NULL;
5222 
5223 		/*
5224 		 * for the poll mode, it is probably some cqe pending in the
5225 		 * cq. So ibd has to poll cq here, otherwise acache probably
5226 		 * may not be recycled.
5227 		 */
5228 		if (ibd_txcomp_poll == 1)
5229 			ibd_poll_compq(state, state->id_scq_hdl);
5230 
5231 		/*
5232 		 * Here if ibd_acache_lookup() returns EFAULT, it means ibd
5233 		 * can not find a path for the specific dest address. We
5234 		 * should get rid of this kind of packet.  We also should get
5235 		 * rid of the packet if we cannot schedule a poll via the
5236 		 * async thread.  For the normal case, ibd will return the
5237 		 * packet to upper layer and wait for AH creating.
5238 		 *
5239 		 * Note that we always queue a work slot entry for the async
5240 		 * thread when we fail AH lookup (even in intr mode); this is
5241 		 * due to the convoluted way the code currently looks for AH.
5242 		 */
5243 		if (ret == EFAULT) {
5244 			dofree = B_TRUE;
5245 			rc = B_TRUE;
5246 		} else if (ibd_sched_poll(state, IBD_RSRC_SWQE, 1) != 0) {
5247 			dofree = B_TRUE;
5248 			rc = B_TRUE;
5249 		} else {
5250 			dofree = B_FALSE;
5251 			rc = B_FALSE;
5252 		}
5253 		goto ibd_send_fail;
5254 	}
5255 
5256 	/*
5257 	 * For ND6 packets, padding is at the front of the source lladdr.
5258 	 * Insert the padding at front.
5259 	 */
5260 	if (ntohs(ipibp->ipib_rhdr.ipoib_type) == IP6_DL_SAP) {
5261 		if (MBLKL(mp) < sizeof (ib_header_info_t) + IPV6_HDR_LEN) {
5262 			if (!pullupmsg(mp, IPV6_HDR_LEN +
5263 			    sizeof (ib_header_info_t))) {
5264 				DPRINT(10, "ibd_send: pullupmsg failure ");
5265 				dofree = B_TRUE;
5266 				rc = B_TRUE;
5267 				goto ibd_send_fail;
5268 			}
5269 			ipibp = (ib_header_info_t *)mp->b_rptr;
5270 		}
5271 		ip6h = (ip6_t *)((uchar_t *)ipibp +
5272 		    sizeof (ib_header_info_t));
5273 		len = ntohs(ip6h->ip6_plen);
5274 		if (ip6h->ip6_nxt == IPPROTO_ICMPV6) {
5275 			mblk_t	*pad;
5276 
5277 			pad = allocb(4, 0);
5278 			pad->b_wptr = (uchar_t *)pad->b_rptr + 4;
5279 			linkb(mp, pad);
5280 			if (MBLKL(mp) < sizeof (ib_header_info_t) +
5281 			    IPV6_HDR_LEN + len + 4) {
5282 				if (!pullupmsg(mp, sizeof (ib_header_info_t) +
5283 				    IPV6_HDR_LEN + len + 4)) {
5284 					DPRINT(10, "ibd_send: pullupmsg "
5285 					    "failure ");
5286 					dofree = B_TRUE;
5287 					rc = B_TRUE;
5288 					goto ibd_send_fail;
5289 				}
5290 				ip6h = (ip6_t *)((uchar_t *)mp->b_rptr +
5291 				    sizeof (ib_header_info_t));
5292 			}
5293 
5294 			/* LINTED: E_CONSTANT_CONDITION */
5295 			IBD_PAD_NSNA(ip6h, len, IBD_SEND);
5296 		}
5297 	}
5298 
5299 	mp->b_rptr += sizeof (ib_addrs_t);
5300 
5301 	/*
5302 	 * Do LSO and checksum related work here.  For LSO send, adjust the
5303 	 * ud destination, the opcode and the LSO header information to the
5304 	 * work request.
5305 	 */
5306 	lso_info_get(mp, &mss, &lsoflags);
5307 	if ((lsoflags & HW_LSO) != HW_LSO) {
5308 		node->w_swr.wr_opcode = IBT_WRC_SEND;
5309 		lsohdr_sz = 0;
5310 	} else {
5311 		if (ibd_setup_lso(node, mp, mss, ace->ac_dest) != 0) {
5312 			/*
5313 			 * The routine can only fail if there's no memory; we
5314 			 * can only drop the packet if this happens
5315 			 */
5316 			ibd_print_warn(state,
5317 			    "ibd_send: no memory, lso posting failed");
5318 			dofree = B_TRUE;
5319 			rc = B_TRUE;
5320 			goto ibd_send_fail;
5321 		}
5322 
5323 		node->w_swr.wr_opcode = IBT_WRC_SEND_LSO;
5324 		lsohdr_sz = (node->w_swr.wr.ud_lso).lso_hdr_sz;
5325 	}
5326 
5327 	hcksum_retrieve(mp, NULL, NULL, NULL, NULL, NULL, NULL, &hckflags);
5328 	if ((hckflags & HCK_FULLCKSUM) == HCK_FULLCKSUM)
5329 		node->w_swr.wr_flags |= IBT_WR_SEND_CKSUM;
5330 	else
5331 		node->w_swr.wr_flags &= ~IBT_WR_SEND_CKSUM;
5332 
5333 	/*
5334 	 * Prepare the sgl for posting; the routine can only fail if there's
5335 	 * no lso buf available for posting. If this is the case, we should
5336 	 * probably resched for lso bufs to become available and then try again.
5337 	 */
5338 	if (ibd_prepare_sgl(state, mp, node, lsohdr_sz) != 0) {
5339 		if (ibd_sched_poll(state, IBD_RSRC_LSOBUF, 1) != 0) {
5340 			dofree = B_TRUE;
5341 			rc = B_TRUE;
5342 		} else {
5343 			dofree = B_FALSE;
5344 			rc = B_FALSE;
5345 		}
5346 		goto ibd_send_fail;
5347 	}
5348 	node->swqe_im_mblk = mp;
5349 
5350 	/*
5351 	 * Queue the wqe to hardware; since we can now simply queue a
5352 	 * post instead of doing it serially, we cannot assume anything
5353 	 * about the 'node' after ibd_post_send() returns.
5354 	 */
5355 	ibd_post_send(state, node);
5356 
5357 	return (B_TRUE);
5358 
5359 ibd_send_fail:
5360 	if (node && mp)
5361 		ibd_free_lsohdr(node, mp);
5362 
5363 	if (dofree)
5364 		freemsg(mp);
5365 
5366 	if (node != NULL)
5367 		ibd_tx_cleanup(state, node);
5368 
5369 	return (rc);
5370 }
5371 
5372 /*
5373  * GLDv3 entry point for transmitting datagram.
5374  */
5375 static mblk_t *
5376 ibd_m_tx(void *arg, mblk_t *mp)
5377 {
5378 	ibd_state_t *state = (ibd_state_t *)arg;
5379 	mblk_t *next;
5380 
5381 	while (mp != NULL) {
5382 		next = mp->b_next;
5383 		mp->b_next = NULL;
5384 		if (ibd_send(state, mp) == B_FALSE) {
5385 			/* Send fail */
5386 			mp->b_next = next;
5387 			break;
5388 		}
5389 		mp = next;
5390 	}
5391 
5392 	return (mp);
5393 }
5394 
5395 /*
5396  * this handles Tx and Rx completions. With separate CQs, this handles
5397  * only Rx completions.
5398  */
5399 static uint_t
5400 ibd_intr(char *arg)
5401 {
5402 	ibd_state_t *state = (ibd_state_t *)arg;
5403 
5404 	ibd_poll_compq(state, state->id_rcq_hdl);
5405 
5406 	return (DDI_INTR_CLAIMED);
5407 }
5408 
5409 /*
5410  * Poll and drain the cq
5411  */
5412 static uint_t
5413 ibd_drain_cq(ibd_state_t *state, ibt_cq_hdl_t cq_hdl, ibt_wc_t *wcs,
5414     uint_t numwcs)
5415 {
5416 	ibd_wqe_t *wqe;
5417 	ibt_wc_t *wc;
5418 	uint_t total_polled = 0;
5419 	uint_t num_polled;
5420 	int i;
5421 
5422 	while (ibt_poll_cq(cq_hdl, wcs, numwcs, &num_polled) == IBT_SUCCESS) {
5423 		total_polled += num_polled;
5424 		for (i = 0, wc = wcs; i < num_polled; i++, wc++) {
5425 			wqe = (ibd_wqe_t *)(uintptr_t)wc->wc_id;
5426 			ASSERT((wqe->w_type == IBD_WQE_SEND) ||
5427 			    (wqe->w_type == IBD_WQE_RECV));
5428 			if (wc->wc_status != IBT_WC_SUCCESS) {
5429 				/*
5430 				 * Channel being torn down.
5431 				 */
5432 				if (wc->wc_status == IBT_WC_WR_FLUSHED_ERR) {
5433 					DPRINT(5, "ibd_drain_cq: flush error");
5434 					/*
5435 					 * Only invoke the Tx handler to
5436 					 * release possibly held resources
5437 					 * like AH refcount etc. Can not
5438 					 * invoke Rx handler because it might
5439 					 * try adding buffers to the Rx pool
5440 					 * when we are trying to deinitialize.
5441 					 */
5442 					if (wqe->w_type == IBD_WQE_RECV) {
5443 						continue;
5444 					} else {
5445 						DPRINT(10, "ibd_drain_cq: Bad "
5446 						    "status %d", wc->wc_status);
5447 					}
5448 				}
5449 			}
5450 			if (wqe->w_type == IBD_WQE_SEND) {
5451 				ibd_tx_cleanup(state, WQE_TO_SWQE(wqe));
5452 			} else {
5453 				ibd_process_rx(state, WQE_TO_RWQE(wqe), wc);
5454 			}
5455 		}
5456 	}
5457 
5458 	return (total_polled);
5459 }
5460 
5461 /*
5462  * Common code for interrupt handling as well as for polling
5463  * for all completed wqe's while detaching.
5464  */
5465 static void
5466 ibd_poll_compq(ibd_state_t *state, ibt_cq_hdl_t cq_hdl)
5467 {
5468 	ibt_wc_t *wcs;
5469 	uint_t numwcs;
5470 	int flag, redo_flag;
5471 	int redo = 1;
5472 	uint_t num_polled = 0;
5473 
5474 	if (ibd_separate_cqs == 1) {
5475 		if (cq_hdl == state->id_rcq_hdl) {
5476 			flag = IBD_RX_CQ_POLLING;
5477 			redo_flag = IBD_REDO_RX_CQ_POLLING;
5478 		} else {
5479 			flag = IBD_TX_CQ_POLLING;
5480 			redo_flag = IBD_REDO_TX_CQ_POLLING;
5481 		}
5482 	} else {
5483 		flag = IBD_RX_CQ_POLLING | IBD_TX_CQ_POLLING;
5484 		redo_flag = IBD_REDO_RX_CQ_POLLING | IBD_REDO_TX_CQ_POLLING;
5485 	}
5486 
5487 	mutex_enter(&state->id_cq_poll_lock);
5488 	if (state->id_cq_poll_busy & flag) {
5489 		state->id_cq_poll_busy |= redo_flag;
5490 		mutex_exit(&state->id_cq_poll_lock);
5491 		return;
5492 	}
5493 	state->id_cq_poll_busy |= flag;
5494 	mutex_exit(&state->id_cq_poll_lock);
5495 
5496 	/*
5497 	 * In some cases (eg detaching), this code can be invoked on
5498 	 * any cpu after disabling cq notification (thus no concurrency
5499 	 * exists). Apart from that, the following applies normally:
5500 	 * The receive completion handling is always on the Rx interrupt
5501 	 * cpu. Transmit completion handling could be from any cpu if
5502 	 * Tx CQ is poll driven, but always on Tx interrupt cpu if Tx CQ
5503 	 * is interrupt driven. Combined completion handling is always
5504 	 * on the interrupt cpu. Thus, lock accordingly and use the
5505 	 * proper completion array.
5506 	 */
5507 	if (ibd_separate_cqs == 1) {
5508 		if (cq_hdl == state->id_rcq_hdl) {
5509 			wcs = state->id_rxwcs;
5510 			numwcs = state->id_rxwcs_size;
5511 		} else {
5512 			wcs = state->id_txwcs;
5513 			numwcs = state->id_txwcs_size;
5514 		}
5515 	} else {
5516 		wcs = state->id_rxwcs;
5517 		numwcs = state->id_rxwcs_size;
5518 	}
5519 
5520 	/*
5521 	 * Poll and drain the CQ
5522 	 */
5523 	num_polled = ibd_drain_cq(state, cq_hdl, wcs, numwcs);
5524 
5525 	/*
5526 	 * Enable CQ notifications and redrain the cq to catch any
5527 	 * completions we might have missed after the ibd_drain_cq()
5528 	 * above and before the ibt_enable_cq_notify() that follows.
5529 	 * Finally, service any new requests to poll the cq that
5530 	 * could've come in after the ibt_enable_cq_notify().
5531 	 */
5532 	do {
5533 		if (ibt_enable_cq_notify(cq_hdl, IBT_NEXT_COMPLETION) !=
5534 		    IBT_SUCCESS) {
5535 			DPRINT(10, "ibd_intr: ibt_enable_cq_notify() failed");
5536 		}
5537 
5538 		num_polled += ibd_drain_cq(state, cq_hdl, wcs, numwcs);
5539 
5540 		mutex_enter(&state->id_cq_poll_lock);
5541 		if (state->id_cq_poll_busy & redo_flag)
5542 			state->id_cq_poll_busy &= ~redo_flag;
5543 		else {
5544 			state->id_cq_poll_busy &= ~flag;
5545 			redo = 0;
5546 		}
5547 		mutex_exit(&state->id_cq_poll_lock);
5548 
5549 	} while (redo);
5550 
5551 	/*
5552 	 * If we polled the receive cq and found anything, we need to flush
5553 	 * it out to the nw layer here.
5554 	 */
5555 	if ((flag & IBD_RX_CQ_POLLING) && (num_polled > 0)) {
5556 		ibd_flush_rx(state, NULL);
5557 	}
5558 }
5559 
5560 /*
5561  * Unmap the memory area associated with a given swqe.
5562  */
5563 static void
5564 ibd_unmap_mem(ibd_state_t *state, ibd_swqe_t *swqe)
5565 {
5566 	ibt_status_t stat;
5567 
5568 	DPRINT(20, "ibd_unmap_mem: wqe=%p, seg=%d\n", swqe, swqe->w_swr.wr_nds);
5569 
5570 	if (swqe->w_mi_hdl) {
5571 		if ((stat = ibt_unmap_mem_iov(state->id_hca_hdl,
5572 		    swqe->w_mi_hdl)) != IBT_SUCCESS) {
5573 			DPRINT(10,
5574 			    "failed in ibt_unmap_mem_iov, ret=%d\n", stat);
5575 		}
5576 		swqe->w_mi_hdl = NULL;
5577 	}
5578 	swqe->w_swr.wr_nds = 0;
5579 }
5580 
5581 /*
5582  * Common code that deals with clean ups after a successful or
5583  * erroneous transmission attempt.
5584  */
5585 static void
5586 ibd_tx_cleanup(ibd_state_t *state, ibd_swqe_t *swqe)
5587 {
5588 	ibd_ace_t *ace = swqe->w_ahandle;
5589 
5590 	DPRINT(20, "ibd_tx_cleanup %p\n", swqe);
5591 
5592 	/*
5593 	 * If this was a dynamic mapping in ibd_send(), we need to
5594 	 * unmap here. If this was an lso buffer we'd used for sending,
5595 	 * we need to release the lso buf to the pool, since the resource
5596 	 * is scarce. However, if this was simply a normal send using
5597 	 * the copybuf (present in each swqe), we don't need to release it.
5598 	 */
5599 	if (swqe->swqe_im_mblk != NULL) {
5600 		if (swqe->w_buftype == IBD_WQE_MAPPED) {
5601 			ibd_unmap_mem(state, swqe);
5602 		} else if (swqe->w_buftype == IBD_WQE_LSOBUF) {
5603 			ibd_release_lsobufs(state,
5604 			    swqe->w_swr.wr_sgl, swqe->w_swr.wr_nds);
5605 		}
5606 		ibd_free_lsohdr(swqe, swqe->swqe_im_mblk);
5607 		freemsg(swqe->swqe_im_mblk);
5608 		swqe->swqe_im_mblk = NULL;
5609 	}
5610 
5611 	/*
5612 	 * Drop the reference count on the AH; it can be reused
5613 	 * now for a different destination if there are no more
5614 	 * posted sends that will use it. This can be eliminated
5615 	 * if we can always associate each Tx buffer with an AH.
5616 	 * The ace can be null if we are cleaning up from the
5617 	 * ibd_send() error path.
5618 	 */
5619 	if (ace != NULL) {
5620 		/*
5621 		 * The recycling logic can be eliminated from here
5622 		 * and put into the async thread if we create another
5623 		 * list to hold ACE's for unjoined mcg's.
5624 		 */
5625 		if (DEC_REF_DO_CYCLE(ace)) {
5626 			ibd_mce_t *mce;
5627 
5628 			/*
5629 			 * Check with the lock taken: we decremented
5630 			 * reference count without the lock, and some
5631 			 * transmitter might alreay have bumped the
5632 			 * reference count (possible in case of multicast
5633 			 * disable when we leave the AH on the active
5634 			 * list). If not still 0, get out, leaving the
5635 			 * recycle bit intact.
5636 			 *
5637 			 * Atomically transition the AH from active
5638 			 * to free list, and queue a work request to
5639 			 * leave the group and destroy the mce. No
5640 			 * transmitter can be looking at the AH or
5641 			 * the MCE in between, since we have the
5642 			 * ac_mutex lock. In the SendOnly reap case,
5643 			 * it is not neccesary to hold the ac_mutex
5644 			 * and recheck the ref count (since the AH was
5645 			 * taken off the active list), we just do it
5646 			 * to have uniform processing with the Full
5647 			 * reap case.
5648 			 */
5649 			mutex_enter(&state->id_ac_mutex);
5650 			mce = ace->ac_mce;
5651 			if (GET_REF_CYCLE(ace) == 0) {
5652 				CLEAR_REFCYCLE(ace);
5653 				/*
5654 				 * Identify the case of fullmember reap as
5655 				 * opposed to mcg trap reap. Also, port up
5656 				 * might set ac_mce to NULL to indicate Tx
5657 				 * cleanup should do no more than put the
5658 				 * AH in the free list (see ibd_async_link).
5659 				 */
5660 				if (mce != NULL) {
5661 					ace->ac_mce = NULL;
5662 					IBD_ACACHE_PULLOUT_ACTIVE(state, ace);
5663 					/*
5664 					 * mc_req was initialized at mce
5665 					 * creation time.
5666 					 */
5667 					ibd_queue_work_slot(state,
5668 					    &mce->mc_req, IBD_ASYNC_REAP);
5669 				}
5670 				IBD_ACACHE_INSERT_FREE(state, ace);
5671 			}
5672 			mutex_exit(&state->id_ac_mutex);
5673 		}
5674 	}
5675 
5676 	/*
5677 	 * Release the send wqe for reuse.
5678 	 */
5679 	ibd_release_swqe(state, swqe);
5680 }
5681 
5682 /*
5683  * Hand off the processed rx mp chain to mac_rx()
5684  */
5685 static void
5686 ibd_flush_rx(ibd_state_t *state, mblk_t *mpc)
5687 {
5688 	if (mpc == NULL) {
5689 		mutex_enter(&state->id_rx_lock);
5690 
5691 		mpc = state->id_rx_mp;
5692 
5693 		state->id_rx_mp = NULL;
5694 		state->id_rx_mp_tail = NULL;
5695 		state->id_rx_mp_len = 0;
5696 
5697 		mutex_exit(&state->id_rx_lock);
5698 	}
5699 
5700 	if (mpc) {
5701 		mac_rx(state->id_mh, state->id_rh, mpc);
5702 	}
5703 }
5704 
5705 /*
5706  * Processing to be done after receipt of a packet; hand off to GLD
5707  * in the format expected by GLD.  The received packet has this
5708  * format: 2b sap :: 00 :: data.
5709  */
5710 static void
5711 ibd_process_rx(ibd_state_t *state, ibd_rwqe_t *rwqe, ibt_wc_t *wc)
5712 {
5713 	ib_header_info_t *phdr;
5714 	mblk_t *mp;
5715 	mblk_t *mpc = NULL;
5716 	ipoib_hdr_t *ipibp;
5717 	ip6_t *ip6h;
5718 	int rxcnt, len;
5719 
5720 	/*
5721 	 * Track number handed to upper layer, and number still
5722 	 * available to receive packets.
5723 	 */
5724 	rxcnt = atomic_add_32_nv(&state->id_rx_list.dl_cnt, -1);
5725 	ASSERT(rxcnt >= 0);
5726 	atomic_add_32(&state->id_rx_list.dl_bufs_outstanding, 1);
5727 
5728 	/*
5729 	 * Adjust write pointer depending on how much data came in.
5730 	 */
5731 	mp = rwqe->rwqe_im_mblk;
5732 	mp->b_wptr = mp->b_rptr + wc->wc_bytes_xfer;
5733 
5734 	/*
5735 	 * Make sure this is NULL or we're in trouble.
5736 	 */
5737 	if (mp->b_next != NULL) {
5738 		ibd_print_warn(state,
5739 		    "ibd_process_rx: got duplicate mp from rcq?");
5740 		mp->b_next = NULL;
5741 	}
5742 
5743 	/*
5744 	 * the IB link will deliver one of the IB link layer
5745 	 * headers called, the Global Routing Header (GRH).
5746 	 * ibd driver uses the information in GRH to build the
5747 	 * Header_info structure and pass it with the datagram up
5748 	 * to GLDv3.
5749 	 * If the GRH is not valid, indicate to GLDv3 by setting
5750 	 * the VerTcFlow field to 0.
5751 	 */
5752 	phdr = (ib_header_info_t *)mp->b_rptr;
5753 	if (wc->wc_flags & IBT_WC_GRH_PRESENT) {
5754 		phdr->ib_grh.ipoib_sqpn = htonl(wc->wc_qpn);
5755 
5756 		/* if it is loop back packet, just drop it. */
5757 		if (bcmp(&phdr->ib_grh.ipoib_sqpn, &state->id_macaddr,
5758 		    IPOIB_ADDRL) == 0) {
5759 			freemsg(mp);
5760 			return;
5761 		}
5762 
5763 		ovbcopy(&phdr->ib_grh.ipoib_sqpn, &phdr->ib_src,
5764 		    sizeof (ipoib_mac_t));
5765 		if (*(uint8_t *)(phdr->ib_grh.ipoib_dgid_pref) == 0xFF) {
5766 			phdr->ib_dst.ipoib_qpn = htonl(IB_MC_QPN);
5767 			IBD_CLEAR_SCOPE_PKEY(&phdr->ib_dst);
5768 		} else {
5769 			phdr->ib_dst.ipoib_qpn = state->id_macaddr.ipoib_qpn;
5770 		}
5771 	} else {
5772 		/*
5773 		 * It can not be a IBA multicast packet. Must have been
5774 		 * unicast for us. Just copy the interface address to dst.
5775 		 */
5776 		phdr->ib_grh.ipoib_vertcflow = 0;
5777 		ovbcopy(&state->id_macaddr, &phdr->ib_dst,
5778 		    sizeof (ipoib_mac_t));
5779 	}
5780 
5781 	/*
5782 	 * For ND6 packets, padding is at the front of the source/target
5783 	 * lladdr. However the inet6 layer is not aware of it, hence remove
5784 	 * the padding from such packets.
5785 	 */
5786 	ipibp = (ipoib_hdr_t *)((uchar_t *)mp->b_rptr + sizeof (ipoib_pgrh_t));
5787 	if (ntohs(ipibp->ipoib_type) == IP6_DL_SAP) {
5788 		if (MBLKL(mp) < sizeof (ipoib_hdr_t) + IPV6_HDR_LEN) {
5789 			if (!pullupmsg(mp, IPV6_HDR_LEN +
5790 			    sizeof (ipoib_hdr_t))) {
5791 				DPRINT(10, "ibd_process_rx: pullupmsg failed");
5792 				freemsg(mp);
5793 				return;
5794 			}
5795 			ipibp = (ipoib_hdr_t *)((uchar_t *)mp->b_rptr +
5796 			    sizeof (ipoib_pgrh_t));
5797 		}
5798 		ip6h = (ip6_t *)((uchar_t *)ipibp + sizeof (ipoib_hdr_t));
5799 		len = ntohs(ip6h->ip6_plen);
5800 		if (ip6h->ip6_nxt == IPPROTO_ICMPV6) {
5801 			if (MBLKL(mp) < sizeof (ipoib_hdr_t) +
5802 			    IPV6_HDR_LEN + len) {
5803 				if (!pullupmsg(mp, sizeof (ipoib_hdr_t) +
5804 				    IPV6_HDR_LEN + len)) {
5805 					DPRINT(10, "ibd_process_rx: pullupmsg"
5806 					    " failed");
5807 					freemsg(mp);
5808 					return;
5809 				}
5810 				ip6h = (ip6_t *)((uchar_t *)mp->b_rptr +
5811 				    sizeof (ipoib_pgrh_t) +
5812 				    sizeof (ipoib_hdr_t));
5813 			}
5814 			/* LINTED: E_CONSTANT_CONDITION */
5815 			IBD_PAD_NSNA(ip6h, len, IBD_RECV);
5816 		}
5817 	}
5818 
5819 	/*
5820 	 * Update statistics
5821 	 */
5822 	atomic_add_64(&state->id_rcv_bytes, wc->wc_bytes_xfer);
5823 	atomic_inc_64(&state->id_rcv_pkt);
5824 	if (bcmp(&phdr->ib_dst, &state->id_bcaddr, IPOIB_ADDRL) == 0)
5825 		atomic_inc_64(&state->id_brd_rcv);
5826 	else if ((ntohl(phdr->ib_dst.ipoib_qpn) & IB_QPN_MASK) == IB_MC_QPN)
5827 		atomic_inc_64(&state->id_multi_rcv);
5828 
5829 	/*
5830 	 * Set receive checksum status in mp
5831 	 */
5832 	if ((wc->wc_flags & IBT_WC_CKSUM_OK) == IBT_WC_CKSUM_OK) {
5833 		(void) hcksum_assoc(mp, NULL, NULL, 0, 0, 0, 0,
5834 		    HCK_FULLCKSUM | HCK_FULLCKSUM_OK, 0);
5835 	}
5836 
5837 	/*
5838 	 * Add this mp to the list of processed mp's to send to
5839 	 * the nw layer
5840 	 */
5841 	mutex_enter(&state->id_rx_lock);
5842 	if (state->id_rx_mp) {
5843 		ASSERT(state->id_rx_mp_tail != NULL);
5844 		state->id_rx_mp_tail->b_next = mp;
5845 	} else {
5846 		ASSERT(state->id_rx_mp_tail == NULL);
5847 		state->id_rx_mp = mp;
5848 	}
5849 
5850 	state->id_rx_mp_tail = mp;
5851 	state->id_rx_mp_len++;
5852 
5853 	if (state->id_rx_mp_len  >= IBD_MAX_RX_MP_LEN) {
5854 		mpc = state->id_rx_mp;
5855 
5856 		state->id_rx_mp = NULL;
5857 		state->id_rx_mp_tail = NULL;
5858 		state->id_rx_mp_len = 0;
5859 	}
5860 
5861 	mutex_exit(&state->id_rx_lock);
5862 
5863 	if (mpc) {
5864 		ibd_flush_rx(state, mpc);
5865 	}
5866 }
5867 
5868 /*
5869  * Callback code invoked from STREAMs when the receive data buffer is
5870  * free for recycling.
5871  */
5872 static void
5873 ibd_freemsg_cb(char *arg)
5874 {
5875 	ibd_rwqe_t *rwqe = (ibd_rwqe_t *)arg;
5876 	ibd_state_t *state = rwqe->w_state;
5877 
5878 	/*
5879 	 * If the wqe is being destructed, do not attempt recycling.
5880 	 */
5881 	if (rwqe->w_freeing_wqe == B_TRUE) {
5882 		DPRINT(6, "ibd_freemsg: wqe being freed");
5883 		return;
5884 	} else {
5885 		/*
5886 		 * Upper layer has released held mblk, so we have
5887 		 * no more use for keeping the old pointer in
5888 		 * our rwqe.
5889 		 */
5890 		rwqe->rwqe_im_mblk = NULL;
5891 	}
5892 
5893 	rwqe->rwqe_im_mblk = desballoc(rwqe->rwqe_copybuf.ic_bufaddr,
5894 	    state->id_mtu + IPOIB_GRH_SIZE, 0, &rwqe->w_freemsg_cb);
5895 	if (rwqe->rwqe_im_mblk == NULL) {
5896 		ibd_delete_rwqe(state, rwqe);
5897 		ibd_free_rwqe(state, rwqe);
5898 		DPRINT(6, "ibd_freemsg: desballoc failed");
5899 		return;
5900 	}
5901 
5902 	if (ibd_post_rwqe(state, rwqe, B_TRUE) == DDI_FAILURE) {
5903 		ibd_delete_rwqe(state, rwqe);
5904 		ibd_free_rwqe(state, rwqe);
5905 		return;
5906 	}
5907 
5908 	atomic_add_32(&state->id_rx_list.dl_bufs_outstanding, -1);
5909 }
5910 
5911 static uint_t
5912 ibd_tx_recycle(char *arg)
5913 {
5914 	ibd_state_t *state = (ibd_state_t *)arg;
5915 
5916 	/*
5917 	 * Poll for completed entries
5918 	 */
5919 	ibd_poll_compq(state, state->id_scq_hdl);
5920 
5921 	/*
5922 	 * Resume any blocked transmissions if possible
5923 	 */
5924 	(void) ibd_resume_transmission(state);
5925 
5926 	return (DDI_INTR_CLAIMED);
5927 }
5928 
5929 #ifdef IBD_LOGGING
5930 static void
5931 ibd_log_init(void)
5932 {
5933 	ibd_lbuf = kmem_zalloc(IBD_LOG_SZ, KM_SLEEP);
5934 	ibd_lbuf_ndx = 0;
5935 }
5936 
5937 static void
5938 ibd_log_fini(void)
5939 {
5940 	if (ibd_lbuf)
5941 		kmem_free(ibd_lbuf, IBD_LOG_SZ);
5942 	ibd_lbuf_ndx = 0;
5943 	ibd_lbuf = NULL;
5944 }
5945 
5946 static void
5947 ibd_log(const char *fmt, ...)
5948 {
5949 	va_list	ap;
5950 	uint32_t off;
5951 	uint32_t msglen;
5952 	char tmpbuf[IBD_DMAX_LINE];
5953 
5954 	if (ibd_lbuf == NULL)
5955 		return;
5956 
5957 	va_start(ap, fmt);
5958 	msglen = vsnprintf(tmpbuf, IBD_DMAX_LINE, fmt, ap);
5959 	va_end(ap);
5960 
5961 	if (msglen >= IBD_DMAX_LINE)
5962 		msglen = IBD_DMAX_LINE - 1;
5963 
5964 	mutex_enter(&ibd_lbuf_lock);
5965 
5966 	off = ibd_lbuf_ndx;		/* current msg should go here */
5967 	if ((ibd_lbuf_ndx) && (ibd_lbuf[ibd_lbuf_ndx-1] != '\n'))
5968 		ibd_lbuf[ibd_lbuf_ndx-1] = '\n';
5969 
5970 	ibd_lbuf_ndx += msglen;		/* place where next msg should start */
5971 	ibd_lbuf[ibd_lbuf_ndx] = 0;	/* current msg should terminate */
5972 
5973 	if (ibd_lbuf_ndx >= (IBD_LOG_SZ - 2 * IBD_DMAX_LINE))
5974 		ibd_lbuf_ndx = 0;
5975 
5976 	mutex_exit(&ibd_lbuf_lock);
5977 
5978 	bcopy(tmpbuf, ibd_lbuf+off, msglen);	/* no lock needed for this */
5979 }
5980 #endif
5981