xref: /illumos-gate/usr/src/uts/common/io/ib/mgt/ibcm/ibcm_arp_link.c (revision a73c0fe4e90b82a478f821ef3adb5cf34f6a9346)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #include <sys/types.h>
27 #include <sys/stream.h>
28 #include <sys/dlpi.h>
29 #include <sys/stropts.h>
30 #include <sys/strsun.h>
31 #include <sys/sysmacros.h>
32 #include <sys/strlog.h>
33 #include <sys/ddi.h>
34 #include <sys/cmn_err.h>
35 #include <sys/socket.h>
36 #include <net/if.h>
37 #include <net/if_types.h>
38 #include <netinet/in.h>
39 #include <sys/ethernet.h>
40 #include <inet/arp.h>
41 #include <inet/ip.h>
42 #include <inet/ip_ire.h>
43 #include <inet/ip_if.h>
44 #include <sys/ib/mgt/ibcm/ibcm_arp.h>
45 #include <inet/ip_ftable.h>
46 
47 static areq_t ibcm_arp_areq_template = {
48 	AR_ENTRY_QUERY,	/* cmd */
49 	sizeof (areq_t) + (2 * IP_ADDR_LEN),	/* name offset */
50 	sizeof (areq_t),	/* name len */
51 	IP_ARP_PROTO_TYPE,	/* protocol, from arps perspective */
52 	sizeof (areq_t),	/* target addr offset */
53 	IP_ADDR_LEN,	/* target ADDR_length */
54 	0,	/* flags */
55 	sizeof (areq_t) + IP_ADDR_LEN,	/* sender addr offset */
56 	IP_ADDR_LEN,	/* sender addr length */
57 	IBCM_ARP_XMIT_COUNT,	/* xmit_count */
58 	IBCM_ARP_XMIT_INTERVAL,	/* (re)xmit_interval in milliseconds */
59 	4	/* max # of requests to buffer */
60 		/*
61 		 * anything else filled in by the code
62 		 */
63 };
64 
65 static area_t ibcm_arp_area_template = {
66 	AR_ENTRY_ADD,			/* cmd */
67 	sizeof (area_t) + IPOIB_ADDRL + (2 * IP_ADDR_LEN), /* name offset */
68 	sizeof (area_t),		/* name len */
69 	IP_ARP_PROTO_TYPE,		/* protocol, from arps perspective */
70 	sizeof (area_t),		/* proto addr offset */
71 	IP_ADDR_LEN,			/* proto ADDR_length */
72 	sizeof (area_t) + (IP_ADDR_LEN),	/* proto mask offset */
73 	0,				/* flags */
74 	sizeof (area_t) + (2 * IP_ADDR_LEN),	/* hw addr offset */
75 	IPOIB_ADDRL				/* hw addr length */
76 };
77 
78 extern char cmlog[];
79 
80 _NOTE(SCHEME_PROTECTS_DATA("Unshared data", msgb))
81 _NOTE(SCHEME_PROTECTS_DATA("Unshared data", area_t))
82 _NOTE(SCHEME_PROTECTS_DATA("Unshared data", ibcm_arp_streams_t))
83 
84 static void ibcm_arp_timeout(void *arg);
85 void ibcm_arp_pr_callback(ibcm_arp_prwqn_t *wqnp, int status);
86 
87 /*
88  * issue a AR_ENTRY_QUERY to arp driver and schedule a timeout.
89  */
90 int
91 ibcm_arp_query_arp(ibcm_arp_prwqn_t *wqnp)
92 {
93 	int len;
94 	int name_len;
95 	int name_offset;
96 	char *cp;
97 	mblk_t *mp;
98 	mblk_t *mp1;
99 	areq_t *areqp;
100 	ibcm_arp_streams_t *ib_s = (ibcm_arp_streams_t *)wqnp->arg;
101 
102 	IBTF_DPRINTF_L4(cmlog, "ibcm_arp_query_arp(ib_s: %p wqnp: %p)",
103 	    ib_s, wqnp);
104 
105 	name_offset = ibcm_arp_areq_template.areq_name_offset;
106 
107 	/*
108 	 * allocate mblk for AR_ENTRY_QUERY
109 	 */
110 	name_len = strlen(wqnp->ifname) + 1;
111 	len = name_len + name_offset;
112 	if ((mp = allocb(len, BPRI_HI)) == NULL) {
113 		return (ENOMEM);
114 	}
115 	bzero(mp->b_rptr, len);
116 	mp->b_wptr += len;
117 
118 	/*
119 	 * allocate a mblk and set wqnp in the data
120 	 */
121 	if ((mp1 = allocb(sizeof (void *), BPRI_HI)) == NULL) {
122 		freeb(mp);
123 		return (ENOMEM);
124 	}
125 
126 	mp1->b_wptr += sizeof (void *);
127 	*(uintptr_t *)(void *)mp1->b_rptr = (uintptr_t)wqnp;	/* store wqnp */
128 
129 	cp = (char *)mp->b_rptr;
130 	bcopy(&ibcm_arp_areq_template, cp, sizeof (areq_t));
131 	areqp = (void *)cp;
132 	areqp->areq_name_length = name_len;
133 
134 	cp = (char *)areqp + areqp->areq_name_offset;
135 	bcopy(wqnp->ifname, cp, name_len);
136 
137 	areqp->areq_proto = wqnp->ifproto;
138 	bcopy(&wqnp->ifproto, areqp->areq_sap, 2);
139 	cp = (char *)areqp + areqp->areq_target_addr_offset;
140 	bcopy(&wqnp->dst_addr.un.ip4addr, cp, IP_ADDR_LEN);
141 	cp = (char *)areqp + areqp->areq_sender_addr_offset;
142 	bcopy(&wqnp->src_addr.un.ip4addr, cp, IP_ADDR_LEN);
143 
144 	mp->b_cont = mp1;
145 
146 	DB_TYPE(mp) = M_PROTO;
147 
148 	/*
149 	 * issue the request to arp
150 	 */
151 	wqnp->flags |= IBCM_ARP_PR_ARP_PENDING;
152 	wqnp->timeout_id = timeout(ibcm_arp_timeout, wqnp,
153 	    drv_usectohz(IBCM_ARP_TIMEOUT * 1000));
154 	if (canputnext(ib_s->arpqueue)) {
155 		putnext(ib_s->arpqueue, mp);
156 	} else {
157 		(void) putq(ib_s->arpqueue, mp);
158 		qenable(ib_s->arpqueue);
159 	}
160 
161 	return (0);
162 }
163 
164 /*
165  * issue AR_ENTRY_SQUERY to arp driver
166  */
167 int
168 ibcm_arp_squery_arp(ibcm_arp_prwqn_t *wqnp)
169 {
170 	int len;
171 	int name_len;
172 	char *cp;
173 	mblk_t *mp;
174 	mblk_t *mp1;
175 	area_t *areap;
176 	uint32_t  proto_mask = 0xffffffff;
177 	struct iocblk *ioc;
178 	ibcm_arp_streams_t *ib_s = (ibcm_arp_streams_t *)wqnp->arg;
179 
180 	IBTF_DPRINTF_L4(cmlog, "ibcm_arp_squery_arp(ib_s: %p wqnp: %p)",
181 	    ib_s, wqnp);
182 
183 	/*
184 	 * allocate mblk for AR_ENTRY_SQUERY
185 	 */
186 	name_len = strlen(wqnp->ifname) + 1;
187 	len = ibcm_arp_area_template.area_name_offset + name_len +
188 	    sizeof (uintptr_t);
189 	if ((mp = allocb(len, BPRI_HI)) == NULL) {
190 		return (ENOMEM);
191 	}
192 	bzero(mp->b_rptr, len);
193 	mp->b_wptr += len + sizeof (uintptr_t);
194 
195 	*(uintptr_t *)(void *)mp->b_rptr = (uintptr_t)wqnp;	/* store wqnp */
196 	mp->b_rptr += sizeof (uintptr_t);
197 
198 
199 	cp = (char *)mp->b_rptr;
200 	bcopy(&ibcm_arp_area_template, cp, sizeof (area_t));
201 
202 	areap = (void *)cp;
203 	areap->area_cmd = AR_ENTRY_SQUERY;
204 	areap->area_name_length = name_len;
205 	cp = (char *)areap + areap->area_name_offset;
206 	bcopy(wqnp->ifname, cp, name_len);
207 
208 	cp = (char *)areap + areap->area_proto_addr_offset;
209 	bcopy(&wqnp->dst_addr.un.ip4addr, cp, IP_ADDR_LEN);
210 
211 	cp = (char *)areap + areap->area_proto_mask_offset;
212 	bcopy(&proto_mask, cp, IP_ADDR_LEN);
213 
214 	mp1 = allocb(sizeof (struct iocblk), BPRI_HI);
215 	if (mp1 == NULL) {
216 		freeb(mp);
217 		return (ENOMEM);
218 	}
219 	ioc = (void *)mp1->b_rptr;
220 	ioc->ioc_cmd = AR_ENTRY_SQUERY;
221 	ioc->ioc_error = 0;
222 	ioc->ioc_cr = NULL;
223 	ioc->ioc_count = msgdsize(mp);
224 	mp1->b_wptr += sizeof (struct iocblk);
225 	mp1->b_cont = mp;
226 
227 	DB_TYPE(mp1) = M_IOCTL;
228 
229 	if (canputnext(ib_s->arpqueue)) {
230 		putnext(ib_s->arpqueue, mp1);
231 	} else {
232 		(void) putq(ib_s->arpqueue, mp1);
233 		qenable(ib_s->arpqueue);
234 	}
235 	return (0);
236 }
237 
238 /*
239  * issue a AR_ENTRY_ADD to arp driver
240  * This is required as arp driver does not maintain a cache.
241  */
242 int
243 ibcm_arp_add(ibcm_arp_prwqn_t *wqnp)
244 {
245 	int len;
246 	int name_len;
247 	char *cp;
248 	mblk_t *mp;
249 	area_t *areap;
250 	uint32_t  proto_mask = 0xffffffff;
251 	ibcm_arp_streams_t *ib_s = (ibcm_arp_streams_t *)wqnp->arg;
252 
253 	IBTF_DPRINTF_L4(cmlog, "ibcm_arp_add(ib_s: %p wqnp: %p)", ib_s, wqnp);
254 
255 	/*
256 	 * allocate mblk for AR_ENTRY_ADD
257 	 */
258 
259 	name_len = strlen(wqnp->ifname) + 1;
260 	len = ibcm_arp_area_template.area_name_offset + name_len;
261 	if ((mp = allocb(len, BPRI_HI)) == NULL) {
262 		return (ENOMEM);
263 	}
264 	bzero(mp->b_rptr, len);
265 	mp->b_wptr += len;
266 
267 	cp = (char *)mp->b_rptr;
268 	bcopy(&ibcm_arp_area_template, cp, sizeof (area_t));
269 
270 	areap = (void *)mp->b_rptr;
271 	areap->area_name_length = name_len;
272 	cp = (char *)areap + areap->area_name_offset;
273 	bcopy(wqnp->ifname, cp, name_len);
274 
275 	cp = (char *)areap + areap->area_proto_addr_offset;
276 	bcopy(&wqnp->dst_addr.un.ip4addr, cp, IP_ADDR_LEN);
277 
278 	cp = (char *)areap + areap->area_proto_mask_offset;
279 	bcopy(&proto_mask, cp, IP_ADDR_LEN);
280 
281 	cp = (char *)areap + areap->area_hw_addr_offset;
282 	bcopy(&wqnp->dst_mac, cp, IPOIB_ADDRL);
283 
284 	DB_TYPE(mp) = M_PROTO;
285 
286 	if (canputnext(ib_s->arpqueue)) {
287 		putnext(ib_s->arpqueue, mp);
288 	} else {
289 		(void) putq(ib_s->arpqueue, mp);
290 		qenable(ib_s->arpqueue);
291 	}
292 	return (0);
293 }
294 
295 
296 /*
297  * timeout routine when there is no response to AR_ENTRY_QUERY
298  */
299 static void
300 ibcm_arp_timeout(void *arg)
301 {
302 	ibcm_arp_prwqn_t *wqnp = (ibcm_arp_prwqn_t *)arg;
303 	ibcm_arp_streams_t *ib_s = (ibcm_arp_streams_t *)wqnp->arg;
304 
305 	IBTF_DPRINTF_L4(cmlog, "ibcm_arp_timeout(ib_s: %p wqnp: %p)",
306 	    ib_s, wqnp);
307 
308 	/*
309 	 * indicate to user
310 	 */
311 	ibcm_arp_pr_callback(wqnp, EHOSTUNREACH);
312 }
313 
314 /*
315  * delete a wait queue node from the list.
316  * assumes mutex is acquired
317  */
318 void
319 ibcm_arp_prwqn_delete(ibcm_arp_prwqn_t *wqnp)
320 {
321 	ibcm_arp_streams_t *ib_s;
322 
323 	IBTF_DPRINTF_L4(cmlog, "ibcm_arp_prwqn_delete(%p)", wqnp);
324 
325 	ib_s = (ibcm_arp_streams_t *)wqnp->arg;
326 	ib_s->wqnp = NULL;
327 	kmem_free(wqnp, sizeof (ibcm_arp_prwqn_t));
328 }
329 
330 /*
331  * allocate a wait queue node, and insert it in the list
332  */
333 ibcm_arp_prwqn_t *
334 ibcm_arp_create_prwqn(ibcm_arp_streams_t *ib_s, ibt_ip_addr_t *dst_addr,
335     ibt_ip_addr_t *src_addr, uint32_t localroute, uint32_t bound_dev_if,
336     ibcm_arp_pr_comp_func_t func)
337 {
338 	ibcm_arp_prwqn_t *wqnp;
339 
340 	IBTF_DPRINTF_L4(cmlog, "ibcm_arp_create_prwqn(ib_s: 0x%p)", ib_s);
341 
342 	if (dst_addr == NULL) {
343 		return (NULL);
344 	}
345 	if ((wqnp = kmem_zalloc(sizeof (ibcm_arp_prwqn_t), KM_NOSLEEP)) ==
346 	    NULL) {
347 		return (NULL);
348 	}
349 	wqnp->dst_addr = *dst_addr;
350 
351 	if (src_addr) {
352 		wqnp->usrc_addr = *src_addr;
353 	}
354 	wqnp->func = func;
355 	wqnp->arg = ib_s;
356 	wqnp->localroute = localroute;
357 	wqnp->bound_dev_if = bound_dev_if;
358 	wqnp->ifproto = ETHERTYPE_IP;
359 
360 	ib_s->wqnp = wqnp;
361 
362 	IBTF_DPRINTF_L4(cmlog, "ibcm_arp_create_prwqn: Return wqnp: %p", wqnp);
363 
364 	return (wqnp);
365 }
366 
367 /*
368  * call the user function
369  * called with lock held
370  */
371 void
372 ibcm_arp_pr_callback(ibcm_arp_prwqn_t *wqnp, int status)
373 {
374 	IBTF_DPRINTF_L4(cmlog, "ibcm_arp_pr_callback(%p, %d)", wqnp, status);
375 
376 	wqnp->func((void *)wqnp, status);
377 }
378 
379 /*
380  * Check if the interface is loopback or IB.
381  */
382 static int
383 ibcm_arp_check_interface(ill_t *ill)
384 {
385 	if (IS_LOOPBACK(ill) || ill->ill_type == IFT_IB)
386 		return (0);
387 
388 	return (ETIMEDOUT);
389 }
390 
391 #define	IBTL_IPV4_ADDR(a)	(a->un.ip4addr)
392 
393 int
394 ibcm_arp_pr_lookup(ibcm_arp_streams_t *ib_s, ibt_ip_addr_t *dst_addr,
395     ibt_ip_addr_t *src_addr, uint8_t localroute, uint32_t bound_dev_if,
396     ibcm_arp_pr_comp_func_t func)
397 {
398 	ibcm_arp_prwqn_t *wqnp;
399 	ire_t	*ire = NULL;
400 	ire_t	*src_ire = NULL;
401 	ipif_t	*ipif;
402 	ill_t	*ill, *hwaddr_ill = NULL;
403 	ip_stack_t *ipst;
404 
405 	IBTF_DPRINTF_L4(cmlog, "ibcm_arp_pr_lookup(src %p dest %p)",
406 	    src_addr, dst_addr);
407 
408 	if (dst_addr->family != AF_INET_OFFLOAD) {
409 		ib_s->status = EAFNOSUPPORT;
410 		return (1);
411 	}
412 
413 	if ((wqnp = ibcm_arp_create_prwqn(ib_s, dst_addr,
414 	    src_addr, localroute, bound_dev_if, func)) == NULL) {
415 		IBTF_DPRINTF_L2(cmlog, "ibcm_arp_pr_lookup: "
416 		    "ibcm_arp_create_prwqn failed");
417 		ib_s->status = ENOMEM;
418 		return (1);
419 	}
420 
421 	ipst = netstack_find_by_zoneid(GLOBAL_ZONEID)->netstack_ip;
422 	/*
423 	 * Get the ire for the local address
424 	 */
425 	IBTF_DPRINTF_L4(cmlog, "ibcm_arp_pr_lookup: srcip %lX destip %lX",
426 	    IBTL_IPV4_ADDR(src_addr), IBTL_IPV4_ADDR(dst_addr));
427 
428 	src_ire = ire_ctable_lookup(IBTL_IPV4_ADDR(src_addr), NULL,
429 	    IRE_LOCAL, NULL, ALL_ZONES, NULL, MATCH_IRE_TYPE, ipst);
430 	if (src_ire == NULL) {
431 		IBTF_DPRINTF_L2(cmlog, "ibcm_arp_pr_lookup: "
432 		    "ire_ctable_lookup failed");
433 		ib_s->status = EFAULT;
434 		goto fail;
435 	}
436 
437 	/*
438 	 * get an ire for the destination adress with the matching source
439 	 * address
440 	 */
441 	ire = ire_ftable_lookup(IBTL_IPV4_ADDR(dst_addr), 0, 0, 0,
442 	    src_ire->ire_ipif, 0, src_ire->ire_zoneid, 0, NULL, MATCH_IRE_SRC,
443 	    ipst);
444 	if (ire == NULL) {
445 		IBTF_DPRINTF_L2(cmlog, "ibcm_arp_pr_lookup: "
446 		    "ire_ftable_lookup failed");
447 		ib_s->status = EFAULT;
448 		goto fail;
449 	}
450 
451 	wqnp->src_addr.un.ip4addr = ire->ire_src_addr;
452 	wqnp->src_addr.family = AF_INET_OFFLOAD;
453 
454 	ipif = src_ire->ire_ipif;
455 	ill = ipif->ipif_ill;
456 	(void) strlcpy(wqnp->ifname, ill->ill_name, sizeof (wqnp->ifname));
457 
458 	/*
459 	 * For IPMP data addresses, we need to use the hardware address of the
460 	 * interface bound to the given address.
461 	 */
462 	if (IS_IPMP(ill)) {
463 		if ((hwaddr_ill = ipmp_ipif_hold_bound_ill(ipif)) == NULL) {
464 			IBTF_DPRINTF_L2(cmlog, "ibcm_arp_pr_lookup: no bound "
465 			    "ill for IPMP interface %s", ill->ill_name);
466 			ib_s->status = EFAULT;
467 			goto fail;
468 		}
469 	} else {
470 		hwaddr_ill = ill;
471 		ill_refhold(hwaddr_ill); 	/* for symmetry */
472 	}
473 
474 	bcopy(hwaddr_ill->ill_phys_addr, &wqnp->src_mac,
475 	    hwaddr_ill->ill_phys_addr_length);
476 
477 	if ((ib_s->status = ibcm_arp_check_interface(hwaddr_ill)) != 0) {
478 		IBTF_DPRINTF_L2(cmlog, "ibcm_arp_pr_lookup: "
479 		    "ibcm_arp_check_interface failed");
480 		goto fail;
481 	}
482 
483 	if ((ib_s->status = ibcm_arp_squery_arp(wqnp)) != 0) {
484 		IBTF_DPRINTF_L2(cmlog, "ibcm_arp_pr_lookup: "
485 		    "ibcm_arp_squery_arp failed");
486 		goto fail;
487 	}
488 
489 	ill_refrele(hwaddr_ill);
490 	IRE_REFRELE(ire);
491 	IRE_REFRELE(src_ire);
492 	netstack_rele(ipst->ips_netstack);
493 
494 	IBTF_DPRINTF_L4(cmlog, "ibcm_arp_pr_lookup: Return: 0x%p", wqnp);
495 	return (0);
496 fail:
497 	if (hwaddr_ill != NULL)
498 		ill_refrele(hwaddr_ill);
499 	if (ire != NULL)
500 		IRE_REFRELE(ire);
501 	if (src_ire != NULL)
502 		IRE_REFRELE(src_ire);
503 	ibcm_arp_prwqn_delete(wqnp);
504 	netstack_rele(ipst->ips_netstack);
505 	return (1);
506 }
507 
508 #define	IBCM_H2N_GID(gid) \
509 { \
510 	uint32_t	*ptr; \
511 	ptr = (uint32_t *)&gid.gid_prefix; \
512 	gid.gid_prefix = (uint64_t)(((uint64_t)ntohl(ptr[0]) << 32) | \
513 			(ntohl(ptr[1]))); \
514 	ptr = (uint32_t *)&gid.gid_guid; \
515 	gid.gid_guid = (uint64_t)(((uint64_t)ntohl(ptr[0]) << 32) | \
516 			(ntohl(ptr[1]))); \
517 }
518 
519 /*
520  * called from lrsrv.
521  * process a AR_ENTRY_QUERY reply from arp
522  * the message should be M_DATA -->> dl_unitdata_req
523  */
524 void
525 ibcm_arp_pr_arp_query_ack(mblk_t *mp)
526 {
527 	ibcm_arp_prwqn_t 	*wqnp;
528 	dl_unitdata_req_t *dlreq;
529 	ibcm_arp_streams_t *ib_s;
530 	char *cp;
531 	int rc;
532 
533 	IBTF_DPRINTF_L4(cmlog, "ibcm_arp_pr_arp_query_ack(%p)", mp);
534 
535 	/*
536 	 * the first mblk contains the wqnp pointer for the request
537 	 */
538 	if (MBLKL(mp) != sizeof (void *)) {
539 		freemsg(mp);
540 		return;
541 	}
542 
543 	wqnp = *(ibcm_arp_prwqn_t **)(void *)mp->b_rptr; /* retrieve wqnp */
544 	ib_s = (ibcm_arp_streams_t *)wqnp->arg;
545 
546 	mutex_enter(&ib_s->lock);
547 
548 	/*
549 	 * cancel the timeout for this request
550 	 */
551 	(void) untimeout(wqnp->timeout_id);
552 
553 	/*
554 	 * sanity checks on the dl_unitdata_req block
555 	 */
556 	if (!mp->b_cont) {
557 		IBTF_DPRINTF_L2(cmlog, "areq_ack: b_cont = NULL\n");
558 		rc = EPROTO;
559 		goto user_callback;
560 	}
561 	if (MBLKL(mp->b_cont) < (sizeof (dl_unitdata_req_t) + IPOIB_ADDRL)) {
562 		IBTF_DPRINTF_L2(cmlog, "areq_ack: invalid len in "
563 		    "dl_unitdatareq_t block\n");
564 		rc = EPROTO;
565 		goto user_callback;
566 	}
567 	dlreq = (void *)mp->b_cont->b_rptr;
568 	if (dlreq->dl_primitive != DL_UNITDATA_REQ) {
569 		IBTF_DPRINTF_L2(cmlog, "areq_ack: invalid dl_primitive "
570 		    "in dl_unitdatareq_t block\n");
571 		rc = EPROTO;
572 		goto user_callback;
573 	}
574 	if (dlreq->dl_dest_addr_length != (IPOIB_ADDRL + 2)) {
575 		IBTF_DPRINTF_L2(cmlog, "areq_ack: invalid hw len in "
576 		    "dl_unitdatareq_t block %d\n", dlreq->dl_dest_addr_length);
577 		rc = EPROTO;
578 		goto user_callback;
579 	}
580 	cp = (char *)mp->b_cont->b_rptr + dlreq->dl_dest_addr_offset;
581 	bcopy(cp, &wqnp->dst_mac, IPOIB_ADDRL);
582 
583 	/*
584 	 * at this point we have src/dst gid's derived from the mac addresses
585 	 * now get the hca, port
586 	 */
587 	bcopy(&wqnp->src_mac.ipoib_gidpref, &wqnp->sgid, sizeof (ib_gid_t));
588 	bcopy(&wqnp->dst_mac.ipoib_gidpref, &wqnp->dgid, sizeof (ib_gid_t));
589 	freemsg(mp);
590 
591 	IBCM_H2N_GID(wqnp->sgid);
592 	IBCM_H2N_GID(wqnp->dgid);
593 
594 	(void) ibcm_arp_add(wqnp);
595 
596 	mutex_exit(&ib_s->lock);
597 	ibcm_arp_pr_callback(wqnp, 0);
598 
599 	return;
600 user_callback:
601 	freemsg(mp);
602 	mutex_exit(&ib_s->lock);
603 
604 	/*
605 	 * indicate to user
606 	 */
607 	ibcm_arp_pr_callback(wqnp, rc);
608 }
609 
610 /*
611  * process a AR_ENTRY_SQUERY reply from arp
612  * the message should be M_IOCACK -->> area_t
613  */
614 void
615 ibcm_arp_pr_arp_squery_ack(mblk_t *mp)
616 {
617 	struct iocblk *ioc;
618 	mblk_t	*mp1;
619 	ibcm_arp_prwqn_t 	*wqnp;
620 	ibcm_arp_streams_t *ib_s;
621 	area_t *areap;
622 	char *cp;
623 
624 	IBTF_DPRINTF_L4(cmlog, "ibcm_arp_pr_arp_squery_ack(%p)", mp);
625 
626 	if (MBLKL(mp) < sizeof (struct iocblk)) {
627 		freemsg(mp);
628 		return;
629 	}
630 
631 	ioc = (void *)mp->b_rptr;
632 	if ((ioc->ioc_cmd != AR_ENTRY_SQUERY) || (mp->b_cont == NULL)) {
633 		freemsg(mp);
634 		return;
635 	}
636 
637 	mp1 = mp->b_cont;
638 
639 	wqnp = *(ibcm_arp_prwqn_t **)((uintptr_t)mp1->b_rptr -
640 	    sizeof (uintptr_t));
641 	ib_s = (ibcm_arp_streams_t *)wqnp->arg;
642 
643 	mutex_enter(&ib_s->lock);
644 
645 	/* If the entry was not in arp cache, ioc_error is set */
646 	if (ioc->ioc_error) {
647 
648 		/*
649 		 * send out AR_ENTRY_QUERY which would send
650 		 * arp-request on wire
651 		 */
652 		IBTF_DPRINTF_L3(cmlog, "Sending a Query_ARP");
653 
654 		(void) ibcm_arp_query_arp(wqnp);
655 		freemsg(mp);
656 		mutex_exit(&ib_s->lock);
657 		return;
658 	}
659 
660 	areap = (void *)mp1->b_rptr;
661 	cp = (char *)areap + areap->area_hw_addr_offset;
662 	bcopy(cp, &wqnp->dst_mac, IPOIB_ADDRL);
663 
664 	/*
665 	 * at this point we have src/dst gid's derived from the mac addresses
666 	 * now get the hca, port
667 	 */
668 	bcopy(&wqnp->src_mac.ipoib_gidpref, &wqnp->sgid, sizeof (ib_gid_t));
669 	bcopy(&wqnp->dst_mac.ipoib_gidpref, &wqnp->dgid, sizeof (ib_gid_t));
670 	freemsg(mp);
671 
672 	IBCM_H2N_GID(wqnp->sgid);
673 	IBCM_H2N_GID(wqnp->dgid);
674 
675 	mutex_exit(&ib_s->lock);
676 	ibcm_arp_pr_callback(wqnp, 0);
677 }
678 
679 /*
680  * Process arp ack's.
681  */
682 void
683 ibcm_arp_pr_arp_ack(mblk_t *mp)
684 {
685 	IBTF_DPRINTF_L4(cmlog, "ibcm_arp_pr_arp_ack(0x%p, DB_TYPE %lX)",
686 	    mp, DB_TYPE(mp));
687 
688 	if (DB_TYPE(mp) == M_DATA) {
689 		ibcm_arp_pr_arp_query_ack(mp);
690 	} else if ((DB_TYPE(mp) == M_IOCACK) ||
691 	    (DB_TYPE(mp) == M_IOCNAK)) {
692 		ibcm_arp_pr_arp_squery_ack(mp);
693 	} else {
694 		freemsg(mp);
695 	}
696 }
697