xref: /titanic_50/usr/src/uts/common/inet/ip/ipsecah.c (revision 530f2c280d739b194cfbb75f25352b75bb99b4b2)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #include <sys/types.h>
27 #include <sys/stream.h>
28 #include <sys/stropts.h>
29 #include <sys/errno.h>
30 #include <sys/strlog.h>
31 #include <sys/tihdr.h>
32 #include <sys/socket.h>
33 #include <sys/ddi.h>
34 #include <sys/sunddi.h>
35 #include <sys/mkdev.h>
36 #include <sys/kmem.h>
37 #include <sys/zone.h>
38 #include <sys/sysmacros.h>
39 #include <sys/cmn_err.h>
40 #include <sys/vtrace.h>
41 #include <sys/debug.h>
42 #include <sys/atomic.h>
43 #include <sys/strsun.h>
44 #include <sys/random.h>
45 #include <netinet/in.h>
46 #include <net/if.h>
47 #include <netinet/ip6.h>
48 #include <netinet/icmp6.h>
49 #include <net/pfkeyv2.h>
50 #include <net/pfpolicy.h>
51 
52 #include <inet/common.h>
53 #include <inet/mi.h>
54 #include <inet/ip.h>
55 #include <inet/ip6.h>
56 #include <inet/nd.h>
57 #include <inet/ipsec_info.h>
58 #include <inet/ipsec_impl.h>
59 #include <inet/sadb.h>
60 #include <inet/ipsecah.h>
61 #include <inet/ipsec_impl.h>
62 #include <inet/ipdrop.h>
63 #include <sys/taskq.h>
64 #include <sys/policy.h>
65 #include <sys/iphada.h>
66 #include <sys/strsun.h>
67 
68 #include <sys/crypto/common.h>
69 #include <sys/crypto/api.h>
70 #include <sys/kstat.h>
71 #include <sys/strsubr.h>
72 
73 /*
74  * Table of ND variables supported by ipsecah. These are loaded into
75  * ipsecah_g_nd in ipsecah_init_nd.
76  * All of these are alterable, within the min/max values given, at run time.
77  */
78 static	ipsecahparam_t	lcl_param_arr[] = {
79 	/* min	max			value	name */
80 	{ 0,	3,			0,	"ipsecah_debug"},
81 	{ 125,	32000, SADB_AGE_INTERVAL_DEFAULT,	"ipsecah_age_interval"},
82 	{ 1,	10,			1,	"ipsecah_reap_delay"},
83 	{ 1,	SADB_MAX_REPLAY,	64,	"ipsecah_replay_size"},
84 	{ 1,	300,			15,	"ipsecah_acquire_timeout"},
85 	{ 1,	1800,			90,	"ipsecah_larval_timeout"},
86 	/* Default lifetime values for ACQUIRE messages. */
87 	{ 0,	0xffffffffU,		0,	"ipsecah_default_soft_bytes"},
88 	{ 0,	0xffffffffU,		0,	"ipsecah_default_hard_bytes"},
89 	{ 0,	0xffffffffU,		24000,	"ipsecah_default_soft_addtime"},
90 	{ 0,	0xffffffffU,		28800,	"ipsecah_default_hard_addtime"},
91 	{ 0,	0xffffffffU,		0,	"ipsecah_default_soft_usetime"},
92 	{ 0,	0xffffffffU,		0,	"ipsecah_default_hard_usetime"},
93 	{ 0,	1,			0,	"ipsecah_log_unknown_spi"},
94 };
95 #define	ipsecah_debug			ipsecah_params[0].ipsecah_param_value
96 #define	ipsecah_age_interval		ipsecah_params[1].ipsecah_param_value
97 #define	ipsecah_age_int_max		ipsecah_params[1].ipsecah_param_max
98 #define	ipsecah_reap_delay		ipsecah_params[2].ipsecah_param_value
99 #define	ipsecah_replay_size		ipsecah_params[3].ipsecah_param_value
100 #define	ipsecah_acquire_timeout		ipsecah_params[4].ipsecah_param_value
101 #define	ipsecah_larval_timeout		ipsecah_params[5].ipsecah_param_value
102 #define	ipsecah_default_soft_bytes	ipsecah_params[6].ipsecah_param_value
103 #define	ipsecah_default_hard_bytes	ipsecah_params[7].ipsecah_param_value
104 #define	ipsecah_default_soft_addtime	ipsecah_params[8].ipsecah_param_value
105 #define	ipsecah_default_hard_addtime	ipsecah_params[9].ipsecah_param_value
106 #define	ipsecah_default_soft_usetime	ipsecah_params[10].ipsecah_param_value
107 #define	ipsecah_default_hard_usetime	ipsecah_params[11].ipsecah_param_value
108 #define	ipsecah_log_unknown_spi		ipsecah_params[12].ipsecah_param_value
109 
110 #define	ah0dbg(a)	printf a
111 /* NOTE:  != 0 instead of > 0 so lint doesn't complain. */
112 #define	ah1dbg(ahstack, a)	if (ahstack->ipsecah_debug != 0) printf a
113 #define	ah2dbg(ahstack, a)	if (ahstack->ipsecah_debug > 1) printf a
114 #define	ah3dbg(ahstack, a)	if (ahstack->ipsecah_debug > 2) printf a
115 
116 /*
117  * XXX This is broken. Padding should be determined dynamically
118  * depending on the ICV size and IP version number so that the
119  * total AH header size is a multiple of 32 bits or 64 bits
120  * for V4 and V6 respectively. For 96bit ICVs we have no problems.
121  * Anything different from that, we need to fix our code.
122  */
123 #define	IPV4_PADDING_ALIGN	0x04	/* Multiple of 32 bits */
124 #define	IPV6_PADDING_ALIGN	0x04	/* Multiple of 32 bits */
125 
126 /*
127  * Helper macro. Avoids a call to msgdsize if there is only one
128  * mblk in the chain.
129  */
130 #define	AH_MSGSIZE(mp) ((mp)->b_cont != NULL ? msgdsize(mp) : MBLKL(mp))
131 
132 
133 static ipsec_status_t ah_auth_out_done(mblk_t *);
134 static ipsec_status_t ah_auth_in_done(mblk_t *);
135 static mblk_t *ah_process_ip_options_v4(mblk_t *, ipsa_t *, int *, uint_t,
136     boolean_t, ipsecah_stack_t *);
137 static mblk_t *ah_process_ip_options_v6(mblk_t *, ipsa_t *, int *, uint_t,
138     boolean_t, ipsecah_stack_t *);
139 static void ah_getspi(mblk_t *, keysock_in_t *, ipsecah_stack_t *);
140 static ipsec_status_t ah_inbound_accelerated(mblk_t *, boolean_t, ipsa_t *,
141     uint32_t);
142 static ipsec_status_t ah_outbound_accelerated_v4(mblk_t *, ipsa_t *);
143 static ipsec_status_t ah_outbound_accelerated_v6(mblk_t *, ipsa_t *);
144 static ipsec_status_t ah_outbound(mblk_t *);
145 
146 static int ipsecah_open(queue_t *, dev_t *, int, int, cred_t *);
147 static int ipsecah_close(queue_t *);
148 static void ipsecah_rput(queue_t *, mblk_t *);
149 static void ipsecah_wput(queue_t *, mblk_t *);
150 static void ah_send_acquire(ipsacq_t *, mblk_t *, netstack_t *);
151 static boolean_t ah_register_out(uint32_t, uint32_t, uint_t, ipsecah_stack_t *);
152 static void	*ipsecah_stack_init(netstackid_t stackid, netstack_t *ns);
153 static void	ipsecah_stack_fini(netstackid_t stackid, void *arg);
154 
155 extern void (*cl_inet_getspi)(netstackid_t, uint8_t, uint8_t *, size_t,
156     void *);
157 
158 /* Setable in /etc/system */
159 uint32_t ah_hash_size = IPSEC_DEFAULT_HASH_SIZE;
160 
161 static taskq_t *ah_taskq;
162 
163 static struct module_info info = {
164 	5136, "ipsecah", 0, INFPSZ, 65536, 1024
165 };
166 
167 static struct qinit rinit = {
168 	(pfi_t)ipsecah_rput, NULL, ipsecah_open, ipsecah_close, NULL, &info,
169 	NULL
170 };
171 
172 static struct qinit winit = {
173 	(pfi_t)ipsecah_wput, NULL, ipsecah_open, ipsecah_close, NULL, &info,
174 	NULL
175 };
176 
177 struct streamtab ipsecahinfo = {
178 	&rinit, &winit, NULL, NULL
179 };
180 
181 static int ah_kstat_update(kstat_t *, int);
182 
183 uint64_t ipsacq_maxpackets = IPSACQ_MAXPACKETS;
184 
185 static boolean_t
186 ah_kstat_init(ipsecah_stack_t *ahstack, netstackid_t stackid)
187 {
188 	ipsec_stack_t	*ipss = ahstack->ipsecah_netstack->netstack_ipsec;
189 
190 	ahstack->ah_ksp = kstat_create_netstack("ipsecah", 0, "ah_stat", "net",
191 	    KSTAT_TYPE_NAMED, sizeof (ah_kstats_t) / sizeof (kstat_named_t),
192 	    KSTAT_FLAG_PERSISTENT, stackid);
193 
194 	if (ahstack->ah_ksp == NULL || ahstack->ah_ksp->ks_data == NULL)
195 		return (B_FALSE);
196 
197 	ahstack->ah_kstats = ahstack->ah_ksp->ks_data;
198 
199 	ahstack->ah_ksp->ks_update = ah_kstat_update;
200 	ahstack->ah_ksp->ks_private = (void *)(uintptr_t)stackid;
201 
202 #define	K64 KSTAT_DATA_UINT64
203 #define	KI(x) kstat_named_init(&(ahstack->ah_kstats->ah_stat_##x), #x, K64)
204 
205 	KI(num_aalgs);
206 	KI(good_auth);
207 	KI(bad_auth);
208 	KI(replay_failures);
209 	KI(replay_early_failures);
210 	KI(keysock_in);
211 	KI(out_requests);
212 	KI(acquire_requests);
213 	KI(bytes_expired);
214 	KI(out_discards);
215 	KI(in_accelerated);
216 	KI(out_accelerated);
217 	KI(noaccel);
218 	KI(crypto_sync);
219 	KI(crypto_async);
220 	KI(crypto_failures);
221 
222 #undef KI
223 #undef K64
224 
225 	kstat_install(ahstack->ah_ksp);
226 	IP_ACQUIRE_STAT(ipss, maxpackets, ipsacq_maxpackets);
227 	return (B_TRUE);
228 }
229 
230 static int
231 ah_kstat_update(kstat_t *kp, int rw)
232 {
233 	ah_kstats_t	*ekp;
234 	netstackid_t	stackid = (netstackid_t)(uintptr_t)kp->ks_private;
235 	netstack_t	*ns;
236 	ipsec_stack_t	*ipss;
237 
238 	if ((kp == NULL) || (kp->ks_data == NULL))
239 		return (EIO);
240 
241 	if (rw == KSTAT_WRITE)
242 		return (EACCES);
243 
244 	ns = netstack_find_by_stackid(stackid);
245 	if (ns == NULL)
246 		return (-1);
247 	ipss = ns->netstack_ipsec;
248 	if (ipss == NULL) {
249 		netstack_rele(ns);
250 		return (-1);
251 	}
252 	ekp = (ah_kstats_t *)kp->ks_data;
253 
254 	mutex_enter(&ipss->ipsec_alg_lock);
255 	ekp->ah_stat_num_aalgs.value.ui64 = ipss->ipsec_nalgs[IPSEC_ALG_AUTH];
256 	mutex_exit(&ipss->ipsec_alg_lock);
257 
258 	netstack_rele(ns);
259 	return (0);
260 }
261 
262 /*
263  * Don't have to lock ipsec_age_interval, as only one thread will access it at
264  * a time, because I control the one function that does a qtimeout() on
265  * ah_pfkey_q.
266  */
267 static void
268 ah_ager(void *arg)
269 {
270 	ipsecah_stack_t *ahstack = (ipsecah_stack_t *)arg;
271 	netstack_t	*ns = ahstack->ipsecah_netstack;
272 	hrtime_t begin = gethrtime();
273 
274 	sadb_ager(&ahstack->ah_sadb.s_v4, ahstack->ah_pfkey_q,
275 	    ahstack->ah_sadb.s_ip_q, ahstack->ipsecah_reap_delay, ns);
276 	sadb_ager(&ahstack->ah_sadb.s_v6, ahstack->ah_pfkey_q,
277 	    ahstack->ah_sadb.s_ip_q, ahstack->ipsecah_reap_delay, ns);
278 
279 	ahstack->ah_event = sadb_retimeout(begin, ahstack->ah_pfkey_q,
280 	    ah_ager, ahstack,
281 	    &ahstack->ipsecah_age_interval, ahstack->ipsecah_age_int_max,
282 	    info.mi_idnum);
283 }
284 
285 /*
286  * Get an AH NDD parameter.
287  */
288 /* ARGSUSED */
289 static int
290 ipsecah_param_get(q, mp, cp, cr)
291 	queue_t	*q;
292 	mblk_t	*mp;
293 	caddr_t	cp;
294 	cred_t *cr;
295 {
296 	ipsecahparam_t	*ipsecahpa = (ipsecahparam_t *)cp;
297 	uint_t value;
298 	ipsecah_stack_t	*ahstack = (ipsecah_stack_t *)q->q_ptr;
299 
300 	mutex_enter(&ahstack->ipsecah_param_lock);
301 	value = ipsecahpa->ipsecah_param_value;
302 	mutex_exit(&ahstack->ipsecah_param_lock);
303 
304 	(void) mi_mpprintf(mp, "%u", value);
305 	return (0);
306 }
307 
308 /*
309  * This routine sets an NDD variable in a ipsecahparam_t structure.
310  */
311 /* ARGSUSED */
312 static int
313 ipsecah_param_set(q, mp, value, cp, cr)
314 	queue_t	*q;
315 	mblk_t	*mp;
316 	char	*value;
317 	caddr_t	cp;
318 	cred_t *cr;
319 {
320 	ulong_t	new_value;
321 	ipsecahparam_t	*ipsecahpa = (ipsecahparam_t *)cp;
322 	ipsecah_stack_t	*ahstack = (ipsecah_stack_t *)q->q_ptr;
323 
324 	/*
325 	 * Fail the request if the new value does not lie within the
326 	 * required bounds.
327 	 */
328 	if (ddi_strtoul(value, NULL, 10, &new_value) != 0 ||
329 	    new_value < ipsecahpa->ipsecah_param_min ||
330 	    new_value > ipsecahpa->ipsecah_param_max) {
331 		    return (EINVAL);
332 	}
333 
334 	/* Set the new value */
335 	mutex_enter(&ahstack->ipsecah_param_lock);
336 	ipsecahpa->ipsecah_param_value = new_value;
337 	mutex_exit(&ahstack->ipsecah_param_lock);
338 	return (0);
339 }
340 
341 /*
342  * Using lifetime NDD variables, fill in an extended combination's
343  * lifetime information.
344  */
345 void
346 ipsecah_fill_defs(sadb_x_ecomb_t *ecomb, netstack_t *ns)
347 {
348 	ipsecah_stack_t	*ahstack = ns->netstack_ipsecah;
349 
350 	ecomb->sadb_x_ecomb_soft_bytes = ahstack->ipsecah_default_soft_bytes;
351 	ecomb->sadb_x_ecomb_hard_bytes = ahstack->ipsecah_default_hard_bytes;
352 	ecomb->sadb_x_ecomb_soft_addtime =
353 	    ahstack->ipsecah_default_soft_addtime;
354 	ecomb->sadb_x_ecomb_hard_addtime =
355 	    ahstack->ipsecah_default_hard_addtime;
356 	ecomb->sadb_x_ecomb_soft_usetime =
357 	    ahstack->ipsecah_default_soft_usetime;
358 	ecomb->sadb_x_ecomb_hard_usetime =
359 	    ahstack->ipsecah_default_hard_usetime;
360 }
361 
362 /*
363  * Initialize things for AH at module load time.
364  */
365 boolean_t
366 ipsecah_ddi_init(void)
367 {
368 	ah_taskq = taskq_create("ah_taskq", 1, minclsyspri,
369 	    IPSEC_TASKQ_MIN, IPSEC_TASKQ_MAX, 0);
370 
371 	/*
372 	 * We want to be informed each time a stack is created or
373 	 * destroyed in the kernel, so we can maintain the
374 	 * set of ipsecah_stack_t's.
375 	 */
376 	netstack_register(NS_IPSECAH, ipsecah_stack_init, NULL,
377 	    ipsecah_stack_fini);
378 
379 	return (B_TRUE);
380 }
381 
382 /*
383  * Walk through the param array specified registering each element with the
384  * named dispatch handler.
385  */
386 static boolean_t
387 ipsecah_param_register(IDP *ndp, ipsecahparam_t *ahp, int cnt)
388 {
389 	for (; cnt-- > 0; ahp++) {
390 		if (ahp->ipsecah_param_name != NULL &&
391 		    ahp->ipsecah_param_name[0]) {
392 			if (!nd_load(ndp,
393 			    ahp->ipsecah_param_name,
394 			    ipsecah_param_get, ipsecah_param_set,
395 			    (caddr_t)ahp)) {
396 				nd_free(ndp);
397 				return (B_FALSE);
398 			}
399 		}
400 	}
401 	return (B_TRUE);
402 }
403 
404 /*
405  * Initialize things for AH for each stack instance
406  */
407 static void *
408 ipsecah_stack_init(netstackid_t stackid, netstack_t *ns)
409 {
410 	ipsecah_stack_t	*ahstack;
411 	ipsecahparam_t	*ahp;
412 
413 	ahstack = (ipsecah_stack_t *)kmem_zalloc(sizeof (*ahstack), KM_SLEEP);
414 	ahstack->ipsecah_netstack = ns;
415 
416 	ahp = (ipsecahparam_t *)kmem_alloc(sizeof (lcl_param_arr), KM_SLEEP);
417 	ahstack->ipsecah_params = ahp;
418 	bcopy(lcl_param_arr, ahp, sizeof (lcl_param_arr));
419 
420 	(void) ipsecah_param_register(&ahstack->ipsecah_g_nd, ahp,
421 	    A_CNT(lcl_param_arr));
422 
423 	(void) ah_kstat_init(ahstack, stackid);
424 
425 	ahstack->ah_sadb.s_acquire_timeout = &ahstack->ipsecah_acquire_timeout;
426 	ahstack->ah_sadb.s_acqfn = ah_send_acquire;
427 	sadbp_init("AH", &ahstack->ah_sadb, SADB_SATYPE_AH, ah_hash_size,
428 	    ahstack->ipsecah_netstack);
429 
430 	mutex_init(&ahstack->ipsecah_param_lock, NULL, MUTEX_DEFAULT, 0);
431 
432 	ip_drop_register(&ahstack->ah_dropper, "IPsec AH");
433 	return (ahstack);
434 }
435 
436 /*
437  * Destroy things for AH at module unload time.
438  */
439 void
440 ipsecah_ddi_destroy(void)
441 {
442 	netstack_unregister(NS_IPSECAH);
443 	taskq_destroy(ah_taskq);
444 }
445 
446 /*
447  * Destroy things for AH for one stack... Never called?
448  */
449 static void
450 ipsecah_stack_fini(netstackid_t stackid, void *arg)
451 {
452 	ipsecah_stack_t *ahstack = (ipsecah_stack_t *)arg;
453 
454 	if (ahstack->ah_pfkey_q != NULL) {
455 		(void) quntimeout(ahstack->ah_pfkey_q, ahstack->ah_event);
456 	}
457 	ahstack->ah_sadb.s_acqfn = NULL;
458 	ahstack->ah_sadb.s_acquire_timeout = NULL;
459 	sadbp_destroy(&ahstack->ah_sadb, ahstack->ipsecah_netstack);
460 	ip_drop_unregister(&ahstack->ah_dropper);
461 	mutex_destroy(&ahstack->ipsecah_param_lock);
462 	nd_free(&ahstack->ipsecah_g_nd);
463 
464 	kmem_free(ahstack->ipsecah_params, sizeof (lcl_param_arr));
465 	ahstack->ipsecah_params = NULL;
466 	kstat_delete_netstack(ahstack->ah_ksp, stackid);
467 	ahstack->ah_ksp = NULL;
468 	ahstack->ah_kstats = NULL;
469 
470 	kmem_free(ahstack, sizeof (*ahstack));
471 }
472 
473 /*
474  * AH module open routine. The module should be opened by keysock.
475  */
476 /* ARGSUSED */
477 static int
478 ipsecah_open(queue_t *q, dev_t *devp, int flag, int sflag, cred_t *credp)
479 {
480 	netstack_t	*ns;
481 	ipsecah_stack_t	*ahstack;
482 
483 	if (secpolicy_ip_config(credp, B_FALSE) != 0)
484 		return (EPERM);
485 
486 	if (q->q_ptr != NULL)
487 		return (0);  /* Re-open of an already open instance. */
488 
489 	if (sflag != MODOPEN)
490 		return (EINVAL);
491 
492 	ns = netstack_find_by_cred(credp);
493 	ASSERT(ns != NULL);
494 	ahstack = ns->netstack_ipsecah;
495 	ASSERT(ahstack != NULL);
496 
497 	/*
498 	 * ASSUMPTIONS (because I'm MT_OCEXCL):
499 	 *
500 	 *	* I'm being pushed on top of IP for all my opens (incl. #1).
501 	 *	* Only ipsecah_open() can write into ah_sadb.s_ip_q.
502 	 *	* Because of this, I can check lazily for ah_sadb.s_ip_q.
503 	 *
504 	 *  If these assumptions are wrong, I'm in BIG trouble...
505 	 */
506 
507 	q->q_ptr = ahstack;
508 	WR(q)->q_ptr = q->q_ptr;
509 
510 	if (ahstack->ah_sadb.s_ip_q == NULL) {
511 		struct T_unbind_req *tur;
512 
513 		ahstack->ah_sadb.s_ip_q = WR(q);
514 		/* Allocate an unbind... */
515 		ahstack->ah_ip_unbind = allocb(sizeof (struct T_unbind_req),
516 		    BPRI_HI);
517 
518 		/*
519 		 * Send down T_BIND_REQ to bind IPPROTO_AH.
520 		 * Handle the ACK here in AH.
521 		 */
522 		qprocson(q);
523 		if (ahstack->ah_ip_unbind == NULL ||
524 		    !sadb_t_bind_req(ahstack->ah_sadb.s_ip_q, IPPROTO_AH)) {
525 			if (ahstack->ah_ip_unbind != NULL) {
526 				freeb(ahstack->ah_ip_unbind);
527 				ahstack->ah_ip_unbind = NULL;
528 			}
529 			q->q_ptr = NULL;
530 			qprocsoff(q);
531 			netstack_rele(ahstack->ipsecah_netstack);
532 			return (ENOMEM);
533 		}
534 
535 		ahstack->ah_ip_unbind->b_datap->db_type = M_PROTO;
536 		tur = (struct T_unbind_req *)ahstack->ah_ip_unbind->b_rptr;
537 		tur->PRIM_type = T_UNBIND_REQ;
538 	} else {
539 		qprocson(q);
540 	}
541 
542 	/*
543 	 * For now, there's not much I can do.  I'll be getting a message
544 	 * passed down to me from keysock (in my wput), and a T_BIND_ACK
545 	 * up from IP (in my rput).
546 	 */
547 
548 	return (0);
549 }
550 
551 /*
552  * AH module close routine.
553  */
554 static int
555 ipsecah_close(queue_t *q)
556 {
557 	ipsecah_stack_t	*ahstack = (ipsecah_stack_t *)q->q_ptr;
558 
559 	/*
560 	 * If ah_sadb.s_ip_q is attached to this instance, send a
561 	 * T_UNBIND_REQ to IP for the instance before doing
562 	 * a qprocsoff().
563 	 */
564 	if (WR(q) == ahstack->ah_sadb.s_ip_q &&
565 	    ahstack->ah_ip_unbind != NULL) {
566 		putnext(WR(q), ahstack->ah_ip_unbind);
567 		ahstack->ah_ip_unbind = NULL;
568 	}
569 
570 	/*
571 	 * Clean up q_ptr, if needed.
572 	 */
573 	qprocsoff(q);
574 
575 	/* Keysock queue check is safe, because of OCEXCL perimeter. */
576 
577 	if (q == ahstack->ah_pfkey_q) {
578 		ah1dbg(ahstack,
579 		    ("ipsecah_close:  Ummm... keysock is closing AH.\n"));
580 		ahstack->ah_pfkey_q = NULL;
581 		/* Detach qtimeouts. */
582 		(void) quntimeout(q, ahstack->ah_event);
583 	}
584 
585 	if (WR(q) == ahstack->ah_sadb.s_ip_q) {
586 		/*
587 		 * If the ah_sadb.s_ip_q is attached to this instance, find
588 		 * another.  The OCEXCL outer perimeter helps us here.
589 		 */
590 
591 		ahstack->ah_sadb.s_ip_q = NULL;
592 
593 		/*
594 		 * Find a replacement queue for ah_sadb.s_ip_q.
595 		 */
596 		if (ahstack->ah_pfkey_q != NULL &&
597 		    ahstack->ah_pfkey_q != RD(q)) {
598 			/*
599 			 * See if we can use the pfkey_q.
600 			 */
601 			ahstack->ah_sadb.s_ip_q = WR(ahstack->ah_pfkey_q);
602 		}
603 
604 		if (ahstack->ah_sadb.s_ip_q == NULL ||
605 		    !sadb_t_bind_req(ahstack->ah_sadb.s_ip_q, IPPROTO_AH)) {
606 			ah1dbg(ahstack,
607 			    ("ipsecah: Can't reassign ah_sadb.s_ip_q.\n"));
608 			ahstack->ah_sadb.s_ip_q = NULL;
609 		} else {
610 			ahstack->ah_ip_unbind =
611 			    allocb(sizeof (struct T_unbind_req), BPRI_HI);
612 
613 			if (ahstack->ah_ip_unbind != NULL) {
614 				struct T_unbind_req *tur;
615 
616 				ahstack->ah_ip_unbind->b_datap->db_type =
617 				    M_PROTO;
618 				tur = (struct T_unbind_req *)
619 				    ahstack->ah_ip_unbind->b_rptr;
620 				tur->PRIM_type = T_UNBIND_REQ;
621 			}
622 			/* If it's NULL, I can't do much here. */
623 		}
624 	}
625 
626 	netstack_rele(ahstack->ipsecah_netstack);
627 	return (0);
628 }
629 
630 /*
631  * AH module read put routine.
632  */
633 /* ARGSUSED */
634 static void
635 ipsecah_rput(queue_t *q, mblk_t *mp)
636 {
637 	ipsecah_stack_t	*ahstack = (ipsecah_stack_t *)q->q_ptr;
638 
639 	ASSERT(mp->b_datap->db_type != M_CTL);	/* No more IRE_DB_REQ. */
640 
641 	switch (mp->b_datap->db_type) {
642 	case M_PROTO:
643 	case M_PCPROTO:
644 		/* TPI message of some sort. */
645 		switch (*((t_scalar_t *)mp->b_rptr)) {
646 		case T_BIND_ACK:
647 			/* We expect this. */
648 			ah3dbg(ahstack,
649 			    ("Thank you IP from AH for T_BIND_ACK\n"));
650 			break;
651 		case T_ERROR_ACK:
652 			cmn_err(CE_WARN,
653 			    "ipsecah:  AH received T_ERROR_ACK from IP.");
654 			break;
655 		case T_OK_ACK:
656 			/* Probably from a (rarely sent) T_UNBIND_REQ. */
657 			break;
658 		default:
659 			ah1dbg(ahstack, ("Unknown M_{,PC}PROTO message.\n"));
660 		}
661 		freemsg(mp);
662 		break;
663 	default:
664 		/* For now, passthru message. */
665 		ah2dbg(ahstack, ("AH got unknown mblk type %d.\n",
666 		    mp->b_datap->db_type));
667 		putnext(q, mp);
668 	}
669 }
670 
671 /*
672  * Construct an SADB_REGISTER message with the current algorithms.
673  */
674 static boolean_t
675 ah_register_out(uint32_t sequence, uint32_t pid, uint_t serial,
676     ipsecah_stack_t *ahstack)
677 {
678 	mblk_t *mp;
679 	boolean_t rc = B_TRUE;
680 	sadb_msg_t *samsg;
681 	sadb_supported_t *sasupp;
682 	sadb_alg_t *saalg;
683 	uint_t allocsize = sizeof (*samsg);
684 	uint_t i, numalgs_snap;
685 	ipsec_alginfo_t **authalgs;
686 	uint_t num_aalgs;
687 	ipsec_stack_t	*ipss = ahstack->ipsecah_netstack->netstack_ipsec;
688 
689 	/* Allocate the KEYSOCK_OUT. */
690 	mp = sadb_keysock_out(serial);
691 	if (mp == NULL) {
692 		ah0dbg(("ah_register_out: couldn't allocate mblk.\n"));
693 		return (B_FALSE);
694 	}
695 
696 	/*
697 	 * Allocate the PF_KEY message that follows KEYSOCK_OUT.
698 	 * The alg reader lock needs to be held while allocating
699 	 * the variable part (i.e. the algorithms) of the message.
700 	 */
701 
702 	mutex_enter(&ipss->ipsec_alg_lock);
703 
704 	/*
705 	 * Return only valid algorithms, so the number of algorithms
706 	 * to send up may be less than the number of algorithm entries
707 	 * in the table.
708 	 */
709 	authalgs = ipss->ipsec_alglists[IPSEC_ALG_AUTH];
710 	for (num_aalgs = 0, i = 0; i < IPSEC_MAX_ALGS; i++)
711 		if (authalgs[i] != NULL && ALG_VALID(authalgs[i]))
712 			num_aalgs++;
713 
714 	/*
715 	 * Fill SADB_REGISTER message's algorithm descriptors.  Hold
716 	 * down the lock while filling it.
717 	 */
718 	if (num_aalgs != 0) {
719 		allocsize += (num_aalgs * sizeof (*saalg));
720 		allocsize += sizeof (*sasupp);
721 	}
722 	mp->b_cont = allocb(allocsize, BPRI_HI);
723 	if (mp->b_cont == NULL) {
724 		mutex_exit(&ipss->ipsec_alg_lock);
725 		freemsg(mp);
726 		return (B_FALSE);
727 	}
728 
729 	mp->b_cont->b_wptr += allocsize;
730 	if (num_aalgs != 0) {
731 
732 		saalg = (sadb_alg_t *)(mp->b_cont->b_rptr + sizeof (*samsg) +
733 		    sizeof (*sasupp));
734 		ASSERT(((ulong_t)saalg & 0x7) == 0);
735 
736 		numalgs_snap = 0;
737 		for (i = 0;
738 		    ((i < IPSEC_MAX_ALGS) && (numalgs_snap < num_aalgs));
739 		    i++) {
740 			if (authalgs[i] == NULL || !ALG_VALID(authalgs[i]))
741 				continue;
742 
743 			saalg->sadb_alg_id = authalgs[i]->alg_id;
744 			saalg->sadb_alg_ivlen = 0;
745 			saalg->sadb_alg_minbits = authalgs[i]->alg_ef_minbits;
746 			saalg->sadb_alg_maxbits = authalgs[i]->alg_ef_maxbits;
747 			saalg->sadb_x_alg_increment =
748 			    authalgs[i]->alg_increment;
749 			/* For now, salt is meaningless in AH. */
750 			ASSERT(authalgs[i]->alg_saltlen == 0);
751 			saalg->sadb_x_alg_saltbits =
752 			    SADB_8TO1(authalgs[i]->alg_saltlen);
753 			numalgs_snap++;
754 			saalg++;
755 		}
756 		ASSERT(numalgs_snap == num_aalgs);
757 #ifdef DEBUG
758 		/*
759 		 * Reality check to make sure I snagged all of the
760 		 * algorithms.
761 		 */
762 		for (; i < IPSEC_MAX_ALGS; i++)
763 			if (authalgs[i] != NULL && ALG_VALID(authalgs[i]))
764 				cmn_err(CE_PANIC,
765 				    "ah_register_out()!  Missed #%d.\n", i);
766 #endif /* DEBUG */
767 	}
768 
769 	mutex_exit(&ipss->ipsec_alg_lock);
770 
771 	/* Now fill the restof the SADB_REGISTER message. */
772 
773 	samsg = (sadb_msg_t *)mp->b_cont->b_rptr;
774 	samsg->sadb_msg_version = PF_KEY_V2;
775 	samsg->sadb_msg_type = SADB_REGISTER;
776 	samsg->sadb_msg_errno = 0;
777 	samsg->sadb_msg_satype = SADB_SATYPE_AH;
778 	samsg->sadb_msg_len = SADB_8TO64(allocsize);
779 	samsg->sadb_msg_reserved = 0;
780 	/*
781 	 * Assume caller has sufficient sequence/pid number info.  If it's one
782 	 * from me over a new alg., I could give two hoots about sequence.
783 	 */
784 	samsg->sadb_msg_seq = sequence;
785 	samsg->sadb_msg_pid = pid;
786 
787 	if (allocsize > sizeof (*samsg)) {
788 		sasupp = (sadb_supported_t *)(samsg + 1);
789 		sasupp->sadb_supported_len =
790 		    SADB_8TO64(allocsize - sizeof (sadb_msg_t));
791 		sasupp->sadb_supported_exttype = SADB_EXT_SUPPORTED_AUTH;
792 		sasupp->sadb_supported_reserved = 0;
793 	}
794 
795 	if (ahstack->ah_pfkey_q != NULL)
796 		putnext(ahstack->ah_pfkey_q, mp);
797 	else {
798 		rc = B_FALSE;
799 		freemsg(mp);
800 	}
801 
802 	return (rc);
803 }
804 
805 /*
806  * Invoked when the algorithm table changes. Causes SADB_REGISTER
807  * messages continaining the current list of algorithms to be
808  * sent up to the AH listeners.
809  */
810 void
811 ipsecah_algs_changed(netstack_t *ns)
812 {
813 	ipsecah_stack_t	*ahstack = ns->netstack_ipsecah;
814 
815 	/*
816 	 * Time to send a PF_KEY SADB_REGISTER message to AH listeners
817 	 * everywhere.  (The function itself checks for NULL ah_pfkey_q.)
818 	 */
819 	(void) ah_register_out(0, 0, 0, ahstack);
820 }
821 
822 /*
823  * Stub function that taskq_dispatch() invokes to take the mblk (in arg)
824  * and put() it into AH and STREAMS again.
825  */
826 static void
827 inbound_task(void *arg)
828 {
829 	ah_t *ah;
830 	mblk_t *mp = (mblk_t *)arg;
831 	ipsec_in_t *ii = (ipsec_in_t *)mp->b_rptr;
832 	int ipsec_rc;
833 	netstack_t *ns;
834 	ipsecah_stack_t	*ahstack;
835 
836 	ns = netstack_find_by_stackid(ii->ipsec_in_stackid);
837 	if (ns == NULL || ns != ii->ipsec_in_ns) {
838 		/* Just freemsg(). */
839 		if (ns != NULL)
840 			netstack_rele(ns);
841 		freemsg(mp);
842 		return;
843 	}
844 
845 	ahstack = ns->netstack_ipsecah;
846 
847 	ah2dbg(ahstack, ("in AH inbound_task"));
848 
849 	ASSERT(ahstack != NULL);
850 	ah = ipsec_inbound_ah_sa(mp, ns);
851 	if (ah != NULL) {
852 		ASSERT(ii->ipsec_in_ah_sa != NULL);
853 		ipsec_rc = ii->ipsec_in_ah_sa->ipsa_input_func(mp, ah);
854 		if (ipsec_rc == IPSEC_STATUS_SUCCESS)
855 			ip_fanout_proto_again(mp, NULL, NULL, NULL);
856 	}
857 	netstack_rele(ns);
858 }
859 
860 /*
861  * Now that weak-key passed, actually ADD the security association, and
862  * send back a reply ADD message.
863  */
864 static int
865 ah_add_sa_finish(mblk_t *mp, sadb_msg_t *samsg, keysock_in_t *ksi,
866     int *diagnostic, ipsecah_stack_t *ahstack)
867 {
868 	isaf_t *primary = NULL, *secondary, *inbound, *outbound;
869 	sadb_sa_t *assoc = (sadb_sa_t *)ksi->ks_in_extv[SADB_EXT_SA];
870 	sadb_address_t *dstext =
871 	    (sadb_address_t *)ksi->ks_in_extv[SADB_EXT_ADDRESS_DST];
872 	struct sockaddr_in *dst;
873 	struct sockaddr_in6 *dst6;
874 	boolean_t is_ipv4, clone = B_FALSE, is_inbound = B_FALSE;
875 	uint32_t *dstaddr;
876 	ipsa_t *larval;
877 	ipsacq_t *acqrec;
878 	iacqf_t *acq_bucket;
879 	mblk_t *acq_msgs = NULL;
880 	mblk_t *lpkt;
881 	int rc;
882 	sadb_t *sp;
883 	int outhash;
884 	netstack_t	*ns = ahstack->ipsecah_netstack;
885 	ipsec_stack_t	*ipss = ns->netstack_ipsec;
886 
887 	/*
888 	 * Locate the appropriate table(s).
889 	 */
890 
891 	dst = (struct sockaddr_in *)(dstext + 1);
892 	dst6 = (struct sockaddr_in6 *)dst;
893 	is_ipv4 = (dst->sin_family == AF_INET);
894 	if (is_ipv4) {
895 		sp = &ahstack->ah_sadb.s_v4;
896 		dstaddr = (uint32_t *)(&dst->sin_addr);
897 		outhash = OUTBOUND_HASH_V4(sp, *(ipaddr_t *)dstaddr);
898 	} else {
899 		ASSERT(dst->sin_family == AF_INET6);
900 		sp = &ahstack->ah_sadb.s_v6;
901 		dstaddr = (uint32_t *)(&dst6->sin6_addr);
902 		outhash = OUTBOUND_HASH_V6(sp, *(in6_addr_t *)dstaddr);
903 	}
904 
905 	inbound = INBOUND_BUCKET(sp, assoc->sadb_sa_spi);
906 	outbound = &sp->sdb_of[outhash];
907 	/*
908 	 * Use the direction flags provided by the KMD to determine
909 	 * if the inbound or outbound table should be the primary
910 	 * for this SA. If these flags were absent then make this
911 	 * decision based on the addresses.
912 	 */
913 	if (assoc->sadb_sa_flags & IPSA_F_INBOUND) {
914 		primary = inbound;
915 		secondary = outbound;
916 		is_inbound = B_TRUE;
917 		if (assoc->sadb_sa_flags & IPSA_F_OUTBOUND)
918 			clone = B_TRUE;
919 	} else {
920 		if (assoc->sadb_sa_flags & IPSA_F_OUTBOUND) {
921 			primary = outbound;
922 			secondary = inbound;
923 		}
924 	}
925 
926 	if (primary == NULL) {
927 		/*
928 		 * The KMD did not set a direction flag, determine which
929 		 * table to insert the SA into based on addresses.
930 		 */
931 		switch (ksi->ks_in_dsttype) {
932 		case KS_IN_ADDR_MBCAST:
933 			clone = B_TRUE;	/* All mcast SAs can be bidirectional */
934 			assoc->sadb_sa_flags |= IPSA_F_OUTBOUND;
935 			/* FALLTHRU */
936 		/*
937 		 * If the source address is either one of mine, or unspecified
938 		 * (which is best summed up by saying "not 'not mine'"),
939 		 * then the association is potentially bi-directional,
940 		 * in that it can be used for inbound traffic and outbound
941 		 * traffic.  The best example of such and SA is a multicast
942 		 * SA (which allows me to receive the outbound traffic).
943 		 */
944 		case KS_IN_ADDR_ME:
945 			assoc->sadb_sa_flags |= IPSA_F_INBOUND;
946 			primary = inbound;
947 			secondary = outbound;
948 			if (ksi->ks_in_srctype != KS_IN_ADDR_NOTME)
949 				clone = B_TRUE;
950 			is_inbound = B_TRUE;
951 			break;
952 		/*
953 		 * If the source address literally not mine (either
954 		 * unspecified or not mine), then this SA may have an
955 		 * address that WILL be mine after some configuration.
956 		 * We pay the price for this by making it a bi-directional
957 		 * SA.
958 		 */
959 		case KS_IN_ADDR_NOTME:
960 			assoc->sadb_sa_flags |= IPSA_F_OUTBOUND;
961 			primary = outbound;
962 			secondary = inbound;
963 			if (ksi->ks_in_srctype != KS_IN_ADDR_ME) {
964 				assoc->sadb_sa_flags |= IPSA_F_INBOUND;
965 				clone = B_TRUE;
966 			}
967 			break;
968 		default:
969 			*diagnostic = SADB_X_DIAGNOSTIC_BAD_DST;
970 			return (EINVAL);
971 		}
972 	}
973 
974 	/*
975 	 * Find a ACQUIRE list entry if possible.  If we've added an SA that
976 	 * suits the needs of an ACQUIRE list entry, we can eliminate the
977 	 * ACQUIRE list entry and transmit the enqueued packets.  Use the
978 	 * high-bit of the sequence number to queue it.  Key off destination
979 	 * addr, and change acqrec's state.
980 	 */
981 
982 	if (samsg->sadb_msg_seq & IACQF_LOWEST_SEQ) {
983 		acq_bucket = &sp->sdb_acq[outhash];
984 		mutex_enter(&acq_bucket->iacqf_lock);
985 		for (acqrec = acq_bucket->iacqf_ipsacq; acqrec != NULL;
986 		    acqrec = acqrec->ipsacq_next) {
987 			mutex_enter(&acqrec->ipsacq_lock);
988 			/*
989 			 * Q:  I only check sequence.  Should I check dst?
990 			 * A: Yes, check dest because those are the packets
991 			 *    that are queued up.
992 			 */
993 			if (acqrec->ipsacq_seq == samsg->sadb_msg_seq &&
994 			    IPSA_ARE_ADDR_EQUAL(dstaddr,
995 			    acqrec->ipsacq_dstaddr, acqrec->ipsacq_addrfam))
996 				break;
997 			mutex_exit(&acqrec->ipsacq_lock);
998 		}
999 		if (acqrec != NULL) {
1000 			/*
1001 			 * AHA!  I found an ACQUIRE record for this SA.
1002 			 * Grab the msg list, and free the acquire record.
1003 			 * I already am holding the lock for this record,
1004 			 * so all I have to do is free it.
1005 			 */
1006 			acq_msgs = acqrec->ipsacq_mp;
1007 			acqrec->ipsacq_mp = NULL;
1008 			mutex_exit(&acqrec->ipsacq_lock);
1009 			sadb_destroy_acquire(acqrec, ns);
1010 		}
1011 		mutex_exit(&acq_bucket->iacqf_lock);
1012 	}
1013 
1014 	/*
1015 	 * Find PF_KEY message, and see if I'm an update.  If so, find entry
1016 	 * in larval list (if there).
1017 	 */
1018 
1019 	larval = NULL;
1020 
1021 	if (samsg->sadb_msg_type == SADB_UPDATE) {
1022 		mutex_enter(&inbound->isaf_lock);
1023 		larval = ipsec_getassocbyspi(inbound, assoc->sadb_sa_spi,
1024 		    ALL_ZEROES_PTR, dstaddr, dst->sin_family);
1025 		mutex_exit(&inbound->isaf_lock);
1026 
1027 		if ((larval == NULL) ||
1028 		    (larval->ipsa_state != IPSA_STATE_LARVAL)) {
1029 			*diagnostic = SADB_X_DIAGNOSTIC_SA_NOTFOUND;
1030 			if (larval != NULL) {
1031 				IPSA_REFRELE(larval);
1032 			}
1033 			ah0dbg(("Larval update, but larval disappeared.\n"));
1034 			return (ESRCH);
1035 		} /* Else sadb_common_add unlinks it for me! */
1036 	}
1037 
1038 	lpkt = NULL;
1039 	if (larval != NULL)
1040 		lpkt = sadb_clear_lpkt(larval);
1041 
1042 	rc = sadb_common_add(ahstack->ah_sadb.s_ip_q, ahstack->ah_pfkey_q, mp,
1043 	    samsg, ksi, primary, secondary, larval, clone, is_inbound,
1044 	    diagnostic, ns, &ahstack->ah_sadb);
1045 
1046 	/*
1047 	 * How much more stack will I create with all of these
1048 	 * ah_inbound_* and ah_outbound_*() calls?
1049 	 */
1050 
1051 	if (rc == 0 && lpkt != NULL)
1052 		rc = !taskq_dispatch(ah_taskq, inbound_task, lpkt, TQ_NOSLEEP);
1053 
1054 	if (rc != 0) {
1055 		ip_drop_packet(lpkt, B_TRUE, NULL, NULL,
1056 		    DROPPER(ipss, ipds_sadb_inlarval_timeout),
1057 		    &ahstack->ah_dropper);
1058 	}
1059 
1060 	while (acq_msgs != NULL) {
1061 		mblk_t *mp = acq_msgs;
1062 
1063 		acq_msgs = acq_msgs->b_next;
1064 		mp->b_next = NULL;
1065 		if (rc == 0) {
1066 			ipsec_out_t *io = (ipsec_out_t *)mp->b_rptr;
1067 
1068 			ASSERT(ahstack->ah_sadb.s_ip_q != NULL);
1069 			if (ipsec_outbound_sa(mp, IPPROTO_AH)) {
1070 				io->ipsec_out_ah_done = B_TRUE;
1071 				if (ah_outbound(mp) == IPSEC_STATUS_SUCCESS) {
1072 					ipha_t *ipha = (ipha_t *)
1073 					    mp->b_cont->b_rptr;
1074 					if (is_ipv4) {
1075 						ip_wput_ipsec_out(NULL, mp,
1076 						    ipha, NULL, NULL);
1077 					} else {
1078 						ip6_t *ip6h = (ip6_t *)ipha;
1079 						ip_wput_ipsec_out_v6(NULL,
1080 						    mp, ip6h, NULL, NULL);
1081 					}
1082 				}
1083 				continue;
1084 			}
1085 		}
1086 		AH_BUMP_STAT(ahstack, out_discards);
1087 		ip_drop_packet(mp, B_FALSE, NULL, NULL,
1088 		    DROPPER(ipss, ipds_sadb_acquire_timeout),
1089 		    &ahstack->ah_dropper);
1090 	}
1091 
1092 	return (rc);
1093 }
1094 
1095 /*
1096  * Add new AH security association.  This may become a generic AH/ESP
1097  * routine eventually.
1098  */
1099 static int
1100 ah_add_sa(mblk_t *mp, keysock_in_t *ksi, int *diagnostic, netstack_t *ns)
1101 {
1102 	sadb_sa_t *assoc = (sadb_sa_t *)ksi->ks_in_extv[SADB_EXT_SA];
1103 	sadb_address_t *srcext =
1104 	    (sadb_address_t *)ksi->ks_in_extv[SADB_EXT_ADDRESS_SRC];
1105 	sadb_address_t *dstext =
1106 	    (sadb_address_t *)ksi->ks_in_extv[SADB_EXT_ADDRESS_DST];
1107 	sadb_address_t *isrcext =
1108 	    (sadb_address_t *)ksi->ks_in_extv[SADB_X_EXT_ADDRESS_INNER_SRC];
1109 	sadb_address_t *idstext =
1110 	    (sadb_address_t *)ksi->ks_in_extv[SADB_X_EXT_ADDRESS_INNER_DST];
1111 	sadb_key_t *key = (sadb_key_t *)ksi->ks_in_extv[SADB_EXT_KEY_AUTH];
1112 	struct sockaddr_in *src, *dst;
1113 	/* We don't need sockaddr_in6 for now. */
1114 	sadb_lifetime_t *soft =
1115 	    (sadb_lifetime_t *)ksi->ks_in_extv[SADB_EXT_LIFETIME_SOFT];
1116 	sadb_lifetime_t *hard =
1117 	    (sadb_lifetime_t *)ksi->ks_in_extv[SADB_EXT_LIFETIME_HARD];
1118 	sadb_lifetime_t *idle =
1119 	    (sadb_lifetime_t *)ksi->ks_in_extv[SADB_X_EXT_LIFETIME_IDLE];
1120 	ipsec_alginfo_t *aalg;
1121 	ipsecah_stack_t	*ahstack = ns->netstack_ipsecah;
1122 	ipsec_stack_t	*ipss = ns->netstack_ipsec;
1123 
1124 	/* I need certain extensions present for an ADD message. */
1125 	if (srcext == NULL) {
1126 		*diagnostic = SADB_X_DIAGNOSTIC_MISSING_SRC;
1127 		return (EINVAL);
1128 	}
1129 	if (dstext == NULL) {
1130 		*diagnostic = SADB_X_DIAGNOSTIC_MISSING_DST;
1131 		return (EINVAL);
1132 	}
1133 	if (isrcext == NULL && idstext != NULL) {
1134 		*diagnostic = SADB_X_DIAGNOSTIC_MISSING_INNER_SRC;
1135 		return (EINVAL);
1136 	}
1137 	if (isrcext != NULL && idstext == NULL) {
1138 		*diagnostic = SADB_X_DIAGNOSTIC_MISSING_INNER_DST;
1139 		return (EINVAL);
1140 	}
1141 	if (assoc == NULL) {
1142 		*diagnostic = SADB_X_DIAGNOSTIC_MISSING_SA;
1143 		return (EINVAL);
1144 	}
1145 	if (key == NULL) {
1146 		*diagnostic = SADB_X_DIAGNOSTIC_MISSING_AKEY;
1147 		return (EINVAL);
1148 	}
1149 
1150 	src = (struct sockaddr_in *)(srcext + 1);
1151 	dst = (struct sockaddr_in *)(dstext + 1);
1152 
1153 	/* Sundry ADD-specific reality checks. */
1154 	/* XXX STATS : Logging/stats here? */
1155 
1156 	if ((assoc->sadb_sa_state != SADB_SASTATE_MATURE) &&
1157 	    (assoc->sadb_sa_state != SADB_X_SASTATE_ACTIVE_ELSEWHERE)) {
1158 		*diagnostic = SADB_X_DIAGNOSTIC_BAD_SASTATE;
1159 		return (EINVAL);
1160 	}
1161 	if (assoc->sadb_sa_encrypt != SADB_EALG_NONE) {
1162 		*diagnostic = SADB_X_DIAGNOSTIC_ENCR_NOTSUPP;
1163 		return (EINVAL);
1164 	}
1165 	if (assoc->sadb_sa_flags & ~ahstack->ah_sadb.s_addflags) {
1166 		*diagnostic = SADB_X_DIAGNOSTIC_BAD_SAFLAGS;
1167 		return (EINVAL);
1168 	}
1169 	if ((*diagnostic = sadb_hardsoftchk(hard, soft, idle)) != 0)
1170 		return (EINVAL);
1171 
1172 	ASSERT(src->sin_family == dst->sin_family);
1173 
1174 	/* Stuff I don't support, for now.  XXX Diagnostic? */
1175 	if (ksi->ks_in_extv[SADB_EXT_LIFETIME_CURRENT] != NULL ||
1176 	    ksi->ks_in_extv[SADB_EXT_SENSITIVITY] != NULL)
1177 		return (EOPNOTSUPP);
1178 
1179 	/*
1180 	 * XXX Policy : I'm not checking identities or sensitivity
1181 	 * labels at this time, but if I did, I'd do them here, before I sent
1182 	 * the weak key check up to the algorithm.
1183 	 */
1184 
1185 	/* verify that there is a mapping for the specified algorithm */
1186 	mutex_enter(&ipss->ipsec_alg_lock);
1187 	aalg = ipss->ipsec_alglists[IPSEC_ALG_AUTH][assoc->sadb_sa_auth];
1188 	if (aalg == NULL || !ALG_VALID(aalg)) {
1189 		mutex_exit(&ipss->ipsec_alg_lock);
1190 		ah1dbg(ahstack, ("Couldn't find auth alg #%d.\n",
1191 		    assoc->sadb_sa_auth));
1192 		*diagnostic = SADB_X_DIAGNOSTIC_BAD_AALG;
1193 		return (EINVAL);
1194 	}
1195 	ASSERT(aalg->alg_mech_type != CRYPTO_MECHANISM_INVALID);
1196 
1197 	/* sanity check key sizes */
1198 	if (!ipsec_valid_key_size(key->sadb_key_bits, aalg)) {
1199 		mutex_exit(&ipss->ipsec_alg_lock);
1200 		*diagnostic = SADB_X_DIAGNOSTIC_BAD_AKEYBITS;
1201 		return (EINVAL);
1202 	}
1203 
1204 	/* check key and fix parity if needed */
1205 	if (ipsec_check_key(aalg->alg_mech_type, key, B_TRUE,
1206 	    diagnostic) != 0) {
1207 		mutex_exit(&ipss->ipsec_alg_lock);
1208 		return (EINVAL);
1209 	}
1210 
1211 	mutex_exit(&ipss->ipsec_alg_lock);
1212 
1213 	return (ah_add_sa_finish(mp, (sadb_msg_t *)mp->b_cont->b_rptr, ksi,
1214 	    diagnostic, ahstack));
1215 }
1216 
1217 /*
1218  * Update a security association.  Updates come in two varieties.  The first
1219  * is an update of lifetimes on a non-larval SA.  The second is an update of
1220  * a larval SA, which ends up looking a lot more like an add.
1221  */
1222 static int
1223 ah_update_sa(mblk_t *mp, keysock_in_t *ksi, int *diagnostic,
1224     ipsecah_stack_t *ahstack, uint8_t sadb_msg_type)
1225 {
1226 	sadb_sa_t *assoc = (sadb_sa_t *)ksi->ks_in_extv[SADB_EXT_SA];
1227 	sadb_address_t *dstext =
1228 	    (sadb_address_t *)ksi->ks_in_extv[SADB_EXT_ADDRESS_DST];
1229 	mblk_t	*buf_pkt;
1230 	int rcode;
1231 
1232 	if (dstext == NULL) {
1233 		*diagnostic = SADB_X_DIAGNOSTIC_MISSING_DST;
1234 		return (EINVAL);
1235 	}
1236 
1237 	rcode = sadb_update_sa(mp, ksi, &buf_pkt, &ahstack->ah_sadb,
1238 	    diagnostic, ahstack->ah_pfkey_q, ah_add_sa,
1239 	    ahstack->ipsecah_netstack, sadb_msg_type);
1240 
1241 	if ((assoc->sadb_sa_state != SADB_X_SASTATE_ACTIVE) ||
1242 	    (rcode != 0)) {
1243 		return (rcode);
1244 	}
1245 
1246 	HANDLE_BUF_PKT(ah_taskq, ahstack->ipsecah_netstack->netstack_ipsec,
1247 	    ahstack->ah_dropper, buf_pkt);
1248 
1249 	return (rcode);
1250 }
1251 
1252 /*
1253  * Delete a security association.  This is REALLY likely to be code common to
1254  * both AH and ESP.  Find the association, then unlink it.
1255  */
1256 static int
1257 ah_del_sa(mblk_t *mp, keysock_in_t *ksi, int *diagnostic,
1258     ipsecah_stack_t *ahstack, uint8_t sadb_msg_type)
1259 {
1260 	sadb_sa_t *assoc = (sadb_sa_t *)ksi->ks_in_extv[SADB_EXT_SA];
1261 	sadb_address_t *dstext =
1262 	    (sadb_address_t *)ksi->ks_in_extv[SADB_EXT_ADDRESS_DST];
1263 	sadb_address_t *srcext =
1264 	    (sadb_address_t *)ksi->ks_in_extv[SADB_EXT_ADDRESS_SRC];
1265 	struct sockaddr_in *sin;
1266 
1267 	if (assoc == NULL) {
1268 		if (dstext != NULL)
1269 			sin = (struct sockaddr_in *)(dstext + 1);
1270 		else if (srcext != NULL)
1271 			sin = (struct sockaddr_in *)(srcext + 1);
1272 		else {
1273 			*diagnostic = SADB_X_DIAGNOSTIC_MISSING_SA;
1274 			return (EINVAL);
1275 		}
1276 		return (sadb_purge_sa(mp, ksi,
1277 		    (sin->sin_family == AF_INET6) ? &ahstack->ah_sadb.s_v6 :
1278 		    &ahstack->ah_sadb.s_v4,
1279 		    ahstack->ah_pfkey_q, ahstack->ah_sadb.s_ip_q));
1280 	}
1281 
1282 	return (sadb_delget_sa(mp, ksi, &ahstack->ah_sadb, diagnostic,
1283 	    ahstack->ah_pfkey_q, sadb_msg_type));
1284 }
1285 
1286 /*
1287  * Convert the entire contents of all of AH's SA tables into PF_KEY SADB_DUMP
1288  * messages.
1289  */
1290 static void
1291 ah_dump(mblk_t *mp, keysock_in_t *ksi, ipsecah_stack_t *ahstack)
1292 {
1293 	int error;
1294 	sadb_msg_t *samsg;
1295 
1296 	/*
1297 	 * Dump each fanout, bailing if error is non-zero.
1298 	 */
1299 
1300 	error = sadb_dump(ahstack->ah_pfkey_q, mp, ksi, &ahstack->ah_sadb.s_v4);
1301 	if (error != 0)
1302 		goto bail;
1303 
1304 	error = sadb_dump(ahstack->ah_pfkey_q, mp, ksi, &ahstack->ah_sadb.s_v6);
1305 bail:
1306 	ASSERT(mp->b_cont != NULL);
1307 	samsg = (sadb_msg_t *)mp->b_cont->b_rptr;
1308 	samsg->sadb_msg_errno = (uint8_t)error;
1309 	sadb_pfkey_echo(ahstack->ah_pfkey_q, mp,
1310 	    (sadb_msg_t *)mp->b_cont->b_rptr, ksi, NULL);
1311 }
1312 
1313 /*
1314  * First-cut reality check for an inbound PF_KEY message.
1315  */
1316 static boolean_t
1317 ah_pfkey_reality_failures(mblk_t *mp, keysock_in_t *ksi,
1318     ipsecah_stack_t *ahstack)
1319 {
1320 	int diagnostic;
1321 
1322 	if (mp->b_cont == NULL) {
1323 		freemsg(mp);
1324 		return (B_TRUE);
1325 	}
1326 
1327 	if (ksi->ks_in_extv[SADB_EXT_KEY_ENCRYPT] != NULL) {
1328 		diagnostic = SADB_X_DIAGNOSTIC_EKEY_PRESENT;
1329 		goto badmsg;
1330 	}
1331 	if (ksi->ks_in_extv[SADB_EXT_PROPOSAL] != NULL) {
1332 		diagnostic = SADB_X_DIAGNOSTIC_PROP_PRESENT;
1333 		goto badmsg;
1334 	}
1335 	if (ksi->ks_in_extv[SADB_EXT_SUPPORTED_AUTH] != NULL ||
1336 	    ksi->ks_in_extv[SADB_EXT_SUPPORTED_ENCRYPT] != NULL) {
1337 		diagnostic = SADB_X_DIAGNOSTIC_SUPP_PRESENT;
1338 		goto badmsg;
1339 	}
1340 	return (B_FALSE);	/* False ==> no failures */
1341 
1342 badmsg:
1343 	sadb_pfkey_error(ahstack->ah_pfkey_q, mp, EINVAL,
1344 	    diagnostic, ksi->ks_in_serial);
1345 	return (B_TRUE);	/* True ==> failures */
1346 }
1347 
1348 /*
1349  * AH parsing of PF_KEY messages.  Keysock did most of the really silly
1350  * error cases.  What I receive is a fully-formed, syntactically legal
1351  * PF_KEY message.  I then need to check semantics...
1352  *
1353  * This code may become common to AH and ESP.  Stay tuned.
1354  *
1355  * I also make the assumption that db_ref's are cool.  If this assumption
1356  * is wrong, this means that someone other than keysock or me has been
1357  * mucking with PF_KEY messages.
1358  */
1359 static void
1360 ah_parse_pfkey(mblk_t *mp, ipsecah_stack_t *ahstack)
1361 {
1362 	mblk_t *msg = mp->b_cont;
1363 	sadb_msg_t *samsg;
1364 	keysock_in_t *ksi;
1365 	int error;
1366 	int diagnostic = SADB_X_DIAGNOSTIC_NONE;
1367 
1368 	ASSERT(msg != NULL);
1369 
1370 	samsg = (sadb_msg_t *)msg->b_rptr;
1371 	ksi = (keysock_in_t *)mp->b_rptr;
1372 
1373 	/*
1374 	 * If applicable, convert unspecified AF_INET6 to unspecified
1375 	 * AF_INET.
1376 	 */
1377 	if (!sadb_addrfix(ksi, ahstack->ah_pfkey_q, mp,
1378 	    ahstack->ipsecah_netstack) ||
1379 	    ah_pfkey_reality_failures(mp, ksi, ahstack)) {
1380 		return;
1381 	}
1382 
1383 	switch (samsg->sadb_msg_type) {
1384 	case SADB_ADD:
1385 		error = ah_add_sa(mp, ksi, &diagnostic,
1386 		    ahstack->ipsecah_netstack);
1387 		if (error != 0) {
1388 			sadb_pfkey_error(ahstack->ah_pfkey_q, mp, error,
1389 			    diagnostic, ksi->ks_in_serial);
1390 		}
1391 		/* else ah_add_sa() took care of things. */
1392 		break;
1393 	case SADB_DELETE:
1394 	case SADB_X_DELPAIR:
1395 	case SADB_X_DELPAIR_STATE:
1396 		error = ah_del_sa(mp, ksi, &diagnostic, ahstack,
1397 		    samsg->sadb_msg_type);
1398 		if (error != 0) {
1399 			sadb_pfkey_error(ahstack->ah_pfkey_q, mp, error,
1400 			    diagnostic, ksi->ks_in_serial);
1401 		}
1402 		/* Else ah_del_sa() took care of things. */
1403 		break;
1404 	case SADB_GET:
1405 		error = sadb_delget_sa(mp, ksi, &ahstack->ah_sadb, &diagnostic,
1406 		    ahstack->ah_pfkey_q, samsg->sadb_msg_type);
1407 		if (error != 0) {
1408 			sadb_pfkey_error(ahstack->ah_pfkey_q, mp, error,
1409 			    diagnostic, ksi->ks_in_serial);
1410 		}
1411 		/* Else sadb_get_sa() took care of things. */
1412 		break;
1413 	case SADB_FLUSH:
1414 		sadbp_flush(&ahstack->ah_sadb, ahstack->ipsecah_netstack);
1415 		sadb_pfkey_echo(ahstack->ah_pfkey_q, mp, samsg, ksi, NULL);
1416 		break;
1417 	case SADB_REGISTER:
1418 		/*
1419 		 * Hmmm, let's do it!  Check for extensions (there should
1420 		 * be none), extract the fields, call ah_register_out(),
1421 		 * then either free or report an error.
1422 		 *
1423 		 * Keysock takes care of the PF_KEY bookkeeping for this.
1424 		 */
1425 		if (ah_register_out(samsg->sadb_msg_seq, samsg->sadb_msg_pid,
1426 		    ksi->ks_in_serial, ahstack)) {
1427 			freemsg(mp);
1428 		} else {
1429 			/*
1430 			 * Only way this path hits is if there is a memory
1431 			 * failure.  It will not return B_FALSE because of
1432 			 * lack of ah_pfkey_q if I am in wput().
1433 			 */
1434 			sadb_pfkey_error(ahstack->ah_pfkey_q, mp, ENOMEM,
1435 			    diagnostic, ksi->ks_in_serial);
1436 		}
1437 		break;
1438 	case SADB_UPDATE:
1439 	case SADB_X_UPDATEPAIR:
1440 		/*
1441 		 * Find a larval, if not there, find a full one and get
1442 		 * strict.
1443 		 */
1444 		error = ah_update_sa(mp, ksi, &diagnostic, ahstack,
1445 		    samsg->sadb_msg_type);
1446 		if (error != 0) {
1447 			sadb_pfkey_error(ahstack->ah_pfkey_q, mp, error,
1448 			    diagnostic, ksi->ks_in_serial);
1449 		}
1450 		/* else ah_update_sa() took care of things. */
1451 		break;
1452 	case SADB_GETSPI:
1453 		/*
1454 		 * Reserve a new larval entry.
1455 		 */
1456 		ah_getspi(mp, ksi, ahstack);
1457 		break;
1458 	case SADB_ACQUIRE:
1459 		/*
1460 		 * Find larval and/or ACQUIRE record and kill it (them), I'm
1461 		 * most likely an error.  Inbound ACQUIRE messages should only
1462 		 * have the base header.
1463 		 */
1464 		sadb_in_acquire(samsg, &ahstack->ah_sadb, ahstack->ah_pfkey_q,
1465 		    ahstack->ipsecah_netstack);
1466 		freemsg(mp);
1467 		break;
1468 	case SADB_DUMP:
1469 		/*
1470 		 * Dump all entries.
1471 		 */
1472 		ah_dump(mp, ksi, ahstack);
1473 		/* ah_dump will take care of the return message, etc. */
1474 		break;
1475 	case SADB_EXPIRE:
1476 		/* Should never reach me. */
1477 		sadb_pfkey_error(ahstack->ah_pfkey_q, mp, EOPNOTSUPP,
1478 		    diagnostic, ksi->ks_in_serial);
1479 		break;
1480 	default:
1481 		sadb_pfkey_error(ahstack->ah_pfkey_q, mp, EINVAL,
1482 		    SADB_X_DIAGNOSTIC_UNKNOWN_MSG, ksi->ks_in_serial);
1483 		break;
1484 	}
1485 }
1486 
1487 /*
1488  * Handle case where PF_KEY says it can't find a keysock for one of my
1489  * ACQUIRE messages.
1490  */
1491 static void
1492 ah_keysock_no_socket(mblk_t *mp, ipsecah_stack_t *ahstack)
1493 {
1494 	sadb_msg_t *samsg;
1495 	keysock_out_err_t *kse = (keysock_out_err_t *)mp->b_rptr;
1496 
1497 	if (mp->b_cont == NULL) {
1498 		freemsg(mp);
1499 		return;
1500 	}
1501 	samsg = (sadb_msg_t *)mp->b_cont->b_rptr;
1502 
1503 	/*
1504 	 * If keysock can't find any registered, delete the acquire record
1505 	 * immediately, and handle errors.
1506 	 */
1507 	if (samsg->sadb_msg_type == SADB_ACQUIRE) {
1508 		samsg->sadb_msg_errno = kse->ks_err_errno;
1509 		samsg->sadb_msg_len = SADB_8TO64(sizeof (*samsg));
1510 		/*
1511 		 * Use the write-side of the ah_pfkey_q, in case there is
1512 		 * no ahstack->ah_sadb.s_ip_q.
1513 		 */
1514 		sadb_in_acquire(samsg, &ahstack->ah_sadb,
1515 		    WR(ahstack->ah_pfkey_q), ahstack->ipsecah_netstack);
1516 	}
1517 
1518 	freemsg(mp);
1519 }
1520 
1521 /*
1522  * AH module write put routine.
1523  */
1524 static void
1525 ipsecah_wput(queue_t *q, mblk_t *mp)
1526 {
1527 	ipsec_info_t *ii;
1528 	struct iocblk *iocp;
1529 	ipsecah_stack_t	*ahstack = (ipsecah_stack_t *)q->q_ptr;
1530 
1531 	ah3dbg(ahstack, ("In ah_wput().\n"));
1532 
1533 	/* NOTE:  Each case must take care of freeing or passing mp. */
1534 	switch (mp->b_datap->db_type) {
1535 	case M_CTL:
1536 		if ((mp->b_wptr - mp->b_rptr) < sizeof (ipsec_info_t)) {
1537 			/* Not big enough message. */
1538 			freemsg(mp);
1539 			break;
1540 		}
1541 		ii = (ipsec_info_t *)mp->b_rptr;
1542 
1543 		switch (ii->ipsec_info_type) {
1544 		case KEYSOCK_OUT_ERR:
1545 			ah1dbg(ahstack, ("Got KEYSOCK_OUT_ERR message.\n"));
1546 			ah_keysock_no_socket(mp, ahstack);
1547 			break;
1548 		case KEYSOCK_IN:
1549 			AH_BUMP_STAT(ahstack, keysock_in);
1550 			ah3dbg(ahstack, ("Got KEYSOCK_IN message.\n"));
1551 
1552 			/* Parse the message. */
1553 			ah_parse_pfkey(mp, ahstack);
1554 			break;
1555 		case KEYSOCK_HELLO:
1556 			sadb_keysock_hello(&ahstack->ah_pfkey_q, q, mp,
1557 			    ah_ager, (void *)ahstack, &ahstack->ah_event,
1558 			    SADB_SATYPE_AH);
1559 			break;
1560 		default:
1561 			ah1dbg(ahstack, ("Got M_CTL from above of 0x%x.\n",
1562 			    ii->ipsec_info_type));
1563 			freemsg(mp);
1564 			break;
1565 		}
1566 		break;
1567 	case M_IOCTL:
1568 		iocp = (struct iocblk *)mp->b_rptr;
1569 		switch (iocp->ioc_cmd) {
1570 		case ND_SET:
1571 		case ND_GET:
1572 			if (nd_getset(q, ahstack->ipsecah_g_nd, mp)) {
1573 				qreply(q, mp);
1574 				return;
1575 			} else {
1576 				iocp->ioc_error = ENOENT;
1577 			}
1578 			/* FALLTHRU */
1579 		default:
1580 			/* We really don't support any other ioctls, do we? */
1581 
1582 			/* Return EINVAL */
1583 			if (iocp->ioc_error != ENOENT)
1584 				iocp->ioc_error = EINVAL;
1585 			iocp->ioc_count = 0;
1586 			mp->b_datap->db_type = M_IOCACK;
1587 			qreply(q, mp);
1588 			return;
1589 		}
1590 	default:
1591 		ah3dbg(ahstack,
1592 		    ("Got default message, type %d, passing to IP.\n",
1593 		    mp->b_datap->db_type));
1594 		putnext(q, mp);
1595 	}
1596 }
1597 
1598 /*
1599  * Updating use times can be tricky business if the ipsa_haspeer flag is
1600  * set.  This function is called once in an SA's lifetime.
1601  *
1602  * Caller has to REFRELE "assoc" which is passed in.  This function has
1603  * to REFRELE any peer SA that is obtained.
1604  */
1605 static void
1606 ah_set_usetime(ipsa_t *assoc, boolean_t inbound)
1607 {
1608 	ipsa_t *inassoc, *outassoc;
1609 	isaf_t *bucket;
1610 	sadb_t *sp;
1611 	int outhash;
1612 	boolean_t isv6;
1613 	netstack_t	*ns = assoc->ipsa_netstack;
1614 	ipsecah_stack_t	*ahstack = ns->netstack_ipsecah;
1615 
1616 	/* No peer?  No problem! */
1617 	if (!assoc->ipsa_haspeer) {
1618 		sadb_set_usetime(assoc);
1619 		return;
1620 	}
1621 
1622 	/*
1623 	 * Otherwise, we want to grab both the original assoc and its peer.
1624 	 * There might be a race for this, but if it's a real race, the times
1625 	 * will be out-of-synch by at most a second, and since our time
1626 	 * granularity is a second, this won't be a problem.
1627 	 *
1628 	 * If we need tight synchronization on the peer SA, then we need to
1629 	 * reconsider.
1630 	 */
1631 
1632 	/* Use address family to select IPv6/IPv4 */
1633 	isv6 = (assoc->ipsa_addrfam == AF_INET6);
1634 	if (isv6) {
1635 		sp = &ahstack->ah_sadb.s_v6;
1636 	} else {
1637 		sp = &ahstack->ah_sadb.s_v4;
1638 		ASSERT(assoc->ipsa_addrfam == AF_INET);
1639 	}
1640 	if (inbound) {
1641 		inassoc = assoc;
1642 		if (isv6)
1643 			outhash = OUTBOUND_HASH_V6(sp,
1644 			    *((in6_addr_t *)&inassoc->ipsa_dstaddr));
1645 		else
1646 			outhash = OUTBOUND_HASH_V4(sp,
1647 			    *((ipaddr_t *)&inassoc->ipsa_dstaddr));
1648 		bucket = &sp->sdb_of[outhash];
1649 
1650 		mutex_enter(&bucket->isaf_lock);
1651 		outassoc = ipsec_getassocbyspi(bucket, inassoc->ipsa_spi,
1652 		    inassoc->ipsa_srcaddr, inassoc->ipsa_dstaddr,
1653 		    inassoc->ipsa_addrfam);
1654 		mutex_exit(&bucket->isaf_lock);
1655 		if (outassoc == NULL) {
1656 			/* Q: Do we wish to set haspeer == B_FALSE? */
1657 			ah0dbg(("ah_set_usetime: "
1658 			    "can't find peer for inbound.\n"));
1659 			sadb_set_usetime(inassoc);
1660 			return;
1661 		}
1662 	} else {
1663 		outassoc = assoc;
1664 		bucket = INBOUND_BUCKET(sp, outassoc->ipsa_spi);
1665 		mutex_enter(&bucket->isaf_lock);
1666 		inassoc = ipsec_getassocbyspi(bucket, outassoc->ipsa_spi,
1667 		    outassoc->ipsa_srcaddr, outassoc->ipsa_dstaddr,
1668 		    outassoc->ipsa_addrfam);
1669 		mutex_exit(&bucket->isaf_lock);
1670 		if (inassoc == NULL) {
1671 			/* Q: Do we wish to set haspeer == B_FALSE? */
1672 			ah0dbg(("ah_set_usetime: "
1673 			    "can't find peer for outbound.\n"));
1674 			sadb_set_usetime(outassoc);
1675 			return;
1676 		}
1677 	}
1678 
1679 	/* Update usetime on both. */
1680 	sadb_set_usetime(inassoc);
1681 	sadb_set_usetime(outassoc);
1682 
1683 	/*
1684 	 * REFRELE any peer SA.
1685 	 *
1686 	 * Because of the multi-line macro nature of IPSA_REFRELE, keep
1687 	 * them in { }.
1688 	 */
1689 	if (inbound) {
1690 		IPSA_REFRELE(outassoc);
1691 	} else {
1692 		IPSA_REFRELE(inassoc);
1693 	}
1694 }
1695 
1696 /*
1697  * Add a number of bytes to what the SA has protected so far.  Return
1698  * B_TRUE if the SA can still protect that many bytes.
1699  *
1700  * Caller must REFRELE the passed-in assoc.  This function must REFRELE
1701  * any obtained peer SA.
1702  */
1703 static boolean_t
1704 ah_age_bytes(ipsa_t *assoc, uint64_t bytes, boolean_t inbound)
1705 {
1706 	ipsa_t *inassoc, *outassoc;
1707 	isaf_t *bucket;
1708 	boolean_t inrc, outrc, isv6;
1709 	sadb_t *sp;
1710 	int outhash;
1711 	netstack_t	*ns = assoc->ipsa_netstack;
1712 	ipsecah_stack_t	*ahstack = ns->netstack_ipsecah;
1713 
1714 	/* No peer?  No problem! */
1715 	if (!assoc->ipsa_haspeer) {
1716 		return (sadb_age_bytes(ahstack->ah_pfkey_q, assoc, bytes,
1717 		    B_TRUE));
1718 	}
1719 
1720 	/*
1721 	 * Otherwise, we want to grab both the original assoc and its peer.
1722 	 * There might be a race for this, but if it's a real race, two
1723 	 * expire messages may occur.  We limit this by only sending the
1724 	 * expire message on one of the peers, we'll pick the inbound
1725 	 * arbitrarily.
1726 	 *
1727 	 * If we need tight synchronization on the peer SA, then we need to
1728 	 * reconsider.
1729 	 */
1730 
1731 	/* Pick v4/v6 bucket based on addrfam. */
1732 	isv6 = (assoc->ipsa_addrfam == AF_INET6);
1733 	if (isv6) {
1734 		sp = &ahstack->ah_sadb.s_v6;
1735 	} else {
1736 		sp = &ahstack->ah_sadb.s_v4;
1737 		ASSERT(assoc->ipsa_addrfam == AF_INET);
1738 	}
1739 	if (inbound) {
1740 		inassoc = assoc;
1741 		if (isv6)
1742 			outhash = OUTBOUND_HASH_V6(sp,
1743 			    *((in6_addr_t *)&inassoc->ipsa_dstaddr));
1744 		else
1745 			outhash = OUTBOUND_HASH_V4(sp,
1746 			    *((ipaddr_t *)&inassoc->ipsa_dstaddr));
1747 		bucket = &sp->sdb_of[outhash];
1748 		mutex_enter(&bucket->isaf_lock);
1749 		outassoc = ipsec_getassocbyspi(bucket, inassoc->ipsa_spi,
1750 		    inassoc->ipsa_srcaddr, inassoc->ipsa_dstaddr,
1751 		    inassoc->ipsa_addrfam);
1752 		mutex_exit(&bucket->isaf_lock);
1753 		if (outassoc == NULL) {
1754 			/* Q: Do we wish to set haspeer == B_FALSE? */
1755 			ah0dbg(("ah_age_bytes: "
1756 			    "can't find peer for inbound.\n"));
1757 			return (sadb_age_bytes(ahstack->ah_pfkey_q, inassoc,
1758 			    bytes, B_TRUE));
1759 		}
1760 	} else {
1761 		outassoc = assoc;
1762 		bucket = INBOUND_BUCKET(sp, outassoc->ipsa_spi);
1763 		mutex_enter(&bucket->isaf_lock);
1764 		inassoc = ipsec_getassocbyspi(bucket, outassoc->ipsa_spi,
1765 		    outassoc->ipsa_srcaddr, outassoc->ipsa_dstaddr,
1766 		    outassoc->ipsa_addrfam);
1767 		mutex_exit(&bucket->isaf_lock);
1768 		if (inassoc == NULL) {
1769 			/* Q: Do we wish to set haspeer == B_FALSE? */
1770 			ah0dbg(("ah_age_bytes: "
1771 			    "can't find peer for outbound.\n"));
1772 			return (sadb_age_bytes(ahstack->ah_pfkey_q, outassoc,
1773 			    bytes, B_TRUE));
1774 		}
1775 	}
1776 
1777 	inrc = sadb_age_bytes(ahstack->ah_pfkey_q, inassoc, bytes, B_TRUE);
1778 	outrc = sadb_age_bytes(ahstack->ah_pfkey_q, outassoc, bytes, B_FALSE);
1779 
1780 	/*
1781 	 * REFRELE any peer SA.
1782 	 *
1783 	 * Because of the multi-line macro nature of IPSA_REFRELE, keep
1784 	 * them in { }.
1785 	 */
1786 	if (inbound) {
1787 		IPSA_REFRELE(outassoc);
1788 	} else {
1789 		IPSA_REFRELE(inassoc);
1790 	}
1791 
1792 	return (inrc && outrc);
1793 }
1794 
1795 /*
1796  * Perform the really difficult work of inserting the proposed situation.
1797  * Called while holding the algorithm lock.
1798  */
1799 static void
1800 ah_insert_prop(sadb_prop_t *prop, ipsacq_t *acqrec, uint_t combs)
1801 {
1802 	sadb_comb_t *comb = (sadb_comb_t *)(prop + 1);
1803 	ipsec_out_t *io;
1804 	ipsec_action_t *ap;
1805 	ipsec_prot_t *prot;
1806 	ipsecah_stack_t	*ahstack;
1807 	netstack_t	*ns;
1808 	ipsec_stack_t	*ipss;
1809 
1810 	io = (ipsec_out_t *)acqrec->ipsacq_mp->b_rptr;
1811 	ASSERT(io->ipsec_out_type == IPSEC_OUT);
1812 
1813 	ns = io->ipsec_out_ns;
1814 	ipss = ns->netstack_ipsec;
1815 	ahstack = ns->netstack_ipsecah;
1816 	ASSERT(MUTEX_HELD(&ipss->ipsec_alg_lock));
1817 
1818 	prop->sadb_prop_exttype = SADB_EXT_PROPOSAL;
1819 	prop->sadb_prop_len = SADB_8TO64(sizeof (sadb_prop_t));
1820 	*(uint32_t *)(&prop->sadb_prop_replay) = 0;	/* Quick zero-out! */
1821 
1822 	prop->sadb_prop_replay = ahstack->ipsecah_replay_size;
1823 
1824 	/*
1825 	 * Based upon algorithm properties, and what-not, prioritize a
1826 	 * proposal, based on the ordering of the ah algorithms in the
1827 	 * alternatives presented in the policy rule passed down
1828 	 * through the ipsec_out_t and attached to the acquire record.
1829 	 */
1830 
1831 	for (ap = acqrec->ipsacq_act; ap != NULL;
1832 	    ap = ap->ipa_next) {
1833 		ipsec_alginfo_t *aalg;
1834 
1835 		if ((ap->ipa_act.ipa_type != IPSEC_POLICY_APPLY) ||
1836 		    (!ap->ipa_act.ipa_apply.ipp_use_ah))
1837 			continue;
1838 
1839 		prot = &ap->ipa_act.ipa_apply;
1840 
1841 		ASSERT(prot->ipp_auth_alg > 0);
1842 
1843 		aalg = ipss->ipsec_alglists[IPSEC_ALG_AUTH]
1844 		    [prot->ipp_auth_alg];
1845 		if (aalg == NULL || !ALG_VALID(aalg))
1846 			continue;
1847 
1848 		/* XXX check aalg for duplicates??.. */
1849 
1850 		comb->sadb_comb_flags = 0;
1851 		comb->sadb_comb_reserved = 0;
1852 		comb->sadb_comb_encrypt = 0;
1853 		comb->sadb_comb_encrypt_minbits = 0;
1854 		comb->sadb_comb_encrypt_maxbits = 0;
1855 
1856 		comb->sadb_comb_auth = aalg->alg_id;
1857 		comb->sadb_comb_auth_minbits =
1858 		    MAX(prot->ipp_ah_minbits, aalg->alg_ef_minbits);
1859 		comb->sadb_comb_auth_maxbits =
1860 		    MIN(prot->ipp_ah_maxbits, aalg->alg_ef_maxbits);
1861 
1862 		/*
1863 		 * The following may be based on algorithm
1864 		 * properties, but in the meantime, we just pick
1865 		 * some good, sensible numbers.  Key mgmt. can
1866 		 * (and perhaps should) be the place to finalize
1867 		 * such decisions.
1868 		 */
1869 
1870 		/*
1871 		 * No limits on allocations, since we really don't
1872 		 * support that concept currently.
1873 		 */
1874 		comb->sadb_comb_soft_allocations = 0;
1875 		comb->sadb_comb_hard_allocations = 0;
1876 
1877 		/*
1878 		 * These may want to come from policy rule..
1879 		 */
1880 		comb->sadb_comb_soft_bytes =
1881 		    ahstack->ipsecah_default_soft_bytes;
1882 		comb->sadb_comb_hard_bytes =
1883 		    ahstack->ipsecah_default_hard_bytes;
1884 		comb->sadb_comb_soft_addtime =
1885 		    ahstack->ipsecah_default_soft_addtime;
1886 		comb->sadb_comb_hard_addtime =
1887 		    ahstack->ipsecah_default_hard_addtime;
1888 		comb->sadb_comb_soft_usetime =
1889 		    ahstack->ipsecah_default_soft_usetime;
1890 		comb->sadb_comb_hard_usetime =
1891 		    ahstack->ipsecah_default_hard_usetime;
1892 
1893 		prop->sadb_prop_len += SADB_8TO64(sizeof (*comb));
1894 		if (--combs == 0)
1895 			return;	/* out of space.. */
1896 		comb++;
1897 	}
1898 }
1899 
1900 /*
1901  * Prepare and actually send the SADB_ACQUIRE message to PF_KEY.
1902  */
1903 static void
1904 ah_send_acquire(ipsacq_t *acqrec, mblk_t *extended, netstack_t *ns)
1905 {
1906 	uint_t combs;
1907 	sadb_msg_t *samsg;
1908 	sadb_prop_t *prop;
1909 	mblk_t *pfkeymp, *msgmp;
1910 	ipsecah_stack_t	*ahstack = ns->netstack_ipsecah;
1911 	ipsec_stack_t	*ipss = ns->netstack_ipsec;
1912 
1913 	AH_BUMP_STAT(ahstack, acquire_requests);
1914 
1915 	if (ahstack->ah_pfkey_q == NULL) {
1916 		mutex_exit(&acqrec->ipsacq_lock);
1917 		return;
1918 	}
1919 
1920 	/* Set up ACQUIRE. */
1921 	pfkeymp = sadb_setup_acquire(acqrec, SADB_SATYPE_AH,
1922 	    ns->netstack_ipsec);
1923 	if (pfkeymp == NULL) {
1924 		ah0dbg(("sadb_setup_acquire failed.\n"));
1925 		mutex_exit(&acqrec->ipsacq_lock);
1926 		return;
1927 	}
1928 	ASSERT(MUTEX_HELD(&ipss->ipsec_alg_lock));
1929 	combs = ipss->ipsec_nalgs[IPSEC_ALG_AUTH];
1930 	msgmp = pfkeymp->b_cont;
1931 	samsg = (sadb_msg_t *)(msgmp->b_rptr);
1932 
1933 	/* Insert proposal here. */
1934 
1935 	prop = (sadb_prop_t *)(((uint64_t *)samsg) + samsg->sadb_msg_len);
1936 	ah_insert_prop(prop, acqrec, combs);
1937 	samsg->sadb_msg_len += prop->sadb_prop_len;
1938 	msgmp->b_wptr += SADB_64TO8(samsg->sadb_msg_len);
1939 
1940 	mutex_exit(&ipss->ipsec_alg_lock);
1941 
1942 	/*
1943 	 * Must mutex_exit() before sending PF_KEY message up, in
1944 	 * order to avoid recursive mutex_enter() if there are no registered
1945 	 * listeners.
1946 	 *
1947 	 * Once I've sent the message, I'm cool anyway.
1948 	 */
1949 	mutex_exit(&acqrec->ipsacq_lock);
1950 	if (extended != NULL) {
1951 		putnext(ahstack->ah_pfkey_q, extended);
1952 	}
1953 	putnext(ahstack->ah_pfkey_q, pfkeymp);
1954 }
1955 
1956 /*
1957  * Handle the SADB_GETSPI message.  Create a larval SA.
1958  */
1959 static void
1960 ah_getspi(mblk_t *mp, keysock_in_t *ksi, ipsecah_stack_t *ahstack)
1961 {
1962 	ipsa_t *newbie, *target;
1963 	isaf_t *outbound, *inbound;
1964 	int rc, diagnostic;
1965 	sadb_sa_t *assoc;
1966 	keysock_out_t *kso;
1967 	uint32_t newspi;
1968 
1969 	/*
1970 	 * Randomly generate a proposed SPI value.
1971 	 */
1972 	if (cl_inet_getspi != NULL) {
1973 		cl_inet_getspi(ahstack->ipsecah_netstack->netstack_stackid,
1974 		    IPPROTO_AH, (uint8_t *)&newspi, sizeof (uint32_t), NULL);
1975 	} else {
1976 		(void) random_get_pseudo_bytes((uint8_t *)&newspi,
1977 		    sizeof (uint32_t));
1978 	}
1979 	newbie = sadb_getspi(ksi, newspi, &diagnostic,
1980 	    ahstack->ipsecah_netstack, IPPROTO_AH);
1981 
1982 	if (newbie == NULL) {
1983 		sadb_pfkey_error(ahstack->ah_pfkey_q, mp, ENOMEM, diagnostic,
1984 		    ksi->ks_in_serial);
1985 		return;
1986 	} else if (newbie == (ipsa_t *)-1) {
1987 		sadb_pfkey_error(ahstack->ah_pfkey_q, mp, EINVAL, diagnostic,
1988 		    ksi->ks_in_serial);
1989 		return;
1990 	}
1991 
1992 	/*
1993 	 * XXX - We may randomly collide.  We really should recover from this.
1994 	 *	 Unfortunately, that could require spending way-too-much-time
1995 	 *	 in here.  For now, let the user retry.
1996 	 */
1997 
1998 	if (newbie->ipsa_addrfam == AF_INET6) {
1999 		outbound = OUTBOUND_BUCKET_V6(&ahstack->ah_sadb.s_v6,
2000 		    *(uint32_t *)(newbie->ipsa_dstaddr));
2001 		inbound = INBOUND_BUCKET(&ahstack->ah_sadb.s_v6,
2002 		    newbie->ipsa_spi);
2003 	} else {
2004 		outbound = OUTBOUND_BUCKET_V4(&ahstack->ah_sadb.s_v4,
2005 		    *(uint32_t *)(newbie->ipsa_dstaddr));
2006 		inbound = INBOUND_BUCKET(&ahstack->ah_sadb.s_v4,
2007 		    newbie->ipsa_spi);
2008 	}
2009 
2010 	mutex_enter(&outbound->isaf_lock);
2011 	mutex_enter(&inbound->isaf_lock);
2012 
2013 	/*
2014 	 * Check for collisions (i.e. did sadb_getspi() return with something
2015 	 * that already exists?).
2016 	 *
2017 	 * Try outbound first.  Even though SADB_GETSPI is traditionally
2018 	 * for inbound SAs, you never know what a user might do.
2019 	 */
2020 	target = ipsec_getassocbyspi(outbound, newbie->ipsa_spi,
2021 	    newbie->ipsa_srcaddr, newbie->ipsa_dstaddr, newbie->ipsa_addrfam);
2022 	if (target == NULL) {
2023 		target = ipsec_getassocbyspi(inbound, newbie->ipsa_spi,
2024 		    newbie->ipsa_srcaddr, newbie->ipsa_dstaddr,
2025 		    newbie->ipsa_addrfam);
2026 	}
2027 
2028 	/*
2029 	 * I don't have collisions elsewhere!
2030 	 * (Nor will I because I'm still holding inbound/outbound locks.)
2031 	 */
2032 
2033 	if (target != NULL) {
2034 		rc = EEXIST;
2035 		IPSA_REFRELE(target);
2036 	} else {
2037 		/*
2038 		 * sadb_insertassoc() also checks for collisions, so
2039 		 * if there's a colliding larval entry, rc will be set
2040 		 * to EEXIST.
2041 		 */
2042 		rc = sadb_insertassoc(newbie, inbound);
2043 		newbie->ipsa_hardexpiretime = gethrestime_sec();
2044 		newbie->ipsa_hardexpiretime += ahstack->ipsecah_larval_timeout;
2045 	}
2046 
2047 	/*
2048 	 * Can exit outbound mutex.  Hold inbound until we're done with
2049 	 * newbie.
2050 	 */
2051 	mutex_exit(&outbound->isaf_lock);
2052 
2053 	if (rc != 0) {
2054 		mutex_exit(&inbound->isaf_lock);
2055 		IPSA_REFRELE(newbie);
2056 		sadb_pfkey_error(ahstack->ah_pfkey_q, mp, rc,
2057 		    SADB_X_DIAGNOSTIC_NONE, ksi->ks_in_serial);
2058 		return;
2059 	}
2060 
2061 	/* Can write here because I'm still holding the bucket lock. */
2062 	newbie->ipsa_type = SADB_SATYPE_AH;
2063 
2064 	/*
2065 	 * Construct successful return message.  We have one thing going
2066 	 * for us in PF_KEY v2.  That's the fact that
2067 	 *	sizeof (sadb_spirange_t) == sizeof (sadb_sa_t)
2068 	 */
2069 	assoc = (sadb_sa_t *)ksi->ks_in_extv[SADB_EXT_SPIRANGE];
2070 	assoc->sadb_sa_exttype = SADB_EXT_SA;
2071 	assoc->sadb_sa_spi = newbie->ipsa_spi;
2072 	*((uint64_t *)(&assoc->sadb_sa_replay)) = 0;
2073 	mutex_exit(&inbound->isaf_lock);
2074 
2075 	/* Convert KEYSOCK_IN to KEYSOCK_OUT. */
2076 	kso = (keysock_out_t *)ksi;
2077 	kso->ks_out_len = sizeof (*kso);
2078 	kso->ks_out_serial = ksi->ks_in_serial;
2079 	kso->ks_out_type = KEYSOCK_OUT;
2080 
2081 	/*
2082 	 * Can safely putnext() to ah_pfkey_q, because this is a turnaround
2083 	 * from the ah_pfkey_q.
2084 	 */
2085 	putnext(ahstack->ah_pfkey_q, mp);
2086 }
2087 
2088 /*
2089  * IPv6 sends up the ICMP errors for validation and the removal of the AH
2090  * header.
2091  */
2092 static ipsec_status_t
2093 ah_icmp_error_v6(mblk_t *ipsec_mp, ipsecah_stack_t *ahstack)
2094 {
2095 	mblk_t *mp;
2096 	ip6_t *ip6h, *oip6h;
2097 	uint16_t hdr_length, ah_length;
2098 	uint8_t *nexthdrp;
2099 	ah_t *ah;
2100 	icmp6_t *icmp6;
2101 	isaf_t *isaf;
2102 	ipsa_t *assoc;
2103 	uint8_t *post_ah_ptr;
2104 	ipsec_stack_t	*ipss = ahstack->ipsecah_netstack->netstack_ipsec;
2105 
2106 	mp = ipsec_mp->b_cont;
2107 	ASSERT(mp->b_datap->db_type == M_CTL);
2108 
2109 	/*
2110 	 * Change the type to M_DATA till we finish pullups.
2111 	 */
2112 	mp->b_datap->db_type = M_DATA;
2113 
2114 	/*
2115 	 * Eat the cost of a pullupmsg() for now.  It makes the rest of this
2116 	 * code far less convoluted.
2117 	 */
2118 	if (!pullupmsg(mp, -1) ||
2119 	    !ip_hdr_length_nexthdr_v6(mp, (ip6_t *)mp->b_rptr, &hdr_length,
2120 	    &nexthdrp) ||
2121 	    mp->b_rptr + hdr_length + sizeof (icmp6_t) + sizeof (ip6_t) +
2122 	    sizeof (ah_t) > mp->b_wptr) {
2123 		IP_AH_BUMP_STAT(ipss, in_discards);
2124 		ip_drop_packet(ipsec_mp, B_TRUE, NULL, NULL,
2125 		    DROPPER(ipss, ipds_ah_nomem),
2126 		    &ahstack->ah_dropper);
2127 		return (IPSEC_STATUS_FAILED);
2128 	}
2129 
2130 	oip6h = (ip6_t *)mp->b_rptr;
2131 	icmp6 = (icmp6_t *)((uint8_t *)oip6h + hdr_length);
2132 	ip6h = (ip6_t *)(icmp6 + 1);
2133 	if (!ip_hdr_length_nexthdr_v6(mp, ip6h, &hdr_length, &nexthdrp)) {
2134 		IP_AH_BUMP_STAT(ipss, in_discards);
2135 		ip_drop_packet(ipsec_mp, B_TRUE, NULL, NULL,
2136 		    DROPPER(ipss, ipds_ah_bad_v6_hdrs),
2137 		    &ahstack->ah_dropper);
2138 		return (IPSEC_STATUS_FAILED);
2139 	}
2140 	ah = (ah_t *)((uint8_t *)ip6h + hdr_length);
2141 
2142 	isaf = OUTBOUND_BUCKET_V6(&ahstack->ah_sadb.s_v6, ip6h->ip6_dst);
2143 	mutex_enter(&isaf->isaf_lock);
2144 	assoc = ipsec_getassocbyspi(isaf, ah->ah_spi,
2145 	    (uint32_t *)&ip6h->ip6_src, (uint32_t *)&ip6h->ip6_dst, AF_INET6);
2146 	mutex_exit(&isaf->isaf_lock);
2147 
2148 	if (assoc == NULL) {
2149 		IP_AH_BUMP_STAT(ipss, lookup_failure);
2150 		IP_AH_BUMP_STAT(ipss, in_discards);
2151 		if (ahstack->ipsecah_log_unknown_spi) {
2152 			ipsec_assocfailure(info.mi_idnum, 0, 0,
2153 			    SL_CONSOLE | SL_WARN | SL_ERROR,
2154 			    "Bad ICMP message - No association for the "
2155 			    "attached AH header whose spi is 0x%x, "
2156 			    "sender is 0x%x\n",
2157 			    ah->ah_spi, &oip6h->ip6_src, AF_INET6,
2158 			    ahstack->ipsecah_netstack);
2159 		}
2160 		ip_drop_packet(ipsec_mp, B_TRUE, NULL, NULL,
2161 		    DROPPER(ipss, ipds_ah_no_sa),
2162 		    &ahstack->ah_dropper);
2163 		return (IPSEC_STATUS_FAILED);
2164 	}
2165 
2166 	IPSA_REFRELE(assoc);
2167 
2168 	/*
2169 	 * There seems to be a valid association. If there is enough of AH
2170 	 * header remove it, otherwise bail.  One could check whether it has
2171 	 * complete AH header plus 8 bytes but it does not make sense if an
2172 	 * icmp error is returned for ICMP messages e.g ICMP time exceeded,
2173 	 * that are being sent up. Let the caller figure out.
2174 	 *
2175 	 * NOTE: ah_length is the number of 32 bit words minus 2.
2176 	 */
2177 	ah_length = (ah->ah_length << 2) + 8;
2178 	post_ah_ptr = (uint8_t *)ah + ah_length;
2179 
2180 	if (post_ah_ptr > mp->b_wptr) {
2181 		IP_AH_BUMP_STAT(ipss, in_discards);
2182 		ip_drop_packet(ipsec_mp, B_TRUE, NULL, NULL,
2183 		    DROPPER(ipss, ipds_ah_bad_length),
2184 		    &ahstack->ah_dropper);
2185 		return (IPSEC_STATUS_FAILED);
2186 	}
2187 
2188 	ip6h->ip6_plen = htons(ntohs(ip6h->ip6_plen) - ah_length);
2189 	*nexthdrp = ah->ah_nexthdr;
2190 	ovbcopy(post_ah_ptr, ah,
2191 	    (size_t)((uintptr_t)mp->b_wptr - (uintptr_t)post_ah_ptr));
2192 	mp->b_wptr -= ah_length;
2193 	/* Rewhack to be an ICMP error. */
2194 	mp->b_datap->db_type = M_CTL;
2195 
2196 	return (IPSEC_STATUS_SUCCESS);
2197 }
2198 
2199 /*
2200  * IP sends up the ICMP errors for validation and the removal of
2201  * the AH header.
2202  */
2203 static ipsec_status_t
2204 ah_icmp_error_v4(mblk_t *ipsec_mp, ipsecah_stack_t *ahstack)
2205 {
2206 	mblk_t *mp;
2207 	mblk_t *mp1;
2208 	icmph_t *icmph;
2209 	int iph_hdr_length;
2210 	int hdr_length;
2211 	isaf_t *hptr;
2212 	ipsa_t *assoc;
2213 	int ah_length;
2214 	ipha_t *ipha;
2215 	ipha_t *oipha;
2216 	ah_t *ah;
2217 	uint32_t length;
2218 	int alloc_size;
2219 	uint8_t nexthdr;
2220 	ipsec_stack_t	*ipss = ahstack->ipsecah_netstack->netstack_ipsec;
2221 
2222 	mp = ipsec_mp->b_cont;
2223 	ASSERT(mp->b_datap->db_type == M_CTL);
2224 
2225 	/*
2226 	 * Change the type to M_DATA till we finish pullups.
2227 	 */
2228 	mp->b_datap->db_type = M_DATA;
2229 
2230 	oipha = ipha = (ipha_t *)mp->b_rptr;
2231 	iph_hdr_length = IPH_HDR_LENGTH(ipha);
2232 	icmph = (icmph_t *)&mp->b_rptr[iph_hdr_length];
2233 
2234 	ipha = (ipha_t *)&icmph[1];
2235 	hdr_length = IPH_HDR_LENGTH(ipha);
2236 
2237 	/*
2238 	 * See if we have enough to locate the SPI
2239 	 */
2240 	if ((uchar_t *)ipha + hdr_length + 8 > mp->b_wptr) {
2241 		if (!pullupmsg(mp, (uchar_t *)ipha + hdr_length + 8 -
2242 		    mp->b_rptr)) {
2243 			ipsec_rl_strlog(ahstack->ipsecah_netstack,
2244 			    info.mi_idnum, 0, 0,
2245 			    SL_WARN | SL_ERROR,
2246 			    "ICMP error: Small AH header\n");
2247 			IP_AH_BUMP_STAT(ipss, in_discards);
2248 			ip_drop_packet(ipsec_mp, B_TRUE, NULL, NULL,
2249 			    DROPPER(ipss, ipds_ah_bad_length),
2250 			    &ahstack->ah_dropper);
2251 			return (IPSEC_STATUS_FAILED);
2252 		}
2253 		icmph = (icmph_t *)&mp->b_rptr[iph_hdr_length];
2254 		ipha = (ipha_t *)&icmph[1];
2255 	}
2256 
2257 	ah = (ah_t *)((uint8_t *)ipha + hdr_length);
2258 	nexthdr = ah->ah_nexthdr;
2259 
2260 	hptr = OUTBOUND_BUCKET_V4(&ahstack->ah_sadb.s_v4, ipha->ipha_dst);
2261 	mutex_enter(&hptr->isaf_lock);
2262 	assoc = ipsec_getassocbyspi(hptr, ah->ah_spi,
2263 	    (uint32_t *)&ipha->ipha_src, (uint32_t *)&ipha->ipha_dst, AF_INET);
2264 	mutex_exit(&hptr->isaf_lock);
2265 
2266 	if (assoc == NULL) {
2267 		IP_AH_BUMP_STAT(ipss, lookup_failure);
2268 		IP_AH_BUMP_STAT(ipss, in_discards);
2269 		if (ahstack->ipsecah_log_unknown_spi) {
2270 			ipsec_assocfailure(info.mi_idnum, 0, 0,
2271 			    SL_CONSOLE | SL_WARN | SL_ERROR,
2272 			    "Bad ICMP message - No association for the "
2273 			    "attached AH header whose spi is 0x%x, "
2274 			    "sender is 0x%x\n",
2275 			    ah->ah_spi, &oipha->ipha_src, AF_INET,
2276 			    ahstack->ipsecah_netstack);
2277 		}
2278 		ip_drop_packet(ipsec_mp, B_TRUE, NULL, NULL,
2279 		    DROPPER(ipss, ipds_ah_no_sa),
2280 		    &ahstack->ah_dropper);
2281 		return (IPSEC_STATUS_FAILED);
2282 	}
2283 
2284 	IPSA_REFRELE(assoc);
2285 	/*
2286 	 * There seems to be a valid association. If there
2287 	 * is enough of AH header remove it, otherwise remove
2288 	 * as much as possible and send it back. One could check
2289 	 * whether it has complete AH header plus 8 bytes but it
2290 	 * does not make sense if an icmp error is returned for
2291 	 * ICMP messages e.g ICMP time exceeded, that are being
2292 	 * sent up. Let the caller figure out.
2293 	 *
2294 	 * NOTE: ah_length is the number of 32 bit words minus 2.
2295 	 */
2296 	ah_length = (ah->ah_length << 2) + 8;
2297 
2298 	if ((uchar_t *)ipha + hdr_length + ah_length > mp->b_wptr) {
2299 		if (mp->b_cont == NULL) {
2300 			/*
2301 			 * There is nothing to pullup. Just remove as
2302 			 * much as possible. This is a common case for
2303 			 * IPV4.
2304 			 */
2305 			ah_length = (mp->b_wptr - ((uchar_t *)ipha +
2306 			    hdr_length));
2307 			goto done;
2308 		}
2309 		/* Pullup the full ah header */
2310 		if (!pullupmsg(mp, (uchar_t *)ah + ah_length - mp->b_rptr)) {
2311 			/*
2312 			 * pullupmsg could have failed if there was not
2313 			 * enough to pullup or memory allocation failed.
2314 			 * We tried hard, give up now.
2315 			 */
2316 			IP_AH_BUMP_STAT(ipss, in_discards);
2317 			ip_drop_packet(ipsec_mp, B_TRUE, NULL, NULL,
2318 			    DROPPER(ipss, ipds_ah_nomem),
2319 			    &ahstack->ah_dropper);
2320 			return (IPSEC_STATUS_FAILED);
2321 		}
2322 		icmph = (icmph_t *)&mp->b_rptr[iph_hdr_length];
2323 		ipha = (ipha_t *)&icmph[1];
2324 	}
2325 done:
2326 	/*
2327 	 * Remove the AH header and change the protocol.
2328 	 * Don't update the spi fields in the ipsec_in
2329 	 * message as we are called just to validate the
2330 	 * message attached to the ICMP message.
2331 	 *
2332 	 * If we never pulled up since all of the message
2333 	 * is in one single mblk, we can't remove the AH header
2334 	 * by just setting the b_wptr to the beginning of the
2335 	 * AH header. We need to allocate a mblk that can hold
2336 	 * up until the inner IP header and copy them.
2337 	 */
2338 	alloc_size = iph_hdr_length + sizeof (icmph_t) + hdr_length;
2339 
2340 	if ((mp1 = allocb(alloc_size, BPRI_LO)) == NULL) {
2341 		IP_AH_BUMP_STAT(ipss, in_discards);
2342 		ip_drop_packet(ipsec_mp, B_TRUE, NULL, NULL,
2343 		    DROPPER(ipss, ipds_ah_nomem),
2344 		    &ahstack->ah_dropper);
2345 		return (IPSEC_STATUS_FAILED);
2346 	}
2347 	/* ICMP errors are M_CTL messages */
2348 	mp1->b_datap->db_type = M_CTL;
2349 	ipsec_mp->b_cont = mp1;
2350 	bcopy(mp->b_rptr, mp1->b_rptr, alloc_size);
2351 	mp1->b_wptr += alloc_size;
2352 
2353 	/*
2354 	 * Skip whatever we have copied and as much of AH header
2355 	 * possible. If we still have something left in the original
2356 	 * message, tag on.
2357 	 */
2358 	mp->b_rptr = (uchar_t *)ipha + hdr_length + ah_length;
2359 
2360 	if (mp->b_rptr != mp->b_wptr) {
2361 		mp1->b_cont = mp;
2362 	} else {
2363 		if (mp->b_cont != NULL)
2364 			mp1->b_cont = mp->b_cont;
2365 		freeb(mp);
2366 	}
2367 
2368 	ipha = (ipha_t *)(mp1->b_rptr + iph_hdr_length + sizeof (icmph_t));
2369 	ipha->ipha_protocol = nexthdr;
2370 	length = ntohs(ipha->ipha_length);
2371 	length -= ah_length;
2372 	ipha->ipha_length = htons((uint16_t)length);
2373 	ipha->ipha_hdr_checksum = 0;
2374 	ipha->ipha_hdr_checksum = (uint16_t)ip_csum_hdr(ipha);
2375 
2376 	return (IPSEC_STATUS_SUCCESS);
2377 }
2378 
2379 /*
2380  * IP calls this to validate the ICMP errors that
2381  * we got from the network.
2382  */
2383 ipsec_status_t
2384 ipsecah_icmp_error(mblk_t *mp)
2385 {
2386 	ipsec_in_t *ii = (ipsec_in_t *)mp->b_rptr;
2387 	netstack_t	*ns = ii->ipsec_in_ns;
2388 	ipsecah_stack_t	*ahstack = ns->netstack_ipsecah;
2389 
2390 	if (ii->ipsec_in_v4)
2391 		return (ah_icmp_error_v4(mp, ahstack));
2392 	else
2393 		return (ah_icmp_error_v6(mp, ahstack));
2394 }
2395 
2396 static int
2397 ah_fix_tlv_options_v6(uint8_t *oi_opt, uint8_t *pi_opt, uint_t ehdrlen,
2398     uint8_t hdr_type, boolean_t copy_always)
2399 {
2400 	uint8_t opt_type;
2401 	uint_t optlen;
2402 
2403 	ASSERT(hdr_type == IPPROTO_DSTOPTS || hdr_type == IPPROTO_HOPOPTS);
2404 
2405 	/*
2406 	 * Copy the next header and hdr ext. len of the HOP-by-HOP
2407 	 * and Destination option.
2408 	 */
2409 	*pi_opt++ = *oi_opt++;
2410 	*pi_opt++ = *oi_opt++;
2411 	ehdrlen -= 2;
2412 
2413 	/*
2414 	 * Now handle all the TLV encoded options.
2415 	 */
2416 	while (ehdrlen != 0) {
2417 		opt_type = *oi_opt;
2418 
2419 		if (opt_type == IP6OPT_PAD1) {
2420 			optlen = 1;
2421 		} else {
2422 			if (ehdrlen < 2)
2423 				goto bad_opt;
2424 			optlen = 2 + oi_opt[1];
2425 			if (optlen > ehdrlen)
2426 				goto bad_opt;
2427 		}
2428 		if (copy_always || !(opt_type & IP6OPT_MUTABLE)) {
2429 			bcopy(oi_opt, pi_opt, optlen);
2430 		} else {
2431 			if (optlen == 1) {
2432 				*pi_opt = 0;
2433 			} else {
2434 				/*
2435 				 * Copy the type and data length fields.
2436 				 * Zero the option data by skipping
2437 				 * option type and option data len
2438 				 * fields.
2439 				 */
2440 				*pi_opt = *oi_opt;
2441 				*(pi_opt + 1) = *(oi_opt + 1);
2442 				bzero(pi_opt + 2, optlen - 2);
2443 			}
2444 		}
2445 		ehdrlen -= optlen;
2446 		oi_opt += optlen;
2447 		pi_opt += optlen;
2448 	}
2449 	return (0);
2450 bad_opt:
2451 	return (-1);
2452 }
2453 
2454 /*
2455  * Construct a pseudo header for AH, processing all the options.
2456  *
2457  * oip6h is the IPv6 header of the incoming or outgoing packet.
2458  * ip6h is the pointer to the pseudo headers IPV6 header. All
2459  * the space needed for the options have been allocated including
2460  * the AH header.
2461  *
2462  * If copy_always is set, all the options that appear before AH are copied
2463  * blindly without checking for IP6OPT_MUTABLE. This is used by
2464  * ah_auth_out_done().  Please refer to that function for details.
2465  *
2466  * NOTE :
2467  *
2468  * *  AH header is never copied in this function even if copy_always
2469  *    is set. It just returns the ah_offset - offset of the AH header
2470  *    and the caller needs to do the copying. This is done so that we
2471  *    don't have pass extra arguments e.g. SA etc. and also,
2472  *    it is not needed when ah_auth_out_done is calling this function.
2473  */
2474 static uint_t
2475 ah_fix_phdr_v6(ip6_t *ip6h, ip6_t *oip6h, boolean_t outbound,
2476     boolean_t copy_always)
2477 {
2478 	uint8_t	*oi_opt;
2479 	uint8_t	*pi_opt;
2480 	uint8_t nexthdr;
2481 	uint8_t *prev_nexthdr;
2482 	ip6_hbh_t *hbhhdr;
2483 	ip6_dest_t *dsthdr = NULL;
2484 	ip6_rthdr0_t *rthdr;
2485 	int ehdrlen;
2486 	ah_t *ah;
2487 	int ret;
2488 
2489 	/*
2490 	 * In the outbound case for source route, ULP has already moved
2491 	 * the first hop, which is now in ip6_dst. We need to re-arrange
2492 	 * the header to make it look like how it would appear in the
2493 	 * receiver i.e
2494 	 *
2495 	 * Because of ip_massage_options_v6 the header looks like
2496 	 * this :
2497 	 *
2498 	 * ip6_src = S, ip6_dst = I1. followed by I2,I3,D.
2499 	 *
2500 	 * When it reaches the receiver, it would look like
2501 	 *
2502 	 * ip6_src = S, ip6_dst = D. followed by I1,I2,I3.
2503 	 *
2504 	 * NOTE : We assume that there are no problems with the options
2505 	 * as IP should have already checked this.
2506 	 */
2507 
2508 	oi_opt = (uchar_t *)&oip6h[1];
2509 	pi_opt = (uchar_t *)&ip6h[1];
2510 
2511 	/*
2512 	 * We set the prev_nexthdr properly in the pseudo header.
2513 	 * After we finish authentication and come back from the
2514 	 * algorithm module, pseudo header will become the real
2515 	 * IP header.
2516 	 */
2517 	prev_nexthdr = (uint8_t *)&ip6h->ip6_nxt;
2518 	nexthdr = oip6h->ip6_nxt;
2519 	/* Assume IP has already stripped it */
2520 	ASSERT(nexthdr != IPPROTO_FRAGMENT && nexthdr != IPPROTO_RAW);
2521 	ah = NULL;
2522 	dsthdr = NULL;
2523 	for (;;) {
2524 		switch (nexthdr) {
2525 		case IPPROTO_HOPOPTS:
2526 			hbhhdr = (ip6_hbh_t *)oi_opt;
2527 			nexthdr = hbhhdr->ip6h_nxt;
2528 			ehdrlen = 8 * (hbhhdr->ip6h_len + 1);
2529 			ret = ah_fix_tlv_options_v6(oi_opt, pi_opt, ehdrlen,
2530 			    IPPROTO_HOPOPTS, copy_always);
2531 			/*
2532 			 * Return a zero offset indicating error if there
2533 			 * was error.
2534 			 */
2535 			if (ret == -1)
2536 				return (0);
2537 			hbhhdr = (ip6_hbh_t *)pi_opt;
2538 			prev_nexthdr = (uint8_t *)&hbhhdr->ip6h_nxt;
2539 			break;
2540 		case IPPROTO_ROUTING:
2541 			rthdr = (ip6_rthdr0_t *)oi_opt;
2542 			nexthdr = rthdr->ip6r0_nxt;
2543 			ehdrlen = 8 * (rthdr->ip6r0_len + 1);
2544 			if (!copy_always && outbound) {
2545 				int i, left;
2546 				ip6_rthdr0_t *prthdr;
2547 				in6_addr_t *ap, *pap;
2548 
2549 				left = rthdr->ip6r0_segleft;
2550 				prthdr = (ip6_rthdr0_t *)pi_opt;
2551 				pap = (in6_addr_t *)(prthdr + 1);
2552 				ap = (in6_addr_t *)(rthdr + 1);
2553 				/*
2554 				 * First eight bytes except seg_left
2555 				 * does not change en route.
2556 				 */
2557 				bcopy(oi_opt, pi_opt, 8);
2558 				prthdr->ip6r0_segleft = 0;
2559 				/*
2560 				 * First address has been moved to
2561 				 * the destination address of the
2562 				 * ip header by ip_massage_options_v6.
2563 				 * And the real destination address is
2564 				 * in the last address part of the
2565 				 * option.
2566 				 */
2567 				*pap = oip6h->ip6_dst;
2568 				for (i = 1; i < left - 1; i++)
2569 					pap[i] = ap[i - 1];
2570 				ip6h->ip6_dst = *(ap + left - 1);
2571 			} else {
2572 				bcopy(oi_opt, pi_opt, ehdrlen);
2573 			}
2574 			rthdr = (ip6_rthdr0_t *)pi_opt;
2575 			prev_nexthdr = (uint8_t *)&rthdr->ip6r0_nxt;
2576 			break;
2577 		case IPPROTO_DSTOPTS:
2578 			/*
2579 			 * Destination options are tricky.  If there is
2580 			 * a terminal (e.g. non-IPv6-extension) header
2581 			 * following the destination options, don't
2582 			 * reset prev_nexthdr or advance the AH insertion
2583 			 * point and just treat this as a terminal header.
2584 			 *
2585 			 * If this is an inbound packet, just deal with
2586 			 * it as is.
2587 			 */
2588 			dsthdr = (ip6_dest_t *)oi_opt;
2589 			/*
2590 			 * XXX I hope common-subexpression elimination
2591 			 * saves us the double-evaluate.
2592 			 */
2593 			if (outbound && dsthdr->ip6d_nxt != IPPROTO_ROUTING &&
2594 			    dsthdr->ip6d_nxt != IPPROTO_HOPOPTS)
2595 				goto terminal_hdr;
2596 			nexthdr = dsthdr->ip6d_nxt;
2597 			ehdrlen = 8 * (dsthdr->ip6d_len + 1);
2598 			ret = ah_fix_tlv_options_v6(oi_opt, pi_opt, ehdrlen,
2599 			    IPPROTO_DSTOPTS, copy_always);
2600 			/*
2601 			 * Return a zero offset indicating error if there
2602 			 * was error.
2603 			 */
2604 			if (ret == -1)
2605 				return (0);
2606 			break;
2607 		case IPPROTO_AH:
2608 			/*
2609 			 * Be conservative in what you send.  We shouldn't
2610 			 * see two same-scoped AH's in one packet.
2611 			 * (Inner-IP-scoped AH will be hit by terminal
2612 			 * header of IP or IPv6.)
2613 			 */
2614 			ASSERT(!outbound);
2615 			return ((uint_t)(pi_opt - (uint8_t *)ip6h));
2616 		default:
2617 			ASSERT(outbound);
2618 terminal_hdr:
2619 			*prev_nexthdr = IPPROTO_AH;
2620 			ah = (ah_t *)pi_opt;
2621 			ah->ah_nexthdr = nexthdr;
2622 			return ((uint_t)(pi_opt - (uint8_t *)ip6h));
2623 		}
2624 		pi_opt += ehdrlen;
2625 		oi_opt += ehdrlen;
2626 	}
2627 	/* NOTREACHED */
2628 }
2629 
2630 static boolean_t
2631 ah_finish_up(ah_t *phdr_ah, ah_t *inbound_ah, ipsa_t *assoc,
2632     int ah_data_sz, int ah_align_sz, ipsecah_stack_t *ahstack)
2633 {
2634 	int i;
2635 
2636 	/*
2637 	 * Padding :
2638 	 *
2639 	 * 1) Authentication data may have to be padded
2640 	 * before ICV calculation if ICV is not a multiple
2641 	 * of 64 bits. This padding is arbitrary and transmitted
2642 	 * with the packet at the end of the authentication data.
2643 	 * Payload length should include the padding bytes.
2644 	 *
2645 	 * 2) Explicit padding of the whole datagram may be
2646 	 * required by the algorithm which need not be
2647 	 * transmitted. It is assumed that this will be taken
2648 	 * care by the algorithm module.
2649 	 */
2650 	bzero(phdr_ah + 1, ah_data_sz);	/* Zero out ICV for pseudo-hdr. */
2651 
2652 	if (inbound_ah == NULL) {
2653 		/* Outbound AH datagram. */
2654 
2655 		phdr_ah->ah_length = (ah_align_sz >> 2) + 1;
2656 		phdr_ah->ah_reserved = 0;
2657 		phdr_ah->ah_spi = assoc->ipsa_spi;
2658 
2659 		phdr_ah->ah_replay =
2660 		    htonl(atomic_add_32_nv(&assoc->ipsa_replay, 1));
2661 		if (phdr_ah->ah_replay == 0 && assoc->ipsa_replay_wsize != 0) {
2662 			/*
2663 			 * XXX We have replay counter wrapping.  We probably
2664 			 * want to nuke this SA (and its peer).
2665 			 */
2666 			ipsec_assocfailure(info.mi_idnum, 0, 0,
2667 			    SL_ERROR | SL_CONSOLE | SL_WARN,
2668 			    "Outbound AH SA (0x%x), dst %s has wrapped "
2669 			    "sequence.\n", phdr_ah->ah_spi,
2670 			    assoc->ipsa_dstaddr, assoc->ipsa_addrfam,
2671 			    ahstack->ipsecah_netstack);
2672 
2673 			sadb_replay_delete(assoc);
2674 			/* Caller will free phdr_mp and return NULL. */
2675 			return (B_FALSE);
2676 		}
2677 
2678 		if (ah_data_sz != ah_align_sz) {
2679 			uchar_t *pad = ((uchar_t *)phdr_ah + sizeof (ah_t) +
2680 			    ah_data_sz);
2681 
2682 			for (i = 0; i < (ah_align_sz - ah_data_sz); i++) {
2683 				pad[i] = (uchar_t)i;	/* Fill the padding */
2684 			}
2685 		}
2686 	} else {
2687 		/* Inbound AH datagram. */
2688 		phdr_ah->ah_nexthdr = inbound_ah->ah_nexthdr;
2689 		phdr_ah->ah_length = inbound_ah->ah_length;
2690 		phdr_ah->ah_reserved = 0;
2691 		ASSERT(inbound_ah->ah_spi == assoc->ipsa_spi);
2692 		phdr_ah->ah_spi = inbound_ah->ah_spi;
2693 		phdr_ah->ah_replay = inbound_ah->ah_replay;
2694 
2695 		if (ah_data_sz != ah_align_sz) {
2696 			uchar_t *opad = ((uchar_t *)inbound_ah +
2697 			    sizeof (ah_t) + ah_data_sz);
2698 			uchar_t *pad = ((uchar_t *)phdr_ah + sizeof (ah_t) +
2699 			    ah_data_sz);
2700 
2701 			for (i = 0; i < (ah_align_sz - ah_data_sz); i++) {
2702 				pad[i] = opad[i];	/* Copy the padding */
2703 			}
2704 		}
2705 	}
2706 
2707 	return (B_TRUE);
2708 }
2709 
2710 /*
2711  * Called upon failing the inbound ICV check. The message passed as
2712  * argument is freed.
2713  */
2714 static void
2715 ah_log_bad_auth(mblk_t *ipsec_in)
2716 {
2717 	mblk_t *mp = ipsec_in->b_cont->b_cont;
2718 	ipsec_in_t *ii = (ipsec_in_t *)ipsec_in->b_rptr;
2719 	boolean_t isv4 = ii->ipsec_in_v4;
2720 	ipsa_t *assoc = ii->ipsec_in_ah_sa;
2721 	int af;
2722 	void *addr;
2723 	netstack_t	*ns = ii->ipsec_in_ns;
2724 	ipsecah_stack_t	*ahstack = ns->netstack_ipsecah;
2725 	ipsec_stack_t	*ipss = ns->netstack_ipsec;
2726 
2727 	mp->b_rptr -= ii->ipsec_in_skip_len;
2728 
2729 	if (isv4) {
2730 		ipha_t *ipha = (ipha_t *)mp->b_rptr;
2731 		addr = &ipha->ipha_dst;
2732 		af = AF_INET;
2733 	} else {
2734 		ip6_t *ip6h = (ip6_t *)mp->b_rptr;
2735 		addr = &ip6h->ip6_dst;
2736 		af = AF_INET6;
2737 	}
2738 
2739 	/*
2740 	 * Log the event. Don't print to the console, block
2741 	 * potential denial-of-service attack.
2742 	 */
2743 	AH_BUMP_STAT(ahstack, bad_auth);
2744 
2745 	ipsec_assocfailure(info.mi_idnum, 0, 0, SL_ERROR | SL_WARN,
2746 	    "AH Authentication failed spi %x, dst_addr %s",
2747 	    assoc->ipsa_spi, addr, af, ahstack->ipsecah_netstack);
2748 
2749 	IP_AH_BUMP_STAT(ipss, in_discards);
2750 	ip_drop_packet(ipsec_in, B_TRUE, NULL, NULL,
2751 	    DROPPER(ipss, ipds_ah_bad_auth),
2752 	    &ahstack->ah_dropper);
2753 }
2754 
2755 /*
2756  * Kernel crypto framework callback invoked after completion of async
2757  * crypto requests.
2758  */
2759 static void
2760 ah_kcf_callback(void *arg, int status)
2761 {
2762 	mblk_t *ipsec_mp = (mblk_t *)arg;
2763 	ipsec_in_t *ii = (ipsec_in_t *)ipsec_mp->b_rptr;
2764 	boolean_t is_inbound = (ii->ipsec_in_type == IPSEC_IN);
2765 	netstackid_t	stackid;
2766 	netstack_t	*ns, *ns_arg;
2767 	ipsec_stack_t	*ipss;
2768 	ipsecah_stack_t	*ahstack;
2769 	ipsec_out_t	*io = (ipsec_out_t *)ii;
2770 
2771 	ASSERT(ipsec_mp->b_cont != NULL);
2772 
2773 	if (is_inbound) {
2774 		stackid = ii->ipsec_in_stackid;
2775 		ns_arg = ii->ipsec_in_ns;
2776 	} else {
2777 		stackid = io->ipsec_out_stackid;
2778 		ns_arg = io->ipsec_out_ns;
2779 	}
2780 	/*
2781 	 * Verify that the netstack is still around; could have vanished
2782 	 * while kEf was doing its work.
2783 	 */
2784 	ns = netstack_find_by_stackid(stackid);
2785 	if (ns == NULL || ns != ns_arg) {
2786 		/* Disappeared on us */
2787 		if (ns != NULL)
2788 			netstack_rele(ns);
2789 		freemsg(ipsec_mp);
2790 		return;
2791 	}
2792 
2793 	ahstack = ns->netstack_ipsecah;
2794 	ipss = ns->netstack_ipsec;
2795 
2796 	if (status == CRYPTO_SUCCESS) {
2797 		if (is_inbound) {
2798 			if (ah_auth_in_done(ipsec_mp) != IPSEC_STATUS_SUCCESS) {
2799 				netstack_rele(ns);
2800 				return;
2801 			}
2802 			/* finish IPsec processing */
2803 			ip_fanout_proto_again(ipsec_mp, NULL, NULL, NULL);
2804 		} else {
2805 			ipha_t *ipha;
2806 
2807 			if (ah_auth_out_done(ipsec_mp) !=
2808 			    IPSEC_STATUS_SUCCESS) {
2809 				netstack_rele(ns);
2810 				return;
2811 			}
2812 
2813 			/* finish IPsec processing */
2814 			ipha = (ipha_t *)ipsec_mp->b_cont->b_rptr;
2815 			if (IPH_HDR_VERSION(ipha) == IP_VERSION) {
2816 				ip_wput_ipsec_out(NULL, ipsec_mp, ipha, NULL,
2817 				    NULL);
2818 			} else {
2819 				ip6_t *ip6h = (ip6_t *)ipha;
2820 				ip_wput_ipsec_out_v6(NULL, ipsec_mp, ip6h,
2821 				    NULL, NULL);
2822 			}
2823 		}
2824 
2825 	} else if (status == CRYPTO_INVALID_MAC) {
2826 		ah_log_bad_auth(ipsec_mp);
2827 	} else {
2828 		ah1dbg(ahstack, ("ah_kcf_callback: crypto failed with 0x%x\n",
2829 		    status));
2830 		AH_BUMP_STAT(ahstack, crypto_failures);
2831 		if (is_inbound)
2832 			IP_AH_BUMP_STAT(ipss, in_discards);
2833 		else
2834 			AH_BUMP_STAT(ahstack, out_discards);
2835 		ip_drop_packet(ipsec_mp, is_inbound, NULL, NULL,
2836 		    DROPPER(ipss, ipds_ah_crypto_failed),
2837 		    &ahstack->ah_dropper);
2838 	}
2839 	netstack_rele(ns);
2840 }
2841 
2842 /*
2843  * Invoked on kernel crypto failure during inbound and outbound processing.
2844  */
2845 static void
2846 ah_crypto_failed(mblk_t *mp, boolean_t is_inbound, int kef_rc,
2847     ipsecah_stack_t *ahstack)
2848 {
2849 	ipsec_stack_t	*ipss = ahstack->ipsecah_netstack->netstack_ipsec;
2850 
2851 	ah1dbg(ahstack, ("crypto failed for %s AH with 0x%x\n",
2852 	    is_inbound ? "inbound" : "outbound", kef_rc));
2853 	ip_drop_packet(mp, is_inbound, NULL, NULL,
2854 	    DROPPER(ipss, ipds_ah_crypto_failed),
2855 	    &ahstack->ah_dropper);
2856 	AH_BUMP_STAT(ahstack, crypto_failures);
2857 	if (is_inbound)
2858 		IP_AH_BUMP_STAT(ipss, in_discards);
2859 	else
2860 		AH_BUMP_STAT(ahstack, out_discards);
2861 }
2862 
2863 /*
2864  * Helper macros for the ah_submit_req_{inbound,outbound}() functions.
2865  */
2866 
2867 #define	AH_INIT_CALLREQ(_cr, _ipss) {					\
2868 	(_cr)->cr_flag = CRYPTO_SKIP_REQID|CRYPTO_RESTRICTED;		\
2869 	if ((_ipss)->ipsec_algs_exec_mode[IPSEC_ALG_AUTH] == 		\
2870 	    IPSEC_ALGS_EXEC_ASYNC)					\
2871 		(_cr)->cr_flag |= CRYPTO_ALWAYS_QUEUE;			\
2872 	(_cr)->cr_callback_arg = ipsec_mp;				\
2873 	(_cr)->cr_callback_func = ah_kcf_callback;			\
2874 }
2875 
2876 #define	AH_INIT_CRYPTO_DATA(data, msglen, mblk) {			\
2877 	(data)->cd_format = CRYPTO_DATA_MBLK;				\
2878 	(data)->cd_mp = mblk;						\
2879 	(data)->cd_offset = 0;						\
2880 	(data)->cd_length = msglen;					\
2881 }
2882 
2883 #define	AH_INIT_CRYPTO_MAC(mac, icvlen, icvbuf) {			\
2884 	(mac)->cd_format = CRYPTO_DATA_RAW;				\
2885 	(mac)->cd_offset = 0;						\
2886 	(mac)->cd_length = icvlen;					\
2887 	(mac)->cd_raw.iov_base = icvbuf;				\
2888 	(mac)->cd_raw.iov_len = icvlen;					\
2889 }
2890 
2891 /*
2892  * Submit an inbound packet for processing by the crypto framework.
2893  */
2894 static ipsec_status_t
2895 ah_submit_req_inbound(mblk_t *ipsec_mp, size_t skip_len, uint32_t ah_offset,
2896     ipsa_t *assoc)
2897 {
2898 	int kef_rc;
2899 	mblk_t *phdr_mp;
2900 	crypto_call_req_t call_req;
2901 	ipsec_in_t *ii = (ipsec_in_t *)ipsec_mp->b_rptr;
2902 	uint_t icv_len = assoc->ipsa_mac_len;
2903 	crypto_ctx_template_t ctx_tmpl;
2904 	netstack_t	*ns = ii->ipsec_in_ns;
2905 	ipsecah_stack_t	*ahstack = ns->netstack_ipsecah;
2906 	ipsec_stack_t	*ipss = ns->netstack_ipsec;
2907 
2908 	phdr_mp = ipsec_mp->b_cont;
2909 	ASSERT(phdr_mp != NULL);
2910 	ASSERT(ii->ipsec_in_type == IPSEC_IN);
2911 
2912 	/*
2913 	 * In case kEF queues and calls back, make sure we have the
2914 	 * netstackid_t for verification that the IP instance is still around
2915 	 * in esp_kcf_callback().
2916 	 */
2917 	ASSERT(ii->ipsec_in_stackid == ns->netstack_stackid);
2918 
2919 	/* init arguments for the crypto framework */
2920 	AH_INIT_CRYPTO_DATA(&ii->ipsec_in_crypto_data, AH_MSGSIZE(phdr_mp),
2921 	    phdr_mp);
2922 
2923 	AH_INIT_CRYPTO_MAC(&ii->ipsec_in_crypto_mac, icv_len,
2924 	    (char *)phdr_mp->b_cont->b_rptr - skip_len + ah_offset +
2925 	    sizeof (ah_t));
2926 
2927 	AH_INIT_CALLREQ(&call_req, ipss);
2928 
2929 	ii->ipsec_in_skip_len = skip_len;
2930 
2931 	IPSEC_CTX_TMPL(assoc, ipsa_authtmpl, IPSEC_ALG_AUTH, ctx_tmpl);
2932 
2933 	/* call KEF to do the MAC operation */
2934 	kef_rc = crypto_mac_verify(&assoc->ipsa_amech,
2935 	    &ii->ipsec_in_crypto_data, &assoc->ipsa_kcfauthkey, ctx_tmpl,
2936 	    &ii->ipsec_in_crypto_mac, &call_req);
2937 
2938 	switch (kef_rc) {
2939 	case CRYPTO_SUCCESS:
2940 		AH_BUMP_STAT(ahstack, crypto_sync);
2941 		return (ah_auth_in_done(ipsec_mp));
2942 	case CRYPTO_QUEUED:
2943 		/* ah_kcf_callback() will be invoked on completion */
2944 		AH_BUMP_STAT(ahstack, crypto_async);
2945 		return (IPSEC_STATUS_PENDING);
2946 	case CRYPTO_INVALID_MAC:
2947 		AH_BUMP_STAT(ahstack, crypto_sync);
2948 		ah_log_bad_auth(ipsec_mp);
2949 		return (IPSEC_STATUS_FAILED);
2950 	}
2951 
2952 	ah_crypto_failed(ipsec_mp, B_TRUE, kef_rc, ahstack);
2953 	return (IPSEC_STATUS_FAILED);
2954 }
2955 
2956 /*
2957  * Submit an outbound packet for processing by the crypto framework.
2958  */
2959 static ipsec_status_t
2960 ah_submit_req_outbound(mblk_t *ipsec_mp, size_t skip_len, ipsa_t *assoc)
2961 {
2962 	int kef_rc;
2963 	mblk_t *phdr_mp;
2964 	crypto_call_req_t call_req;
2965 	ipsec_out_t *io = (ipsec_out_t *)ipsec_mp->b_rptr;
2966 	uint_t icv_len = assoc->ipsa_mac_len;
2967 	netstack_t	*ns = io->ipsec_out_ns;
2968 	ipsecah_stack_t	*ahstack = ns->netstack_ipsecah;
2969 	ipsec_stack_t	*ipss = ns->netstack_ipsec;
2970 
2971 	phdr_mp = ipsec_mp->b_cont;
2972 	ASSERT(phdr_mp != NULL);
2973 	ASSERT(io->ipsec_out_type == IPSEC_OUT);
2974 
2975 	/*
2976 	 * In case kEF queues and calls back, keep netstackid_t for
2977 	 * verification that the IP instance is still around in
2978 	 * ah_kcf_callback().
2979 	 */
2980 	io->ipsec_out_stackid = ns->netstack_stackid;
2981 
2982 	/* init arguments for the crypto framework */
2983 	AH_INIT_CRYPTO_DATA(&io->ipsec_out_crypto_data, AH_MSGSIZE(phdr_mp),
2984 	    phdr_mp);
2985 
2986 	AH_INIT_CRYPTO_MAC(&io->ipsec_out_crypto_mac, icv_len,
2987 	    (char *)phdr_mp->b_wptr);
2988 
2989 	AH_INIT_CALLREQ(&call_req, ipss);
2990 
2991 	io->ipsec_out_skip_len = skip_len;
2992 
2993 	ASSERT(io->ipsec_out_ah_sa != NULL);
2994 
2995 	/* call KEF to do the MAC operation */
2996 	kef_rc = crypto_mac(&assoc->ipsa_amech, &io->ipsec_out_crypto_data,
2997 	    &assoc->ipsa_kcfauthkey, assoc->ipsa_authtmpl,
2998 	    &io->ipsec_out_crypto_mac, &call_req);
2999 
3000 	switch (kef_rc) {
3001 	case CRYPTO_SUCCESS:
3002 		AH_BUMP_STAT(ahstack, crypto_sync);
3003 		return (ah_auth_out_done(ipsec_mp));
3004 	case CRYPTO_QUEUED:
3005 		/* ah_kcf_callback() will be invoked on completion */
3006 		AH_BUMP_STAT(ahstack, crypto_async);
3007 		return (IPSEC_STATUS_PENDING);
3008 	}
3009 
3010 	ah_crypto_failed(ipsec_mp, B_FALSE, kef_rc, ahstack);
3011 	return (IPSEC_STATUS_FAILED);
3012 }
3013 
3014 /*
3015  * This function constructs a pseudo header by looking at the IP header
3016  * and options if any. This is called for both outbound and inbound,
3017  * before computing the ICV.
3018  */
3019 static mblk_t *
3020 ah_process_ip_options_v6(mblk_t *mp, ipsa_t *assoc, int *length_to_skip,
3021     uint_t ah_data_sz, boolean_t outbound, ipsecah_stack_t *ahstack)
3022 {
3023 	ip6_t	*ip6h;
3024 	ip6_t	*oip6h;
3025 	mblk_t 	*phdr_mp;
3026 	int option_length;
3027 	uint_t	ah_align_sz;
3028 	uint_t ah_offset;
3029 	int hdr_size;
3030 	ipsec_stack_t	*ipss = ahstack->ipsecah_netstack->netstack_ipsec;
3031 
3032 	/*
3033 	 * Allocate space for the authentication data also. It is
3034 	 * useful both during the ICV calculation where we need to
3035 	 * feed in zeroes and while sending the datagram back to IP
3036 	 * where we will be using the same space.
3037 	 *
3038 	 * We need to allocate space for padding bytes if it is not
3039 	 * a multiple of IPV6_PADDING_ALIGN.
3040 	 *
3041 	 * In addition, we allocate space for the ICV computed by
3042 	 * the kernel crypto framework, saving us a separate kmem
3043 	 * allocation down the road.
3044 	 */
3045 
3046 	ah_align_sz = P2ALIGN(ah_data_sz + IPV6_PADDING_ALIGN - 1,
3047 	    IPV6_PADDING_ALIGN);
3048 
3049 	ASSERT(ah_align_sz >= ah_data_sz);
3050 
3051 	hdr_size = ipsec_ah_get_hdr_size_v6(mp, B_FALSE);
3052 	option_length = hdr_size - IPV6_HDR_LEN;
3053 
3054 	/* This was not included in ipsec_ah_get_hdr_size_v6() */
3055 	hdr_size += (sizeof (ah_t) + ah_align_sz);
3056 
3057 	if (!outbound && (MBLKL(mp) < hdr_size)) {
3058 		/*
3059 		 * We have post-AH header options in a separate mblk,
3060 		 * a pullup is required.
3061 		 */
3062 		if (!pullupmsg(mp, hdr_size))
3063 			return (NULL);
3064 	}
3065 
3066 	if ((phdr_mp = allocb_tmpl(hdr_size + ah_data_sz, mp)) == NULL) {
3067 		return (NULL);
3068 	}
3069 
3070 	oip6h = (ip6_t *)mp->b_rptr;
3071 
3072 	/*
3073 	 * Form the basic IP header first. Zero out the header
3074 	 * so that the mutable fields are zeroed out.
3075 	 */
3076 	ip6h = (ip6_t *)phdr_mp->b_rptr;
3077 	bzero(ip6h, sizeof (ip6_t));
3078 	ip6h->ip6_vcf = IPV6_DEFAULT_VERS_AND_FLOW;
3079 
3080 	if (outbound) {
3081 		/*
3082 		 * Include the size of AH and authentication data.
3083 		 * This is how our recipient would compute the
3084 		 * authentication data. Look at what we do in the
3085 		 * inbound case below.
3086 		 */
3087 		ip6h->ip6_plen = htons(ntohs(oip6h->ip6_plen) +
3088 		    sizeof (ah_t) + ah_align_sz);
3089 	} else {
3090 		ip6h->ip6_plen = oip6h->ip6_plen;
3091 	}
3092 
3093 	ip6h->ip6_src = oip6h->ip6_src;
3094 	ip6h->ip6_dst = oip6h->ip6_dst;
3095 
3096 	*length_to_skip = IPV6_HDR_LEN;
3097 	if (option_length == 0) {
3098 		/* Form the AH header */
3099 		ip6h->ip6_nxt = IPPROTO_AH;
3100 		((ah_t *)(ip6h + 1))->ah_nexthdr = oip6h->ip6_nxt;
3101 		ah_offset = *length_to_skip;
3102 	} else {
3103 		ip6h->ip6_nxt = oip6h->ip6_nxt;
3104 		/* option_length does not include the AH header's size */
3105 		*length_to_skip += option_length;
3106 
3107 		ah_offset = ah_fix_phdr_v6(ip6h, oip6h, outbound, B_FALSE);
3108 		if (ah_offset == 0) {
3109 			ip_drop_packet(phdr_mp, !outbound, NULL, NULL,
3110 			    DROPPER(ipss, ipds_ah_bad_v6_hdrs),
3111 			    &ahstack->ah_dropper);
3112 			return (NULL);
3113 		}
3114 	}
3115 
3116 	if (!ah_finish_up(((ah_t *)((uint8_t *)ip6h + ah_offset)),
3117 	    (outbound ? NULL : ((ah_t *)((uint8_t *)oip6h + ah_offset))),
3118 	    assoc, ah_data_sz, ah_align_sz, ahstack)) {
3119 		freeb(phdr_mp);
3120 		/*
3121 		 * Returning NULL will tell the caller to
3122 		 * IPSA_REFELE(), free the memory, etc.
3123 		 */
3124 		return (NULL);
3125 	}
3126 
3127 	phdr_mp->b_wptr = ((uint8_t *)ip6h + ah_offset + sizeof (ah_t) +
3128 	    ah_align_sz);
3129 	if (!outbound)
3130 		*length_to_skip += sizeof (ah_t) + ah_align_sz;
3131 	return (phdr_mp);
3132 }
3133 
3134 /*
3135  * This function constructs a pseudo header by looking at the IP header
3136  * and options if any. This is called for both outbound and inbound,
3137  * before computing the ICV.
3138  */
3139 static mblk_t *
3140 ah_process_ip_options_v4(mblk_t *mp, ipsa_t *assoc, int *length_to_skip,
3141     uint_t ah_data_sz, boolean_t outbound, ipsecah_stack_t *ahstack)
3142 {
3143 	ipoptp_t opts;
3144 	uint32_t option_length;
3145 	ipha_t	*ipha;
3146 	ipha_t	*oipha;
3147 	mblk_t 	*phdr_mp;
3148 	int	 size;
3149 	uchar_t	*optptr;
3150 	uint8_t optval;
3151 	uint8_t optlen;
3152 	ipaddr_t dst;
3153 	uint32_t v_hlen_tos_len;
3154 	int ip_hdr_length;
3155 	uint_t	ah_align_sz;
3156 	uint32_t off;
3157 
3158 #ifdef	_BIG_ENDIAN
3159 #define	V_HLEN	(v_hlen_tos_len >> 24)
3160 #else
3161 #define	V_HLEN	(v_hlen_tos_len & 0xFF)
3162 #endif
3163 
3164 	oipha = (ipha_t *)mp->b_rptr;
3165 	v_hlen_tos_len = ((uint32_t *)oipha)[0];
3166 
3167 	/*
3168 	 * Allocate space for the authentication data also. It is
3169 	 * useful both during the ICV calculation where we need to
3170 	 * feed in zeroes and while sending the datagram back to IP
3171 	 * where we will be using the same space.
3172 	 *
3173 	 * We need to allocate space for padding bytes if it is not
3174 	 * a multiple of IPV4_PADDING_ALIGN.
3175 	 *
3176 	 * In addition, we allocate space for the ICV computed by
3177 	 * the kernel crypto framework, saving us a separate kmem
3178 	 * allocation down the road.
3179 	 */
3180 
3181 	ah_align_sz = P2ALIGN(ah_data_sz + IPV4_PADDING_ALIGN - 1,
3182 	    IPV4_PADDING_ALIGN);
3183 
3184 	ASSERT(ah_align_sz >= ah_data_sz);
3185 
3186 	size = IP_SIMPLE_HDR_LENGTH + sizeof (ah_t) + ah_align_sz +
3187 	    ah_data_sz;
3188 
3189 	if (V_HLEN != IP_SIMPLE_HDR_VERSION) {
3190 		option_length = oipha->ipha_version_and_hdr_length -
3191 		    (uint8_t)((IP_VERSION << 4) +
3192 		    IP_SIMPLE_HDR_LENGTH_IN_WORDS);
3193 		option_length <<= 2;
3194 		size += option_length;
3195 	}
3196 
3197 	if ((phdr_mp = allocb_tmpl(size, mp)) == NULL) {
3198 		return (NULL);
3199 	}
3200 
3201 	/*
3202 	 * Form the basic IP header first.
3203 	 */
3204 	ipha = (ipha_t *)phdr_mp->b_rptr;
3205 	ipha->ipha_version_and_hdr_length = oipha->ipha_version_and_hdr_length;
3206 	ipha->ipha_type_of_service = 0;
3207 
3208 	if (outbound) {
3209 		/*
3210 		 * Include the size of AH and authentication data.
3211 		 * This is how our recipient would compute the
3212 		 * authentication data. Look at what we do in the
3213 		 * inbound case below.
3214 		 */
3215 		ipha->ipha_length = ntohs(htons(oipha->ipha_length) +
3216 		    sizeof (ah_t) + ah_align_sz);
3217 	} else {
3218 		ipha->ipha_length = oipha->ipha_length;
3219 	}
3220 
3221 	ipha->ipha_ident = oipha->ipha_ident;
3222 	ipha->ipha_fragment_offset_and_flags = 0;
3223 	ipha->ipha_ttl = 0;
3224 	ipha->ipha_protocol = IPPROTO_AH;
3225 	ipha->ipha_hdr_checksum = 0;
3226 	ipha->ipha_src = oipha->ipha_src;
3227 	ipha->ipha_dst = dst = oipha->ipha_dst;
3228 
3229 	/*
3230 	 * If there is no option to process return now.
3231 	 */
3232 	ip_hdr_length = IP_SIMPLE_HDR_LENGTH;
3233 
3234 	if (V_HLEN == IP_SIMPLE_HDR_VERSION) {
3235 		/* Form the AH header */
3236 		goto ah_hdr;
3237 	}
3238 
3239 	ip_hdr_length += option_length;
3240 
3241 	/*
3242 	 * We have options. In the outbound case for source route,
3243 	 * ULP has already moved the first hop, which is now in
3244 	 * ipha_dst. We need the final destination for the calculation
3245 	 * of authentication data. And also make sure that mutable
3246 	 * and experimental fields are zeroed out in the IP options.
3247 	 */
3248 
3249 	bcopy(&oipha[1], &ipha[1], option_length);
3250 
3251 	for (optval = ipoptp_first(&opts, ipha);
3252 	    optval != IPOPT_EOL;
3253 	    optval = ipoptp_next(&opts)) {
3254 		optptr = opts.ipoptp_cur;
3255 		optlen = opts.ipoptp_len;
3256 		switch (optval) {
3257 		case IPOPT_EXTSEC:
3258 		case IPOPT_COMSEC:
3259 		case IPOPT_RA:
3260 		case IPOPT_SDMDD:
3261 		case IPOPT_SECURITY:
3262 			/*
3263 			 * These options are Immutable, leave them as-is.
3264 			 * Note that IPOPT_NOP is also Immutable, but it
3265 			 * was skipped by ipoptp_next() and thus remains
3266 			 * intact in the header.
3267 			 */
3268 			break;
3269 		case IPOPT_SSRR:
3270 		case IPOPT_LSRR:
3271 			if ((opts.ipoptp_flags & IPOPTP_ERROR) != 0)
3272 				goto bad_ipv4opt;
3273 			/*
3274 			 * These two are mutable and will be zeroed, but
3275 			 * first get the final destination.
3276 			 */
3277 			off = optptr[IPOPT_OFFSET];
3278 			/*
3279 			 * If one of the conditions is true, it means
3280 			 * end of options and dst already has the right
3281 			 * value. So, just fall through.
3282 			 */
3283 			if (!(optlen < IP_ADDR_LEN || off > optlen - 3)) {
3284 				off = optlen - IP_ADDR_LEN;
3285 				bcopy(&optptr[off], &dst, IP_ADDR_LEN);
3286 			}
3287 			/* FALLTHRU */
3288 		case IPOPT_RR:
3289 		case IPOPT_TS:
3290 		case IPOPT_SATID:
3291 		default:
3292 			/*
3293 			 * optlen should include from the beginning of an
3294 			 * option.
3295 			 * NOTE : Stream Identifier Option (SID): RFC 791
3296 			 * shows the bit pattern of optlen as 2 and documents
3297 			 * the length as 4. We assume it to be 2 here.
3298 			 */
3299 			bzero(optptr, optlen);
3300 			break;
3301 		}
3302 	}
3303 
3304 	if ((opts.ipoptp_flags & IPOPTP_ERROR) != 0) {
3305 bad_ipv4opt:
3306 		ah1dbg(ahstack, ("AH : bad IPv4 option"));
3307 		freeb(phdr_mp);
3308 		return (NULL);
3309 	}
3310 
3311 	/*
3312 	 * Don't change ipha_dst for an inbound datagram as it points
3313 	 * to the right value. Only for the outbound with LSRR/SSRR,
3314 	 * because of ip_massage_options called by the ULP, ipha_dst
3315 	 * points to the first hop and we need to use the final
3316 	 * destination for computing the ICV.
3317 	 */
3318 
3319 	if (outbound)
3320 		ipha->ipha_dst = dst;
3321 ah_hdr:
3322 	((ah_t *)((uint8_t *)ipha + ip_hdr_length))->ah_nexthdr =
3323 	    oipha->ipha_protocol;
3324 	if (!ah_finish_up(((ah_t *)((uint8_t *)ipha + ip_hdr_length)),
3325 	    (outbound ? NULL : ((ah_t *)((uint8_t *)oipha + ip_hdr_length))),
3326 	    assoc, ah_data_sz, ah_align_sz, ahstack)) {
3327 		freeb(phdr_mp);
3328 		/*
3329 		 * Returning NULL will tell the caller to IPSA_REFELE(), free
3330 		 * the memory, etc.
3331 		 */
3332 		return (NULL);
3333 	}
3334 
3335 	phdr_mp->b_wptr = ((uchar_t *)ipha + ip_hdr_length +
3336 	    sizeof (ah_t) + ah_align_sz);
3337 
3338 	ASSERT(phdr_mp->b_wptr <= phdr_mp->b_datap->db_lim);
3339 	if (outbound)
3340 		*length_to_skip = ip_hdr_length;
3341 	else
3342 		*length_to_skip = ip_hdr_length + sizeof (ah_t) + ah_align_sz;
3343 	return (phdr_mp);
3344 }
3345 
3346 /*
3347  * Authenticate an outbound datagram. This function is called
3348  * whenever IP sends an outbound datagram that needs authentication.
3349  */
3350 static ipsec_status_t
3351 ah_outbound(mblk_t *ipsec_out)
3352 {
3353 	mblk_t *mp;
3354 	mblk_t *phdr_mp;
3355 	ipsec_out_t *oi;
3356 	ipsa_t *assoc;
3357 	int length_to_skip;
3358 	uint_t ah_align_sz;
3359 	uint_t age_bytes;
3360 	netstack_t	*ns;
3361 	ipsec_stack_t	*ipss;
3362 	ipsecah_stack_t	*ahstack;
3363 
3364 	/*
3365 	 * Construct the chain of mblks
3366 	 *
3367 	 * IPSEC_OUT->PSEUDO_HDR->DATA
3368 	 *
3369 	 * one by one.
3370 	 */
3371 
3372 	ASSERT(ipsec_out->b_datap->db_type == M_CTL);
3373 
3374 	ASSERT(MBLKL(ipsec_out) >= sizeof (ipsec_info_t));
3375 
3376 	mp = ipsec_out->b_cont;
3377 	oi = (ipsec_out_t *)ipsec_out->b_rptr;
3378 	ns = oi->ipsec_out_ns;
3379 	ipss = ns->netstack_ipsec;
3380 	ahstack = ns->netstack_ipsecah;
3381 
3382 	AH_BUMP_STAT(ahstack, out_requests);
3383 
3384 	ASSERT(mp->b_datap->db_type == M_DATA);
3385 
3386 	assoc = oi->ipsec_out_ah_sa;
3387 	ASSERT(assoc != NULL);
3388 
3389 	/*
3390 	 * Age SA according to number of bytes that will be sent after
3391 	 * adding the AH header, ICV, and padding to the packet.
3392 	 */
3393 
3394 	if (oi->ipsec_out_v4) {
3395 		ipha_t *ipha = (ipha_t *)mp->b_rptr;
3396 		ah_align_sz = P2ALIGN(assoc->ipsa_mac_len +
3397 		    IPV4_PADDING_ALIGN - 1, IPV4_PADDING_ALIGN);
3398 		age_bytes = ntohs(ipha->ipha_length) + sizeof (ah_t) +
3399 		    ah_align_sz;
3400 	} else {
3401 		ip6_t *ip6h = (ip6_t *)mp->b_rptr;
3402 		ah_align_sz = P2ALIGN(assoc->ipsa_mac_len +
3403 		    IPV6_PADDING_ALIGN - 1, IPV6_PADDING_ALIGN);
3404 		age_bytes = sizeof (ip6_t) + ntohs(ip6h->ip6_plen) +
3405 		    sizeof (ah_t) + ah_align_sz;
3406 	}
3407 
3408 	if (!ah_age_bytes(assoc, age_bytes, B_FALSE)) {
3409 		/* rig things as if ipsec_getassocbyconn() failed */
3410 		ipsec_assocfailure(info.mi_idnum, 0, 0, SL_ERROR | SL_WARN,
3411 		    "AH association 0x%x, dst %s had bytes expire.\n",
3412 		    ntohl(assoc->ipsa_spi), assoc->ipsa_dstaddr, AF_INET,
3413 		    ahstack->ipsecah_netstack);
3414 		freemsg(ipsec_out);
3415 		return (IPSEC_STATUS_FAILED);
3416 	}
3417 
3418 	if (oi->ipsec_out_is_capab_ill) {
3419 		ah3dbg(ahstack, ("ah_outbound: pkt can be accelerated\n"));
3420 		if (oi->ipsec_out_v4)
3421 			return (ah_outbound_accelerated_v4(ipsec_out, assoc));
3422 		else
3423 			return (ah_outbound_accelerated_v6(ipsec_out, assoc));
3424 	}
3425 	AH_BUMP_STAT(ahstack, noaccel);
3426 
3427 	/*
3428 	 * Insert pseudo header:
3429 	 * IPSEC_INFO -> [IP, ULP] => IPSEC_INFO -> [IP, AH, ICV] -> ULP
3430 	 */
3431 
3432 	if (oi->ipsec_out_v4) {
3433 		phdr_mp = ah_process_ip_options_v4(mp, assoc, &length_to_skip,
3434 		    assoc->ipsa_mac_len, B_TRUE, ahstack);
3435 	} else {
3436 		phdr_mp = ah_process_ip_options_v6(mp, assoc, &length_to_skip,
3437 		    assoc->ipsa_mac_len, B_TRUE, ahstack);
3438 	}
3439 
3440 	if (phdr_mp == NULL) {
3441 		AH_BUMP_STAT(ahstack, out_discards);
3442 		ip_drop_packet(ipsec_out, B_FALSE, NULL, NULL,
3443 		    DROPPER(ipss, ipds_ah_bad_v4_opts),
3444 		    &ahstack->ah_dropper);
3445 		return (IPSEC_STATUS_FAILED);
3446 	}
3447 
3448 	ipsec_out->b_cont = phdr_mp;
3449 	phdr_mp->b_cont = mp;
3450 	mp->b_rptr += length_to_skip;
3451 
3452 	/*
3453 	 * At this point ipsec_out points to the IPSEC_OUT, new_mp
3454 	 * points to an mblk containing the pseudo header (IP header,
3455 	 * AH header, and ICV with mutable fields zero'ed out).
3456 	 * mp points to the mblk containing the ULP data. The original
3457 	 * IP header is kept before the ULP data in mp.
3458 	 */
3459 
3460 	/* submit MAC request to KCF */
3461 	return (ah_submit_req_outbound(ipsec_out, length_to_skip, assoc));
3462 }
3463 
3464 static ipsec_status_t
3465 ah_inbound(mblk_t *ipsec_in_mp, void *arg)
3466 {
3467 	mblk_t *data_mp = ipsec_in_mp->b_cont;
3468 	ipsec_in_t *ii = (ipsec_in_t *)ipsec_in_mp->b_rptr;
3469 	ah_t *ah = (ah_t *)arg;
3470 	ipsa_t *assoc = ii->ipsec_in_ah_sa;
3471 	int length_to_skip;
3472 	int ah_length;
3473 	mblk_t *phdr_mp;
3474 	uint32_t ah_offset;
3475 	netstack_t	*ns = ii->ipsec_in_ns;
3476 	ipsecah_stack_t	*ahstack = ns->netstack_ipsecah;
3477 	ipsec_stack_t	*ipss = ns->netstack_ipsec;
3478 
3479 	ASSERT(assoc != NULL);
3480 
3481 	/*
3482 	 * We may wish to check replay in-range-only here as an optimization.
3483 	 * Include the reality check of ipsa->ipsa_replay >
3484 	 * ipsa->ipsa_replay_wsize for times when it's the first N packets,
3485 	 * where N == ipsa->ipsa_replay_wsize.
3486 	 *
3487 	 * Another check that may come here later is the "collision" check.
3488 	 * If legitimate packets flow quickly enough, this won't be a problem,
3489 	 * but collisions may cause authentication algorithm crunching to
3490 	 * take place when it doesn't need to.
3491 	 */
3492 	if (!sadb_replay_peek(assoc, ah->ah_replay)) {
3493 		AH_BUMP_STAT(ahstack, replay_early_failures);
3494 		IP_AH_BUMP_STAT(ipss, in_discards);
3495 		ip_drop_packet(ipsec_in_mp, B_TRUE, NULL, NULL,
3496 		    DROPPER(ipss, ipds_ah_early_replay),
3497 		    &ahstack->ah_dropper);
3498 		return (IPSEC_STATUS_FAILED);
3499 	}
3500 
3501 	/*
3502 	 * The offset of the AH header can be computed from its pointer
3503 	 * within the data mblk, which was pulled up until the AH header
3504 	 * by ipsec_inbound_ah_sa() during SA selection.
3505 	 */
3506 	ah_offset = (uchar_t *)ah - data_mp->b_rptr;
3507 
3508 	/*
3509 	 * Has this packet already been processed by a hardware
3510 	 * IPsec accelerator?
3511 	 */
3512 	if (ii->ipsec_in_accelerated) {
3513 		ah3dbg(ahstack,
3514 		    ("ah_inbound_v6: pkt processed by ill=%d isv6=%d\n",
3515 		    ii->ipsec_in_ill_index, !ii->ipsec_in_v4));
3516 		return (ah_inbound_accelerated(ipsec_in_mp, ii->ipsec_in_v4,
3517 		    assoc, ah_offset));
3518 	}
3519 	AH_BUMP_STAT(ahstack, noaccel);
3520 
3521 	/*
3522 	 * We need to pullup until the ICV before we call
3523 	 * ah_process_ip_options_v6.
3524 	 */
3525 	ah_length = (ah->ah_length << 2) + 8;
3526 
3527 	/*
3528 	 * NOTE : If we want to use any field of IP/AH header, you need
3529 	 * to re-assign following the pullup.
3530 	 */
3531 	if (((uchar_t *)ah + ah_length) > data_mp->b_wptr) {
3532 		if (!pullupmsg(data_mp, (uchar_t *)ah + ah_length -
3533 		    data_mp->b_rptr)) {
3534 			(void) ipsec_rl_strlog(ns, info.mi_idnum, 0, 0,
3535 			    SL_WARN | SL_ERROR,
3536 			    "ah_inbound: Small AH header\n");
3537 			IP_AH_BUMP_STAT(ipss, in_discards);
3538 			ip_drop_packet(ipsec_in_mp, B_TRUE, NULL, NULL,
3539 			    DROPPER(ipss, ipds_ah_nomem),
3540 			    &ahstack->ah_dropper);
3541 			return (IPSEC_STATUS_FAILED);
3542 		}
3543 	}
3544 
3545 	/*
3546 	 * Insert pseudo header:
3547 	 * IPSEC_INFO -> [IP, ULP] => IPSEC_INFO -> [IP, AH, ICV] -> ULP
3548 	 */
3549 	if (ii->ipsec_in_v4) {
3550 		phdr_mp = ah_process_ip_options_v4(data_mp, assoc,
3551 		    &length_to_skip, assoc->ipsa_mac_len, B_FALSE, ahstack);
3552 	} else {
3553 		phdr_mp = ah_process_ip_options_v6(data_mp, assoc,
3554 		    &length_to_skip, assoc->ipsa_mac_len, B_FALSE, ahstack);
3555 	}
3556 
3557 	if (phdr_mp == NULL) {
3558 		IP_AH_BUMP_STAT(ipss, in_discards);
3559 		ip_drop_packet(ipsec_in_mp, B_TRUE, NULL, NULL,
3560 		    (ii->ipsec_in_v4 ?
3561 		    DROPPER(ipss, ipds_ah_bad_v4_opts) :
3562 		    DROPPER(ipss, ipds_ah_bad_v6_hdrs)),
3563 		    &ahstack->ah_dropper);
3564 		return (IPSEC_STATUS_FAILED);
3565 	}
3566 
3567 	ipsec_in_mp->b_cont = phdr_mp;
3568 	phdr_mp->b_cont = data_mp;
3569 	data_mp->b_rptr += length_to_skip;
3570 
3571 	/* submit request to KCF */
3572 	return (ah_submit_req_inbound(ipsec_in_mp, length_to_skip, ah_offset,
3573 	    assoc));
3574 }
3575 
3576 /*
3577  * ah_inbound_accelerated:
3578  * Called from ah_inbound() to process IPsec packets that have been
3579  * accelerated by hardware.
3580  *
3581  * Basically does what ah_auth_in_done() with some changes since
3582  * no pseudo-headers are involved, i.e. the passed message is a
3583  * IPSEC_INFO->DATA.
3584  *
3585  * It is assumed that only packets that have been successfully
3586  * processed by the adapter come here.
3587  *
3588  * 1. get algorithm structure corresponding to association
3589  * 2. calculate pointers to authentication header and ICV
3590  * 3. compare ICV in AH header with ICV in data attributes
3591  *    3.1 if different:
3592  *	  3.1.1 generate error
3593  *        3.1.2 discard message
3594  *    3.2 if ICV matches:
3595  *	  3.2.1 check replay
3596  *        3.2.2 remove AH header
3597  *        3.2.3 age SA byte
3598  *        3.2.4 send to IP
3599  */
3600 ipsec_status_t
3601 ah_inbound_accelerated(mblk_t *ipsec_in, boolean_t isv4, ipsa_t *assoc,
3602     uint32_t ah_offset)
3603 {
3604 	mblk_t *mp;
3605 	ipha_t *ipha;
3606 	ah_t *ah;
3607 	ipsec_in_t *ii;
3608 	uint32_t icv_len;
3609 	uint32_t align_len;
3610 	uint32_t age_bytes;
3611 	ip6_t *ip6h;
3612 	uint8_t *in_icv;
3613 	mblk_t *hada_mp;
3614 	uint32_t next_hdr;
3615 	da_ipsec_t *hada;
3616 	kstat_named_t *counter;
3617 	ipsecah_stack_t	*ahstack;
3618 	netstack_t	*ns;
3619 	ipsec_stack_t	*ipss;
3620 
3621 	ii = (ipsec_in_t *)ipsec_in->b_rptr;
3622 	ns = ii->ipsec_in_ns;
3623 	ahstack = ns->netstack_ipsecah;
3624 	ipss = ns->netstack_ipsec;
3625 
3626 	mp = ipsec_in->b_cont;
3627 	hada_mp = ii->ipsec_in_da;
3628 	ASSERT(hada_mp != NULL);
3629 	hada = (da_ipsec_t *)hada_mp->b_rptr;
3630 
3631 	AH_BUMP_STAT(ahstack, in_accelerated);
3632 
3633 	/*
3634 	 * We only support one level of decapsulation in hardware, so
3635 	 * nuke the pointer.
3636 	 */
3637 	ii->ipsec_in_da = NULL;
3638 	ii->ipsec_in_accelerated = B_FALSE;
3639 
3640 	/*
3641 	 * Extract ICV length from attributes M_CTL and sanity check
3642 	 * its value. We allow the mblk to be smaller than da_ipsec_t
3643 	 * for a small ICV, as long as the entire ICV fits within the mblk.
3644 	 * Also ensures that the ICV length computed by Provider
3645 	 * corresponds to the ICV length of the algorithm specified by the SA.
3646 	 */
3647 	icv_len = hada->da_icv_len;
3648 	if ((icv_len != assoc->ipsa_mac_len) ||
3649 	    (icv_len > DA_ICV_MAX_LEN) || (MBLKL(hada_mp) <
3650 	    (sizeof (da_ipsec_t) - DA_ICV_MAX_LEN + icv_len))) {
3651 		ah0dbg(("ah_inbound_accelerated: "
3652 		    "ICV len (%u) incorrect or mblk too small (%u)\n",
3653 		    icv_len, (uint32_t)(MBLKL(hada_mp))));
3654 		counter = DROPPER(ipss, ipds_ah_bad_length);
3655 		goto ah_in_discard;
3656 	}
3657 	ASSERT(icv_len != 0);
3658 
3659 	/* compute the padded AH ICV len */
3660 	if (isv4) {
3661 		ipha = (ipha_t *)mp->b_rptr;
3662 		align_len = (icv_len + IPV4_PADDING_ALIGN - 1) &
3663 		    -IPV4_PADDING_ALIGN;
3664 	} else {
3665 		ip6h = (ip6_t *)mp->b_rptr;
3666 		align_len = (icv_len + IPV6_PADDING_ALIGN - 1) &
3667 		    -IPV6_PADDING_ALIGN;
3668 	}
3669 
3670 	ah = (ah_t *)(mp->b_rptr + ah_offset);
3671 	in_icv = (uint8_t *)ah + sizeof (ah_t);
3672 
3673 	/* compare ICV in AH header vs ICV computed by adapter */
3674 	if (bcmp(hada->da_icv, in_icv, icv_len)) {
3675 		int af;
3676 		void *addr;
3677 
3678 		if (isv4) {
3679 			addr = &ipha->ipha_dst;
3680 			af = AF_INET;
3681 		} else {
3682 			addr = &ip6h->ip6_dst;
3683 			af = AF_INET6;
3684 		}
3685 
3686 		/*
3687 		 * Log the event. Don't print to the console, block
3688 		 * potential denial-of-service attack.
3689 		 */
3690 		AH_BUMP_STAT(ahstack, bad_auth);
3691 		ipsec_assocfailure(info.mi_idnum, 0, 0, SL_ERROR | SL_WARN,
3692 		    "AH Authentication failed spi %x, dst_addr %s",
3693 		    assoc->ipsa_spi, addr, af, ahstack->ipsecah_netstack);
3694 		counter = DROPPER(ipss, ipds_ah_bad_auth);
3695 		goto ah_in_discard;
3696 	}
3697 
3698 	ah3dbg(ahstack, ("AH succeeded, checking replay\n"));
3699 	AH_BUMP_STAT(ahstack, good_auth);
3700 
3701 	if (!sadb_replay_check(assoc, ah->ah_replay)) {
3702 		int af;
3703 		void *addr;
3704 
3705 		if (isv4) {
3706 			addr = &ipha->ipha_dst;
3707 			af = AF_INET;
3708 		} else {
3709 			addr = &ip6h->ip6_dst;
3710 			af = AF_INET6;
3711 		}
3712 
3713 		/*
3714 		 * Log the event. As of now we print out an event.
3715 		 * Do not print the replay failure number, or else
3716 		 * syslog cannot collate the error messages.  Printing
3717 		 * the replay number that failed (or printing to the
3718 		 * console) opens a denial-of-service attack.
3719 		 */
3720 		AH_BUMP_STAT(ahstack, replay_failures);
3721 		ipsec_assocfailure(info.mi_idnum, 0, 0,
3722 		    SL_ERROR | SL_WARN,
3723 		    "Replay failed for AH spi %x, dst_addr %s",
3724 		    assoc->ipsa_spi, addr, af, ahstack->ipsecah_netstack);
3725 		counter = DROPPER(ipss, ipds_ah_replay);
3726 		goto ah_in_discard;
3727 	}
3728 
3729 	/*
3730 	 * Remove AH header. We do this by copying everything before
3731 	 * the AH header onto the AH header+ICV.
3732 	 */
3733 	/* overwrite AH with what was preceeding it (IP header) */
3734 	next_hdr = ah->ah_nexthdr;
3735 	ovbcopy(mp->b_rptr, mp->b_rptr + sizeof (ah_t) + align_len,
3736 	    ah_offset);
3737 	mp->b_rptr += sizeof (ah_t) + align_len;
3738 	if (isv4) {
3739 		/* adjust IP header next protocol */
3740 		ipha = (ipha_t *)mp->b_rptr;
3741 		ipha->ipha_protocol = next_hdr;
3742 
3743 		age_bytes = ipha->ipha_length;
3744 
3745 		/* adjust length in IP header */
3746 		ipha->ipha_length -= (sizeof (ah_t) + align_len);
3747 
3748 		/* recalculate checksum */
3749 		ipha->ipha_hdr_checksum = 0;
3750 		ipha->ipha_hdr_checksum = (uint16_t)ip_csum_hdr(ipha);
3751 	} else {
3752 		/* adjust IP header next protocol */
3753 		ip6h = (ip6_t *)mp->b_rptr;
3754 		ip6h->ip6_nxt = next_hdr;
3755 
3756 		age_bytes = sizeof (ip6_t) + ntohs(ip6h->ip6_plen) +
3757 		    sizeof (ah_t);
3758 
3759 		/* adjust length in IP header */
3760 		ip6h->ip6_plen = htons(ntohs(ip6h->ip6_plen) -
3761 		    (sizeof (ah_t) + align_len));
3762 	}
3763 
3764 	/* age SA */
3765 	if (!ah_age_bytes(assoc, age_bytes, B_TRUE)) {
3766 		/* The ipsa has hit hard expiration, LOG and AUDIT. */
3767 		ipsec_assocfailure(info.mi_idnum, 0, 0,
3768 		    SL_ERROR | SL_WARN,
3769 		    "AH Association 0x%x, dst %s had bytes expire.\n",
3770 		    assoc->ipsa_spi, assoc->ipsa_dstaddr,
3771 		    AF_INET, ahstack->ipsecah_netstack);
3772 		AH_BUMP_STAT(ahstack, bytes_expired);
3773 		counter = DROPPER(ipss, ipds_ah_bytes_expire);
3774 		goto ah_in_discard;
3775 	}
3776 
3777 	freeb(hada_mp);
3778 	return (IPSEC_STATUS_SUCCESS);
3779 
3780 ah_in_discard:
3781 	IP_AH_BUMP_STAT(ipss, in_discards);
3782 	freeb(hada_mp);
3783 	ip_drop_packet(ipsec_in, B_TRUE, NULL, NULL, counter,
3784 	    &ahstack->ah_dropper);
3785 	return (IPSEC_STATUS_FAILED);
3786 }
3787 
3788 /*
3789  * ah_outbound_accelerated_v4:
3790  * Called from ah_outbound_v4() and once it is determined that the
3791  * packet is elligible for hardware acceleration.
3792  *
3793  * We proceed as follows:
3794  * 1. allocate and initialize attributes mblk
3795  * 2. mark IPSEC_OUT to indicate that pkt is accelerated
3796  * 3. insert AH header
3797  */
3798 static ipsec_status_t
3799 ah_outbound_accelerated_v4(mblk_t *ipsec_mp, ipsa_t *assoc)
3800 {
3801 	mblk_t *mp, *new_mp;
3802 	ipsec_out_t *oi;
3803 	uint_t ah_data_sz;	/* ICV length, algorithm dependent */
3804 	uint_t ah_align_sz;	/* ICV length + padding */
3805 	uint32_t v_hlen_tos_len; /* from original IP header */
3806 	ipha_t	*oipha;		/* original IP header */
3807 	ipha_t	*nipha;		/* new IP header */
3808 	uint_t option_length = 0;
3809 	uint_t new_hdr_len;	/* new header length */
3810 	uint_t iphdr_length;
3811 	ah_t *ah_hdr;		/* ptr to AH header */
3812 	netstack_t	*ns;
3813 	ipsec_stack_t	*ipss;
3814 	ipsecah_stack_t	*ahstack;
3815 
3816 	oi = (ipsec_out_t *)ipsec_mp->b_rptr;
3817 	ns = oi->ipsec_out_ns;
3818 	ipss = ns->netstack_ipsec;
3819 	ahstack = ns->netstack_ipsecah;
3820 
3821 	mp = ipsec_mp->b_cont;
3822 
3823 	AH_BUMP_STAT(ahstack, out_accelerated);
3824 
3825 	oipha = (ipha_t *)mp->b_rptr;
3826 	v_hlen_tos_len = ((uint32_t *)oipha)[0];
3827 
3828 	/* mark packet as being accelerated in IPSEC_OUT */
3829 	ASSERT(oi->ipsec_out_accelerated == B_FALSE);
3830 	oi->ipsec_out_accelerated = B_TRUE;
3831 
3832 	/* calculate authentication data length, i.e. ICV + padding */
3833 	ah_data_sz = assoc->ipsa_mac_len;
3834 	ah_align_sz = (ah_data_sz + IPV4_PADDING_ALIGN - 1) &
3835 	    -IPV4_PADDING_ALIGN;
3836 
3837 	/*
3838 	 * Insert pseudo header:
3839 	 * IPSEC_INFO -> [IP, ULP] => IPSEC_INFO -> [IP, AH, ICV] -> ULP
3840 	 */
3841 
3842 	/* IP + AH + authentication + padding data length */
3843 	new_hdr_len = IP_SIMPLE_HDR_LENGTH + sizeof (ah_t) + ah_align_sz;
3844 	if (V_HLEN != IP_SIMPLE_HDR_VERSION) {
3845 		option_length = oipha->ipha_version_and_hdr_length -
3846 		    (uint8_t)((IP_VERSION << 4) +
3847 		    IP_SIMPLE_HDR_LENGTH_IN_WORDS);
3848 		option_length <<= 2;
3849 		new_hdr_len += option_length;
3850 	}
3851 
3852 	/* allocate pseudo-header mblk */
3853 	if ((new_mp = allocb(new_hdr_len, BPRI_HI)) == NULL) {
3854 		/* IPsec kstats: bump bean counter here */
3855 		ip_drop_packet(ipsec_mp, B_FALSE, NULL, NULL,
3856 		    DROPPER(ipss, ipds_ah_nomem),
3857 		    &ahstack->ah_dropper);
3858 		return (IPSEC_STATUS_FAILED);
3859 	}
3860 
3861 	new_mp->b_cont = mp;
3862 	ipsec_mp->b_cont = new_mp;
3863 	new_mp->b_wptr += new_hdr_len;
3864 
3865 	/* copy original IP header to new header */
3866 	bcopy(mp->b_rptr, new_mp->b_rptr, IP_SIMPLE_HDR_LENGTH +
3867 	    option_length);
3868 
3869 	/* update IP header */
3870 	nipha = (ipha_t *)new_mp->b_rptr;
3871 	nipha->ipha_protocol = IPPROTO_AH;
3872 	iphdr_length = ntohs(nipha->ipha_length);
3873 	iphdr_length += sizeof (ah_t) + ah_align_sz;
3874 	nipha->ipha_length = htons(iphdr_length);
3875 	nipha->ipha_hdr_checksum = 0;
3876 	nipha->ipha_hdr_checksum = (uint16_t)ip_csum_hdr(nipha);
3877 
3878 	/* skip original IP header in mp */
3879 	mp->b_rptr += IP_SIMPLE_HDR_LENGTH + option_length;
3880 
3881 	/* initialize AH header */
3882 	ah_hdr = (ah_t *)(new_mp->b_rptr + IP_SIMPLE_HDR_LENGTH +
3883 	    option_length);
3884 	ah_hdr->ah_nexthdr = oipha->ipha_protocol;
3885 	if (!ah_finish_up(ah_hdr, NULL, assoc, ah_data_sz, ah_align_sz,
3886 	    ahstack)) {
3887 		/* Only way this fails is if outbound replay counter wraps. */
3888 		ip_drop_packet(ipsec_mp, B_FALSE, NULL, NULL,
3889 		    DROPPER(ipss, ipds_ah_replay),
3890 		    &ahstack->ah_dropper);
3891 		return (IPSEC_STATUS_FAILED);
3892 	}
3893 
3894 	return (IPSEC_STATUS_SUCCESS);
3895 }
3896 
3897 /*
3898  * ah_outbound_accelerated_v6:
3899  *
3900  * Called from ah_outbound_v6() once it is determined that the packet
3901  * is eligible for hardware acceleration.
3902  *
3903  * We proceed as follows:
3904  * 1. allocate and initialize attributes mblk
3905  * 2. mark IPSEC_OUT to indicate that pkt is accelerated
3906  * 3. insert AH header
3907  */
3908 static ipsec_status_t
3909 ah_outbound_accelerated_v6(mblk_t *ipsec_mp, ipsa_t *assoc)
3910 {
3911 	mblk_t *mp, *phdr_mp;
3912 	ipsec_out_t *oi;
3913 	uint_t ah_data_sz;	/* ICV length, algorithm dependent */
3914 	uint_t ah_align_sz;	/* ICV length + padding */
3915 	ip6_t	*oip6h;		/* original IP header */
3916 	ip6_t	*ip6h;		/* new IP header */
3917 	uint_t option_length = 0;
3918 	uint_t hdr_size;
3919 	uint_t ah_offset;
3920 	ah_t *ah_hdr;		/* ptr to AH header */
3921 	netstack_t	*ns;
3922 	ipsec_stack_t	*ipss;
3923 	ipsecah_stack_t	*ahstack;
3924 
3925 	oi = (ipsec_out_t *)ipsec_mp->b_rptr;
3926 	ns = oi->ipsec_out_ns;
3927 	ipss = ns->netstack_ipsec;
3928 	ahstack = ns->netstack_ipsecah;
3929 
3930 	mp = ipsec_mp->b_cont;
3931 
3932 	AH_BUMP_STAT(ahstack, out_accelerated);
3933 
3934 	oip6h = (ip6_t *)mp->b_rptr;
3935 
3936 	/* mark packet as being accelerated in IPSEC_OUT */
3937 	ASSERT(oi->ipsec_out_accelerated == B_FALSE);
3938 	oi->ipsec_out_accelerated = B_TRUE;
3939 
3940 	/* calculate authentication data length, i.e. ICV + padding */
3941 	ah_data_sz = assoc->ipsa_mac_len;
3942 	ah_align_sz = (ah_data_sz + IPV4_PADDING_ALIGN - 1) &
3943 	    -IPV4_PADDING_ALIGN;
3944 
3945 	ASSERT(ah_align_sz >= ah_data_sz);
3946 
3947 	hdr_size = ipsec_ah_get_hdr_size_v6(mp, B_FALSE);
3948 	option_length = hdr_size - IPV6_HDR_LEN;
3949 
3950 	/* This was not included in ipsec_ah_get_hdr_size_v6() */
3951 	hdr_size += (sizeof (ah_t) + ah_align_sz);
3952 
3953 	if ((phdr_mp = allocb(hdr_size, BPRI_HI)) == NULL) {
3954 		ip_drop_packet(ipsec_mp, B_FALSE, NULL, NULL,
3955 		    DROPPER(ipss, ipds_ah_nomem),
3956 		    &ahstack->ah_dropper);
3957 		return (IPSEC_STATUS_FAILED);
3958 	}
3959 	phdr_mp->b_wptr += hdr_size;
3960 
3961 	/*
3962 	 * Form the basic IP header first.  We always assign every bit
3963 	 * of the v6 basic header, so a separate bzero is unneeded.
3964 	 */
3965 	ip6h = (ip6_t *)phdr_mp->b_rptr;
3966 	ip6h->ip6_vcf = oip6h->ip6_vcf;
3967 	ip6h->ip6_hlim = oip6h->ip6_hlim;
3968 	ip6h->ip6_src = oip6h->ip6_src;
3969 	ip6h->ip6_dst = oip6h->ip6_dst;
3970 	/*
3971 	 * Include the size of AH and authentication data.
3972 	 * This is how our recipient would compute the
3973 	 * authentication data. Look at what we do in the
3974 	 * inbound case below.
3975 	 */
3976 	ip6h->ip6_plen = htons(ntohs(oip6h->ip6_plen) + sizeof (ah_t) +
3977 	    ah_align_sz);
3978 
3979 	/*
3980 	 * Insert pseudo header:
3981 	 * IPSEC_INFO -> [IP6, LLH, ULP] =>
3982 	 *	IPSEC_INFO -> [IP, LLH, AH, ICV] -> ULP
3983 	 */
3984 
3985 	if (option_length == 0) {
3986 		/* Form the AH header */
3987 		ip6h->ip6_nxt = IPPROTO_AH;
3988 		((ah_t *)(ip6h + 1))->ah_nexthdr = oip6h->ip6_nxt;
3989 		ah_offset = IPV6_HDR_LEN;
3990 	} else {
3991 		ip6h->ip6_nxt = oip6h->ip6_nxt;
3992 		/* option_length does not include the AH header's size */
3993 		ah_offset = ah_fix_phdr_v6(ip6h, oip6h, B_TRUE, B_FALSE);
3994 		if (ah_offset == 0) {
3995 			freemsg(phdr_mp);
3996 			ip_drop_packet(ipsec_mp, B_FALSE, NULL, NULL,
3997 			    DROPPER(ipss, ipds_ah_bad_v6_hdrs),
3998 			    &ahstack->ah_dropper);
3999 			return (IPSEC_STATUS_FAILED);
4000 		}
4001 	}
4002 
4003 	phdr_mp->b_cont = mp;
4004 	ipsec_mp->b_cont = phdr_mp;
4005 
4006 	/* skip original IP header in mp */
4007 	mp->b_rptr += IPV6_HDR_LEN + option_length;
4008 
4009 	/* initialize AH header */
4010 	ah_hdr = (ah_t *)(phdr_mp->b_rptr + IPV6_HDR_LEN + option_length);
4011 	ah_hdr->ah_nexthdr = oip6h->ip6_nxt;
4012 
4013 	if (!ah_finish_up(((ah_t *)((uint8_t *)ip6h + ah_offset)), NULL,
4014 	    assoc, ah_data_sz, ah_align_sz, ahstack)) {
4015 		/* Only way this fails is if outbound replay counter wraps. */
4016 		ip_drop_packet(ipsec_mp, B_FALSE, NULL, NULL,
4017 		    DROPPER(ipss, ipds_ah_replay),
4018 		    &ahstack->ah_dropper);
4019 		return (IPSEC_STATUS_FAILED);
4020 	}
4021 
4022 	return (IPSEC_STATUS_SUCCESS);
4023 }
4024 
4025 /*
4026  * Invoked after processing of an inbound packet by the
4027  * kernel crypto framework. Called by ah_submit_req() for a sync request,
4028  * or by the kcf callback for an async request.
4029  * Returns IPSEC_STATUS_SUCCESS on success, IPSEC_STATUS_FAILED on failure.
4030  * On failure, the mblk chain ipsec_in is freed by this function.
4031  */
4032 static ipsec_status_t
4033 ah_auth_in_done(mblk_t *ipsec_in)
4034 {
4035 	mblk_t *phdr_mp;
4036 	ipha_t *ipha;
4037 	uint_t ah_offset = 0;
4038 	mblk_t *mp;
4039 	int align_len, newpos;
4040 	ah_t *ah;
4041 	uint32_t length;
4042 	uint32_t *dest32;
4043 	uint8_t *dest;
4044 	ipsec_in_t *ii;
4045 	boolean_t isv4;
4046 	ip6_t *ip6h;
4047 	uint_t icv_len;
4048 	ipsa_t *assoc;
4049 	kstat_named_t *counter;
4050 	netstack_t	*ns;
4051 	ipsecah_stack_t	*ahstack;
4052 	ipsec_stack_t	*ipss;
4053 
4054 	ii = (ipsec_in_t *)ipsec_in->b_rptr;
4055 	ns = ii->ipsec_in_ns;
4056 	ahstack = ns->netstack_ipsecah;
4057 	ipss = ns->netstack_ipsec;
4058 
4059 	isv4 = ii->ipsec_in_v4;
4060 	assoc = ii->ipsec_in_ah_sa;
4061 	icv_len = (uint_t)ii->ipsec_in_crypto_mac.cd_raw.iov_len;
4062 
4063 	phdr_mp = ipsec_in->b_cont;
4064 	if (phdr_mp == NULL) {
4065 		ip_drop_packet(ipsec_in, B_TRUE, NULL, NULL,
4066 		    DROPPER(ipss, ipds_ah_nomem),
4067 		    &ahstack->ah_dropper);
4068 		return (IPSEC_STATUS_FAILED);
4069 	}
4070 
4071 	mp = phdr_mp->b_cont;
4072 	if (mp == NULL) {
4073 		ip_drop_packet(ipsec_in, B_TRUE, NULL, NULL,
4074 		    DROPPER(ipss, ipds_ah_nomem),
4075 		    &ahstack->ah_dropper);
4076 		return (IPSEC_STATUS_FAILED);
4077 	}
4078 	mp->b_rptr -= ii->ipsec_in_skip_len;
4079 
4080 	ah_set_usetime(assoc, B_TRUE);
4081 
4082 	if (isv4) {
4083 		ipha = (ipha_t *)mp->b_rptr;
4084 		ah_offset = ipha->ipha_version_and_hdr_length -
4085 		    (uint8_t)((IP_VERSION << 4));
4086 		ah_offset <<= 2;
4087 		align_len = P2ALIGN(icv_len + IPV4_PADDING_ALIGN - 1,
4088 		    IPV4_PADDING_ALIGN);
4089 	} else {
4090 		ip6h = (ip6_t *)mp->b_rptr;
4091 		ah_offset = ipsec_ah_get_hdr_size_v6(mp, B_TRUE);
4092 		ASSERT((mp->b_wptr - mp->b_rptr) >= ah_offset);
4093 		align_len = P2ALIGN(icv_len + IPV6_PADDING_ALIGN - 1,
4094 		    IPV6_PADDING_ALIGN);
4095 	}
4096 
4097 	ah = (ah_t *)(mp->b_rptr + ah_offset);
4098 	newpos = sizeof (ah_t) + align_len;
4099 
4100 	/*
4101 	 * We get here only when authentication passed.
4102 	 */
4103 
4104 	ah3dbg(ahstack, ("AH succeeded, checking replay\n"));
4105 	AH_BUMP_STAT(ahstack, good_auth);
4106 
4107 	if (!sadb_replay_check(assoc, ah->ah_replay)) {
4108 		int af;
4109 		void *addr;
4110 
4111 		if (isv4) {
4112 			addr = &ipha->ipha_dst;
4113 			af = AF_INET;
4114 		} else {
4115 			addr = &ip6h->ip6_dst;
4116 			af = AF_INET6;
4117 		}
4118 
4119 		/*
4120 		 * Log the event. As of now we print out an event.
4121 		 * Do not print the replay failure number, or else
4122 		 * syslog cannot collate the error messages.  Printing
4123 		 * the replay number that failed (or printing to the
4124 		 * console) opens a denial-of-service attack.
4125 		 */
4126 		AH_BUMP_STAT(ahstack, replay_failures);
4127 		ipsec_assocfailure(info.mi_idnum, 0, 0,
4128 		    SL_ERROR | SL_WARN,
4129 		    "Replay failed for AH spi %x, dst_addr %s",
4130 		    assoc->ipsa_spi, addr, af, ahstack->ipsecah_netstack);
4131 		counter = DROPPER(ipss, ipds_ah_replay);
4132 		goto ah_in_discard;
4133 	}
4134 
4135 	/*
4136 	 * We need to remove the AH header from the original
4137 	 * datagram. Best way to do this is to move the pre-AH headers
4138 	 * forward in the (relatively simple) IPv4 case.  In IPv6, it's
4139 	 * a bit more complicated because of IPv6's next-header chaining,
4140 	 * but it's doable.
4141 	 */
4142 	if (isv4) {
4143 		/*
4144 		 * Assign the right protocol, adjust the length as we
4145 		 * are removing the AH header and adjust the checksum to
4146 		 * account for the protocol and length.
4147 		 */
4148 		length = ntohs(ipha->ipha_length);
4149 		if (!ah_age_bytes(assoc, length, B_TRUE)) {
4150 			/* The ipsa has hit hard expiration, LOG and AUDIT. */
4151 			ipsec_assocfailure(info.mi_idnum, 0, 0,
4152 			    SL_ERROR | SL_WARN,
4153 			    "AH Association 0x%x, dst %s had bytes expire.\n",
4154 			    assoc->ipsa_spi, assoc->ipsa_dstaddr,
4155 			    AF_INET, ahstack->ipsecah_netstack);
4156 			AH_BUMP_STAT(ahstack, bytes_expired);
4157 			counter = DROPPER(ipss, ipds_ah_bytes_expire);
4158 			goto ah_in_discard;
4159 		}
4160 		ipha->ipha_protocol = ah->ah_nexthdr;
4161 		length -= newpos;
4162 
4163 		ipha->ipha_length = htons((uint16_t)length);
4164 		ipha->ipha_hdr_checksum = 0;
4165 		ipha->ipha_hdr_checksum = (uint16_t)ip_csum_hdr(ipha);
4166 	} else {
4167 		uchar_t *whereptr;
4168 		int hdrlen;
4169 		uint8_t *nexthdr;
4170 		ip6_hbh_t *hbhhdr;
4171 		ip6_dest_t *dsthdr;
4172 		ip6_rthdr0_t *rthdr;
4173 
4174 		/*
4175 		 * Make phdr_mp hold until the AH header and make
4176 		 * mp hold everything past AH header.
4177 		 */
4178 		length = ntohs(ip6h->ip6_plen);
4179 		if (!ah_age_bytes(assoc, length + sizeof (ip6_t), B_TRUE)) {
4180 			/* The ipsa has hit hard expiration, LOG and AUDIT. */
4181 			ipsec_assocfailure(info.mi_idnum, 0, 0,
4182 			    SL_ERROR | SL_WARN,
4183 			    "AH Association 0x%x, dst %s had bytes "
4184 			    "expire.\n", assoc->ipsa_spi, &ip6h->ip6_dst,
4185 			    AF_INET6, ahstack->ipsecah_netstack);
4186 			AH_BUMP_STAT(ahstack, bytes_expired);
4187 			counter = DROPPER(ipss, ipds_ah_bytes_expire);
4188 			goto ah_in_discard;
4189 		}
4190 
4191 		/*
4192 		 * Update the next header field of the header preceding
4193 		 * AH with the next header field of AH. Start with the
4194 		 * IPv6 header and proceed with the extension headers
4195 		 * until we find what we're looking for.
4196 		 */
4197 		nexthdr = &ip6h->ip6_nxt;
4198 		whereptr =  (uchar_t *)ip6h;
4199 		hdrlen = sizeof (ip6_t);
4200 
4201 		while (*nexthdr != IPPROTO_AH) {
4202 			whereptr += hdrlen;
4203 			/* Assume IP has already stripped it */
4204 			ASSERT(*nexthdr != IPPROTO_FRAGMENT &&
4205 			    *nexthdr != IPPROTO_RAW);
4206 			switch (*nexthdr) {
4207 			case IPPROTO_HOPOPTS:
4208 				hbhhdr = (ip6_hbh_t *)whereptr;
4209 				nexthdr = &hbhhdr->ip6h_nxt;
4210 				hdrlen = 8 * (hbhhdr->ip6h_len + 1);
4211 				break;
4212 			case IPPROTO_DSTOPTS:
4213 				dsthdr = (ip6_dest_t *)whereptr;
4214 				nexthdr = &dsthdr->ip6d_nxt;
4215 				hdrlen = 8 * (dsthdr->ip6d_len + 1);
4216 				break;
4217 			case IPPROTO_ROUTING:
4218 				rthdr = (ip6_rthdr0_t *)whereptr;
4219 				nexthdr = &rthdr->ip6r0_nxt;
4220 				hdrlen = 8 * (rthdr->ip6r0_len + 1);
4221 				break;
4222 			}
4223 		}
4224 		*nexthdr = ah->ah_nexthdr;
4225 		length -= newpos;
4226 		ip6h->ip6_plen = htons((uint16_t)length);
4227 	}
4228 
4229 	/* Now that we've fixed the IP header, move it forward. */
4230 	mp->b_rptr += newpos;
4231 	if (IS_P2ALIGNED(mp->b_rptr, sizeof (uint32_t))) {
4232 		dest32 = (uint32_t *)(mp->b_rptr + ah_offset);
4233 		while (--dest32 >= (uint32_t *)mp->b_rptr)
4234 			*dest32 = *(dest32 - (newpos >> 2));
4235 	} else {
4236 		dest = mp->b_rptr + ah_offset;
4237 		while (--dest >= mp->b_rptr)
4238 			*dest = *(dest - newpos);
4239 	}
4240 	/*
4241 	 * If a db_credp exists in phdr_mp, it must also exist in mp.
4242 	 */
4243 	ASSERT(DB_CRED(phdr_mp) == NULL ||
4244 	    msg_getcred(mp, NULL) != NULL);
4245 
4246 	freeb(phdr_mp);
4247 	ipsec_in->b_cont = mp;
4248 	if (assoc->ipsa_state == IPSA_STATE_IDLE) {
4249 		/*
4250 		 * Cluster buffering case.  Tell caller that we're
4251 		 * handling the packet.
4252 		 */
4253 		sadb_buf_pkt(assoc, ipsec_in, ns);
4254 		return (IPSEC_STATUS_PENDING);
4255 	}
4256 	return (IPSEC_STATUS_SUCCESS);
4257 
4258 ah_in_discard:
4259 	IP_AH_BUMP_STAT(ipss, in_discards);
4260 	ip_drop_packet(ipsec_in, B_TRUE, NULL, NULL, counter,
4261 	    &ahstack->ah_dropper);
4262 	return (IPSEC_STATUS_FAILED);
4263 }
4264 
4265 /*
4266  * Invoked after processing of an outbound packet by the
4267  * kernel crypto framework, either by ah_submit_req() for a request
4268  * executed syncrhonously, or by the KEF callback for a request
4269  * executed asynchronously.
4270  */
4271 static ipsec_status_t
4272 ah_auth_out_done(mblk_t *ipsec_out)
4273 {
4274 	mblk_t *phdr_mp;
4275 	mblk_t *mp;
4276 	int align_len;
4277 	uint32_t hdrs_length;
4278 	uchar_t *ptr;
4279 	uint32_t length;
4280 	boolean_t isv4;
4281 	ipsec_out_t *io;
4282 	size_t icv_len;
4283 	netstack_t	*ns;
4284 	ipsec_stack_t	*ipss;
4285 	ipsecah_stack_t	*ahstack;
4286 
4287 	io = (ipsec_out_t *)ipsec_out->b_rptr;
4288 	ns = io->ipsec_out_ns;
4289 	ipss = ns->netstack_ipsec;
4290 	ahstack = ns->netstack_ipsecah;
4291 
4292 	isv4 = io->ipsec_out_v4;
4293 	icv_len = io->ipsec_out_crypto_mac.cd_raw.iov_len;
4294 
4295 	phdr_mp = ipsec_out->b_cont;
4296 	if (phdr_mp == NULL) {
4297 		ip_drop_packet(ipsec_out, B_FALSE, NULL, NULL,
4298 		    DROPPER(ipss, ipds_ah_nomem),
4299 		    &ahstack->ah_dropper);
4300 		return (IPSEC_STATUS_FAILED);
4301 	}
4302 
4303 	mp = phdr_mp->b_cont;
4304 	if (mp == NULL) {
4305 		ip_drop_packet(ipsec_out, B_FALSE, NULL, NULL,
4306 		    DROPPER(ipss, ipds_ah_nomem),
4307 		    &ahstack->ah_dropper);
4308 		return (IPSEC_STATUS_FAILED);
4309 	}
4310 	mp->b_rptr -= io->ipsec_out_skip_len;
4311 
4312 	ASSERT(io->ipsec_out_ah_sa != NULL);
4313 	ah_set_usetime(io->ipsec_out_ah_sa, B_FALSE);
4314 
4315 	if (isv4) {
4316 		ipha_t *ipha;
4317 		ipha_t *nipha;
4318 
4319 		ipha = (ipha_t *)mp->b_rptr;
4320 		hdrs_length = ipha->ipha_version_and_hdr_length -
4321 		    (uint8_t)((IP_VERSION << 4));
4322 		hdrs_length <<= 2;
4323 		align_len = P2ALIGN(icv_len + IPV4_PADDING_ALIGN - 1,
4324 		    IPV4_PADDING_ALIGN);
4325 		/*
4326 		 * phdr_mp must have the right amount of space for the
4327 		 * combined IP and AH header. Copy the IP header and
4328 		 * the ack_data onto AH. Note that the AH header was
4329 		 * already formed before the ICV calculation and hence
4330 		 * you don't have to copy it here.
4331 		 */
4332 		bcopy(mp->b_rptr, phdr_mp->b_rptr, hdrs_length);
4333 
4334 		ptr = phdr_mp->b_rptr + hdrs_length + sizeof (ah_t);
4335 		bcopy(phdr_mp->b_wptr, ptr, icv_len);
4336 
4337 		/*
4338 		 * Compute the new header checksum as we are assigning
4339 		 * IPPROTO_AH and adjusting the length here.
4340 		 */
4341 		nipha = (ipha_t *)phdr_mp->b_rptr;
4342 
4343 		nipha->ipha_protocol = IPPROTO_AH;
4344 		length = ntohs(nipha->ipha_length);
4345 		length += (sizeof (ah_t) + align_len);
4346 		nipha->ipha_length = htons((uint16_t)length);
4347 		nipha->ipha_hdr_checksum = 0;
4348 		nipha->ipha_hdr_checksum = (uint16_t)ip_csum_hdr(nipha);
4349 	} else {
4350 		ip6_t *ip6h;
4351 		ip6_t *nip6h;
4352 		uint_t ah_offset;
4353 
4354 		ip6h = (ip6_t *)mp->b_rptr;
4355 		nip6h = (ip6_t *)phdr_mp->b_rptr;
4356 		align_len = P2ALIGN(icv_len + IPV6_PADDING_ALIGN - 1,
4357 		    IPV6_PADDING_ALIGN);
4358 		/*
4359 		 * phdr_mp must have the right amount of space for the
4360 		 * combined IP and AH header. Copy the IP header with
4361 		 * options into the pseudo header. When we constructed
4362 		 * a pseudo header, we did not copy some of the mutable
4363 		 * fields. We do it now by calling ah_fix_phdr_v6()
4364 		 * with the last argument B_TRUE. It returns the
4365 		 * ah_offset into the pseudo header.
4366 		 */
4367 
4368 		bcopy(ip6h, nip6h, IPV6_HDR_LEN);
4369 		ah_offset = ah_fix_phdr_v6(nip6h, ip6h, B_TRUE, B_TRUE);
4370 		ASSERT(ah_offset != 0);
4371 		/*
4372 		 * phdr_mp can hold exactly the whole IP header with options
4373 		 * plus the AH header also. Thus subtracting the AH header's
4374 		 * size should give exactly how much of the original header
4375 		 * should be skipped.
4376 		 */
4377 		hdrs_length = (phdr_mp->b_wptr - phdr_mp->b_rptr) -
4378 		    sizeof (ah_t) - icv_len;
4379 		bcopy(phdr_mp->b_wptr, ((uint8_t *)nip6h + ah_offset +
4380 		    sizeof (ah_t)), icv_len);
4381 		length = ntohs(nip6h->ip6_plen);
4382 		length += (sizeof (ah_t) + align_len);
4383 		nip6h->ip6_plen = htons((uint16_t)length);
4384 	}
4385 
4386 	/* Skip the original IP header */
4387 	mp->b_rptr += hdrs_length;
4388 	if (mp->b_rptr == mp->b_wptr) {
4389 		phdr_mp->b_cont = mp->b_cont;
4390 		freeb(mp);
4391 	}
4392 
4393 	return (IPSEC_STATUS_SUCCESS);
4394 }
4395 
4396 /*
4397  * Wrapper to allow IP to trigger an AH association failure message
4398  * during SA inbound selection.
4399  */
4400 void
4401 ipsecah_in_assocfailure(mblk_t *mp, char level, ushort_t sl, char *fmt,
4402     uint32_t spi, void *addr, int af, ipsecah_stack_t *ahstack)
4403 {
4404 	ipsec_stack_t	*ipss = ahstack->ipsecah_netstack->netstack_ipsec;
4405 
4406 	if (ahstack->ipsecah_log_unknown_spi) {
4407 		ipsec_assocfailure(info.mi_idnum, 0, level, sl, fmt, spi,
4408 		    addr, af, ahstack->ipsecah_netstack);
4409 	}
4410 
4411 	ip_drop_packet(mp, B_TRUE, NULL, NULL,
4412 	    DROPPER(ipss, ipds_ah_no_sa),
4413 	    &ahstack->ah_dropper);
4414 }
4415 
4416 /*
4417  * Initialize the AH input and output processing functions.
4418  */
4419 void
4420 ipsecah_init_funcs(ipsa_t *sa)
4421 {
4422 	if (sa->ipsa_output_func == NULL)
4423 		sa->ipsa_output_func = ah_outbound;
4424 	if (sa->ipsa_input_func == NULL)
4425 		sa->ipsa_input_func = ah_inbound;
4426 }
4427