xref: /freebsd/sys/netipsec/ipsec_offload.c (revision c4a0ee9b97bfc7407366567aaa2c09313b3e6bd2)
1ef2a572bSKonstantin Belousov /*-
2ef2a572bSKonstantin Belousov  * Copyright (c) 2021,2022 NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED.
3ef2a572bSKonstantin Belousov  *
4ef2a572bSKonstantin Belousov  * Redistribution and use in source and binary forms, with or without
5ef2a572bSKonstantin Belousov  * modification, are permitted provided that the following conditions
6ef2a572bSKonstantin Belousov  * are met:
7ef2a572bSKonstantin Belousov  * 1. Redistributions of source code must retain the above copyright
8ef2a572bSKonstantin Belousov  *    notice, this list of conditions and the following disclaimer.
9ef2a572bSKonstantin Belousov  * 2. Redistributions in binary form must reproduce the above copyright
10ef2a572bSKonstantin Belousov  *    notice, this list of conditions and the following disclaimer in the
11ef2a572bSKonstantin Belousov  *    documentation and/or other materials provided with the distribution.
12ef2a572bSKonstantin Belousov  *
13ef2a572bSKonstantin Belousov  * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
14ef2a572bSKonstantin Belousov  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15ef2a572bSKonstantin Belousov  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16ef2a572bSKonstantin Belousov  * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
17ef2a572bSKonstantin Belousov  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18ef2a572bSKonstantin Belousov  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19ef2a572bSKonstantin Belousov  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20ef2a572bSKonstantin Belousov  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21ef2a572bSKonstantin Belousov  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22ef2a572bSKonstantin Belousov  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23ef2a572bSKonstantin Belousov  * SUCH DAMAGE.
24ef2a572bSKonstantin Belousov  */
25ef2a572bSKonstantin Belousov 
26ef2a572bSKonstantin Belousov #include "opt_inet.h"
27ef2a572bSKonstantin Belousov #include "opt_inet6.h"
28ef2a572bSKonstantin Belousov #include "opt_ipsec.h"
29ef2a572bSKonstantin Belousov 
30ef2a572bSKonstantin Belousov #include <sys/param.h>
31ef2a572bSKonstantin Belousov #include <sys/systm.h>
32ef2a572bSKonstantin Belousov #include <sys/ck.h>
33*c4a0ee9bSKonstantin Belousov #include <sys/eventhandler.h>
34ef2a572bSKonstantin Belousov #include <sys/kernel.h>
35ef2a572bSKonstantin Belousov #include <sys/mbuf.h>
36ef2a572bSKonstantin Belousov #include <sys/pctrie.h>
37ef2a572bSKonstantin Belousov #include <sys/proc.h>
38ef2a572bSKonstantin Belousov #include <sys/socket.h>
399a7096ffSKonstantin Belousov #include <sys/sysctl.h>
40ef2a572bSKonstantin Belousov #include <sys/protosw.h>
41ef2a572bSKonstantin Belousov #include <sys/taskqueue.h>
42ef2a572bSKonstantin Belousov 
439a7096ffSKonstantin Belousov #include <machine/stdarg.h>
449a7096ffSKonstantin Belousov 
45ef2a572bSKonstantin Belousov #include <net/if.h>
46ef2a572bSKonstantin Belousov #include <net/if_var.h>
47ef2a572bSKonstantin Belousov #include <net/vnet.h>
48ef2a572bSKonstantin Belousov #include <netinet/in.h>
49ef2a572bSKonstantin Belousov #include <netinet/ip.h>
50ef2a572bSKonstantin Belousov #include <netinet/ip_var.h>
51ef2a572bSKonstantin Belousov #include <netinet/ip6.h>
52ef2a572bSKonstantin Belousov #include <netinet6/ip6_var.h>
53ef2a572bSKonstantin Belousov #include <netinet/in_pcb.h>
54e6e2c0a5SKonstantin Belousov #include <netinet/tcp_var.h>
55ef2a572bSKonstantin Belousov 
56ef2a572bSKonstantin Belousov #include <netipsec/key.h>
57ef2a572bSKonstantin Belousov #include <netipsec/keydb.h>
58ef2a572bSKonstantin Belousov #include <netipsec/key_debug.h>
59ef2a572bSKonstantin Belousov #include <netipsec/xform.h>
60ef2a572bSKonstantin Belousov #include <netipsec/ipsec.h>
61ef2a572bSKonstantin Belousov #include <netipsec/ipsec_offload.h>
62ef2a572bSKonstantin Belousov #include <netipsec/ah_var.h>
63ef2a572bSKonstantin Belousov #include <netipsec/esp.h>
64ef2a572bSKonstantin Belousov #include <netipsec/esp_var.h>
65ef2a572bSKonstantin Belousov #include <netipsec/ipcomp_var.h>
66ef2a572bSKonstantin Belousov 
67ef2a572bSKonstantin Belousov #ifdef IPSEC_OFFLOAD
68ef2a572bSKonstantin Belousov 
69ef2a572bSKonstantin Belousov static struct mtx ipsec_accel_sav_tmp;
70ef2a572bSKonstantin Belousov static struct unrhdr *drv_spi_unr;
71ef2a572bSKonstantin Belousov static struct mtx ipsec_accel_cnt_lock;
72ef2a572bSKonstantin Belousov 
73ef2a572bSKonstantin Belousov struct ipsec_accel_install_newkey_tq {
74ef2a572bSKonstantin Belousov 	struct secasvar *sav;
75ef2a572bSKonstantin Belousov 	struct vnet *install_vnet;
76ef2a572bSKonstantin Belousov 	struct task install_task;
77ef2a572bSKonstantin Belousov };
78ef2a572bSKonstantin Belousov 
79ef2a572bSKonstantin Belousov struct ipsec_accel_forget_tq {
80ef2a572bSKonstantin Belousov 	struct vnet *forget_vnet;
81ef2a572bSKonstantin Belousov 	struct task forget_task;
82ef2a572bSKonstantin Belousov 	struct secasvar *sav;
83ef2a572bSKonstantin Belousov };
84ef2a572bSKonstantin Belousov 
85ef2a572bSKonstantin Belousov struct ifp_handle_sav {
86ef2a572bSKonstantin Belousov 	CK_LIST_ENTRY(ifp_handle_sav) sav_link;
87ef2a572bSKonstantin Belousov 	CK_LIST_ENTRY(ifp_handle_sav) sav_allh_link;
88ef2a572bSKonstantin Belousov 	struct secasvar *sav;
89ef2a572bSKonstantin Belousov 	struct ifnet *ifp;
90ef2a572bSKonstantin Belousov 	void *ifdata;
91ef2a572bSKonstantin Belousov 	uint64_t drv_spi;
92ef2a572bSKonstantin Belousov 	uint32_t flags;
93ef2a572bSKonstantin Belousov 	size_t hdr_ext_size;
94ef2a572bSKonstantin Belousov 	uint64_t cnt_octets;
95ef2a572bSKonstantin Belousov 	uint64_t cnt_allocs;
96ef2a572bSKonstantin Belousov };
97ef2a572bSKonstantin Belousov 
98ef2a572bSKonstantin Belousov #define	IFP_HS_HANDLED	0x00000001
99ef2a572bSKonstantin Belousov #define	IFP_HS_REJECTED	0x00000002
100ef2a572bSKonstantin Belousov #define	IFP_HS_INPUT	0x00000004
101ef2a572bSKonstantin Belousov #define	IFP_HS_OUTPUT	0x00000008
102ef2a572bSKonstantin Belousov #define	IFP_HS_MARKER	0x00000010
103ef2a572bSKonstantin Belousov 
104ef2a572bSKonstantin Belousov static CK_LIST_HEAD(, ifp_handle_sav) ipsec_accel_all_sav_handles;
105ef2a572bSKonstantin Belousov 
106ef2a572bSKonstantin Belousov struct ifp_handle_sp {
107ef2a572bSKonstantin Belousov 	CK_LIST_ENTRY(ifp_handle_sp) sp_link;
108ef2a572bSKonstantin Belousov 	CK_LIST_ENTRY(ifp_handle_sp) sp_allh_link;
109ef2a572bSKonstantin Belousov 	struct secpolicy *sp;
110ef2a572bSKonstantin Belousov 	struct ifnet *ifp;
111ef2a572bSKonstantin Belousov 	void *ifdata;
112ef2a572bSKonstantin Belousov 	uint32_t flags;
113ef2a572bSKonstantin Belousov };
114ef2a572bSKonstantin Belousov 
115ef2a572bSKonstantin Belousov #define	IFP_HP_HANDLED	0x00000001
116ef2a572bSKonstantin Belousov #define	IFP_HP_REJECTED	0x00000002
117ef2a572bSKonstantin Belousov #define	IFP_HP_MARKER	0x00000004
118ef2a572bSKonstantin Belousov 
119ef2a572bSKonstantin Belousov static CK_LIST_HEAD(, ifp_handle_sp) ipsec_accel_all_sp_handles;
120ef2a572bSKonstantin Belousov 
121ef2a572bSKonstantin Belousov static void *
122ef2a572bSKonstantin Belousov drvspi_sa_trie_alloc(struct pctrie *ptree)
123ef2a572bSKonstantin Belousov {
124ef2a572bSKonstantin Belousov 	void *res;
125ef2a572bSKonstantin Belousov 
126ef2a572bSKonstantin Belousov 	res = malloc(pctrie_node_size(), M_IPSEC_MISC, M_ZERO | M_NOWAIT);
127ef2a572bSKonstantin Belousov 	if (res != NULL)
128ef2a572bSKonstantin Belousov 		pctrie_zone_init(res, 0, 0);
129ef2a572bSKonstantin Belousov 	return (res);
130ef2a572bSKonstantin Belousov }
131ef2a572bSKonstantin Belousov 
132ef2a572bSKonstantin Belousov static void
133ef2a572bSKonstantin Belousov drvspi_sa_trie_free(struct pctrie *ptree, void *node)
134ef2a572bSKonstantin Belousov {
135ef2a572bSKonstantin Belousov 	free(node, M_IPSEC_MISC);
136ef2a572bSKonstantin Belousov }
137ef2a572bSKonstantin Belousov 
138ef2a572bSKonstantin Belousov PCTRIE_DEFINE(DRVSPI_SA, ifp_handle_sav, drv_spi,
139ef2a572bSKonstantin Belousov     drvspi_sa_trie_alloc, drvspi_sa_trie_free);
140ef2a572bSKonstantin Belousov static struct pctrie drv_spi_pctrie;
141ef2a572bSKonstantin Belousov 
142*c4a0ee9bSKonstantin Belousov static eventhandler_tag ipsec_accel_ifdetach_event_tag;
143*c4a0ee9bSKonstantin Belousov 
144ef2a572bSKonstantin Belousov static void ipsec_accel_sa_newkey_impl(struct secasvar *sav);
145ef2a572bSKonstantin Belousov static int ipsec_accel_handle_sav(struct secasvar *sav, struct ifnet *ifp,
146ef2a572bSKonstantin Belousov     u_int drv_spi, void *priv, uint32_t flags, struct ifp_handle_sav **ires);
147ef2a572bSKonstantin Belousov static void ipsec_accel_forget_sav_clear(struct secasvar *sav);
148ef2a572bSKonstantin Belousov static struct ifp_handle_sav *ipsec_accel_is_accel_sav_ptr(struct secasvar *sav,
149ef2a572bSKonstantin Belousov     struct ifnet *ifp);
150ef2a572bSKonstantin Belousov static int ipsec_accel_sa_lifetime_op_impl(struct secasvar *sav,
151ef2a572bSKonstantin Belousov     struct seclifetime *lft_c, if_t ifp, enum IF_SA_CNT_WHICH op,
152ef2a572bSKonstantin Belousov     struct rm_priotracker *sahtree_trackerp);
153ef2a572bSKonstantin Belousov static void ipsec_accel_sa_recordxfer(struct secasvar *sav, struct mbuf *m);
154ef2a572bSKonstantin Belousov static void ipsec_accel_sync_imp(void);
155ef2a572bSKonstantin Belousov static bool ipsec_accel_is_accel_sav_impl(struct secasvar *sav);
156ef2a572bSKonstantin Belousov static struct mbuf *ipsec_accel_key_setaccelif_impl(struct secasvar *sav);
15765f264dcSKonstantin Belousov static void ipsec_accel_on_ifdown_impl(struct ifnet *ifp);
15865f264dcSKonstantin Belousov static void ipsec_accel_drv_sa_lifetime_update_impl(struct secasvar *sav,
15965f264dcSKonstantin Belousov     if_t ifp, u_int drv_spi, uint64_t octets, uint64_t allocs);
160*c4a0ee9bSKonstantin Belousov static void ipsec_accel_ifdetach_event(void *arg, struct ifnet *ifp);
161ef2a572bSKonstantin Belousov 
162ef2a572bSKonstantin Belousov static void
163ef2a572bSKonstantin Belousov ipsec_accel_init(void *arg)
164ef2a572bSKonstantin Belousov {
165ef2a572bSKonstantin Belousov 	mtx_init(&ipsec_accel_sav_tmp, "ipasat", MTX_DEF, 0);
166ef2a572bSKonstantin Belousov 	mtx_init(&ipsec_accel_cnt_lock, "ipascn", MTX_DEF, 0);
167ef2a572bSKonstantin Belousov 	drv_spi_unr = new_unrhdr(IPSEC_ACCEL_DRV_SPI_MIN,
168ef2a572bSKonstantin Belousov 	    IPSEC_ACCEL_DRV_SPI_MAX, &ipsec_accel_sav_tmp);
169ef2a572bSKonstantin Belousov 	ipsec_accel_sa_newkey_p = ipsec_accel_sa_newkey_impl;
170ef2a572bSKonstantin Belousov 	ipsec_accel_forget_sav_p = ipsec_accel_forget_sav_impl;
171ef2a572bSKonstantin Belousov 	ipsec_accel_spdadd_p = ipsec_accel_spdadd_impl;
172ef2a572bSKonstantin Belousov 	ipsec_accel_spddel_p = ipsec_accel_spddel_impl;
173ef2a572bSKonstantin Belousov 	ipsec_accel_sa_lifetime_op_p = ipsec_accel_sa_lifetime_op_impl;
174ef2a572bSKonstantin Belousov 	ipsec_accel_sync_p = ipsec_accel_sync_imp;
175ef2a572bSKonstantin Belousov 	ipsec_accel_is_accel_sav_p = ipsec_accel_is_accel_sav_impl;
176ef2a572bSKonstantin Belousov 	ipsec_accel_key_setaccelif_p = ipsec_accel_key_setaccelif_impl;
17765f264dcSKonstantin Belousov 	ipsec_accel_on_ifdown_p = ipsec_accel_on_ifdown_impl;
17865f264dcSKonstantin Belousov 	ipsec_accel_drv_sa_lifetime_update_p =
17965f264dcSKonstantin Belousov 	    ipsec_accel_drv_sa_lifetime_update_impl;
180ef2a572bSKonstantin Belousov 	pctrie_init(&drv_spi_pctrie);
181*c4a0ee9bSKonstantin Belousov 	ipsec_accel_ifdetach_event_tag = EVENTHANDLER_REGISTER(
182*c4a0ee9bSKonstantin Belousov 	    ifnet_departure_event, ipsec_accel_ifdetach_event, NULL,
183*c4a0ee9bSKonstantin Belousov 	    EVENTHANDLER_PRI_ANY);
184ef2a572bSKonstantin Belousov }
185ef2a572bSKonstantin Belousov SYSINIT(ipsec_accel_init, SI_SUB_VNET_DONE, SI_ORDER_ANY,
186ef2a572bSKonstantin Belousov     ipsec_accel_init, NULL);
187ef2a572bSKonstantin Belousov 
188ef2a572bSKonstantin Belousov static void
189ef2a572bSKonstantin Belousov ipsec_accel_fini(void *arg)
190ef2a572bSKonstantin Belousov {
191*c4a0ee9bSKonstantin Belousov 	EVENTHANDLER_DEREGISTER(ifnet_departure_event,
192*c4a0ee9bSKonstantin Belousov 	    ipsec_accel_ifdetach_event_tag);
193ef2a572bSKonstantin Belousov 	ipsec_accel_sa_newkey_p = NULL;
194ef2a572bSKonstantin Belousov 	ipsec_accel_forget_sav_p = NULL;
195ef2a572bSKonstantin Belousov 	ipsec_accel_spdadd_p = NULL;
196ef2a572bSKonstantin Belousov 	ipsec_accel_spddel_p = NULL;
197ef2a572bSKonstantin Belousov 	ipsec_accel_sa_lifetime_op_p = NULL;
198ef2a572bSKonstantin Belousov 	ipsec_accel_sync_p = NULL;
199ef2a572bSKonstantin Belousov 	ipsec_accel_is_accel_sav_p = NULL;
200ef2a572bSKonstantin Belousov 	ipsec_accel_key_setaccelif_p = NULL;
20165f264dcSKonstantin Belousov 	ipsec_accel_on_ifdown_p = NULL;
20265f264dcSKonstantin Belousov 	ipsec_accel_drv_sa_lifetime_update_p = NULL;
203ef2a572bSKonstantin Belousov 	ipsec_accel_sync_imp();
204ef2a572bSKonstantin Belousov 	clean_unrhdr(drv_spi_unr);	/* avoid panic, should go later */
205ef2a572bSKonstantin Belousov 	clear_unrhdr(drv_spi_unr);
206ef2a572bSKonstantin Belousov 	delete_unrhdr(drv_spi_unr);
207ef2a572bSKonstantin Belousov 	mtx_destroy(&ipsec_accel_sav_tmp);
208ef2a572bSKonstantin Belousov 	mtx_destroy(&ipsec_accel_cnt_lock);
209ef2a572bSKonstantin Belousov }
210ef2a572bSKonstantin Belousov SYSUNINIT(ipsec_accel_fini, SI_SUB_VNET_DONE, SI_ORDER_ANY,
211ef2a572bSKonstantin Belousov     ipsec_accel_fini, NULL);
212ef2a572bSKonstantin Belousov 
2139a7096ffSKonstantin Belousov SYSCTL_NODE(_net_inet_ipsec, OID_AUTO, offload, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
2149a7096ffSKonstantin Belousov     "");
2159a7096ffSKonstantin Belousov 
2169a7096ffSKonstantin Belousov static bool ipsec_offload_verbose = false;
2179a7096ffSKonstantin Belousov SYSCTL_BOOL(_net_inet_ipsec_offload, OID_AUTO, verbose, CTLFLAG_RW,
2189a7096ffSKonstantin Belousov     &ipsec_offload_verbose, 0,
2199a7096ffSKonstantin Belousov     "Verbose SA/SP offload install and deinstall");
2209a7096ffSKonstantin Belousov 
2219a7096ffSKonstantin Belousov static void
2229a7096ffSKonstantin Belousov dprintf(const char *fmt, ...)
2239a7096ffSKonstantin Belousov {
2249a7096ffSKonstantin Belousov 	va_list ap;
2259a7096ffSKonstantin Belousov 
2269a7096ffSKonstantin Belousov 	if (!ipsec_offload_verbose)
2279a7096ffSKonstantin Belousov 		return;
2289a7096ffSKonstantin Belousov 
2299a7096ffSKonstantin Belousov 	va_start(ap, fmt);
2309a7096ffSKonstantin Belousov 	vprintf(fmt, ap);
2319a7096ffSKonstantin Belousov 	va_end(ap);
2329a7096ffSKonstantin Belousov }
2339a7096ffSKonstantin Belousov 
234ef2a572bSKonstantin Belousov static void
235ef2a572bSKonstantin Belousov ipsec_accel_alloc_forget_tq(struct secasvar *sav)
236ef2a572bSKonstantin Belousov {
237ef2a572bSKonstantin Belousov 	void *ftq;
238ef2a572bSKonstantin Belousov 
239ef2a572bSKonstantin Belousov 	if (sav->accel_forget_tq != 0)
240ef2a572bSKonstantin Belousov 		return;
241ef2a572bSKonstantin Belousov 
242ef2a572bSKonstantin Belousov 	ftq = malloc(sizeof(struct ipsec_accel_forget_tq), M_TEMP, M_WAITOK);
243ef2a572bSKonstantin Belousov 	if (!atomic_cmpset_ptr(&sav->accel_forget_tq, 0, (uintptr_t)ftq))
244ef2a572bSKonstantin Belousov 		free(ftq, M_TEMP);
245ef2a572bSKonstantin Belousov }
246ef2a572bSKonstantin Belousov 
247ef2a572bSKonstantin Belousov static bool
248ef2a572bSKonstantin Belousov ipsec_accel_sa_install_match(if_t ifp, void *arg)
249ef2a572bSKonstantin Belousov {
250ef2a572bSKonstantin Belousov 	if ((ifp->if_capenable2 & IFCAP2_BIT(IFCAP2_IPSEC_OFFLOAD)) == 0)
251ef2a572bSKonstantin Belousov 		return (false);
252ef2a572bSKonstantin Belousov 	if (ifp->if_ipsec_accel_m->if_sa_newkey == NULL) {
2539a7096ffSKonstantin Belousov 		dprintf("driver bug ifp %s if_sa_newkey NULL\n",
254ef2a572bSKonstantin Belousov 		    if_name(ifp));
255ef2a572bSKonstantin Belousov 		return (false);
256ef2a572bSKonstantin Belousov 	}
257ef2a572bSKonstantin Belousov 	return (true);
258ef2a572bSKonstantin Belousov }
259ef2a572bSKonstantin Belousov 
260ef2a572bSKonstantin Belousov static int
261ef2a572bSKonstantin Belousov ipsec_accel_sa_newkey_cb(if_t ifp, void *arg)
262ef2a572bSKonstantin Belousov {
263ef2a572bSKonstantin Belousov 	struct ipsec_accel_install_newkey_tq *tq;
264ef2a572bSKonstantin Belousov 	void *priv;
265ef2a572bSKonstantin Belousov 	u_int drv_spi;
266ef2a572bSKonstantin Belousov 	int error;
267ef2a572bSKonstantin Belousov 
268ef2a572bSKonstantin Belousov 	tq = arg;
269ef2a572bSKonstantin Belousov 
2709a7096ffSKonstantin Belousov 	dprintf("ipsec_accel_sa_newkey_act: ifp %s h %p spi %#x "
271ef2a572bSKonstantin Belousov 	    "flags %#x seq %d\n",
272ef2a572bSKonstantin Belousov 	    if_name(ifp), ifp->if_ipsec_accel_m->if_sa_newkey,
273ef2a572bSKonstantin Belousov 	    be32toh(tq->sav->spi), tq->sav->flags, tq->sav->seq);
274ef2a572bSKonstantin Belousov 	priv = NULL;
275ef2a572bSKonstantin Belousov 	drv_spi = alloc_unr(drv_spi_unr);
276ef2a572bSKonstantin Belousov 	if (tq->sav->accel_ifname != NULL &&
277ef2a572bSKonstantin Belousov 	    strcmp(tq->sav->accel_ifname, if_name(ifp)) != 0) {
278ef2a572bSKonstantin Belousov 		error = ipsec_accel_handle_sav(tq->sav,
279ef2a572bSKonstantin Belousov 		    ifp, drv_spi, priv, IFP_HS_REJECTED, NULL);
280ef2a572bSKonstantin Belousov 		goto out;
281ef2a572bSKonstantin Belousov 	}
282ef2a572bSKonstantin Belousov 	if (drv_spi == -1) {
283ef2a572bSKonstantin Belousov 		/* XXXKIB */
2849a7096ffSKonstantin Belousov 		dprintf("ipsec_accel_sa_install_newkey: cannot alloc "
285ef2a572bSKonstantin Belousov 		    "drv_spi if %s spi %#x\n", if_name(ifp),
286ef2a572bSKonstantin Belousov 		    be32toh(tq->sav->spi));
287ef2a572bSKonstantin Belousov 		return (ENOMEM);
288ef2a572bSKonstantin Belousov 	}
289ef2a572bSKonstantin Belousov 	error = ifp->if_ipsec_accel_m->if_sa_newkey(ifp, tq->sav,
290ef2a572bSKonstantin Belousov 	    drv_spi, &priv);
291ef2a572bSKonstantin Belousov 	if (error != 0) {
292ef2a572bSKonstantin Belousov 		if (error == EOPNOTSUPP) {
2939a7096ffSKonstantin Belousov 			dprintf("ipsec_accel_sa_newkey: driver "
294ef2a572bSKonstantin Belousov 			    "refused sa if %s spi %#x\n",
295ef2a572bSKonstantin Belousov 			    if_name(ifp), be32toh(tq->sav->spi));
296ef2a572bSKonstantin Belousov 			error = ipsec_accel_handle_sav(tq->sav,
297ef2a572bSKonstantin Belousov 			    ifp, drv_spi, priv, IFP_HS_REJECTED, NULL);
298ef2a572bSKonstantin Belousov 			/* XXXKIB */
299ef2a572bSKonstantin Belousov 		} else {
3009a7096ffSKonstantin Belousov 			dprintf("ipsec_accel_sa_newkey: driver "
301ef2a572bSKonstantin Belousov 			    "error %d if %s spi %#x\n",
302ef2a572bSKonstantin Belousov 			    error, if_name(ifp), be32toh(tq->sav->spi));
303ef2a572bSKonstantin Belousov 			/* XXXKIB */
304ef2a572bSKonstantin Belousov 		}
305ef2a572bSKonstantin Belousov 	} else {
306ef2a572bSKonstantin Belousov 		error = ipsec_accel_handle_sav(tq->sav, ifp,
307ef2a572bSKonstantin Belousov 		    drv_spi, priv, IFP_HS_HANDLED, NULL);
308ef2a572bSKonstantin Belousov 		if (error != 0) {
309ef2a572bSKonstantin Belousov 			/* XXXKIB */
3109a7096ffSKonstantin Belousov 			dprintf("ipsec_accel_sa_newkey: handle_sav "
311ef2a572bSKonstantin Belousov 			    "err %d if %s spi %#x\n", error,
312ef2a572bSKonstantin Belousov 			    if_name(ifp), be32toh(tq->sav->spi));
313ef2a572bSKonstantin Belousov 		}
314ef2a572bSKonstantin Belousov 	}
315ef2a572bSKonstantin Belousov out:
316ef2a572bSKonstantin Belousov 	return (error);
317ef2a572bSKonstantin Belousov }
318ef2a572bSKonstantin Belousov 
319ef2a572bSKonstantin Belousov static void
320ef2a572bSKonstantin Belousov ipsec_accel_sa_newkey_act(void *context, int pending)
321ef2a572bSKonstantin Belousov {
322ef2a572bSKonstantin Belousov 	struct ipsec_accel_install_newkey_tq *tq;
323ef2a572bSKonstantin Belousov 	void *tqf;
324ef2a572bSKonstantin Belousov 	struct secasvar *sav;
325ef2a572bSKonstantin Belousov 
326ef2a572bSKonstantin Belousov 	tq = context;
327ef2a572bSKonstantin Belousov 	tqf = NULL;
328ef2a572bSKonstantin Belousov 	sav = tq->sav;
329ef2a572bSKonstantin Belousov 	CURVNET_SET(tq->install_vnet);
330ef2a572bSKonstantin Belousov 	mtx_lock(&ipsec_accel_sav_tmp);
331ef2a572bSKonstantin Belousov 	if ((sav->accel_flags & (SADB_KEY_ACCEL_INST |
332ef2a572bSKonstantin Belousov 	    SADB_KEY_ACCEL_DEINST)) == 0 &&
333ef2a572bSKonstantin Belousov 	    sav->state == SADB_SASTATE_MATURE) {
334ef2a572bSKonstantin Belousov 		sav->accel_flags |= SADB_KEY_ACCEL_INST;
335ef2a572bSKonstantin Belousov 		mtx_unlock(&ipsec_accel_sav_tmp);
336ef2a572bSKonstantin Belousov 		if_foreach_sleep(ipsec_accel_sa_install_match, context,
337ef2a572bSKonstantin Belousov 		    ipsec_accel_sa_newkey_cb, context);
338ef2a572bSKonstantin Belousov 		ipsec_accel_alloc_forget_tq(sav);
339ef2a572bSKonstantin Belousov 		mtx_lock(&ipsec_accel_sav_tmp);
340ef2a572bSKonstantin Belousov 
341ef2a572bSKonstantin Belousov 		/*
342ef2a572bSKonstantin Belousov 		 * If ipsec_accel_forget_sav() raced with us and set
343ef2a572bSKonstantin Belousov 		 * the flag, do its work.  Its task cannot execute in
344ef2a572bSKonstantin Belousov 		 * parallel since taskqueue_thread is single-threaded.
345ef2a572bSKonstantin Belousov 		 */
346ef2a572bSKonstantin Belousov 		if ((sav->accel_flags & SADB_KEY_ACCEL_DEINST) != 0) {
347ef2a572bSKonstantin Belousov 			tqf = (void *)sav->accel_forget_tq;
348ef2a572bSKonstantin Belousov 			sav->accel_forget_tq = 0;
349ef2a572bSKonstantin Belousov 			ipsec_accel_forget_sav_clear(sav);
350ef2a572bSKonstantin Belousov 		}
351ef2a572bSKonstantin Belousov 	}
352ef2a572bSKonstantin Belousov 	mtx_unlock(&ipsec_accel_sav_tmp);
353ef2a572bSKonstantin Belousov 	key_freesav(&tq->sav);
354ef2a572bSKonstantin Belousov 	CURVNET_RESTORE();
355ef2a572bSKonstantin Belousov 	free(tq, M_TEMP);
356ef2a572bSKonstantin Belousov 	free(tqf, M_TEMP);
357ef2a572bSKonstantin Belousov }
358ef2a572bSKonstantin Belousov 
359ef2a572bSKonstantin Belousov static void
360ef2a572bSKonstantin Belousov ipsec_accel_sa_newkey_impl(struct secasvar *sav)
361ef2a572bSKonstantin Belousov {
362ef2a572bSKonstantin Belousov 	struct ipsec_accel_install_newkey_tq *tq;
363ef2a572bSKonstantin Belousov 
364ef2a572bSKonstantin Belousov 	if ((sav->accel_flags & (SADB_KEY_ACCEL_INST |
365ef2a572bSKonstantin Belousov 	    SADB_KEY_ACCEL_DEINST)) != 0)
366ef2a572bSKonstantin Belousov 		return;
367ef2a572bSKonstantin Belousov 
3689a7096ffSKonstantin Belousov 	dprintf(
369ef2a572bSKonstantin Belousov 	    "ipsec_accel_sa_install_newkey: spi %#x flags %#x seq %d\n",
370ef2a572bSKonstantin Belousov 	    be32toh(sav->spi), sav->flags, sav->seq);
371ef2a572bSKonstantin Belousov 
372ef2a572bSKonstantin Belousov 	tq = malloc(sizeof(*tq), M_TEMP, M_NOWAIT);
373ef2a572bSKonstantin Belousov 	if (tq == NULL) {
3749a7096ffSKonstantin Belousov 		dprintf("ipsec_accel_sa_install_newkey: no memory for tq, "
375ef2a572bSKonstantin Belousov 		    "spi %#x\n", be32toh(sav->spi));
376ef2a572bSKonstantin Belousov 		/* XXXKIB */
377ef2a572bSKonstantin Belousov 		return;
378ef2a572bSKonstantin Belousov 	}
379ef2a572bSKonstantin Belousov 
380ef2a572bSKonstantin Belousov 	refcount_acquire(&sav->refcnt);
381ef2a572bSKonstantin Belousov 
382ef2a572bSKonstantin Belousov 	TASK_INIT(&tq->install_task, 0, ipsec_accel_sa_newkey_act, tq);
383ef2a572bSKonstantin Belousov 	tq->sav = sav;
384ef2a572bSKonstantin Belousov 	tq->install_vnet = curthread->td_vnet;	/* XXXKIB liveness */
385ef2a572bSKonstantin Belousov 	taskqueue_enqueue(taskqueue_thread, &tq->install_task);
386ef2a572bSKonstantin Belousov }
387ef2a572bSKonstantin Belousov 
388ef2a572bSKonstantin Belousov static int
389ef2a572bSKonstantin Belousov ipsec_accel_handle_sav(struct secasvar *sav, struct ifnet *ifp,
390ef2a572bSKonstantin Belousov     u_int drv_spi, void *priv, uint32_t flags, struct ifp_handle_sav **ires)
391ef2a572bSKonstantin Belousov {
392ef2a572bSKonstantin Belousov 	struct ifp_handle_sav *ihs, *i;
393ef2a572bSKonstantin Belousov 	int error;
394ef2a572bSKonstantin Belousov 
395ef2a572bSKonstantin Belousov 	MPASS(__bitcount(flags & (IFP_HS_HANDLED | IFP_HS_REJECTED)) == 1);
396ef2a572bSKonstantin Belousov 
397ef2a572bSKonstantin Belousov 	ihs = malloc(sizeof(*ihs), M_IPSEC_MISC, M_WAITOK | M_ZERO);
398ef2a572bSKonstantin Belousov 	ihs->ifp = ifp;
399ef2a572bSKonstantin Belousov 	ihs->sav = sav;
400ef2a572bSKonstantin Belousov 	ihs->drv_spi = drv_spi;
401ef2a572bSKonstantin Belousov 	ihs->ifdata = priv;
402ef2a572bSKonstantin Belousov 	ihs->flags = flags;
403ef2a572bSKonstantin Belousov 	if ((flags & IFP_HS_OUTPUT) != 0)
404ef2a572bSKonstantin Belousov 		ihs->hdr_ext_size = esp_hdrsiz(sav);
405ef2a572bSKonstantin Belousov 	mtx_lock(&ipsec_accel_sav_tmp);
406ef2a572bSKonstantin Belousov 	CK_LIST_FOREACH(i, &sav->accel_ifps, sav_link) {
407ef2a572bSKonstantin Belousov 		if (i->ifp == ifp) {
408ef2a572bSKonstantin Belousov 			error = EALREADY;
409ef2a572bSKonstantin Belousov 			goto errout;
410ef2a572bSKonstantin Belousov 		}
411ef2a572bSKonstantin Belousov 	}
412ef2a572bSKonstantin Belousov 	error = DRVSPI_SA_PCTRIE_INSERT(&drv_spi_pctrie, ihs);
413ef2a572bSKonstantin Belousov 	if (error != 0)
414ef2a572bSKonstantin Belousov 		goto errout;
415ef2a572bSKonstantin Belousov 	if_ref(ihs->ifp);
416ef2a572bSKonstantin Belousov 	CK_LIST_INSERT_HEAD(&sav->accel_ifps, ihs, sav_link);
417ef2a572bSKonstantin Belousov 	CK_LIST_INSERT_HEAD(&ipsec_accel_all_sav_handles, ihs, sav_allh_link);
418ef2a572bSKonstantin Belousov 	mtx_unlock(&ipsec_accel_sav_tmp);
419ef2a572bSKonstantin Belousov 	if (ires != NULL)
420ef2a572bSKonstantin Belousov 		*ires = ihs;
421ef2a572bSKonstantin Belousov 	return (0);
422ef2a572bSKonstantin Belousov errout:
423ef2a572bSKonstantin Belousov 	mtx_unlock(&ipsec_accel_sav_tmp);
424ef2a572bSKonstantin Belousov 	free(ihs, M_IPSEC_MISC);
425ef2a572bSKonstantin Belousov 	if (ires != NULL)
426ef2a572bSKonstantin Belousov 		*ires = NULL;
427ef2a572bSKonstantin Belousov 	return (error);
428ef2a572bSKonstantin Belousov }
429ef2a572bSKonstantin Belousov 
430ef2a572bSKonstantin Belousov static void
431ef2a572bSKonstantin Belousov ipsec_accel_forget_handle_sav(struct ifp_handle_sav *i, bool freesav)
432ef2a572bSKonstantin Belousov {
433ef2a572bSKonstantin Belousov 	struct ifnet *ifp;
434ef2a572bSKonstantin Belousov 	struct secasvar *sav;
435ef2a572bSKonstantin Belousov 
436ef2a572bSKonstantin Belousov 	mtx_assert(&ipsec_accel_sav_tmp, MA_OWNED);
437ef2a572bSKonstantin Belousov 
438ef2a572bSKonstantin Belousov 	CK_LIST_REMOVE(i, sav_link);
439ef2a572bSKonstantin Belousov 	CK_LIST_REMOVE(i, sav_allh_link);
440ef2a572bSKonstantin Belousov 	DRVSPI_SA_PCTRIE_REMOVE(&drv_spi_pctrie, i->drv_spi);
441ef2a572bSKonstantin Belousov 	mtx_unlock(&ipsec_accel_sav_tmp);
442ef2a572bSKonstantin Belousov 	NET_EPOCH_WAIT();
443ef2a572bSKonstantin Belousov 	ifp = i->ifp;
444ef2a572bSKonstantin Belousov 	sav = i->sav;
445ef2a572bSKonstantin Belousov 	if ((i->flags & (IFP_HS_HANDLED | IFP_HS_REJECTED)) ==
446ef2a572bSKonstantin Belousov 	    IFP_HS_HANDLED) {
4479a7096ffSKonstantin Belousov 		dprintf("sa deinstall %s %p spi %#x ifl %#x\n",
448ef2a572bSKonstantin Belousov 		    if_name(ifp), sav, be32toh(sav->spi), i->flags);
449ef2a572bSKonstantin Belousov 		ifp->if_ipsec_accel_m->if_sa_deinstall(ifp,
450ef2a572bSKonstantin Belousov 		    i->drv_spi, i->ifdata);
451ef2a572bSKonstantin Belousov 	}
452ef2a572bSKonstantin Belousov 	if_rele(ifp);
453ef2a572bSKonstantin Belousov 	free_unr(drv_spi_unr, i->drv_spi);
454ef2a572bSKonstantin Belousov 	free(i, M_IPSEC_MISC);
455ef2a572bSKonstantin Belousov 	if (freesav)
456ef2a572bSKonstantin Belousov 		key_freesav(&sav);
457ef2a572bSKonstantin Belousov 	mtx_lock(&ipsec_accel_sav_tmp);
458ef2a572bSKonstantin Belousov }
459ef2a572bSKonstantin Belousov 
460ef2a572bSKonstantin Belousov static void
461ef2a572bSKonstantin Belousov ipsec_accel_forget_sav_clear(struct secasvar *sav)
462ef2a572bSKonstantin Belousov {
463ef2a572bSKonstantin Belousov 	struct ifp_handle_sav *i;
464ef2a572bSKonstantin Belousov 
465ef2a572bSKonstantin Belousov 	for (;;) {
466ef2a572bSKonstantin Belousov 		i = CK_LIST_FIRST(&sav->accel_ifps);
467ef2a572bSKonstantin Belousov 		if (i == NULL)
468ef2a572bSKonstantin Belousov 			break;
469ef2a572bSKonstantin Belousov 		ipsec_accel_forget_handle_sav(i, false);
470ef2a572bSKonstantin Belousov 	}
471ef2a572bSKonstantin Belousov }
472ef2a572bSKonstantin Belousov 
473ef2a572bSKonstantin Belousov static void
474ef2a572bSKonstantin Belousov ipsec_accel_forget_sav_act(void *arg, int pending)
475ef2a572bSKonstantin Belousov {
476ef2a572bSKonstantin Belousov 	struct ipsec_accel_forget_tq *tq;
477ef2a572bSKonstantin Belousov 	struct secasvar *sav;
478ef2a572bSKonstantin Belousov 
479ef2a572bSKonstantin Belousov 	tq = arg;
480ef2a572bSKonstantin Belousov 	sav = tq->sav;
481ef2a572bSKonstantin Belousov 	CURVNET_SET(tq->forget_vnet);
482ef2a572bSKonstantin Belousov 	mtx_lock(&ipsec_accel_sav_tmp);
483ef2a572bSKonstantin Belousov 	ipsec_accel_forget_sav_clear(sav);
484ef2a572bSKonstantin Belousov 	mtx_unlock(&ipsec_accel_sav_tmp);
485ef2a572bSKonstantin Belousov 	key_freesav(&sav);
486ef2a572bSKonstantin Belousov 	CURVNET_RESTORE();
487ef2a572bSKonstantin Belousov 	free(tq, M_TEMP);
488ef2a572bSKonstantin Belousov }
489ef2a572bSKonstantin Belousov 
490ef2a572bSKonstantin Belousov void
491ef2a572bSKonstantin Belousov ipsec_accel_forget_sav_impl(struct secasvar *sav)
492ef2a572bSKonstantin Belousov {
493ef2a572bSKonstantin Belousov 	struct ipsec_accel_forget_tq *tq;
494ef2a572bSKonstantin Belousov 
495ef2a572bSKonstantin Belousov 	mtx_lock(&ipsec_accel_sav_tmp);
496ef2a572bSKonstantin Belousov 	sav->accel_flags |= SADB_KEY_ACCEL_DEINST;
497ef2a572bSKonstantin Belousov 	tq = (void *)atomic_load_ptr(&sav->accel_forget_tq);
498ef2a572bSKonstantin Belousov 	if (tq == NULL || !atomic_cmpset_ptr(&sav->accel_forget_tq,
499ef2a572bSKonstantin Belousov 	    (uintptr_t)tq, 0)) {
500ef2a572bSKonstantin Belousov 		mtx_unlock(&ipsec_accel_sav_tmp);
501ef2a572bSKonstantin Belousov 		return;
502ef2a572bSKonstantin Belousov 	}
503ef2a572bSKonstantin Belousov 	mtx_unlock(&ipsec_accel_sav_tmp);
504ef2a572bSKonstantin Belousov 
505ef2a572bSKonstantin Belousov 	refcount_acquire(&sav->refcnt);
506ef2a572bSKonstantin Belousov 	TASK_INIT(&tq->forget_task, 0, ipsec_accel_forget_sav_act, tq);
507ef2a572bSKonstantin Belousov 	tq->forget_vnet = curthread->td_vnet;
508ef2a572bSKonstantin Belousov 	tq->sav = sav;
509ef2a572bSKonstantin Belousov 	taskqueue_enqueue(taskqueue_thread, &tq->forget_task);
510ef2a572bSKonstantin Belousov }
511ef2a572bSKonstantin Belousov 
512ef2a572bSKonstantin Belousov static void
513ef2a572bSKonstantin Belousov ipsec_accel_on_ifdown_sav(struct ifnet *ifp)
514ef2a572bSKonstantin Belousov {
515ef2a572bSKonstantin Belousov 	struct ifp_handle_sav *i, *marker;
516ef2a572bSKonstantin Belousov 
517ef2a572bSKonstantin Belousov 	marker = malloc(sizeof(*marker), M_IPSEC_MISC, M_WAITOK | M_ZERO);
518ef2a572bSKonstantin Belousov 	marker->flags = IFP_HS_MARKER;
519ef2a572bSKonstantin Belousov 
520ef2a572bSKonstantin Belousov 	mtx_lock(&ipsec_accel_sav_tmp);
521ef2a572bSKonstantin Belousov 	CK_LIST_INSERT_HEAD(&ipsec_accel_all_sav_handles, marker,
522ef2a572bSKonstantin Belousov 	    sav_allh_link);
523ef2a572bSKonstantin Belousov 	for (;;) {
524ef2a572bSKonstantin Belousov 		i = CK_LIST_NEXT(marker, sav_allh_link);
525ef2a572bSKonstantin Belousov 		if (i == NULL)
526ef2a572bSKonstantin Belousov 			break;
527ef2a572bSKonstantin Belousov 		CK_LIST_REMOVE(marker, sav_allh_link);
528ef2a572bSKonstantin Belousov 		CK_LIST_INSERT_AFTER(i, marker, sav_allh_link);
529ef2a572bSKonstantin Belousov 		if (i->ifp == ifp) {
530ef2a572bSKonstantin Belousov 			refcount_acquire(&i->sav->refcnt); /* XXXKIB wrap ? */
531ef2a572bSKonstantin Belousov 			ipsec_accel_forget_handle_sav(i, true);
532ef2a572bSKonstantin Belousov 		}
533ef2a572bSKonstantin Belousov 	}
534ef2a572bSKonstantin Belousov 	CK_LIST_REMOVE(marker, sav_allh_link);
535ef2a572bSKonstantin Belousov 	mtx_unlock(&ipsec_accel_sav_tmp);
536ef2a572bSKonstantin Belousov 	free(marker, M_IPSEC_MISC);
537ef2a572bSKonstantin Belousov }
538ef2a572bSKonstantin Belousov 
539ef2a572bSKonstantin Belousov static struct ifp_handle_sav *
540ef2a572bSKonstantin Belousov ipsec_accel_is_accel_sav_ptr_raw(struct secasvar *sav, struct ifnet *ifp)
541ef2a572bSKonstantin Belousov {
542ef2a572bSKonstantin Belousov 	struct ifp_handle_sav *i;
543ef2a572bSKonstantin Belousov 
544ef2a572bSKonstantin Belousov 	if ((ifp->if_capenable2 & IFCAP2_BIT(IFCAP2_IPSEC_OFFLOAD)) == 0)
545ef2a572bSKonstantin Belousov 		return (NULL);
546ef2a572bSKonstantin Belousov 	CK_LIST_FOREACH(i, &sav->accel_ifps, sav_link) {
547ef2a572bSKonstantin Belousov 		if (i->ifp == ifp)
548ef2a572bSKonstantin Belousov 			return (i);
549ef2a572bSKonstantin Belousov 	}
550ef2a572bSKonstantin Belousov 	return (NULL);
551ef2a572bSKonstantin Belousov }
552ef2a572bSKonstantin Belousov 
553ef2a572bSKonstantin Belousov static struct ifp_handle_sav *
554ef2a572bSKonstantin Belousov ipsec_accel_is_accel_sav_ptr(struct secasvar *sav, struct ifnet *ifp)
555ef2a572bSKonstantin Belousov {
556ef2a572bSKonstantin Belousov 	NET_EPOCH_ASSERT();
557ef2a572bSKonstantin Belousov 	return (ipsec_accel_is_accel_sav_ptr_raw(sav, ifp));
558ef2a572bSKonstantin Belousov }
559ef2a572bSKonstantin Belousov 
560ef2a572bSKonstantin Belousov static bool
561ef2a572bSKonstantin Belousov ipsec_accel_is_accel_sav_impl(struct secasvar *sav)
562ef2a572bSKonstantin Belousov {
563ef2a572bSKonstantin Belousov 	return (!CK_LIST_EMPTY(&sav->accel_ifps));
564ef2a572bSKonstantin Belousov }
565ef2a572bSKonstantin Belousov 
566ef2a572bSKonstantin Belousov static struct secasvar *
567ef2a572bSKonstantin Belousov ipsec_accel_drvspi_to_sa(u_int drv_spi)
568ef2a572bSKonstantin Belousov {
569ef2a572bSKonstantin Belousov 	struct ifp_handle_sav *i;
570ef2a572bSKonstantin Belousov 
571ef2a572bSKonstantin Belousov 	i = DRVSPI_SA_PCTRIE_LOOKUP(&drv_spi_pctrie, drv_spi);
572ef2a572bSKonstantin Belousov 	if (i == NULL)
573ef2a572bSKonstantin Belousov 		return (NULL);
574ef2a572bSKonstantin Belousov 	return (i->sav);
575ef2a572bSKonstantin Belousov }
576ef2a572bSKonstantin Belousov 
577ef2a572bSKonstantin Belousov static struct ifp_handle_sp *
578ef2a572bSKonstantin Belousov ipsec_accel_find_accel_sp(struct secpolicy *sp, if_t ifp)
579ef2a572bSKonstantin Belousov {
580ef2a572bSKonstantin Belousov 	struct ifp_handle_sp *i;
581ef2a572bSKonstantin Belousov 
582ef2a572bSKonstantin Belousov 	CK_LIST_FOREACH(i, &sp->accel_ifps, sp_link) {
583ef2a572bSKonstantin Belousov 		if (i->ifp == ifp)
584ef2a572bSKonstantin Belousov 			return (i);
585ef2a572bSKonstantin Belousov 	}
586ef2a572bSKonstantin Belousov 	return (NULL);
587ef2a572bSKonstantin Belousov }
588ef2a572bSKonstantin Belousov 
589ef2a572bSKonstantin Belousov static bool
590ef2a572bSKonstantin Belousov ipsec_accel_is_accel_sp(struct secpolicy *sp, if_t ifp)
591ef2a572bSKonstantin Belousov {
592ef2a572bSKonstantin Belousov 	return (ipsec_accel_find_accel_sp(sp, ifp) != NULL);
593ef2a572bSKonstantin Belousov }
594ef2a572bSKonstantin Belousov 
595ef2a572bSKonstantin Belousov static int
596ef2a572bSKonstantin Belousov ipsec_accel_remember_sp(struct secpolicy *sp, if_t ifp,
597ef2a572bSKonstantin Belousov     struct ifp_handle_sp **ip)
598ef2a572bSKonstantin Belousov {
599ef2a572bSKonstantin Belousov 	struct ifp_handle_sp *i;
600ef2a572bSKonstantin Belousov 
601ef2a572bSKonstantin Belousov 	i = malloc(sizeof(*i), M_IPSEC_MISC, M_WAITOK | M_ZERO);
602ef2a572bSKonstantin Belousov 	i->sp = sp;
603ef2a572bSKonstantin Belousov 	i->ifp = ifp;
604ef2a572bSKonstantin Belousov 	if_ref(ifp);
605ef2a572bSKonstantin Belousov 	i->flags = IFP_HP_HANDLED;
606ef2a572bSKonstantin Belousov 	mtx_lock(&ipsec_accel_sav_tmp);
607ef2a572bSKonstantin Belousov 	CK_LIST_INSERT_HEAD(&sp->accel_ifps, i, sp_link);
608ef2a572bSKonstantin Belousov 	CK_LIST_INSERT_HEAD(&ipsec_accel_all_sp_handles, i, sp_allh_link);
609ef2a572bSKonstantin Belousov 	mtx_unlock(&ipsec_accel_sav_tmp);
610ef2a572bSKonstantin Belousov 	*ip = i;
611ef2a572bSKonstantin Belousov 	return (0);
612ef2a572bSKonstantin Belousov }
613ef2a572bSKonstantin Belousov 
614ef2a572bSKonstantin Belousov static bool
615ef2a572bSKonstantin Belousov ipsec_accel_spdadd_match(if_t ifp, void *arg)
616ef2a572bSKonstantin Belousov {
617ef2a572bSKonstantin Belousov 	struct secpolicy *sp;
618ef2a572bSKonstantin Belousov 
619ef2a572bSKonstantin Belousov 	if ((ifp->if_capenable2 & IFCAP2_BIT(IFCAP2_IPSEC_OFFLOAD)) == 0 ||
620ef2a572bSKonstantin Belousov 	    ifp->if_ipsec_accel_m->if_spdadd == NULL)
621ef2a572bSKonstantin Belousov 		return (false);
622ef2a572bSKonstantin Belousov 	sp = arg;
623ef2a572bSKonstantin Belousov 	if (sp->accel_ifname != NULL &&
624ef2a572bSKonstantin Belousov 	    strcmp(sp->accel_ifname, if_name(ifp)) != 0)
625ef2a572bSKonstantin Belousov 		return (false);
626ef2a572bSKonstantin Belousov 	if (ipsec_accel_is_accel_sp(sp, ifp))
627ef2a572bSKonstantin Belousov 		return (false);
628ef2a572bSKonstantin Belousov 	return (true);
629ef2a572bSKonstantin Belousov }
630ef2a572bSKonstantin Belousov 
631ef2a572bSKonstantin Belousov static int
632ef2a572bSKonstantin Belousov ipsec_accel_spdadd_cb(if_t ifp, void *arg)
633ef2a572bSKonstantin Belousov {
634ef2a572bSKonstantin Belousov 	struct secpolicy *sp;
635ef2a572bSKonstantin Belousov 	struct inpcb *inp;
636ef2a572bSKonstantin Belousov 	struct ifp_handle_sp *i;
637ef2a572bSKonstantin Belousov 	int error;
638ef2a572bSKonstantin Belousov 
639ef2a572bSKonstantin Belousov 	sp = arg;
640ef2a572bSKonstantin Belousov 	inp = sp->ipsec_accel_add_sp_inp;
6419a7096ffSKonstantin Belousov 	dprintf("ipsec_accel_spdadd_cb: ifp %s m %p sp %p inp %p\n",
642ef2a572bSKonstantin Belousov 	    if_name(ifp), ifp->if_ipsec_accel_m->if_spdadd, sp, inp);
643ef2a572bSKonstantin Belousov 	error = ipsec_accel_remember_sp(sp, ifp, &i);
644ef2a572bSKonstantin Belousov 	if (error != 0) {
6459a7096ffSKonstantin Belousov 		dprintf("ipsec_accel_spdadd: %s if_spdadd %p remember res %d\n",
646ef2a572bSKonstantin Belousov 		    if_name(ifp), sp, error);
647ef2a572bSKonstantin Belousov 		return (error);
648ef2a572bSKonstantin Belousov 	}
649ef2a572bSKonstantin Belousov 	error = ifp->if_ipsec_accel_m->if_spdadd(ifp, sp, inp, &i->ifdata);
650ef2a572bSKonstantin Belousov 	if (error != 0) {
651ef2a572bSKonstantin Belousov 		i->flags |= IFP_HP_REJECTED;
6529a7096ffSKonstantin Belousov 		dprintf("ipsec_accel_spdadd: %s if_spdadd %p res %d\n",
653ef2a572bSKonstantin Belousov 		    if_name(ifp), sp, error);
654ef2a572bSKonstantin Belousov 	}
655ef2a572bSKonstantin Belousov 	return (error);
656ef2a572bSKonstantin Belousov }
657ef2a572bSKonstantin Belousov 
658ef2a572bSKonstantin Belousov static void
659ef2a572bSKonstantin Belousov ipsec_accel_spdadd_act(void *arg, int pending)
660ef2a572bSKonstantin Belousov {
661ef2a572bSKonstantin Belousov 	struct secpolicy *sp;
662ef2a572bSKonstantin Belousov 	struct inpcb *inp;
663ef2a572bSKonstantin Belousov 
664ef2a572bSKonstantin Belousov 	sp = arg;
665ef2a572bSKonstantin Belousov 	CURVNET_SET(sp->accel_add_tq.adddel_vnet);
666ef2a572bSKonstantin Belousov 	if_foreach_sleep(ipsec_accel_spdadd_match, arg,
667ef2a572bSKonstantin Belousov 	    ipsec_accel_spdadd_cb, arg);
668ef2a572bSKonstantin Belousov 	inp = sp->ipsec_accel_add_sp_inp;
669ef2a572bSKonstantin Belousov 	if (inp != NULL) {
670ef2a572bSKonstantin Belousov 		INP_WLOCK(inp);
671ef2a572bSKonstantin Belousov 		if (!in_pcbrele_wlocked(inp))
672ef2a572bSKonstantin Belousov 			INP_WUNLOCK(inp);
673ef2a572bSKonstantin Belousov 		sp->ipsec_accel_add_sp_inp = NULL;
674ef2a572bSKonstantin Belousov 	}
675ef2a572bSKonstantin Belousov 	CURVNET_RESTORE();
676ef2a572bSKonstantin Belousov 	key_freesp(&sp);
677ef2a572bSKonstantin Belousov }
678ef2a572bSKonstantin Belousov 
679ef2a572bSKonstantin Belousov void
680ef2a572bSKonstantin Belousov ipsec_accel_spdadd_impl(struct secpolicy *sp, struct inpcb *inp)
681ef2a572bSKonstantin Belousov {
682ef2a572bSKonstantin Belousov 	struct ipsec_accel_adddel_sp_tq *tq;
683ef2a572bSKonstantin Belousov 
684ef2a572bSKonstantin Belousov 	if (sp == NULL)
685ef2a572bSKonstantin Belousov 		return;
686ef2a572bSKonstantin Belousov 	if (sp->tcount == 0 && inp == NULL)
687ef2a572bSKonstantin Belousov 		return;
688ef2a572bSKonstantin Belousov 	tq = &sp->accel_add_tq;
689ef2a572bSKonstantin Belousov 	if (atomic_cmpset_int(&tq->adddel_scheduled, 0, 1) == 0)
690ef2a572bSKonstantin Belousov 		return;
691ef2a572bSKonstantin Belousov 	tq->adddel_vnet = curthread->td_vnet;
692ef2a572bSKonstantin Belousov 	sp->ipsec_accel_add_sp_inp = inp;
693ef2a572bSKonstantin Belousov 	if (inp != NULL)
694ef2a572bSKonstantin Belousov 		in_pcbref(inp);
695ef2a572bSKonstantin Belousov 	TASK_INIT(&tq->adddel_task, 0, ipsec_accel_spdadd_act, sp);
696ef2a572bSKonstantin Belousov 	key_addref(sp);
697ef2a572bSKonstantin Belousov 	taskqueue_enqueue(taskqueue_thread, &tq->adddel_task);
698ef2a572bSKonstantin Belousov }
699ef2a572bSKonstantin Belousov 
700ef2a572bSKonstantin Belousov static void
701ef2a572bSKonstantin Belousov ipsec_accel_spddel_act(void *arg, int pending)
702ef2a572bSKonstantin Belousov {
703ef2a572bSKonstantin Belousov 	struct ifp_handle_sp *i;
704ef2a572bSKonstantin Belousov 	struct secpolicy *sp;
705ef2a572bSKonstantin Belousov 	int error;
706ef2a572bSKonstantin Belousov 
707ef2a572bSKonstantin Belousov 	sp = arg;
708ef2a572bSKonstantin Belousov 	CURVNET_SET(sp->accel_del_tq.adddel_vnet);
709ef2a572bSKonstantin Belousov 	mtx_lock(&ipsec_accel_sav_tmp);
710ef2a572bSKonstantin Belousov 	for (;;) {
711ef2a572bSKonstantin Belousov 		i = CK_LIST_FIRST(&sp->accel_ifps);
712ef2a572bSKonstantin Belousov 		if (i == NULL)
713ef2a572bSKonstantin Belousov 			break;
714ef2a572bSKonstantin Belousov 		CK_LIST_REMOVE(i, sp_link);
715ef2a572bSKonstantin Belousov 		CK_LIST_REMOVE(i, sp_allh_link);
716ef2a572bSKonstantin Belousov 		mtx_unlock(&ipsec_accel_sav_tmp);
717ef2a572bSKonstantin Belousov 		NET_EPOCH_WAIT();
718ef2a572bSKonstantin Belousov 		if ((i->flags & (IFP_HP_HANDLED | IFP_HP_REJECTED)) ==
719ef2a572bSKonstantin Belousov 		    IFP_HP_HANDLED) {
7209a7096ffSKonstantin Belousov 			dprintf("spd deinstall %s %p\n", if_name(i->ifp), sp);
721ef2a572bSKonstantin Belousov 			error = i->ifp->if_ipsec_accel_m->if_spddel(i->ifp,
722ef2a572bSKonstantin Belousov 			    sp, i->ifdata);
723ef2a572bSKonstantin Belousov 			if (error != 0) {
7249a7096ffSKonstantin Belousov 				dprintf(
725ef2a572bSKonstantin Belousov 		    "ipsec_accel_spddel: %s if_spddel %p res %d\n",
726ef2a572bSKonstantin Belousov 				    if_name(i->ifp), sp, error);
727ef2a572bSKonstantin Belousov 			}
728ef2a572bSKonstantin Belousov 		}
729ef2a572bSKonstantin Belousov 		if_rele(i->ifp);
730ef2a572bSKonstantin Belousov 		free(i, M_IPSEC_MISC);
731ef2a572bSKonstantin Belousov 		mtx_lock(&ipsec_accel_sav_tmp);
732ef2a572bSKonstantin Belousov 	}
733ef2a572bSKonstantin Belousov 	mtx_unlock(&ipsec_accel_sav_tmp);
734ef2a572bSKonstantin Belousov 	key_freesp(&sp);
735ef2a572bSKonstantin Belousov 	CURVNET_RESTORE();
736ef2a572bSKonstantin Belousov }
737ef2a572bSKonstantin Belousov 
738ef2a572bSKonstantin Belousov void
739ef2a572bSKonstantin Belousov ipsec_accel_spddel_impl(struct secpolicy *sp)
740ef2a572bSKonstantin Belousov {
741ef2a572bSKonstantin Belousov 	struct ipsec_accel_adddel_sp_tq *tq;
742ef2a572bSKonstantin Belousov 
743ef2a572bSKonstantin Belousov 	if (sp == NULL)
744ef2a572bSKonstantin Belousov 		return;
745ef2a572bSKonstantin Belousov 
746ef2a572bSKonstantin Belousov 	tq = &sp->accel_del_tq;
747ef2a572bSKonstantin Belousov 	if (atomic_cmpset_int(&tq->adddel_scheduled, 0, 1) == 0)
748ef2a572bSKonstantin Belousov 		return;
749ef2a572bSKonstantin Belousov 	tq->adddel_vnet = curthread->td_vnet;
750ef2a572bSKonstantin Belousov 	TASK_INIT(&tq->adddel_task, 0, ipsec_accel_spddel_act, sp);
751ef2a572bSKonstantin Belousov 	key_addref(sp);
752ef2a572bSKonstantin Belousov 	taskqueue_enqueue(taskqueue_thread, &tq->adddel_task);
753ef2a572bSKonstantin Belousov }
754ef2a572bSKonstantin Belousov 
755ef2a572bSKonstantin Belousov static void
756ef2a572bSKonstantin Belousov ipsec_accel_on_ifdown_sp(struct ifnet *ifp)
757ef2a572bSKonstantin Belousov {
758ef2a572bSKonstantin Belousov 	struct ifp_handle_sp *i, *marker;
759ef2a572bSKonstantin Belousov 	struct secpolicy *sp;
760ef2a572bSKonstantin Belousov 	int error;
761ef2a572bSKonstantin Belousov 
762ef2a572bSKonstantin Belousov 	marker = malloc(sizeof(*marker), M_IPSEC_MISC, M_WAITOK | M_ZERO);
763ef2a572bSKonstantin Belousov 	marker->flags = IFP_HS_MARKER;
764ef2a572bSKonstantin Belousov 
765ef2a572bSKonstantin Belousov 	mtx_lock(&ipsec_accel_sav_tmp);
766ef2a572bSKonstantin Belousov 	CK_LIST_INSERT_HEAD(&ipsec_accel_all_sp_handles, marker,
767ef2a572bSKonstantin Belousov 	    sp_allh_link);
768ef2a572bSKonstantin Belousov 	for (;;) {
769ef2a572bSKonstantin Belousov 		i = CK_LIST_NEXT(marker, sp_allh_link);
770ef2a572bSKonstantin Belousov 		if (i == NULL)
771ef2a572bSKonstantin Belousov 			break;
772ef2a572bSKonstantin Belousov 		CK_LIST_REMOVE(marker, sp_allh_link);
773ef2a572bSKonstantin Belousov 		CK_LIST_INSERT_AFTER(i, marker, sp_allh_link);
774ef2a572bSKonstantin Belousov 		if (i->ifp != ifp)
775ef2a572bSKonstantin Belousov 			continue;
776ef2a572bSKonstantin Belousov 
777ef2a572bSKonstantin Belousov 		sp = i->sp;
778ef2a572bSKonstantin Belousov 		key_addref(sp);
779ef2a572bSKonstantin Belousov 		CK_LIST_REMOVE(i, sp_link);
780ef2a572bSKonstantin Belousov 		CK_LIST_REMOVE(i, sp_allh_link);
781ef2a572bSKonstantin Belousov 		mtx_unlock(&ipsec_accel_sav_tmp);
782ef2a572bSKonstantin Belousov 		NET_EPOCH_WAIT();
783ef2a572bSKonstantin Belousov 		if ((i->flags & (IFP_HP_HANDLED | IFP_HP_REJECTED)) ==
784ef2a572bSKonstantin Belousov 		    IFP_HP_HANDLED) {
7859a7096ffSKonstantin Belousov 			dprintf("spd deinstall %s %p\n", if_name(ifp), sp);
786ef2a572bSKonstantin Belousov 			error = ifp->if_ipsec_accel_m->if_spddel(ifp,
787ef2a572bSKonstantin Belousov 			    sp, i->ifdata);
788ef2a572bSKonstantin Belousov 		}
789ef2a572bSKonstantin Belousov 		if (error != 0) {
7909a7096ffSKonstantin Belousov 			dprintf(
791ef2a572bSKonstantin Belousov 		    "ipsec_accel_on_ifdown_sp: %s if_spddel %p res %d\n",
792ef2a572bSKonstantin Belousov 			    if_name(ifp), sp, error);
793ef2a572bSKonstantin Belousov 		}
794ef2a572bSKonstantin Belousov 		key_freesp(&sp);
795ef2a572bSKonstantin Belousov 		if_rele(ifp);
796ef2a572bSKonstantin Belousov 		free(i, M_IPSEC_MISC);
797ef2a572bSKonstantin Belousov 		mtx_lock(&ipsec_accel_sav_tmp);
798ef2a572bSKonstantin Belousov 	}
799ef2a572bSKonstantin Belousov 	CK_LIST_REMOVE(marker, sp_allh_link);
800ef2a572bSKonstantin Belousov 	mtx_unlock(&ipsec_accel_sav_tmp);
801ef2a572bSKonstantin Belousov 	free(marker, M_IPSEC_MISC);
802ef2a572bSKonstantin Belousov }
803ef2a572bSKonstantin Belousov 
80465f264dcSKonstantin Belousov static void
80565f264dcSKonstantin Belousov ipsec_accel_on_ifdown_impl(struct ifnet *ifp)
806ef2a572bSKonstantin Belousov {
807ef2a572bSKonstantin Belousov 	ipsec_accel_on_ifdown_sp(ifp);
808ef2a572bSKonstantin Belousov 	ipsec_accel_on_ifdown_sav(ifp);
809ef2a572bSKonstantin Belousov }
810ef2a572bSKonstantin Belousov 
811*c4a0ee9bSKonstantin Belousov static void
812*c4a0ee9bSKonstantin Belousov ipsec_accel_ifdetach_event(void *arg __unused, struct ifnet *ifp)
813*c4a0ee9bSKonstantin Belousov {
814*c4a0ee9bSKonstantin Belousov 	if ((ifp->if_flags & IFF_RENAMING) != 0)
815*c4a0ee9bSKonstantin Belousov 		return;
816*c4a0ee9bSKonstantin Belousov 	ipsec_accel_on_ifdown_impl(ifp);
817*c4a0ee9bSKonstantin Belousov }
818*c4a0ee9bSKonstantin Belousov 
819ef2a572bSKonstantin Belousov static bool
820ef2a572bSKonstantin Belousov ipsec_accel_output_pad(struct mbuf *m, struct secasvar *sav, int skip, int mtu)
821ef2a572bSKonstantin Belousov {
822ef2a572bSKonstantin Belousov 	int alen, blks, hlen, padding, rlen;
823ef2a572bSKonstantin Belousov 
824ef2a572bSKonstantin Belousov 	rlen = m->m_pkthdr.len - skip;
825ef2a572bSKonstantin Belousov 	hlen = ((sav->flags & SADB_X_EXT_OLD) != 0 ? sizeof(struct esp) :
826ef2a572bSKonstantin Belousov 	    sizeof(struct newesp)) + sav->ivlen;
827ef2a572bSKonstantin Belousov 	blks = MAX(4, SAV_ISCTR(sav) && VNET(esp_ctr_compatibility) ?
828ef2a572bSKonstantin Belousov 	    sav->tdb_encalgxform->native_blocksize :
829ef2a572bSKonstantin Belousov 	    sav->tdb_encalgxform->blocksize);
830ef2a572bSKonstantin Belousov 	padding = ((blks - ((rlen + 2) % blks)) % blks) + 2;
831ef2a572bSKonstantin Belousov 	alen = xform_ah_authsize(sav->tdb_authalgxform);
832ef2a572bSKonstantin Belousov 
833ef2a572bSKonstantin Belousov 	return (skip + hlen + rlen + padding + alen <= mtu);
834ef2a572bSKonstantin Belousov }
835ef2a572bSKonstantin Belousov 
836ef2a572bSKonstantin Belousov static bool
837ef2a572bSKonstantin Belousov ipsec_accel_output_tag(struct mbuf *m, u_int drv_spi)
838ef2a572bSKonstantin Belousov {
839ef2a572bSKonstantin Belousov 	struct ipsec_accel_out_tag *tag;
840ef2a572bSKonstantin Belousov 
841ef2a572bSKonstantin Belousov 	tag = (struct ipsec_accel_out_tag *)m_tag_get(
842ef2a572bSKonstantin Belousov 	    PACKET_TAG_IPSEC_ACCEL_OUT, sizeof(*tag), M_NOWAIT);
843ef2a572bSKonstantin Belousov 	if (tag == NULL)
844ef2a572bSKonstantin Belousov 		return (false);
845ef2a572bSKonstantin Belousov 	tag->drv_spi = drv_spi;
846ef2a572bSKonstantin Belousov 	m_tag_prepend(m, &tag->tag);
847ef2a572bSKonstantin Belousov 	return (true);
848ef2a572bSKonstantin Belousov }
849ef2a572bSKonstantin Belousov 
850ef2a572bSKonstantin Belousov bool
851ef2a572bSKonstantin Belousov ipsec_accel_output(struct ifnet *ifp, struct mbuf *m, struct inpcb *inp,
852240b7bfeSKonstantin Belousov     struct secpolicy *sp, struct secasvar *sav, int af, int mtu, int *hwassist)
853ef2a572bSKonstantin Belousov {
854ef2a572bSKonstantin Belousov 	struct ifp_handle_sav *i;
855ef2a572bSKonstantin Belousov 	struct ip *ip;
856e6e2c0a5SKonstantin Belousov 	struct tcpcb *tp;
857ef2a572bSKonstantin Belousov 	u_long ip_len, skip;
858e6e2c0a5SKonstantin Belousov 	bool res;
859ef2a572bSKonstantin Belousov 
860240b7bfeSKonstantin Belousov 	*hwassist = 0;
861e6e2c0a5SKonstantin Belousov 	res = false;
862ef2a572bSKonstantin Belousov 	if (ifp == NULL)
863e6e2c0a5SKonstantin Belousov 		return (res);
864ef2a572bSKonstantin Belousov 
865ef2a572bSKonstantin Belousov 	M_ASSERTPKTHDR(m);
866ef2a572bSKonstantin Belousov 	NET_EPOCH_ASSERT();
867ef2a572bSKonstantin Belousov 
868e6e2c0a5SKonstantin Belousov 	if (sav == NULL) {
869e6e2c0a5SKonstantin Belousov 		res = ipsec_accel_output_tag(m, IPSEC_ACCEL_DRV_SPI_BYPASS);
870e6e2c0a5SKonstantin Belousov 		goto out;
871e6e2c0a5SKonstantin Belousov 	}
872ef2a572bSKonstantin Belousov 
873ef2a572bSKonstantin Belousov 	i = ipsec_accel_is_accel_sav_ptr(sav, ifp);
874ef2a572bSKonstantin Belousov 	if (i == NULL)
875e6e2c0a5SKonstantin Belousov 		goto out;
876ef2a572bSKonstantin Belousov 
877ef2a572bSKonstantin Belousov 	if ((m->m_pkthdr.csum_flags & CSUM_TSO) == 0) {
878ef2a572bSKonstantin Belousov 		ip_len = m->m_pkthdr.len;
879ef2a572bSKonstantin Belousov 		if (ip_len + i->hdr_ext_size > mtu)
880e6e2c0a5SKonstantin Belousov 			goto out;
881ef2a572bSKonstantin Belousov 		switch (af) {
882ef2a572bSKonstantin Belousov 		case AF_INET:
883ef2a572bSKonstantin Belousov 			ip = mtod(m, struct ip *);
884ef2a572bSKonstantin Belousov 			skip = ip->ip_hl << 2;
885ef2a572bSKonstantin Belousov 			break;
886ef2a572bSKonstantin Belousov 		case AF_INET6:
887ef2a572bSKonstantin Belousov 			skip = sizeof(struct ip6_hdr);
888ef2a572bSKonstantin Belousov 			break;
889ef2a572bSKonstantin Belousov 		default:
890ef2a572bSKonstantin Belousov 			__unreachable();
891ef2a572bSKonstantin Belousov 		}
892ef2a572bSKonstantin Belousov 		if (!ipsec_accel_output_pad(m, sav, skip, mtu))
893e6e2c0a5SKonstantin Belousov 			goto out;
894ef2a572bSKonstantin Belousov 	}
895ef2a572bSKonstantin Belousov 
896ef2a572bSKonstantin Belousov 	if (!ipsec_accel_output_tag(m, i->drv_spi))
897e6e2c0a5SKonstantin Belousov 		goto out;
898ef2a572bSKonstantin Belousov 
899ef2a572bSKonstantin Belousov 	ipsec_accel_sa_recordxfer(sav, m);
900ef2a572bSKonstantin Belousov 	key_freesav(&sav);
901ef2a572bSKonstantin Belousov 	if (sp != NULL)
902ef2a572bSKonstantin Belousov 		key_freesp(&sp);
903ef2a572bSKonstantin Belousov 
904240b7bfeSKonstantin Belousov 	*hwassist = ifp->if_ipsec_accel_m->if_hwassist(ifp, sav,
905240b7bfeSKonstantin Belousov 	    i->drv_spi, i->ifdata);
906e6e2c0a5SKonstantin Belousov 	res = true;
907e6e2c0a5SKonstantin Belousov out:
908e6e2c0a5SKonstantin Belousov 	if (inp != NULL && inp->inp_pcbinfo == &V_tcbinfo) {
909e6e2c0a5SKonstantin Belousov 		INP_WLOCK_ASSERT(inp);
910e6e2c0a5SKonstantin Belousov 		tp = (struct tcpcb *)inp;
911e6e2c0a5SKonstantin Belousov 		if (res && (*hwassist & (CSUM_TSO | CSUM_IP6_TSO)) != 0) {
912e6e2c0a5SKonstantin Belousov 			tp->t_flags2 |= TF2_IPSEC_TSO;
913e6e2c0a5SKonstantin Belousov 		} else {
914e6e2c0a5SKonstantin Belousov 			tp->t_flags2 &= ~TF2_IPSEC_TSO;
915e6e2c0a5SKonstantin Belousov 		}
916e6e2c0a5SKonstantin Belousov 	}
917e6e2c0a5SKonstantin Belousov 	return (res);
918ef2a572bSKonstantin Belousov }
919ef2a572bSKonstantin Belousov 
920ef2a572bSKonstantin Belousov struct ipsec_accel_in_tag *
921ef2a572bSKonstantin Belousov ipsec_accel_input_tag_lookup(const struct mbuf *m)
922ef2a572bSKonstantin Belousov {
923ef2a572bSKonstantin Belousov 	struct ipsec_accel_in_tag *tag;
924ef2a572bSKonstantin Belousov 	struct m_tag *xtag;
925ef2a572bSKonstantin Belousov 
926ef2a572bSKonstantin Belousov 	xtag = m_tag_find(__DECONST(struct mbuf *, m),
927ef2a572bSKonstantin Belousov 	    PACKET_TAG_IPSEC_ACCEL_IN, NULL);
928ef2a572bSKonstantin Belousov 	if (xtag == NULL)
929ef2a572bSKonstantin Belousov 		return (NULL);
930ef2a572bSKonstantin Belousov 	tag = __containerof(xtag, struct ipsec_accel_in_tag, tag);
931ef2a572bSKonstantin Belousov 	return (tag);
932ef2a572bSKonstantin Belousov }
933ef2a572bSKonstantin Belousov 
934ef2a572bSKonstantin Belousov int
935ef2a572bSKonstantin Belousov ipsec_accel_input(struct mbuf *m, int offset, int proto)
936ef2a572bSKonstantin Belousov {
937ef2a572bSKonstantin Belousov 	struct secasvar *sav;
938ef2a572bSKonstantin Belousov 	struct ipsec_accel_in_tag *tag;
939ef2a572bSKonstantin Belousov 
940ef2a572bSKonstantin Belousov 	tag = ipsec_accel_input_tag_lookup(m);
941ef2a572bSKonstantin Belousov 	if (tag == NULL)
942ef2a572bSKonstantin Belousov 		return (ENXIO);
943ef2a572bSKonstantin Belousov 
944ef2a572bSKonstantin Belousov 	if (tag->drv_spi < IPSEC_ACCEL_DRV_SPI_MIN ||
945ef2a572bSKonstantin Belousov 	    tag->drv_spi > IPSEC_ACCEL_DRV_SPI_MAX) {
9469a7096ffSKonstantin Belousov 		dprintf("if %s mbuf %p drv_spi %d invalid, packet dropped\n",
947ef2a572bSKonstantin Belousov 		    (m->m_flags & M_PKTHDR) != 0 ? if_name(m->m_pkthdr.rcvif) :
948ef2a572bSKonstantin Belousov 		    "<unknwn>", m, tag->drv_spi);
949ef2a572bSKonstantin Belousov 		m_freem(m);
950ef2a572bSKonstantin Belousov 		return (EINPROGRESS);
951ef2a572bSKonstantin Belousov 	}
952ef2a572bSKonstantin Belousov 
953ef2a572bSKonstantin Belousov 	sav = ipsec_accel_drvspi_to_sa(tag->drv_spi);
954ef2a572bSKonstantin Belousov 	if (sav != NULL)
955ef2a572bSKonstantin Belousov 		ipsec_accel_sa_recordxfer(sav, m);
956ef2a572bSKonstantin Belousov 	return (0);
957ef2a572bSKonstantin Belousov }
958ef2a572bSKonstantin Belousov 
959ef2a572bSKonstantin Belousov static void
960ef2a572bSKonstantin Belousov ipsec_accel_sa_recordxfer(struct secasvar *sav, struct mbuf *m)
961ef2a572bSKonstantin Belousov {
962ef2a572bSKonstantin Belousov 	counter_u64_add(sav->accel_lft_sw, 1);
963ef2a572bSKonstantin Belousov 	counter_u64_add(sav->accel_lft_sw + 1, m->m_pkthdr.len);
964ef2a572bSKonstantin Belousov 	if (sav->accel_firstused == 0)
965ef2a572bSKonstantin Belousov 		sav->accel_firstused = time_second;
966ef2a572bSKonstantin Belousov }
967ef2a572bSKonstantin Belousov 
968ef2a572bSKonstantin Belousov static void
969ef2a572bSKonstantin Belousov ipsec_accel_sa_lifetime_update(struct seclifetime *lft_c,
970ef2a572bSKonstantin Belousov     const struct seclifetime *lft_l)
971ef2a572bSKonstantin Belousov {
972ef2a572bSKonstantin Belousov 	lft_c->allocations += lft_l->allocations;
973ef2a572bSKonstantin Belousov 	lft_c->bytes += lft_l->bytes;
974ef2a572bSKonstantin Belousov 	lft_c->usetime = min(lft_c->usetime, lft_l->usetime);
975ef2a572bSKonstantin Belousov }
976ef2a572bSKonstantin Belousov 
97765f264dcSKonstantin Belousov static void
97865f264dcSKonstantin Belousov ipsec_accel_drv_sa_lifetime_update_impl(struct secasvar *sav, if_t ifp,
979ef2a572bSKonstantin Belousov     u_int drv_spi, uint64_t octets, uint64_t allocs)
980ef2a572bSKonstantin Belousov {
981ef2a572bSKonstantin Belousov 	struct epoch_tracker et;
982ef2a572bSKonstantin Belousov 	struct ifp_handle_sav *i;
983ef2a572bSKonstantin Belousov 	uint64_t odiff, adiff;
984ef2a572bSKonstantin Belousov 
985ef2a572bSKonstantin Belousov 	NET_EPOCH_ENTER(et);
986ef2a572bSKonstantin Belousov 	mtx_lock(&ipsec_accel_cnt_lock);
987ef2a572bSKonstantin Belousov 
988ef2a572bSKonstantin Belousov 	if (allocs != 0) {
989ef2a572bSKonstantin Belousov 		if (sav->firstused == 0)
990ef2a572bSKonstantin Belousov 			sav->firstused = time_second;
991ef2a572bSKonstantin Belousov 		if (sav->accel_firstused == 0)
992ef2a572bSKonstantin Belousov 			sav->accel_firstused = time_second;
993ef2a572bSKonstantin Belousov 	}
994ef2a572bSKonstantin Belousov 
995ef2a572bSKonstantin Belousov 	CK_LIST_FOREACH(i, &sav->accel_ifps, sav_link) {
996ef2a572bSKonstantin Belousov 		if (i->ifp == ifp && i->drv_spi == drv_spi)
997ef2a572bSKonstantin Belousov 			break;
998ef2a572bSKonstantin Belousov 	}
999ef2a572bSKonstantin Belousov 	if (i == NULL)
1000ef2a572bSKonstantin Belousov 		goto out;
1001ef2a572bSKonstantin Belousov 
1002ef2a572bSKonstantin Belousov 	odiff = octets - i->cnt_octets;
1003ef2a572bSKonstantin Belousov 	adiff = allocs - i->cnt_allocs;
1004ef2a572bSKonstantin Belousov 
1005ef2a572bSKonstantin Belousov 	if (sav->lft_c != NULL) {
1006ef2a572bSKonstantin Belousov 		counter_u64_add(sav->lft_c_bytes, odiff);
1007ef2a572bSKonstantin Belousov 		counter_u64_add(sav->lft_c_allocations, adiff);
1008ef2a572bSKonstantin Belousov 	}
1009ef2a572bSKonstantin Belousov 
1010ef2a572bSKonstantin Belousov 	i->cnt_octets = octets;
1011ef2a572bSKonstantin Belousov 	i->cnt_allocs = allocs;
1012ef2a572bSKonstantin Belousov 	sav->accel_hw_octets += odiff;
1013ef2a572bSKonstantin Belousov 	sav->accel_hw_allocs += adiff;
1014ef2a572bSKonstantin Belousov 
1015ef2a572bSKonstantin Belousov out:
1016ef2a572bSKonstantin Belousov 	mtx_unlock(&ipsec_accel_cnt_lock);
1017ef2a572bSKonstantin Belousov 	NET_EPOCH_EXIT(et);
1018ef2a572bSKonstantin Belousov }
1019ef2a572bSKonstantin Belousov 
1020ef2a572bSKonstantin Belousov static void
1021ef2a572bSKonstantin Belousov ipsec_accel_sa_lifetime_hw(struct secasvar *sav, if_t ifp,
1022ef2a572bSKonstantin Belousov     struct seclifetime *lft)
1023ef2a572bSKonstantin Belousov {
1024ef2a572bSKonstantin Belousov 	struct ifp_handle_sav *i;
1025ef2a572bSKonstantin Belousov 	if_sa_cnt_fn_t p;
1026ef2a572bSKonstantin Belousov 
1027ef2a572bSKonstantin Belousov 	IFNET_RLOCK_ASSERT();
1028ef2a572bSKonstantin Belousov 
1029ef2a572bSKonstantin Belousov 	i = ipsec_accel_is_accel_sav_ptr(sav, ifp);
1030ef2a572bSKonstantin Belousov 	if (i != NULL && (i->flags & (IFP_HS_HANDLED | IFP_HS_REJECTED)) ==
1031ef2a572bSKonstantin Belousov 	    IFP_HS_HANDLED) {
1032ef2a572bSKonstantin Belousov 		p = ifp->if_ipsec_accel_m->if_sa_cnt;
1033ef2a572bSKonstantin Belousov 		if (p != NULL)
1034ef2a572bSKonstantin Belousov 			p(ifp, sav, i->drv_spi, i->ifdata, lft);
1035ef2a572bSKonstantin Belousov 	}
1036ef2a572bSKonstantin Belousov }
1037ef2a572bSKonstantin Belousov 
1038ef2a572bSKonstantin Belousov static int
1039ef2a572bSKonstantin Belousov ipsec_accel_sa_lifetime_op_impl(struct secasvar *sav,
1040ef2a572bSKonstantin Belousov     struct seclifetime *lft_c, if_t ifp, enum IF_SA_CNT_WHICH op,
1041ef2a572bSKonstantin Belousov     struct rm_priotracker *sahtree_trackerp)
1042ef2a572bSKonstantin Belousov {
1043ef2a572bSKonstantin Belousov 	struct seclifetime lft_l, lft_s;
1044ef2a572bSKonstantin Belousov 	struct ifp_handle_sav *i;
1045ef2a572bSKonstantin Belousov 	if_t ifp1;
1046ef2a572bSKonstantin Belousov 	if_sa_cnt_fn_t p;
1047ef2a572bSKonstantin Belousov 	int error;
1048ef2a572bSKonstantin Belousov 
1049ef2a572bSKonstantin Belousov 	error = 0;
1050ef2a572bSKonstantin Belousov 	memset(&lft_l, 0, sizeof(lft_l));
1051ef2a572bSKonstantin Belousov 	memset(&lft_s, 0, sizeof(lft_s));
1052ef2a572bSKonstantin Belousov 
1053ef2a572bSKonstantin Belousov 	switch (op & ~IF_SA_CNT_UPD) {
1054ef2a572bSKonstantin Belousov 	case IF_SA_CNT_IFP_HW_VAL:
1055ef2a572bSKonstantin Belousov 		ipsec_accel_sa_lifetime_hw(sav, ifp, &lft_l);
1056ef2a572bSKonstantin Belousov 		ipsec_accel_sa_lifetime_update(&lft_l, &lft_s);
1057ef2a572bSKonstantin Belousov 		break;
1058ef2a572bSKonstantin Belousov 
1059ef2a572bSKonstantin Belousov 	case IF_SA_CNT_TOTAL_SW_VAL:
1060ef2a572bSKonstantin Belousov 		lft_l.allocations = (uint32_t)counter_u64_fetch(
1061ef2a572bSKonstantin Belousov 		    sav->accel_lft_sw);
1062ef2a572bSKonstantin Belousov 		lft_l.bytes = counter_u64_fetch(sav->accel_lft_sw + 1);
1063ef2a572bSKonstantin Belousov 		lft_l.usetime = sav->accel_firstused;
1064ef2a572bSKonstantin Belousov 		break;
1065ef2a572bSKonstantin Belousov 
1066ef2a572bSKonstantin Belousov 	case IF_SA_CNT_TOTAL_HW_VAL:
1067ef2a572bSKonstantin Belousov 		IFNET_RLOCK_ASSERT();
1068ef2a572bSKonstantin Belousov 		CK_LIST_FOREACH(i, &sav->accel_ifps, sav_link) {
1069ef2a572bSKonstantin Belousov 			if ((i->flags & (IFP_HS_HANDLED | IFP_HS_REJECTED)) !=
1070ef2a572bSKonstantin Belousov 			    IFP_HS_HANDLED)
1071ef2a572bSKonstantin Belousov 				continue;
1072ef2a572bSKonstantin Belousov 			ifp1 = i->ifp;
1073ef2a572bSKonstantin Belousov 			p = ifp1->if_ipsec_accel_m->if_sa_cnt;
1074ef2a572bSKonstantin Belousov 			if (p == NULL)
1075ef2a572bSKonstantin Belousov 				continue;
1076ef2a572bSKonstantin Belousov 			memset(&lft_s, 0, sizeof(lft_s));
1077ef2a572bSKonstantin Belousov 			if (sahtree_trackerp != NULL)
1078ef2a572bSKonstantin Belousov 				ipsec_sahtree_runlock(sahtree_trackerp);
1079ef2a572bSKonstantin Belousov 			error = p(ifp1, sav, i->drv_spi, i->ifdata, &lft_s);
1080ef2a572bSKonstantin Belousov 			if (sahtree_trackerp != NULL)
1081ef2a572bSKonstantin Belousov 				ipsec_sahtree_rlock(sahtree_trackerp);
1082ef2a572bSKonstantin Belousov 			if (error == 0)
1083ef2a572bSKonstantin Belousov 				ipsec_accel_sa_lifetime_update(&lft_l, &lft_s);
1084ef2a572bSKonstantin Belousov 		}
1085ef2a572bSKonstantin Belousov 		break;
1086ef2a572bSKonstantin Belousov 	}
1087ef2a572bSKonstantin Belousov 
1088ef2a572bSKonstantin Belousov 	if (error == 0) {
1089ef2a572bSKonstantin Belousov 		if ((op & IF_SA_CNT_UPD) == 0)
1090ef2a572bSKonstantin Belousov 			memset(lft_c, 0, sizeof(*lft_c));
1091ef2a572bSKonstantin Belousov 		ipsec_accel_sa_lifetime_update(lft_c, &lft_l);
1092ef2a572bSKonstantin Belousov 	}
1093ef2a572bSKonstantin Belousov 
1094ef2a572bSKonstantin Belousov 	return (error);
1095ef2a572bSKonstantin Belousov }
1096ef2a572bSKonstantin Belousov 
1097ef2a572bSKonstantin Belousov static void
1098ef2a572bSKonstantin Belousov ipsec_accel_sync_imp(void)
1099ef2a572bSKonstantin Belousov {
1100ef2a572bSKonstantin Belousov 	taskqueue_drain_all(taskqueue_thread);
1101ef2a572bSKonstantin Belousov }
1102ef2a572bSKonstantin Belousov 
1103ef2a572bSKonstantin Belousov static struct mbuf *
1104ef2a572bSKonstantin Belousov ipsec_accel_key_setaccelif_impl(struct secasvar *sav)
1105ef2a572bSKonstantin Belousov {
1106ef2a572bSKonstantin Belousov 	struct mbuf *m, *m1;
1107ef2a572bSKonstantin Belousov 	struct ifp_handle_sav *i;
1108ef2a572bSKonstantin Belousov 	struct epoch_tracker et;
1109ef2a572bSKonstantin Belousov 
1110ef2a572bSKonstantin Belousov 	if (sav->accel_ifname != NULL)
1111ef2a572bSKonstantin Belousov 		return (key_setaccelif(sav->accel_ifname));
1112ef2a572bSKonstantin Belousov 
1113ef2a572bSKonstantin Belousov 	m = m1 = NULL;
1114ef2a572bSKonstantin Belousov 
1115ef2a572bSKonstantin Belousov 	NET_EPOCH_ENTER(et);
1116ef2a572bSKonstantin Belousov 	CK_LIST_FOREACH(i, &sav->accel_ifps, sav_link) {
1117ef2a572bSKonstantin Belousov 		if ((i->flags & (IFP_HS_HANDLED | IFP_HS_REJECTED)) ==
1118ef2a572bSKonstantin Belousov 		    IFP_HS_HANDLED) {
1119ef2a572bSKonstantin Belousov 			m1 = key_setaccelif(if_name(i->ifp));
1120ef2a572bSKonstantin Belousov 			if (m == NULL)
1121ef2a572bSKonstantin Belousov 				m = m1;
1122ef2a572bSKonstantin Belousov 			else if (m1 != NULL)
1123ef2a572bSKonstantin Belousov 				m_cat(m, m1);
1124ef2a572bSKonstantin Belousov 		}
1125ef2a572bSKonstantin Belousov 	}
1126ef2a572bSKonstantin Belousov 	NET_EPOCH_EXIT(et);
1127ef2a572bSKonstantin Belousov 	return (m);
1128ef2a572bSKonstantin Belousov }
1129ef2a572bSKonstantin Belousov 
1130ef2a572bSKonstantin Belousov #endif	/* IPSEC_OFFLOAD */
1131