1ef2a572bSKonstantin Belousov /*- 2ef2a572bSKonstantin Belousov * Copyright (c) 2021,2022 NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED. 3ef2a572bSKonstantin Belousov * 4ef2a572bSKonstantin Belousov * Redistribution and use in source and binary forms, with or without 5ef2a572bSKonstantin Belousov * modification, are permitted provided that the following conditions 6ef2a572bSKonstantin Belousov * are met: 7ef2a572bSKonstantin Belousov * 1. Redistributions of source code must retain the above copyright 8ef2a572bSKonstantin Belousov * notice, this list of conditions and the following disclaimer. 9ef2a572bSKonstantin Belousov * 2. Redistributions in binary form must reproduce the above copyright 10ef2a572bSKonstantin Belousov * notice, this list of conditions and the following disclaimer in the 11ef2a572bSKonstantin Belousov * documentation and/or other materials provided with the distribution. 12ef2a572bSKonstantin Belousov * 13ef2a572bSKonstantin Belousov * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND 14ef2a572bSKonstantin Belousov * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15ef2a572bSKonstantin Belousov * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16ef2a572bSKonstantin Belousov * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 17ef2a572bSKonstantin Belousov * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18ef2a572bSKonstantin Belousov * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19ef2a572bSKonstantin Belousov * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20ef2a572bSKonstantin Belousov * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21ef2a572bSKonstantin Belousov * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22ef2a572bSKonstantin Belousov * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23ef2a572bSKonstantin Belousov * SUCH DAMAGE. 24ef2a572bSKonstantin Belousov */ 25ef2a572bSKonstantin Belousov 26ef2a572bSKonstantin Belousov #include "opt_inet.h" 27ef2a572bSKonstantin Belousov #include "opt_inet6.h" 28ef2a572bSKonstantin Belousov #include "opt_ipsec.h" 29ef2a572bSKonstantin Belousov 30ef2a572bSKonstantin Belousov #include <sys/param.h> 31ef2a572bSKonstantin Belousov #include <sys/systm.h> 32ef2a572bSKonstantin Belousov #include <sys/ck.h> 33*c4a0ee9bSKonstantin Belousov #include <sys/eventhandler.h> 34ef2a572bSKonstantin Belousov #include <sys/kernel.h> 35ef2a572bSKonstantin Belousov #include <sys/mbuf.h> 36ef2a572bSKonstantin Belousov #include <sys/pctrie.h> 37ef2a572bSKonstantin Belousov #include <sys/proc.h> 38ef2a572bSKonstantin Belousov #include <sys/socket.h> 399a7096ffSKonstantin Belousov #include <sys/sysctl.h> 40ef2a572bSKonstantin Belousov #include <sys/protosw.h> 41ef2a572bSKonstantin Belousov #include <sys/taskqueue.h> 42ef2a572bSKonstantin Belousov 439a7096ffSKonstantin Belousov #include <machine/stdarg.h> 449a7096ffSKonstantin Belousov 45ef2a572bSKonstantin Belousov #include <net/if.h> 46ef2a572bSKonstantin Belousov #include <net/if_var.h> 47ef2a572bSKonstantin Belousov #include <net/vnet.h> 48ef2a572bSKonstantin Belousov #include <netinet/in.h> 49ef2a572bSKonstantin Belousov #include <netinet/ip.h> 50ef2a572bSKonstantin Belousov #include <netinet/ip_var.h> 51ef2a572bSKonstantin Belousov #include <netinet/ip6.h> 52ef2a572bSKonstantin Belousov #include <netinet6/ip6_var.h> 53ef2a572bSKonstantin Belousov #include <netinet/in_pcb.h> 54e6e2c0a5SKonstantin Belousov #include <netinet/tcp_var.h> 55ef2a572bSKonstantin Belousov 56ef2a572bSKonstantin Belousov #include <netipsec/key.h> 57ef2a572bSKonstantin Belousov #include <netipsec/keydb.h> 58ef2a572bSKonstantin Belousov #include <netipsec/key_debug.h> 59ef2a572bSKonstantin Belousov #include <netipsec/xform.h> 60ef2a572bSKonstantin Belousov #include <netipsec/ipsec.h> 61ef2a572bSKonstantin Belousov #include <netipsec/ipsec_offload.h> 62ef2a572bSKonstantin Belousov #include <netipsec/ah_var.h> 63ef2a572bSKonstantin Belousov #include <netipsec/esp.h> 64ef2a572bSKonstantin Belousov #include <netipsec/esp_var.h> 65ef2a572bSKonstantin Belousov #include <netipsec/ipcomp_var.h> 66ef2a572bSKonstantin Belousov 67ef2a572bSKonstantin Belousov #ifdef IPSEC_OFFLOAD 68ef2a572bSKonstantin Belousov 69ef2a572bSKonstantin Belousov static struct mtx ipsec_accel_sav_tmp; 70ef2a572bSKonstantin Belousov static struct unrhdr *drv_spi_unr; 71ef2a572bSKonstantin Belousov static struct mtx ipsec_accel_cnt_lock; 72ef2a572bSKonstantin Belousov 73ef2a572bSKonstantin Belousov struct ipsec_accel_install_newkey_tq { 74ef2a572bSKonstantin Belousov struct secasvar *sav; 75ef2a572bSKonstantin Belousov struct vnet *install_vnet; 76ef2a572bSKonstantin Belousov struct task install_task; 77ef2a572bSKonstantin Belousov }; 78ef2a572bSKonstantin Belousov 79ef2a572bSKonstantin Belousov struct ipsec_accel_forget_tq { 80ef2a572bSKonstantin Belousov struct vnet *forget_vnet; 81ef2a572bSKonstantin Belousov struct task forget_task; 82ef2a572bSKonstantin Belousov struct secasvar *sav; 83ef2a572bSKonstantin Belousov }; 84ef2a572bSKonstantin Belousov 85ef2a572bSKonstantin Belousov struct ifp_handle_sav { 86ef2a572bSKonstantin Belousov CK_LIST_ENTRY(ifp_handle_sav) sav_link; 87ef2a572bSKonstantin Belousov CK_LIST_ENTRY(ifp_handle_sav) sav_allh_link; 88ef2a572bSKonstantin Belousov struct secasvar *sav; 89ef2a572bSKonstantin Belousov struct ifnet *ifp; 90ef2a572bSKonstantin Belousov void *ifdata; 91ef2a572bSKonstantin Belousov uint64_t drv_spi; 92ef2a572bSKonstantin Belousov uint32_t flags; 93ef2a572bSKonstantin Belousov size_t hdr_ext_size; 94ef2a572bSKonstantin Belousov uint64_t cnt_octets; 95ef2a572bSKonstantin Belousov uint64_t cnt_allocs; 96ef2a572bSKonstantin Belousov }; 97ef2a572bSKonstantin Belousov 98ef2a572bSKonstantin Belousov #define IFP_HS_HANDLED 0x00000001 99ef2a572bSKonstantin Belousov #define IFP_HS_REJECTED 0x00000002 100ef2a572bSKonstantin Belousov #define IFP_HS_INPUT 0x00000004 101ef2a572bSKonstantin Belousov #define IFP_HS_OUTPUT 0x00000008 102ef2a572bSKonstantin Belousov #define IFP_HS_MARKER 0x00000010 103ef2a572bSKonstantin Belousov 104ef2a572bSKonstantin Belousov static CK_LIST_HEAD(, ifp_handle_sav) ipsec_accel_all_sav_handles; 105ef2a572bSKonstantin Belousov 106ef2a572bSKonstantin Belousov struct ifp_handle_sp { 107ef2a572bSKonstantin Belousov CK_LIST_ENTRY(ifp_handle_sp) sp_link; 108ef2a572bSKonstantin Belousov CK_LIST_ENTRY(ifp_handle_sp) sp_allh_link; 109ef2a572bSKonstantin Belousov struct secpolicy *sp; 110ef2a572bSKonstantin Belousov struct ifnet *ifp; 111ef2a572bSKonstantin Belousov void *ifdata; 112ef2a572bSKonstantin Belousov uint32_t flags; 113ef2a572bSKonstantin Belousov }; 114ef2a572bSKonstantin Belousov 115ef2a572bSKonstantin Belousov #define IFP_HP_HANDLED 0x00000001 116ef2a572bSKonstantin Belousov #define IFP_HP_REJECTED 0x00000002 117ef2a572bSKonstantin Belousov #define IFP_HP_MARKER 0x00000004 118ef2a572bSKonstantin Belousov 119ef2a572bSKonstantin Belousov static CK_LIST_HEAD(, ifp_handle_sp) ipsec_accel_all_sp_handles; 120ef2a572bSKonstantin Belousov 121ef2a572bSKonstantin Belousov static void * 122ef2a572bSKonstantin Belousov drvspi_sa_trie_alloc(struct pctrie *ptree) 123ef2a572bSKonstantin Belousov { 124ef2a572bSKonstantin Belousov void *res; 125ef2a572bSKonstantin Belousov 126ef2a572bSKonstantin Belousov res = malloc(pctrie_node_size(), M_IPSEC_MISC, M_ZERO | M_NOWAIT); 127ef2a572bSKonstantin Belousov if (res != NULL) 128ef2a572bSKonstantin Belousov pctrie_zone_init(res, 0, 0); 129ef2a572bSKonstantin Belousov return (res); 130ef2a572bSKonstantin Belousov } 131ef2a572bSKonstantin Belousov 132ef2a572bSKonstantin Belousov static void 133ef2a572bSKonstantin Belousov drvspi_sa_trie_free(struct pctrie *ptree, void *node) 134ef2a572bSKonstantin Belousov { 135ef2a572bSKonstantin Belousov free(node, M_IPSEC_MISC); 136ef2a572bSKonstantin Belousov } 137ef2a572bSKonstantin Belousov 138ef2a572bSKonstantin Belousov PCTRIE_DEFINE(DRVSPI_SA, ifp_handle_sav, drv_spi, 139ef2a572bSKonstantin Belousov drvspi_sa_trie_alloc, drvspi_sa_trie_free); 140ef2a572bSKonstantin Belousov static struct pctrie drv_spi_pctrie; 141ef2a572bSKonstantin Belousov 142*c4a0ee9bSKonstantin Belousov static eventhandler_tag ipsec_accel_ifdetach_event_tag; 143*c4a0ee9bSKonstantin Belousov 144ef2a572bSKonstantin Belousov static void ipsec_accel_sa_newkey_impl(struct secasvar *sav); 145ef2a572bSKonstantin Belousov static int ipsec_accel_handle_sav(struct secasvar *sav, struct ifnet *ifp, 146ef2a572bSKonstantin Belousov u_int drv_spi, void *priv, uint32_t flags, struct ifp_handle_sav **ires); 147ef2a572bSKonstantin Belousov static void ipsec_accel_forget_sav_clear(struct secasvar *sav); 148ef2a572bSKonstantin Belousov static struct ifp_handle_sav *ipsec_accel_is_accel_sav_ptr(struct secasvar *sav, 149ef2a572bSKonstantin Belousov struct ifnet *ifp); 150ef2a572bSKonstantin Belousov static int ipsec_accel_sa_lifetime_op_impl(struct secasvar *sav, 151ef2a572bSKonstantin Belousov struct seclifetime *lft_c, if_t ifp, enum IF_SA_CNT_WHICH op, 152ef2a572bSKonstantin Belousov struct rm_priotracker *sahtree_trackerp); 153ef2a572bSKonstantin Belousov static void ipsec_accel_sa_recordxfer(struct secasvar *sav, struct mbuf *m); 154ef2a572bSKonstantin Belousov static void ipsec_accel_sync_imp(void); 155ef2a572bSKonstantin Belousov static bool ipsec_accel_is_accel_sav_impl(struct secasvar *sav); 156ef2a572bSKonstantin Belousov static struct mbuf *ipsec_accel_key_setaccelif_impl(struct secasvar *sav); 15765f264dcSKonstantin Belousov static void ipsec_accel_on_ifdown_impl(struct ifnet *ifp); 15865f264dcSKonstantin Belousov static void ipsec_accel_drv_sa_lifetime_update_impl(struct secasvar *sav, 15965f264dcSKonstantin Belousov if_t ifp, u_int drv_spi, uint64_t octets, uint64_t allocs); 160*c4a0ee9bSKonstantin Belousov static void ipsec_accel_ifdetach_event(void *arg, struct ifnet *ifp); 161ef2a572bSKonstantin Belousov 162ef2a572bSKonstantin Belousov static void 163ef2a572bSKonstantin Belousov ipsec_accel_init(void *arg) 164ef2a572bSKonstantin Belousov { 165ef2a572bSKonstantin Belousov mtx_init(&ipsec_accel_sav_tmp, "ipasat", MTX_DEF, 0); 166ef2a572bSKonstantin Belousov mtx_init(&ipsec_accel_cnt_lock, "ipascn", MTX_DEF, 0); 167ef2a572bSKonstantin Belousov drv_spi_unr = new_unrhdr(IPSEC_ACCEL_DRV_SPI_MIN, 168ef2a572bSKonstantin Belousov IPSEC_ACCEL_DRV_SPI_MAX, &ipsec_accel_sav_tmp); 169ef2a572bSKonstantin Belousov ipsec_accel_sa_newkey_p = ipsec_accel_sa_newkey_impl; 170ef2a572bSKonstantin Belousov ipsec_accel_forget_sav_p = ipsec_accel_forget_sav_impl; 171ef2a572bSKonstantin Belousov ipsec_accel_spdadd_p = ipsec_accel_spdadd_impl; 172ef2a572bSKonstantin Belousov ipsec_accel_spddel_p = ipsec_accel_spddel_impl; 173ef2a572bSKonstantin Belousov ipsec_accel_sa_lifetime_op_p = ipsec_accel_sa_lifetime_op_impl; 174ef2a572bSKonstantin Belousov ipsec_accel_sync_p = ipsec_accel_sync_imp; 175ef2a572bSKonstantin Belousov ipsec_accel_is_accel_sav_p = ipsec_accel_is_accel_sav_impl; 176ef2a572bSKonstantin Belousov ipsec_accel_key_setaccelif_p = ipsec_accel_key_setaccelif_impl; 17765f264dcSKonstantin Belousov ipsec_accel_on_ifdown_p = ipsec_accel_on_ifdown_impl; 17865f264dcSKonstantin Belousov ipsec_accel_drv_sa_lifetime_update_p = 17965f264dcSKonstantin Belousov ipsec_accel_drv_sa_lifetime_update_impl; 180ef2a572bSKonstantin Belousov pctrie_init(&drv_spi_pctrie); 181*c4a0ee9bSKonstantin Belousov ipsec_accel_ifdetach_event_tag = EVENTHANDLER_REGISTER( 182*c4a0ee9bSKonstantin Belousov ifnet_departure_event, ipsec_accel_ifdetach_event, NULL, 183*c4a0ee9bSKonstantin Belousov EVENTHANDLER_PRI_ANY); 184ef2a572bSKonstantin Belousov } 185ef2a572bSKonstantin Belousov SYSINIT(ipsec_accel_init, SI_SUB_VNET_DONE, SI_ORDER_ANY, 186ef2a572bSKonstantin Belousov ipsec_accel_init, NULL); 187ef2a572bSKonstantin Belousov 188ef2a572bSKonstantin Belousov static void 189ef2a572bSKonstantin Belousov ipsec_accel_fini(void *arg) 190ef2a572bSKonstantin Belousov { 191*c4a0ee9bSKonstantin Belousov EVENTHANDLER_DEREGISTER(ifnet_departure_event, 192*c4a0ee9bSKonstantin Belousov ipsec_accel_ifdetach_event_tag); 193ef2a572bSKonstantin Belousov ipsec_accel_sa_newkey_p = NULL; 194ef2a572bSKonstantin Belousov ipsec_accel_forget_sav_p = NULL; 195ef2a572bSKonstantin Belousov ipsec_accel_spdadd_p = NULL; 196ef2a572bSKonstantin Belousov ipsec_accel_spddel_p = NULL; 197ef2a572bSKonstantin Belousov ipsec_accel_sa_lifetime_op_p = NULL; 198ef2a572bSKonstantin Belousov ipsec_accel_sync_p = NULL; 199ef2a572bSKonstantin Belousov ipsec_accel_is_accel_sav_p = NULL; 200ef2a572bSKonstantin Belousov ipsec_accel_key_setaccelif_p = NULL; 20165f264dcSKonstantin Belousov ipsec_accel_on_ifdown_p = NULL; 20265f264dcSKonstantin Belousov ipsec_accel_drv_sa_lifetime_update_p = NULL; 203ef2a572bSKonstantin Belousov ipsec_accel_sync_imp(); 204ef2a572bSKonstantin Belousov clean_unrhdr(drv_spi_unr); /* avoid panic, should go later */ 205ef2a572bSKonstantin Belousov clear_unrhdr(drv_spi_unr); 206ef2a572bSKonstantin Belousov delete_unrhdr(drv_spi_unr); 207ef2a572bSKonstantin Belousov mtx_destroy(&ipsec_accel_sav_tmp); 208ef2a572bSKonstantin Belousov mtx_destroy(&ipsec_accel_cnt_lock); 209ef2a572bSKonstantin Belousov } 210ef2a572bSKonstantin Belousov SYSUNINIT(ipsec_accel_fini, SI_SUB_VNET_DONE, SI_ORDER_ANY, 211ef2a572bSKonstantin Belousov ipsec_accel_fini, NULL); 212ef2a572bSKonstantin Belousov 2139a7096ffSKonstantin Belousov SYSCTL_NODE(_net_inet_ipsec, OID_AUTO, offload, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 2149a7096ffSKonstantin Belousov ""); 2159a7096ffSKonstantin Belousov 2169a7096ffSKonstantin Belousov static bool ipsec_offload_verbose = false; 2179a7096ffSKonstantin Belousov SYSCTL_BOOL(_net_inet_ipsec_offload, OID_AUTO, verbose, CTLFLAG_RW, 2189a7096ffSKonstantin Belousov &ipsec_offload_verbose, 0, 2199a7096ffSKonstantin Belousov "Verbose SA/SP offload install and deinstall"); 2209a7096ffSKonstantin Belousov 2219a7096ffSKonstantin Belousov static void 2229a7096ffSKonstantin Belousov dprintf(const char *fmt, ...) 2239a7096ffSKonstantin Belousov { 2249a7096ffSKonstantin Belousov va_list ap; 2259a7096ffSKonstantin Belousov 2269a7096ffSKonstantin Belousov if (!ipsec_offload_verbose) 2279a7096ffSKonstantin Belousov return; 2289a7096ffSKonstantin Belousov 2299a7096ffSKonstantin Belousov va_start(ap, fmt); 2309a7096ffSKonstantin Belousov vprintf(fmt, ap); 2319a7096ffSKonstantin Belousov va_end(ap); 2329a7096ffSKonstantin Belousov } 2339a7096ffSKonstantin Belousov 234ef2a572bSKonstantin Belousov static void 235ef2a572bSKonstantin Belousov ipsec_accel_alloc_forget_tq(struct secasvar *sav) 236ef2a572bSKonstantin Belousov { 237ef2a572bSKonstantin Belousov void *ftq; 238ef2a572bSKonstantin Belousov 239ef2a572bSKonstantin Belousov if (sav->accel_forget_tq != 0) 240ef2a572bSKonstantin Belousov return; 241ef2a572bSKonstantin Belousov 242ef2a572bSKonstantin Belousov ftq = malloc(sizeof(struct ipsec_accel_forget_tq), M_TEMP, M_WAITOK); 243ef2a572bSKonstantin Belousov if (!atomic_cmpset_ptr(&sav->accel_forget_tq, 0, (uintptr_t)ftq)) 244ef2a572bSKonstantin Belousov free(ftq, M_TEMP); 245ef2a572bSKonstantin Belousov } 246ef2a572bSKonstantin Belousov 247ef2a572bSKonstantin Belousov static bool 248ef2a572bSKonstantin Belousov ipsec_accel_sa_install_match(if_t ifp, void *arg) 249ef2a572bSKonstantin Belousov { 250ef2a572bSKonstantin Belousov if ((ifp->if_capenable2 & IFCAP2_BIT(IFCAP2_IPSEC_OFFLOAD)) == 0) 251ef2a572bSKonstantin Belousov return (false); 252ef2a572bSKonstantin Belousov if (ifp->if_ipsec_accel_m->if_sa_newkey == NULL) { 2539a7096ffSKonstantin Belousov dprintf("driver bug ifp %s if_sa_newkey NULL\n", 254ef2a572bSKonstantin Belousov if_name(ifp)); 255ef2a572bSKonstantin Belousov return (false); 256ef2a572bSKonstantin Belousov } 257ef2a572bSKonstantin Belousov return (true); 258ef2a572bSKonstantin Belousov } 259ef2a572bSKonstantin Belousov 260ef2a572bSKonstantin Belousov static int 261ef2a572bSKonstantin Belousov ipsec_accel_sa_newkey_cb(if_t ifp, void *arg) 262ef2a572bSKonstantin Belousov { 263ef2a572bSKonstantin Belousov struct ipsec_accel_install_newkey_tq *tq; 264ef2a572bSKonstantin Belousov void *priv; 265ef2a572bSKonstantin Belousov u_int drv_spi; 266ef2a572bSKonstantin Belousov int error; 267ef2a572bSKonstantin Belousov 268ef2a572bSKonstantin Belousov tq = arg; 269ef2a572bSKonstantin Belousov 2709a7096ffSKonstantin Belousov dprintf("ipsec_accel_sa_newkey_act: ifp %s h %p spi %#x " 271ef2a572bSKonstantin Belousov "flags %#x seq %d\n", 272ef2a572bSKonstantin Belousov if_name(ifp), ifp->if_ipsec_accel_m->if_sa_newkey, 273ef2a572bSKonstantin Belousov be32toh(tq->sav->spi), tq->sav->flags, tq->sav->seq); 274ef2a572bSKonstantin Belousov priv = NULL; 275ef2a572bSKonstantin Belousov drv_spi = alloc_unr(drv_spi_unr); 276ef2a572bSKonstantin Belousov if (tq->sav->accel_ifname != NULL && 277ef2a572bSKonstantin Belousov strcmp(tq->sav->accel_ifname, if_name(ifp)) != 0) { 278ef2a572bSKonstantin Belousov error = ipsec_accel_handle_sav(tq->sav, 279ef2a572bSKonstantin Belousov ifp, drv_spi, priv, IFP_HS_REJECTED, NULL); 280ef2a572bSKonstantin Belousov goto out; 281ef2a572bSKonstantin Belousov } 282ef2a572bSKonstantin Belousov if (drv_spi == -1) { 283ef2a572bSKonstantin Belousov /* XXXKIB */ 2849a7096ffSKonstantin Belousov dprintf("ipsec_accel_sa_install_newkey: cannot alloc " 285ef2a572bSKonstantin Belousov "drv_spi if %s spi %#x\n", if_name(ifp), 286ef2a572bSKonstantin Belousov be32toh(tq->sav->spi)); 287ef2a572bSKonstantin Belousov return (ENOMEM); 288ef2a572bSKonstantin Belousov } 289ef2a572bSKonstantin Belousov error = ifp->if_ipsec_accel_m->if_sa_newkey(ifp, tq->sav, 290ef2a572bSKonstantin Belousov drv_spi, &priv); 291ef2a572bSKonstantin Belousov if (error != 0) { 292ef2a572bSKonstantin Belousov if (error == EOPNOTSUPP) { 2939a7096ffSKonstantin Belousov dprintf("ipsec_accel_sa_newkey: driver " 294ef2a572bSKonstantin Belousov "refused sa if %s spi %#x\n", 295ef2a572bSKonstantin Belousov if_name(ifp), be32toh(tq->sav->spi)); 296ef2a572bSKonstantin Belousov error = ipsec_accel_handle_sav(tq->sav, 297ef2a572bSKonstantin Belousov ifp, drv_spi, priv, IFP_HS_REJECTED, NULL); 298ef2a572bSKonstantin Belousov /* XXXKIB */ 299ef2a572bSKonstantin Belousov } else { 3009a7096ffSKonstantin Belousov dprintf("ipsec_accel_sa_newkey: driver " 301ef2a572bSKonstantin Belousov "error %d if %s spi %#x\n", 302ef2a572bSKonstantin Belousov error, if_name(ifp), be32toh(tq->sav->spi)); 303ef2a572bSKonstantin Belousov /* XXXKIB */ 304ef2a572bSKonstantin Belousov } 305ef2a572bSKonstantin Belousov } else { 306ef2a572bSKonstantin Belousov error = ipsec_accel_handle_sav(tq->sav, ifp, 307ef2a572bSKonstantin Belousov drv_spi, priv, IFP_HS_HANDLED, NULL); 308ef2a572bSKonstantin Belousov if (error != 0) { 309ef2a572bSKonstantin Belousov /* XXXKIB */ 3109a7096ffSKonstantin Belousov dprintf("ipsec_accel_sa_newkey: handle_sav " 311ef2a572bSKonstantin Belousov "err %d if %s spi %#x\n", error, 312ef2a572bSKonstantin Belousov if_name(ifp), be32toh(tq->sav->spi)); 313ef2a572bSKonstantin Belousov } 314ef2a572bSKonstantin Belousov } 315ef2a572bSKonstantin Belousov out: 316ef2a572bSKonstantin Belousov return (error); 317ef2a572bSKonstantin Belousov } 318ef2a572bSKonstantin Belousov 319ef2a572bSKonstantin Belousov static void 320ef2a572bSKonstantin Belousov ipsec_accel_sa_newkey_act(void *context, int pending) 321ef2a572bSKonstantin Belousov { 322ef2a572bSKonstantin Belousov struct ipsec_accel_install_newkey_tq *tq; 323ef2a572bSKonstantin Belousov void *tqf; 324ef2a572bSKonstantin Belousov struct secasvar *sav; 325ef2a572bSKonstantin Belousov 326ef2a572bSKonstantin Belousov tq = context; 327ef2a572bSKonstantin Belousov tqf = NULL; 328ef2a572bSKonstantin Belousov sav = tq->sav; 329ef2a572bSKonstantin Belousov CURVNET_SET(tq->install_vnet); 330ef2a572bSKonstantin Belousov mtx_lock(&ipsec_accel_sav_tmp); 331ef2a572bSKonstantin Belousov if ((sav->accel_flags & (SADB_KEY_ACCEL_INST | 332ef2a572bSKonstantin Belousov SADB_KEY_ACCEL_DEINST)) == 0 && 333ef2a572bSKonstantin Belousov sav->state == SADB_SASTATE_MATURE) { 334ef2a572bSKonstantin Belousov sav->accel_flags |= SADB_KEY_ACCEL_INST; 335ef2a572bSKonstantin Belousov mtx_unlock(&ipsec_accel_sav_tmp); 336ef2a572bSKonstantin Belousov if_foreach_sleep(ipsec_accel_sa_install_match, context, 337ef2a572bSKonstantin Belousov ipsec_accel_sa_newkey_cb, context); 338ef2a572bSKonstantin Belousov ipsec_accel_alloc_forget_tq(sav); 339ef2a572bSKonstantin Belousov mtx_lock(&ipsec_accel_sav_tmp); 340ef2a572bSKonstantin Belousov 341ef2a572bSKonstantin Belousov /* 342ef2a572bSKonstantin Belousov * If ipsec_accel_forget_sav() raced with us and set 343ef2a572bSKonstantin Belousov * the flag, do its work. Its task cannot execute in 344ef2a572bSKonstantin Belousov * parallel since taskqueue_thread is single-threaded. 345ef2a572bSKonstantin Belousov */ 346ef2a572bSKonstantin Belousov if ((sav->accel_flags & SADB_KEY_ACCEL_DEINST) != 0) { 347ef2a572bSKonstantin Belousov tqf = (void *)sav->accel_forget_tq; 348ef2a572bSKonstantin Belousov sav->accel_forget_tq = 0; 349ef2a572bSKonstantin Belousov ipsec_accel_forget_sav_clear(sav); 350ef2a572bSKonstantin Belousov } 351ef2a572bSKonstantin Belousov } 352ef2a572bSKonstantin Belousov mtx_unlock(&ipsec_accel_sav_tmp); 353ef2a572bSKonstantin Belousov key_freesav(&tq->sav); 354ef2a572bSKonstantin Belousov CURVNET_RESTORE(); 355ef2a572bSKonstantin Belousov free(tq, M_TEMP); 356ef2a572bSKonstantin Belousov free(tqf, M_TEMP); 357ef2a572bSKonstantin Belousov } 358ef2a572bSKonstantin Belousov 359ef2a572bSKonstantin Belousov static void 360ef2a572bSKonstantin Belousov ipsec_accel_sa_newkey_impl(struct secasvar *sav) 361ef2a572bSKonstantin Belousov { 362ef2a572bSKonstantin Belousov struct ipsec_accel_install_newkey_tq *tq; 363ef2a572bSKonstantin Belousov 364ef2a572bSKonstantin Belousov if ((sav->accel_flags & (SADB_KEY_ACCEL_INST | 365ef2a572bSKonstantin Belousov SADB_KEY_ACCEL_DEINST)) != 0) 366ef2a572bSKonstantin Belousov return; 367ef2a572bSKonstantin Belousov 3689a7096ffSKonstantin Belousov dprintf( 369ef2a572bSKonstantin Belousov "ipsec_accel_sa_install_newkey: spi %#x flags %#x seq %d\n", 370ef2a572bSKonstantin Belousov be32toh(sav->spi), sav->flags, sav->seq); 371ef2a572bSKonstantin Belousov 372ef2a572bSKonstantin Belousov tq = malloc(sizeof(*tq), M_TEMP, M_NOWAIT); 373ef2a572bSKonstantin Belousov if (tq == NULL) { 3749a7096ffSKonstantin Belousov dprintf("ipsec_accel_sa_install_newkey: no memory for tq, " 375ef2a572bSKonstantin Belousov "spi %#x\n", be32toh(sav->spi)); 376ef2a572bSKonstantin Belousov /* XXXKIB */ 377ef2a572bSKonstantin Belousov return; 378ef2a572bSKonstantin Belousov } 379ef2a572bSKonstantin Belousov 380ef2a572bSKonstantin Belousov refcount_acquire(&sav->refcnt); 381ef2a572bSKonstantin Belousov 382ef2a572bSKonstantin Belousov TASK_INIT(&tq->install_task, 0, ipsec_accel_sa_newkey_act, tq); 383ef2a572bSKonstantin Belousov tq->sav = sav; 384ef2a572bSKonstantin Belousov tq->install_vnet = curthread->td_vnet; /* XXXKIB liveness */ 385ef2a572bSKonstantin Belousov taskqueue_enqueue(taskqueue_thread, &tq->install_task); 386ef2a572bSKonstantin Belousov } 387ef2a572bSKonstantin Belousov 388ef2a572bSKonstantin Belousov static int 389ef2a572bSKonstantin Belousov ipsec_accel_handle_sav(struct secasvar *sav, struct ifnet *ifp, 390ef2a572bSKonstantin Belousov u_int drv_spi, void *priv, uint32_t flags, struct ifp_handle_sav **ires) 391ef2a572bSKonstantin Belousov { 392ef2a572bSKonstantin Belousov struct ifp_handle_sav *ihs, *i; 393ef2a572bSKonstantin Belousov int error; 394ef2a572bSKonstantin Belousov 395ef2a572bSKonstantin Belousov MPASS(__bitcount(flags & (IFP_HS_HANDLED | IFP_HS_REJECTED)) == 1); 396ef2a572bSKonstantin Belousov 397ef2a572bSKonstantin Belousov ihs = malloc(sizeof(*ihs), M_IPSEC_MISC, M_WAITOK | M_ZERO); 398ef2a572bSKonstantin Belousov ihs->ifp = ifp; 399ef2a572bSKonstantin Belousov ihs->sav = sav; 400ef2a572bSKonstantin Belousov ihs->drv_spi = drv_spi; 401ef2a572bSKonstantin Belousov ihs->ifdata = priv; 402ef2a572bSKonstantin Belousov ihs->flags = flags; 403ef2a572bSKonstantin Belousov if ((flags & IFP_HS_OUTPUT) != 0) 404ef2a572bSKonstantin Belousov ihs->hdr_ext_size = esp_hdrsiz(sav); 405ef2a572bSKonstantin Belousov mtx_lock(&ipsec_accel_sav_tmp); 406ef2a572bSKonstantin Belousov CK_LIST_FOREACH(i, &sav->accel_ifps, sav_link) { 407ef2a572bSKonstantin Belousov if (i->ifp == ifp) { 408ef2a572bSKonstantin Belousov error = EALREADY; 409ef2a572bSKonstantin Belousov goto errout; 410ef2a572bSKonstantin Belousov } 411ef2a572bSKonstantin Belousov } 412ef2a572bSKonstantin Belousov error = DRVSPI_SA_PCTRIE_INSERT(&drv_spi_pctrie, ihs); 413ef2a572bSKonstantin Belousov if (error != 0) 414ef2a572bSKonstantin Belousov goto errout; 415ef2a572bSKonstantin Belousov if_ref(ihs->ifp); 416ef2a572bSKonstantin Belousov CK_LIST_INSERT_HEAD(&sav->accel_ifps, ihs, sav_link); 417ef2a572bSKonstantin Belousov CK_LIST_INSERT_HEAD(&ipsec_accel_all_sav_handles, ihs, sav_allh_link); 418ef2a572bSKonstantin Belousov mtx_unlock(&ipsec_accel_sav_tmp); 419ef2a572bSKonstantin Belousov if (ires != NULL) 420ef2a572bSKonstantin Belousov *ires = ihs; 421ef2a572bSKonstantin Belousov return (0); 422ef2a572bSKonstantin Belousov errout: 423ef2a572bSKonstantin Belousov mtx_unlock(&ipsec_accel_sav_tmp); 424ef2a572bSKonstantin Belousov free(ihs, M_IPSEC_MISC); 425ef2a572bSKonstantin Belousov if (ires != NULL) 426ef2a572bSKonstantin Belousov *ires = NULL; 427ef2a572bSKonstantin Belousov return (error); 428ef2a572bSKonstantin Belousov } 429ef2a572bSKonstantin Belousov 430ef2a572bSKonstantin Belousov static void 431ef2a572bSKonstantin Belousov ipsec_accel_forget_handle_sav(struct ifp_handle_sav *i, bool freesav) 432ef2a572bSKonstantin Belousov { 433ef2a572bSKonstantin Belousov struct ifnet *ifp; 434ef2a572bSKonstantin Belousov struct secasvar *sav; 435ef2a572bSKonstantin Belousov 436ef2a572bSKonstantin Belousov mtx_assert(&ipsec_accel_sav_tmp, MA_OWNED); 437ef2a572bSKonstantin Belousov 438ef2a572bSKonstantin Belousov CK_LIST_REMOVE(i, sav_link); 439ef2a572bSKonstantin Belousov CK_LIST_REMOVE(i, sav_allh_link); 440ef2a572bSKonstantin Belousov DRVSPI_SA_PCTRIE_REMOVE(&drv_spi_pctrie, i->drv_spi); 441ef2a572bSKonstantin Belousov mtx_unlock(&ipsec_accel_sav_tmp); 442ef2a572bSKonstantin Belousov NET_EPOCH_WAIT(); 443ef2a572bSKonstantin Belousov ifp = i->ifp; 444ef2a572bSKonstantin Belousov sav = i->sav; 445ef2a572bSKonstantin Belousov if ((i->flags & (IFP_HS_HANDLED | IFP_HS_REJECTED)) == 446ef2a572bSKonstantin Belousov IFP_HS_HANDLED) { 4479a7096ffSKonstantin Belousov dprintf("sa deinstall %s %p spi %#x ifl %#x\n", 448ef2a572bSKonstantin Belousov if_name(ifp), sav, be32toh(sav->spi), i->flags); 449ef2a572bSKonstantin Belousov ifp->if_ipsec_accel_m->if_sa_deinstall(ifp, 450ef2a572bSKonstantin Belousov i->drv_spi, i->ifdata); 451ef2a572bSKonstantin Belousov } 452ef2a572bSKonstantin Belousov if_rele(ifp); 453ef2a572bSKonstantin Belousov free_unr(drv_spi_unr, i->drv_spi); 454ef2a572bSKonstantin Belousov free(i, M_IPSEC_MISC); 455ef2a572bSKonstantin Belousov if (freesav) 456ef2a572bSKonstantin Belousov key_freesav(&sav); 457ef2a572bSKonstantin Belousov mtx_lock(&ipsec_accel_sav_tmp); 458ef2a572bSKonstantin Belousov } 459ef2a572bSKonstantin Belousov 460ef2a572bSKonstantin Belousov static void 461ef2a572bSKonstantin Belousov ipsec_accel_forget_sav_clear(struct secasvar *sav) 462ef2a572bSKonstantin Belousov { 463ef2a572bSKonstantin Belousov struct ifp_handle_sav *i; 464ef2a572bSKonstantin Belousov 465ef2a572bSKonstantin Belousov for (;;) { 466ef2a572bSKonstantin Belousov i = CK_LIST_FIRST(&sav->accel_ifps); 467ef2a572bSKonstantin Belousov if (i == NULL) 468ef2a572bSKonstantin Belousov break; 469ef2a572bSKonstantin Belousov ipsec_accel_forget_handle_sav(i, false); 470ef2a572bSKonstantin Belousov } 471ef2a572bSKonstantin Belousov } 472ef2a572bSKonstantin Belousov 473ef2a572bSKonstantin Belousov static void 474ef2a572bSKonstantin Belousov ipsec_accel_forget_sav_act(void *arg, int pending) 475ef2a572bSKonstantin Belousov { 476ef2a572bSKonstantin Belousov struct ipsec_accel_forget_tq *tq; 477ef2a572bSKonstantin Belousov struct secasvar *sav; 478ef2a572bSKonstantin Belousov 479ef2a572bSKonstantin Belousov tq = arg; 480ef2a572bSKonstantin Belousov sav = tq->sav; 481ef2a572bSKonstantin Belousov CURVNET_SET(tq->forget_vnet); 482ef2a572bSKonstantin Belousov mtx_lock(&ipsec_accel_sav_tmp); 483ef2a572bSKonstantin Belousov ipsec_accel_forget_sav_clear(sav); 484ef2a572bSKonstantin Belousov mtx_unlock(&ipsec_accel_sav_tmp); 485ef2a572bSKonstantin Belousov key_freesav(&sav); 486ef2a572bSKonstantin Belousov CURVNET_RESTORE(); 487ef2a572bSKonstantin Belousov free(tq, M_TEMP); 488ef2a572bSKonstantin Belousov } 489ef2a572bSKonstantin Belousov 490ef2a572bSKonstantin Belousov void 491ef2a572bSKonstantin Belousov ipsec_accel_forget_sav_impl(struct secasvar *sav) 492ef2a572bSKonstantin Belousov { 493ef2a572bSKonstantin Belousov struct ipsec_accel_forget_tq *tq; 494ef2a572bSKonstantin Belousov 495ef2a572bSKonstantin Belousov mtx_lock(&ipsec_accel_sav_tmp); 496ef2a572bSKonstantin Belousov sav->accel_flags |= SADB_KEY_ACCEL_DEINST; 497ef2a572bSKonstantin Belousov tq = (void *)atomic_load_ptr(&sav->accel_forget_tq); 498ef2a572bSKonstantin Belousov if (tq == NULL || !atomic_cmpset_ptr(&sav->accel_forget_tq, 499ef2a572bSKonstantin Belousov (uintptr_t)tq, 0)) { 500ef2a572bSKonstantin Belousov mtx_unlock(&ipsec_accel_sav_tmp); 501ef2a572bSKonstantin Belousov return; 502ef2a572bSKonstantin Belousov } 503ef2a572bSKonstantin Belousov mtx_unlock(&ipsec_accel_sav_tmp); 504ef2a572bSKonstantin Belousov 505ef2a572bSKonstantin Belousov refcount_acquire(&sav->refcnt); 506ef2a572bSKonstantin Belousov TASK_INIT(&tq->forget_task, 0, ipsec_accel_forget_sav_act, tq); 507ef2a572bSKonstantin Belousov tq->forget_vnet = curthread->td_vnet; 508ef2a572bSKonstantin Belousov tq->sav = sav; 509ef2a572bSKonstantin Belousov taskqueue_enqueue(taskqueue_thread, &tq->forget_task); 510ef2a572bSKonstantin Belousov } 511ef2a572bSKonstantin Belousov 512ef2a572bSKonstantin Belousov static void 513ef2a572bSKonstantin Belousov ipsec_accel_on_ifdown_sav(struct ifnet *ifp) 514ef2a572bSKonstantin Belousov { 515ef2a572bSKonstantin Belousov struct ifp_handle_sav *i, *marker; 516ef2a572bSKonstantin Belousov 517ef2a572bSKonstantin Belousov marker = malloc(sizeof(*marker), M_IPSEC_MISC, M_WAITOK | M_ZERO); 518ef2a572bSKonstantin Belousov marker->flags = IFP_HS_MARKER; 519ef2a572bSKonstantin Belousov 520ef2a572bSKonstantin Belousov mtx_lock(&ipsec_accel_sav_tmp); 521ef2a572bSKonstantin Belousov CK_LIST_INSERT_HEAD(&ipsec_accel_all_sav_handles, marker, 522ef2a572bSKonstantin Belousov sav_allh_link); 523ef2a572bSKonstantin Belousov for (;;) { 524ef2a572bSKonstantin Belousov i = CK_LIST_NEXT(marker, sav_allh_link); 525ef2a572bSKonstantin Belousov if (i == NULL) 526ef2a572bSKonstantin Belousov break; 527ef2a572bSKonstantin Belousov CK_LIST_REMOVE(marker, sav_allh_link); 528ef2a572bSKonstantin Belousov CK_LIST_INSERT_AFTER(i, marker, sav_allh_link); 529ef2a572bSKonstantin Belousov if (i->ifp == ifp) { 530ef2a572bSKonstantin Belousov refcount_acquire(&i->sav->refcnt); /* XXXKIB wrap ? */ 531ef2a572bSKonstantin Belousov ipsec_accel_forget_handle_sav(i, true); 532ef2a572bSKonstantin Belousov } 533ef2a572bSKonstantin Belousov } 534ef2a572bSKonstantin Belousov CK_LIST_REMOVE(marker, sav_allh_link); 535ef2a572bSKonstantin Belousov mtx_unlock(&ipsec_accel_sav_tmp); 536ef2a572bSKonstantin Belousov free(marker, M_IPSEC_MISC); 537ef2a572bSKonstantin Belousov } 538ef2a572bSKonstantin Belousov 539ef2a572bSKonstantin Belousov static struct ifp_handle_sav * 540ef2a572bSKonstantin Belousov ipsec_accel_is_accel_sav_ptr_raw(struct secasvar *sav, struct ifnet *ifp) 541ef2a572bSKonstantin Belousov { 542ef2a572bSKonstantin Belousov struct ifp_handle_sav *i; 543ef2a572bSKonstantin Belousov 544ef2a572bSKonstantin Belousov if ((ifp->if_capenable2 & IFCAP2_BIT(IFCAP2_IPSEC_OFFLOAD)) == 0) 545ef2a572bSKonstantin Belousov return (NULL); 546ef2a572bSKonstantin Belousov CK_LIST_FOREACH(i, &sav->accel_ifps, sav_link) { 547ef2a572bSKonstantin Belousov if (i->ifp == ifp) 548ef2a572bSKonstantin Belousov return (i); 549ef2a572bSKonstantin Belousov } 550ef2a572bSKonstantin Belousov return (NULL); 551ef2a572bSKonstantin Belousov } 552ef2a572bSKonstantin Belousov 553ef2a572bSKonstantin Belousov static struct ifp_handle_sav * 554ef2a572bSKonstantin Belousov ipsec_accel_is_accel_sav_ptr(struct secasvar *sav, struct ifnet *ifp) 555ef2a572bSKonstantin Belousov { 556ef2a572bSKonstantin Belousov NET_EPOCH_ASSERT(); 557ef2a572bSKonstantin Belousov return (ipsec_accel_is_accel_sav_ptr_raw(sav, ifp)); 558ef2a572bSKonstantin Belousov } 559ef2a572bSKonstantin Belousov 560ef2a572bSKonstantin Belousov static bool 561ef2a572bSKonstantin Belousov ipsec_accel_is_accel_sav_impl(struct secasvar *sav) 562ef2a572bSKonstantin Belousov { 563ef2a572bSKonstantin Belousov return (!CK_LIST_EMPTY(&sav->accel_ifps)); 564ef2a572bSKonstantin Belousov } 565ef2a572bSKonstantin Belousov 566ef2a572bSKonstantin Belousov static struct secasvar * 567ef2a572bSKonstantin Belousov ipsec_accel_drvspi_to_sa(u_int drv_spi) 568ef2a572bSKonstantin Belousov { 569ef2a572bSKonstantin Belousov struct ifp_handle_sav *i; 570ef2a572bSKonstantin Belousov 571ef2a572bSKonstantin Belousov i = DRVSPI_SA_PCTRIE_LOOKUP(&drv_spi_pctrie, drv_spi); 572ef2a572bSKonstantin Belousov if (i == NULL) 573ef2a572bSKonstantin Belousov return (NULL); 574ef2a572bSKonstantin Belousov return (i->sav); 575ef2a572bSKonstantin Belousov } 576ef2a572bSKonstantin Belousov 577ef2a572bSKonstantin Belousov static struct ifp_handle_sp * 578ef2a572bSKonstantin Belousov ipsec_accel_find_accel_sp(struct secpolicy *sp, if_t ifp) 579ef2a572bSKonstantin Belousov { 580ef2a572bSKonstantin Belousov struct ifp_handle_sp *i; 581ef2a572bSKonstantin Belousov 582ef2a572bSKonstantin Belousov CK_LIST_FOREACH(i, &sp->accel_ifps, sp_link) { 583ef2a572bSKonstantin Belousov if (i->ifp == ifp) 584ef2a572bSKonstantin Belousov return (i); 585ef2a572bSKonstantin Belousov } 586ef2a572bSKonstantin Belousov return (NULL); 587ef2a572bSKonstantin Belousov } 588ef2a572bSKonstantin Belousov 589ef2a572bSKonstantin Belousov static bool 590ef2a572bSKonstantin Belousov ipsec_accel_is_accel_sp(struct secpolicy *sp, if_t ifp) 591ef2a572bSKonstantin Belousov { 592ef2a572bSKonstantin Belousov return (ipsec_accel_find_accel_sp(sp, ifp) != NULL); 593ef2a572bSKonstantin Belousov } 594ef2a572bSKonstantin Belousov 595ef2a572bSKonstantin Belousov static int 596ef2a572bSKonstantin Belousov ipsec_accel_remember_sp(struct secpolicy *sp, if_t ifp, 597ef2a572bSKonstantin Belousov struct ifp_handle_sp **ip) 598ef2a572bSKonstantin Belousov { 599ef2a572bSKonstantin Belousov struct ifp_handle_sp *i; 600ef2a572bSKonstantin Belousov 601ef2a572bSKonstantin Belousov i = malloc(sizeof(*i), M_IPSEC_MISC, M_WAITOK | M_ZERO); 602ef2a572bSKonstantin Belousov i->sp = sp; 603ef2a572bSKonstantin Belousov i->ifp = ifp; 604ef2a572bSKonstantin Belousov if_ref(ifp); 605ef2a572bSKonstantin Belousov i->flags = IFP_HP_HANDLED; 606ef2a572bSKonstantin Belousov mtx_lock(&ipsec_accel_sav_tmp); 607ef2a572bSKonstantin Belousov CK_LIST_INSERT_HEAD(&sp->accel_ifps, i, sp_link); 608ef2a572bSKonstantin Belousov CK_LIST_INSERT_HEAD(&ipsec_accel_all_sp_handles, i, sp_allh_link); 609ef2a572bSKonstantin Belousov mtx_unlock(&ipsec_accel_sav_tmp); 610ef2a572bSKonstantin Belousov *ip = i; 611ef2a572bSKonstantin Belousov return (0); 612ef2a572bSKonstantin Belousov } 613ef2a572bSKonstantin Belousov 614ef2a572bSKonstantin Belousov static bool 615ef2a572bSKonstantin Belousov ipsec_accel_spdadd_match(if_t ifp, void *arg) 616ef2a572bSKonstantin Belousov { 617ef2a572bSKonstantin Belousov struct secpolicy *sp; 618ef2a572bSKonstantin Belousov 619ef2a572bSKonstantin Belousov if ((ifp->if_capenable2 & IFCAP2_BIT(IFCAP2_IPSEC_OFFLOAD)) == 0 || 620ef2a572bSKonstantin Belousov ifp->if_ipsec_accel_m->if_spdadd == NULL) 621ef2a572bSKonstantin Belousov return (false); 622ef2a572bSKonstantin Belousov sp = arg; 623ef2a572bSKonstantin Belousov if (sp->accel_ifname != NULL && 624ef2a572bSKonstantin Belousov strcmp(sp->accel_ifname, if_name(ifp)) != 0) 625ef2a572bSKonstantin Belousov return (false); 626ef2a572bSKonstantin Belousov if (ipsec_accel_is_accel_sp(sp, ifp)) 627ef2a572bSKonstantin Belousov return (false); 628ef2a572bSKonstantin Belousov return (true); 629ef2a572bSKonstantin Belousov } 630ef2a572bSKonstantin Belousov 631ef2a572bSKonstantin Belousov static int 632ef2a572bSKonstantin Belousov ipsec_accel_spdadd_cb(if_t ifp, void *arg) 633ef2a572bSKonstantin Belousov { 634ef2a572bSKonstantin Belousov struct secpolicy *sp; 635ef2a572bSKonstantin Belousov struct inpcb *inp; 636ef2a572bSKonstantin Belousov struct ifp_handle_sp *i; 637ef2a572bSKonstantin Belousov int error; 638ef2a572bSKonstantin Belousov 639ef2a572bSKonstantin Belousov sp = arg; 640ef2a572bSKonstantin Belousov inp = sp->ipsec_accel_add_sp_inp; 6419a7096ffSKonstantin Belousov dprintf("ipsec_accel_spdadd_cb: ifp %s m %p sp %p inp %p\n", 642ef2a572bSKonstantin Belousov if_name(ifp), ifp->if_ipsec_accel_m->if_spdadd, sp, inp); 643ef2a572bSKonstantin Belousov error = ipsec_accel_remember_sp(sp, ifp, &i); 644ef2a572bSKonstantin Belousov if (error != 0) { 6459a7096ffSKonstantin Belousov dprintf("ipsec_accel_spdadd: %s if_spdadd %p remember res %d\n", 646ef2a572bSKonstantin Belousov if_name(ifp), sp, error); 647ef2a572bSKonstantin Belousov return (error); 648ef2a572bSKonstantin Belousov } 649ef2a572bSKonstantin Belousov error = ifp->if_ipsec_accel_m->if_spdadd(ifp, sp, inp, &i->ifdata); 650ef2a572bSKonstantin Belousov if (error != 0) { 651ef2a572bSKonstantin Belousov i->flags |= IFP_HP_REJECTED; 6529a7096ffSKonstantin Belousov dprintf("ipsec_accel_spdadd: %s if_spdadd %p res %d\n", 653ef2a572bSKonstantin Belousov if_name(ifp), sp, error); 654ef2a572bSKonstantin Belousov } 655ef2a572bSKonstantin Belousov return (error); 656ef2a572bSKonstantin Belousov } 657ef2a572bSKonstantin Belousov 658ef2a572bSKonstantin Belousov static void 659ef2a572bSKonstantin Belousov ipsec_accel_spdadd_act(void *arg, int pending) 660ef2a572bSKonstantin Belousov { 661ef2a572bSKonstantin Belousov struct secpolicy *sp; 662ef2a572bSKonstantin Belousov struct inpcb *inp; 663ef2a572bSKonstantin Belousov 664ef2a572bSKonstantin Belousov sp = arg; 665ef2a572bSKonstantin Belousov CURVNET_SET(sp->accel_add_tq.adddel_vnet); 666ef2a572bSKonstantin Belousov if_foreach_sleep(ipsec_accel_spdadd_match, arg, 667ef2a572bSKonstantin Belousov ipsec_accel_spdadd_cb, arg); 668ef2a572bSKonstantin Belousov inp = sp->ipsec_accel_add_sp_inp; 669ef2a572bSKonstantin Belousov if (inp != NULL) { 670ef2a572bSKonstantin Belousov INP_WLOCK(inp); 671ef2a572bSKonstantin Belousov if (!in_pcbrele_wlocked(inp)) 672ef2a572bSKonstantin Belousov INP_WUNLOCK(inp); 673ef2a572bSKonstantin Belousov sp->ipsec_accel_add_sp_inp = NULL; 674ef2a572bSKonstantin Belousov } 675ef2a572bSKonstantin Belousov CURVNET_RESTORE(); 676ef2a572bSKonstantin Belousov key_freesp(&sp); 677ef2a572bSKonstantin Belousov } 678ef2a572bSKonstantin Belousov 679ef2a572bSKonstantin Belousov void 680ef2a572bSKonstantin Belousov ipsec_accel_spdadd_impl(struct secpolicy *sp, struct inpcb *inp) 681ef2a572bSKonstantin Belousov { 682ef2a572bSKonstantin Belousov struct ipsec_accel_adddel_sp_tq *tq; 683ef2a572bSKonstantin Belousov 684ef2a572bSKonstantin Belousov if (sp == NULL) 685ef2a572bSKonstantin Belousov return; 686ef2a572bSKonstantin Belousov if (sp->tcount == 0 && inp == NULL) 687ef2a572bSKonstantin Belousov return; 688ef2a572bSKonstantin Belousov tq = &sp->accel_add_tq; 689ef2a572bSKonstantin Belousov if (atomic_cmpset_int(&tq->adddel_scheduled, 0, 1) == 0) 690ef2a572bSKonstantin Belousov return; 691ef2a572bSKonstantin Belousov tq->adddel_vnet = curthread->td_vnet; 692ef2a572bSKonstantin Belousov sp->ipsec_accel_add_sp_inp = inp; 693ef2a572bSKonstantin Belousov if (inp != NULL) 694ef2a572bSKonstantin Belousov in_pcbref(inp); 695ef2a572bSKonstantin Belousov TASK_INIT(&tq->adddel_task, 0, ipsec_accel_spdadd_act, sp); 696ef2a572bSKonstantin Belousov key_addref(sp); 697ef2a572bSKonstantin Belousov taskqueue_enqueue(taskqueue_thread, &tq->adddel_task); 698ef2a572bSKonstantin Belousov } 699ef2a572bSKonstantin Belousov 700ef2a572bSKonstantin Belousov static void 701ef2a572bSKonstantin Belousov ipsec_accel_spddel_act(void *arg, int pending) 702ef2a572bSKonstantin Belousov { 703ef2a572bSKonstantin Belousov struct ifp_handle_sp *i; 704ef2a572bSKonstantin Belousov struct secpolicy *sp; 705ef2a572bSKonstantin Belousov int error; 706ef2a572bSKonstantin Belousov 707ef2a572bSKonstantin Belousov sp = arg; 708ef2a572bSKonstantin Belousov CURVNET_SET(sp->accel_del_tq.adddel_vnet); 709ef2a572bSKonstantin Belousov mtx_lock(&ipsec_accel_sav_tmp); 710ef2a572bSKonstantin Belousov for (;;) { 711ef2a572bSKonstantin Belousov i = CK_LIST_FIRST(&sp->accel_ifps); 712ef2a572bSKonstantin Belousov if (i == NULL) 713ef2a572bSKonstantin Belousov break; 714ef2a572bSKonstantin Belousov CK_LIST_REMOVE(i, sp_link); 715ef2a572bSKonstantin Belousov CK_LIST_REMOVE(i, sp_allh_link); 716ef2a572bSKonstantin Belousov mtx_unlock(&ipsec_accel_sav_tmp); 717ef2a572bSKonstantin Belousov NET_EPOCH_WAIT(); 718ef2a572bSKonstantin Belousov if ((i->flags & (IFP_HP_HANDLED | IFP_HP_REJECTED)) == 719ef2a572bSKonstantin Belousov IFP_HP_HANDLED) { 7209a7096ffSKonstantin Belousov dprintf("spd deinstall %s %p\n", if_name(i->ifp), sp); 721ef2a572bSKonstantin Belousov error = i->ifp->if_ipsec_accel_m->if_spddel(i->ifp, 722ef2a572bSKonstantin Belousov sp, i->ifdata); 723ef2a572bSKonstantin Belousov if (error != 0) { 7249a7096ffSKonstantin Belousov dprintf( 725ef2a572bSKonstantin Belousov "ipsec_accel_spddel: %s if_spddel %p res %d\n", 726ef2a572bSKonstantin Belousov if_name(i->ifp), sp, error); 727ef2a572bSKonstantin Belousov } 728ef2a572bSKonstantin Belousov } 729ef2a572bSKonstantin Belousov if_rele(i->ifp); 730ef2a572bSKonstantin Belousov free(i, M_IPSEC_MISC); 731ef2a572bSKonstantin Belousov mtx_lock(&ipsec_accel_sav_tmp); 732ef2a572bSKonstantin Belousov } 733ef2a572bSKonstantin Belousov mtx_unlock(&ipsec_accel_sav_tmp); 734ef2a572bSKonstantin Belousov key_freesp(&sp); 735ef2a572bSKonstantin Belousov CURVNET_RESTORE(); 736ef2a572bSKonstantin Belousov } 737ef2a572bSKonstantin Belousov 738ef2a572bSKonstantin Belousov void 739ef2a572bSKonstantin Belousov ipsec_accel_spddel_impl(struct secpolicy *sp) 740ef2a572bSKonstantin Belousov { 741ef2a572bSKonstantin Belousov struct ipsec_accel_adddel_sp_tq *tq; 742ef2a572bSKonstantin Belousov 743ef2a572bSKonstantin Belousov if (sp == NULL) 744ef2a572bSKonstantin Belousov return; 745ef2a572bSKonstantin Belousov 746ef2a572bSKonstantin Belousov tq = &sp->accel_del_tq; 747ef2a572bSKonstantin Belousov if (atomic_cmpset_int(&tq->adddel_scheduled, 0, 1) == 0) 748ef2a572bSKonstantin Belousov return; 749ef2a572bSKonstantin Belousov tq->adddel_vnet = curthread->td_vnet; 750ef2a572bSKonstantin Belousov TASK_INIT(&tq->adddel_task, 0, ipsec_accel_spddel_act, sp); 751ef2a572bSKonstantin Belousov key_addref(sp); 752ef2a572bSKonstantin Belousov taskqueue_enqueue(taskqueue_thread, &tq->adddel_task); 753ef2a572bSKonstantin Belousov } 754ef2a572bSKonstantin Belousov 755ef2a572bSKonstantin Belousov static void 756ef2a572bSKonstantin Belousov ipsec_accel_on_ifdown_sp(struct ifnet *ifp) 757ef2a572bSKonstantin Belousov { 758ef2a572bSKonstantin Belousov struct ifp_handle_sp *i, *marker; 759ef2a572bSKonstantin Belousov struct secpolicy *sp; 760ef2a572bSKonstantin Belousov int error; 761ef2a572bSKonstantin Belousov 762ef2a572bSKonstantin Belousov marker = malloc(sizeof(*marker), M_IPSEC_MISC, M_WAITOK | M_ZERO); 763ef2a572bSKonstantin Belousov marker->flags = IFP_HS_MARKER; 764ef2a572bSKonstantin Belousov 765ef2a572bSKonstantin Belousov mtx_lock(&ipsec_accel_sav_tmp); 766ef2a572bSKonstantin Belousov CK_LIST_INSERT_HEAD(&ipsec_accel_all_sp_handles, marker, 767ef2a572bSKonstantin Belousov sp_allh_link); 768ef2a572bSKonstantin Belousov for (;;) { 769ef2a572bSKonstantin Belousov i = CK_LIST_NEXT(marker, sp_allh_link); 770ef2a572bSKonstantin Belousov if (i == NULL) 771ef2a572bSKonstantin Belousov break; 772ef2a572bSKonstantin Belousov CK_LIST_REMOVE(marker, sp_allh_link); 773ef2a572bSKonstantin Belousov CK_LIST_INSERT_AFTER(i, marker, sp_allh_link); 774ef2a572bSKonstantin Belousov if (i->ifp != ifp) 775ef2a572bSKonstantin Belousov continue; 776ef2a572bSKonstantin Belousov 777ef2a572bSKonstantin Belousov sp = i->sp; 778ef2a572bSKonstantin Belousov key_addref(sp); 779ef2a572bSKonstantin Belousov CK_LIST_REMOVE(i, sp_link); 780ef2a572bSKonstantin Belousov CK_LIST_REMOVE(i, sp_allh_link); 781ef2a572bSKonstantin Belousov mtx_unlock(&ipsec_accel_sav_tmp); 782ef2a572bSKonstantin Belousov NET_EPOCH_WAIT(); 783ef2a572bSKonstantin Belousov if ((i->flags & (IFP_HP_HANDLED | IFP_HP_REJECTED)) == 784ef2a572bSKonstantin Belousov IFP_HP_HANDLED) { 7859a7096ffSKonstantin Belousov dprintf("spd deinstall %s %p\n", if_name(ifp), sp); 786ef2a572bSKonstantin Belousov error = ifp->if_ipsec_accel_m->if_spddel(ifp, 787ef2a572bSKonstantin Belousov sp, i->ifdata); 788ef2a572bSKonstantin Belousov } 789ef2a572bSKonstantin Belousov if (error != 0) { 7909a7096ffSKonstantin Belousov dprintf( 791ef2a572bSKonstantin Belousov "ipsec_accel_on_ifdown_sp: %s if_spddel %p res %d\n", 792ef2a572bSKonstantin Belousov if_name(ifp), sp, error); 793ef2a572bSKonstantin Belousov } 794ef2a572bSKonstantin Belousov key_freesp(&sp); 795ef2a572bSKonstantin Belousov if_rele(ifp); 796ef2a572bSKonstantin Belousov free(i, M_IPSEC_MISC); 797ef2a572bSKonstantin Belousov mtx_lock(&ipsec_accel_sav_tmp); 798ef2a572bSKonstantin Belousov } 799ef2a572bSKonstantin Belousov CK_LIST_REMOVE(marker, sp_allh_link); 800ef2a572bSKonstantin Belousov mtx_unlock(&ipsec_accel_sav_tmp); 801ef2a572bSKonstantin Belousov free(marker, M_IPSEC_MISC); 802ef2a572bSKonstantin Belousov } 803ef2a572bSKonstantin Belousov 80465f264dcSKonstantin Belousov static void 80565f264dcSKonstantin Belousov ipsec_accel_on_ifdown_impl(struct ifnet *ifp) 806ef2a572bSKonstantin Belousov { 807ef2a572bSKonstantin Belousov ipsec_accel_on_ifdown_sp(ifp); 808ef2a572bSKonstantin Belousov ipsec_accel_on_ifdown_sav(ifp); 809ef2a572bSKonstantin Belousov } 810ef2a572bSKonstantin Belousov 811*c4a0ee9bSKonstantin Belousov static void 812*c4a0ee9bSKonstantin Belousov ipsec_accel_ifdetach_event(void *arg __unused, struct ifnet *ifp) 813*c4a0ee9bSKonstantin Belousov { 814*c4a0ee9bSKonstantin Belousov if ((ifp->if_flags & IFF_RENAMING) != 0) 815*c4a0ee9bSKonstantin Belousov return; 816*c4a0ee9bSKonstantin Belousov ipsec_accel_on_ifdown_impl(ifp); 817*c4a0ee9bSKonstantin Belousov } 818*c4a0ee9bSKonstantin Belousov 819ef2a572bSKonstantin Belousov static bool 820ef2a572bSKonstantin Belousov ipsec_accel_output_pad(struct mbuf *m, struct secasvar *sav, int skip, int mtu) 821ef2a572bSKonstantin Belousov { 822ef2a572bSKonstantin Belousov int alen, blks, hlen, padding, rlen; 823ef2a572bSKonstantin Belousov 824ef2a572bSKonstantin Belousov rlen = m->m_pkthdr.len - skip; 825ef2a572bSKonstantin Belousov hlen = ((sav->flags & SADB_X_EXT_OLD) != 0 ? sizeof(struct esp) : 826ef2a572bSKonstantin Belousov sizeof(struct newesp)) + sav->ivlen; 827ef2a572bSKonstantin Belousov blks = MAX(4, SAV_ISCTR(sav) && VNET(esp_ctr_compatibility) ? 828ef2a572bSKonstantin Belousov sav->tdb_encalgxform->native_blocksize : 829ef2a572bSKonstantin Belousov sav->tdb_encalgxform->blocksize); 830ef2a572bSKonstantin Belousov padding = ((blks - ((rlen + 2) % blks)) % blks) + 2; 831ef2a572bSKonstantin Belousov alen = xform_ah_authsize(sav->tdb_authalgxform); 832ef2a572bSKonstantin Belousov 833ef2a572bSKonstantin Belousov return (skip + hlen + rlen + padding + alen <= mtu); 834ef2a572bSKonstantin Belousov } 835ef2a572bSKonstantin Belousov 836ef2a572bSKonstantin Belousov static bool 837ef2a572bSKonstantin Belousov ipsec_accel_output_tag(struct mbuf *m, u_int drv_spi) 838ef2a572bSKonstantin Belousov { 839ef2a572bSKonstantin Belousov struct ipsec_accel_out_tag *tag; 840ef2a572bSKonstantin Belousov 841ef2a572bSKonstantin Belousov tag = (struct ipsec_accel_out_tag *)m_tag_get( 842ef2a572bSKonstantin Belousov PACKET_TAG_IPSEC_ACCEL_OUT, sizeof(*tag), M_NOWAIT); 843ef2a572bSKonstantin Belousov if (tag == NULL) 844ef2a572bSKonstantin Belousov return (false); 845ef2a572bSKonstantin Belousov tag->drv_spi = drv_spi; 846ef2a572bSKonstantin Belousov m_tag_prepend(m, &tag->tag); 847ef2a572bSKonstantin Belousov return (true); 848ef2a572bSKonstantin Belousov } 849ef2a572bSKonstantin Belousov 850ef2a572bSKonstantin Belousov bool 851ef2a572bSKonstantin Belousov ipsec_accel_output(struct ifnet *ifp, struct mbuf *m, struct inpcb *inp, 852240b7bfeSKonstantin Belousov struct secpolicy *sp, struct secasvar *sav, int af, int mtu, int *hwassist) 853ef2a572bSKonstantin Belousov { 854ef2a572bSKonstantin Belousov struct ifp_handle_sav *i; 855ef2a572bSKonstantin Belousov struct ip *ip; 856e6e2c0a5SKonstantin Belousov struct tcpcb *tp; 857ef2a572bSKonstantin Belousov u_long ip_len, skip; 858e6e2c0a5SKonstantin Belousov bool res; 859ef2a572bSKonstantin Belousov 860240b7bfeSKonstantin Belousov *hwassist = 0; 861e6e2c0a5SKonstantin Belousov res = false; 862ef2a572bSKonstantin Belousov if (ifp == NULL) 863e6e2c0a5SKonstantin Belousov return (res); 864ef2a572bSKonstantin Belousov 865ef2a572bSKonstantin Belousov M_ASSERTPKTHDR(m); 866ef2a572bSKonstantin Belousov NET_EPOCH_ASSERT(); 867ef2a572bSKonstantin Belousov 868e6e2c0a5SKonstantin Belousov if (sav == NULL) { 869e6e2c0a5SKonstantin Belousov res = ipsec_accel_output_tag(m, IPSEC_ACCEL_DRV_SPI_BYPASS); 870e6e2c0a5SKonstantin Belousov goto out; 871e6e2c0a5SKonstantin Belousov } 872ef2a572bSKonstantin Belousov 873ef2a572bSKonstantin Belousov i = ipsec_accel_is_accel_sav_ptr(sav, ifp); 874ef2a572bSKonstantin Belousov if (i == NULL) 875e6e2c0a5SKonstantin Belousov goto out; 876ef2a572bSKonstantin Belousov 877ef2a572bSKonstantin Belousov if ((m->m_pkthdr.csum_flags & CSUM_TSO) == 0) { 878ef2a572bSKonstantin Belousov ip_len = m->m_pkthdr.len; 879ef2a572bSKonstantin Belousov if (ip_len + i->hdr_ext_size > mtu) 880e6e2c0a5SKonstantin Belousov goto out; 881ef2a572bSKonstantin Belousov switch (af) { 882ef2a572bSKonstantin Belousov case AF_INET: 883ef2a572bSKonstantin Belousov ip = mtod(m, struct ip *); 884ef2a572bSKonstantin Belousov skip = ip->ip_hl << 2; 885ef2a572bSKonstantin Belousov break; 886ef2a572bSKonstantin Belousov case AF_INET6: 887ef2a572bSKonstantin Belousov skip = sizeof(struct ip6_hdr); 888ef2a572bSKonstantin Belousov break; 889ef2a572bSKonstantin Belousov default: 890ef2a572bSKonstantin Belousov __unreachable(); 891ef2a572bSKonstantin Belousov } 892ef2a572bSKonstantin Belousov if (!ipsec_accel_output_pad(m, sav, skip, mtu)) 893e6e2c0a5SKonstantin Belousov goto out; 894ef2a572bSKonstantin Belousov } 895ef2a572bSKonstantin Belousov 896ef2a572bSKonstantin Belousov if (!ipsec_accel_output_tag(m, i->drv_spi)) 897e6e2c0a5SKonstantin Belousov goto out; 898ef2a572bSKonstantin Belousov 899ef2a572bSKonstantin Belousov ipsec_accel_sa_recordxfer(sav, m); 900ef2a572bSKonstantin Belousov key_freesav(&sav); 901ef2a572bSKonstantin Belousov if (sp != NULL) 902ef2a572bSKonstantin Belousov key_freesp(&sp); 903ef2a572bSKonstantin Belousov 904240b7bfeSKonstantin Belousov *hwassist = ifp->if_ipsec_accel_m->if_hwassist(ifp, sav, 905240b7bfeSKonstantin Belousov i->drv_spi, i->ifdata); 906e6e2c0a5SKonstantin Belousov res = true; 907e6e2c0a5SKonstantin Belousov out: 908e6e2c0a5SKonstantin Belousov if (inp != NULL && inp->inp_pcbinfo == &V_tcbinfo) { 909e6e2c0a5SKonstantin Belousov INP_WLOCK_ASSERT(inp); 910e6e2c0a5SKonstantin Belousov tp = (struct tcpcb *)inp; 911e6e2c0a5SKonstantin Belousov if (res && (*hwassist & (CSUM_TSO | CSUM_IP6_TSO)) != 0) { 912e6e2c0a5SKonstantin Belousov tp->t_flags2 |= TF2_IPSEC_TSO; 913e6e2c0a5SKonstantin Belousov } else { 914e6e2c0a5SKonstantin Belousov tp->t_flags2 &= ~TF2_IPSEC_TSO; 915e6e2c0a5SKonstantin Belousov } 916e6e2c0a5SKonstantin Belousov } 917e6e2c0a5SKonstantin Belousov return (res); 918ef2a572bSKonstantin Belousov } 919ef2a572bSKonstantin Belousov 920ef2a572bSKonstantin Belousov struct ipsec_accel_in_tag * 921ef2a572bSKonstantin Belousov ipsec_accel_input_tag_lookup(const struct mbuf *m) 922ef2a572bSKonstantin Belousov { 923ef2a572bSKonstantin Belousov struct ipsec_accel_in_tag *tag; 924ef2a572bSKonstantin Belousov struct m_tag *xtag; 925ef2a572bSKonstantin Belousov 926ef2a572bSKonstantin Belousov xtag = m_tag_find(__DECONST(struct mbuf *, m), 927ef2a572bSKonstantin Belousov PACKET_TAG_IPSEC_ACCEL_IN, NULL); 928ef2a572bSKonstantin Belousov if (xtag == NULL) 929ef2a572bSKonstantin Belousov return (NULL); 930ef2a572bSKonstantin Belousov tag = __containerof(xtag, struct ipsec_accel_in_tag, tag); 931ef2a572bSKonstantin Belousov return (tag); 932ef2a572bSKonstantin Belousov } 933ef2a572bSKonstantin Belousov 934ef2a572bSKonstantin Belousov int 935ef2a572bSKonstantin Belousov ipsec_accel_input(struct mbuf *m, int offset, int proto) 936ef2a572bSKonstantin Belousov { 937ef2a572bSKonstantin Belousov struct secasvar *sav; 938ef2a572bSKonstantin Belousov struct ipsec_accel_in_tag *tag; 939ef2a572bSKonstantin Belousov 940ef2a572bSKonstantin Belousov tag = ipsec_accel_input_tag_lookup(m); 941ef2a572bSKonstantin Belousov if (tag == NULL) 942ef2a572bSKonstantin Belousov return (ENXIO); 943ef2a572bSKonstantin Belousov 944ef2a572bSKonstantin Belousov if (tag->drv_spi < IPSEC_ACCEL_DRV_SPI_MIN || 945ef2a572bSKonstantin Belousov tag->drv_spi > IPSEC_ACCEL_DRV_SPI_MAX) { 9469a7096ffSKonstantin Belousov dprintf("if %s mbuf %p drv_spi %d invalid, packet dropped\n", 947ef2a572bSKonstantin Belousov (m->m_flags & M_PKTHDR) != 0 ? if_name(m->m_pkthdr.rcvif) : 948ef2a572bSKonstantin Belousov "<unknwn>", m, tag->drv_spi); 949ef2a572bSKonstantin Belousov m_freem(m); 950ef2a572bSKonstantin Belousov return (EINPROGRESS); 951ef2a572bSKonstantin Belousov } 952ef2a572bSKonstantin Belousov 953ef2a572bSKonstantin Belousov sav = ipsec_accel_drvspi_to_sa(tag->drv_spi); 954ef2a572bSKonstantin Belousov if (sav != NULL) 955ef2a572bSKonstantin Belousov ipsec_accel_sa_recordxfer(sav, m); 956ef2a572bSKonstantin Belousov return (0); 957ef2a572bSKonstantin Belousov } 958ef2a572bSKonstantin Belousov 959ef2a572bSKonstantin Belousov static void 960ef2a572bSKonstantin Belousov ipsec_accel_sa_recordxfer(struct secasvar *sav, struct mbuf *m) 961ef2a572bSKonstantin Belousov { 962ef2a572bSKonstantin Belousov counter_u64_add(sav->accel_lft_sw, 1); 963ef2a572bSKonstantin Belousov counter_u64_add(sav->accel_lft_sw + 1, m->m_pkthdr.len); 964ef2a572bSKonstantin Belousov if (sav->accel_firstused == 0) 965ef2a572bSKonstantin Belousov sav->accel_firstused = time_second; 966ef2a572bSKonstantin Belousov } 967ef2a572bSKonstantin Belousov 968ef2a572bSKonstantin Belousov static void 969ef2a572bSKonstantin Belousov ipsec_accel_sa_lifetime_update(struct seclifetime *lft_c, 970ef2a572bSKonstantin Belousov const struct seclifetime *lft_l) 971ef2a572bSKonstantin Belousov { 972ef2a572bSKonstantin Belousov lft_c->allocations += lft_l->allocations; 973ef2a572bSKonstantin Belousov lft_c->bytes += lft_l->bytes; 974ef2a572bSKonstantin Belousov lft_c->usetime = min(lft_c->usetime, lft_l->usetime); 975ef2a572bSKonstantin Belousov } 976ef2a572bSKonstantin Belousov 97765f264dcSKonstantin Belousov static void 97865f264dcSKonstantin Belousov ipsec_accel_drv_sa_lifetime_update_impl(struct secasvar *sav, if_t ifp, 979ef2a572bSKonstantin Belousov u_int drv_spi, uint64_t octets, uint64_t allocs) 980ef2a572bSKonstantin Belousov { 981ef2a572bSKonstantin Belousov struct epoch_tracker et; 982ef2a572bSKonstantin Belousov struct ifp_handle_sav *i; 983ef2a572bSKonstantin Belousov uint64_t odiff, adiff; 984ef2a572bSKonstantin Belousov 985ef2a572bSKonstantin Belousov NET_EPOCH_ENTER(et); 986ef2a572bSKonstantin Belousov mtx_lock(&ipsec_accel_cnt_lock); 987ef2a572bSKonstantin Belousov 988ef2a572bSKonstantin Belousov if (allocs != 0) { 989ef2a572bSKonstantin Belousov if (sav->firstused == 0) 990ef2a572bSKonstantin Belousov sav->firstused = time_second; 991ef2a572bSKonstantin Belousov if (sav->accel_firstused == 0) 992ef2a572bSKonstantin Belousov sav->accel_firstused = time_second; 993ef2a572bSKonstantin Belousov } 994ef2a572bSKonstantin Belousov 995ef2a572bSKonstantin Belousov CK_LIST_FOREACH(i, &sav->accel_ifps, sav_link) { 996ef2a572bSKonstantin Belousov if (i->ifp == ifp && i->drv_spi == drv_spi) 997ef2a572bSKonstantin Belousov break; 998ef2a572bSKonstantin Belousov } 999ef2a572bSKonstantin Belousov if (i == NULL) 1000ef2a572bSKonstantin Belousov goto out; 1001ef2a572bSKonstantin Belousov 1002ef2a572bSKonstantin Belousov odiff = octets - i->cnt_octets; 1003ef2a572bSKonstantin Belousov adiff = allocs - i->cnt_allocs; 1004ef2a572bSKonstantin Belousov 1005ef2a572bSKonstantin Belousov if (sav->lft_c != NULL) { 1006ef2a572bSKonstantin Belousov counter_u64_add(sav->lft_c_bytes, odiff); 1007ef2a572bSKonstantin Belousov counter_u64_add(sav->lft_c_allocations, adiff); 1008ef2a572bSKonstantin Belousov } 1009ef2a572bSKonstantin Belousov 1010ef2a572bSKonstantin Belousov i->cnt_octets = octets; 1011ef2a572bSKonstantin Belousov i->cnt_allocs = allocs; 1012ef2a572bSKonstantin Belousov sav->accel_hw_octets += odiff; 1013ef2a572bSKonstantin Belousov sav->accel_hw_allocs += adiff; 1014ef2a572bSKonstantin Belousov 1015ef2a572bSKonstantin Belousov out: 1016ef2a572bSKonstantin Belousov mtx_unlock(&ipsec_accel_cnt_lock); 1017ef2a572bSKonstantin Belousov NET_EPOCH_EXIT(et); 1018ef2a572bSKonstantin Belousov } 1019ef2a572bSKonstantin Belousov 1020ef2a572bSKonstantin Belousov static void 1021ef2a572bSKonstantin Belousov ipsec_accel_sa_lifetime_hw(struct secasvar *sav, if_t ifp, 1022ef2a572bSKonstantin Belousov struct seclifetime *lft) 1023ef2a572bSKonstantin Belousov { 1024ef2a572bSKonstantin Belousov struct ifp_handle_sav *i; 1025ef2a572bSKonstantin Belousov if_sa_cnt_fn_t p; 1026ef2a572bSKonstantin Belousov 1027ef2a572bSKonstantin Belousov IFNET_RLOCK_ASSERT(); 1028ef2a572bSKonstantin Belousov 1029ef2a572bSKonstantin Belousov i = ipsec_accel_is_accel_sav_ptr(sav, ifp); 1030ef2a572bSKonstantin Belousov if (i != NULL && (i->flags & (IFP_HS_HANDLED | IFP_HS_REJECTED)) == 1031ef2a572bSKonstantin Belousov IFP_HS_HANDLED) { 1032ef2a572bSKonstantin Belousov p = ifp->if_ipsec_accel_m->if_sa_cnt; 1033ef2a572bSKonstantin Belousov if (p != NULL) 1034ef2a572bSKonstantin Belousov p(ifp, sav, i->drv_spi, i->ifdata, lft); 1035ef2a572bSKonstantin Belousov } 1036ef2a572bSKonstantin Belousov } 1037ef2a572bSKonstantin Belousov 1038ef2a572bSKonstantin Belousov static int 1039ef2a572bSKonstantin Belousov ipsec_accel_sa_lifetime_op_impl(struct secasvar *sav, 1040ef2a572bSKonstantin Belousov struct seclifetime *lft_c, if_t ifp, enum IF_SA_CNT_WHICH op, 1041ef2a572bSKonstantin Belousov struct rm_priotracker *sahtree_trackerp) 1042ef2a572bSKonstantin Belousov { 1043ef2a572bSKonstantin Belousov struct seclifetime lft_l, lft_s; 1044ef2a572bSKonstantin Belousov struct ifp_handle_sav *i; 1045ef2a572bSKonstantin Belousov if_t ifp1; 1046ef2a572bSKonstantin Belousov if_sa_cnt_fn_t p; 1047ef2a572bSKonstantin Belousov int error; 1048ef2a572bSKonstantin Belousov 1049ef2a572bSKonstantin Belousov error = 0; 1050ef2a572bSKonstantin Belousov memset(&lft_l, 0, sizeof(lft_l)); 1051ef2a572bSKonstantin Belousov memset(&lft_s, 0, sizeof(lft_s)); 1052ef2a572bSKonstantin Belousov 1053ef2a572bSKonstantin Belousov switch (op & ~IF_SA_CNT_UPD) { 1054ef2a572bSKonstantin Belousov case IF_SA_CNT_IFP_HW_VAL: 1055ef2a572bSKonstantin Belousov ipsec_accel_sa_lifetime_hw(sav, ifp, &lft_l); 1056ef2a572bSKonstantin Belousov ipsec_accel_sa_lifetime_update(&lft_l, &lft_s); 1057ef2a572bSKonstantin Belousov break; 1058ef2a572bSKonstantin Belousov 1059ef2a572bSKonstantin Belousov case IF_SA_CNT_TOTAL_SW_VAL: 1060ef2a572bSKonstantin Belousov lft_l.allocations = (uint32_t)counter_u64_fetch( 1061ef2a572bSKonstantin Belousov sav->accel_lft_sw); 1062ef2a572bSKonstantin Belousov lft_l.bytes = counter_u64_fetch(sav->accel_lft_sw + 1); 1063ef2a572bSKonstantin Belousov lft_l.usetime = sav->accel_firstused; 1064ef2a572bSKonstantin Belousov break; 1065ef2a572bSKonstantin Belousov 1066ef2a572bSKonstantin Belousov case IF_SA_CNT_TOTAL_HW_VAL: 1067ef2a572bSKonstantin Belousov IFNET_RLOCK_ASSERT(); 1068ef2a572bSKonstantin Belousov CK_LIST_FOREACH(i, &sav->accel_ifps, sav_link) { 1069ef2a572bSKonstantin Belousov if ((i->flags & (IFP_HS_HANDLED | IFP_HS_REJECTED)) != 1070ef2a572bSKonstantin Belousov IFP_HS_HANDLED) 1071ef2a572bSKonstantin Belousov continue; 1072ef2a572bSKonstantin Belousov ifp1 = i->ifp; 1073ef2a572bSKonstantin Belousov p = ifp1->if_ipsec_accel_m->if_sa_cnt; 1074ef2a572bSKonstantin Belousov if (p == NULL) 1075ef2a572bSKonstantin Belousov continue; 1076ef2a572bSKonstantin Belousov memset(&lft_s, 0, sizeof(lft_s)); 1077ef2a572bSKonstantin Belousov if (sahtree_trackerp != NULL) 1078ef2a572bSKonstantin Belousov ipsec_sahtree_runlock(sahtree_trackerp); 1079ef2a572bSKonstantin Belousov error = p(ifp1, sav, i->drv_spi, i->ifdata, &lft_s); 1080ef2a572bSKonstantin Belousov if (sahtree_trackerp != NULL) 1081ef2a572bSKonstantin Belousov ipsec_sahtree_rlock(sahtree_trackerp); 1082ef2a572bSKonstantin Belousov if (error == 0) 1083ef2a572bSKonstantin Belousov ipsec_accel_sa_lifetime_update(&lft_l, &lft_s); 1084ef2a572bSKonstantin Belousov } 1085ef2a572bSKonstantin Belousov break; 1086ef2a572bSKonstantin Belousov } 1087ef2a572bSKonstantin Belousov 1088ef2a572bSKonstantin Belousov if (error == 0) { 1089ef2a572bSKonstantin Belousov if ((op & IF_SA_CNT_UPD) == 0) 1090ef2a572bSKonstantin Belousov memset(lft_c, 0, sizeof(*lft_c)); 1091ef2a572bSKonstantin Belousov ipsec_accel_sa_lifetime_update(lft_c, &lft_l); 1092ef2a572bSKonstantin Belousov } 1093ef2a572bSKonstantin Belousov 1094ef2a572bSKonstantin Belousov return (error); 1095ef2a572bSKonstantin Belousov } 1096ef2a572bSKonstantin Belousov 1097ef2a572bSKonstantin Belousov static void 1098ef2a572bSKonstantin Belousov ipsec_accel_sync_imp(void) 1099ef2a572bSKonstantin Belousov { 1100ef2a572bSKonstantin Belousov taskqueue_drain_all(taskqueue_thread); 1101ef2a572bSKonstantin Belousov } 1102ef2a572bSKonstantin Belousov 1103ef2a572bSKonstantin Belousov static struct mbuf * 1104ef2a572bSKonstantin Belousov ipsec_accel_key_setaccelif_impl(struct secasvar *sav) 1105ef2a572bSKonstantin Belousov { 1106ef2a572bSKonstantin Belousov struct mbuf *m, *m1; 1107ef2a572bSKonstantin Belousov struct ifp_handle_sav *i; 1108ef2a572bSKonstantin Belousov struct epoch_tracker et; 1109ef2a572bSKonstantin Belousov 1110ef2a572bSKonstantin Belousov if (sav->accel_ifname != NULL) 1111ef2a572bSKonstantin Belousov return (key_setaccelif(sav->accel_ifname)); 1112ef2a572bSKonstantin Belousov 1113ef2a572bSKonstantin Belousov m = m1 = NULL; 1114ef2a572bSKonstantin Belousov 1115ef2a572bSKonstantin Belousov NET_EPOCH_ENTER(et); 1116ef2a572bSKonstantin Belousov CK_LIST_FOREACH(i, &sav->accel_ifps, sav_link) { 1117ef2a572bSKonstantin Belousov if ((i->flags & (IFP_HS_HANDLED | IFP_HS_REJECTED)) == 1118ef2a572bSKonstantin Belousov IFP_HS_HANDLED) { 1119ef2a572bSKonstantin Belousov m1 = key_setaccelif(if_name(i->ifp)); 1120ef2a572bSKonstantin Belousov if (m == NULL) 1121ef2a572bSKonstantin Belousov m = m1; 1122ef2a572bSKonstantin Belousov else if (m1 != NULL) 1123ef2a572bSKonstantin Belousov m_cat(m, m1); 1124ef2a572bSKonstantin Belousov } 1125ef2a572bSKonstantin Belousov } 1126ef2a572bSKonstantin Belousov NET_EPOCH_EXIT(et); 1127ef2a572bSKonstantin Belousov return (m); 1128ef2a572bSKonstantin Belousov } 1129ef2a572bSKonstantin Belousov 1130ef2a572bSKonstantin Belousov #endif /* IPSEC_OFFLOAD */ 1131