1 /*- 2 * Copyright (c) 2021,2022 NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 */ 25 26 #include "opt_inet.h" 27 #include "opt_inet6.h" 28 #include "opt_ipsec.h" 29 30 #include <sys/param.h> 31 #include <sys/systm.h> 32 #include <sys/ck.h> 33 #include <sys/eventhandler.h> 34 #include <sys/kernel.h> 35 #include <sys/mbuf.h> 36 #include <sys/pctrie.h> 37 #include <sys/proc.h> 38 #include <sys/socket.h> 39 #include <sys/sysctl.h> 40 #include <sys/protosw.h> 41 #include <sys/taskqueue.h> 42 43 #include <machine/stdarg.h> 44 45 #include <net/if.h> 46 #include <net/if_var.h> 47 #include <net/vnet.h> 48 #include <netinet/in.h> 49 #include <netinet/ip.h> 50 #include <netinet/ip_var.h> 51 #include <netinet/ip6.h> 52 #include <netinet6/ip6_var.h> 53 #include <netinet/in_pcb.h> 54 #include <netinet/tcp_var.h> 55 56 #include <netipsec/key.h> 57 #include <netipsec/keydb.h> 58 #include <netipsec/key_debug.h> 59 #include <netipsec/xform.h> 60 #include <netipsec/ipsec.h> 61 #include <netipsec/ipsec_offload.h> 62 #include <netipsec/ah_var.h> 63 #include <netipsec/esp.h> 64 #include <netipsec/esp_var.h> 65 #include <netipsec/ipcomp_var.h> 66 67 #ifdef IPSEC_OFFLOAD 68 69 static struct mtx ipsec_accel_sav_tmp; 70 static struct unrhdr *drv_spi_unr; 71 static struct mtx ipsec_accel_cnt_lock; 72 73 struct ipsec_accel_install_newkey_tq { 74 struct secasvar *sav; 75 struct vnet *install_vnet; 76 struct task install_task; 77 }; 78 79 struct ipsec_accel_forget_tq { 80 struct vnet *forget_vnet; 81 struct task forget_task; 82 struct secasvar *sav; 83 }; 84 85 struct ifp_handle_sav { 86 CK_LIST_ENTRY(ifp_handle_sav) sav_link; 87 CK_LIST_ENTRY(ifp_handle_sav) sav_allh_link; 88 struct secasvar *sav; 89 struct ifnet *ifp; 90 void *ifdata; 91 uint64_t drv_spi; 92 uint32_t flags; 93 size_t hdr_ext_size; 94 uint64_t cnt_octets; 95 uint64_t cnt_allocs; 96 }; 97 98 #define IFP_HS_HANDLED 0x00000001 99 #define IFP_HS_REJECTED 0x00000002 100 #define IFP_HS_INPUT 0x00000004 101 #define IFP_HS_OUTPUT 0x00000008 102 #define IFP_HS_MARKER 0x00000010 103 104 static CK_LIST_HEAD(, ifp_handle_sav) ipsec_accel_all_sav_handles; 105 106 struct ifp_handle_sp { 107 CK_LIST_ENTRY(ifp_handle_sp) sp_link; 108 CK_LIST_ENTRY(ifp_handle_sp) sp_allh_link; 109 struct secpolicy *sp; 110 struct ifnet *ifp; 111 void *ifdata; 112 uint32_t flags; 113 }; 114 115 #define IFP_HP_HANDLED 0x00000001 116 #define IFP_HP_REJECTED 0x00000002 117 #define IFP_HP_MARKER 0x00000004 118 119 static CK_LIST_HEAD(, ifp_handle_sp) ipsec_accel_all_sp_handles; 120 121 static void * 122 drvspi_sa_trie_alloc(struct pctrie *ptree) 123 { 124 void *res; 125 126 res = malloc(pctrie_node_size(), M_IPSEC_MISC, M_ZERO | M_NOWAIT); 127 if (res != NULL) 128 pctrie_zone_init(res, 0, 0); 129 return (res); 130 } 131 132 static void 133 drvspi_sa_trie_free(struct pctrie *ptree, void *node) 134 { 135 free(node, M_IPSEC_MISC); 136 } 137 138 PCTRIE_DEFINE(DRVSPI_SA, ifp_handle_sav, drv_spi, 139 drvspi_sa_trie_alloc, drvspi_sa_trie_free); 140 static struct pctrie drv_spi_pctrie; 141 142 static eventhandler_tag ipsec_accel_ifdetach_event_tag; 143 144 static void ipsec_accel_sa_newkey_impl(struct secasvar *sav); 145 static int ipsec_accel_handle_sav(struct secasvar *sav, struct ifnet *ifp, 146 u_int drv_spi, void *priv, uint32_t flags, struct ifp_handle_sav **ires); 147 static void ipsec_accel_forget_sav_clear(struct secasvar *sav); 148 static struct ifp_handle_sav *ipsec_accel_is_accel_sav_ptr(struct secasvar *sav, 149 struct ifnet *ifp); 150 static int ipsec_accel_sa_lifetime_op_impl(struct secasvar *sav, 151 struct seclifetime *lft_c, if_t ifp, enum IF_SA_CNT_WHICH op, 152 struct rm_priotracker *sahtree_trackerp); 153 static void ipsec_accel_sa_recordxfer(struct secasvar *sav, struct mbuf *m); 154 static void ipsec_accel_sync_imp(void); 155 static bool ipsec_accel_is_accel_sav_impl(struct secasvar *sav); 156 static struct mbuf *ipsec_accel_key_setaccelif_impl(struct secasvar *sav); 157 static void ipsec_accel_on_ifdown_impl(struct ifnet *ifp); 158 static void ipsec_accel_drv_sa_lifetime_update_impl(struct secasvar *sav, 159 if_t ifp, u_int drv_spi, uint64_t octets, uint64_t allocs); 160 static int ipsec_accel_drv_sa_lifetime_fetch_impl(struct secasvar *sav, 161 if_t ifp, u_int drv_spi, uint64_t *octets, uint64_t *allocs); 162 static void ipsec_accel_ifdetach_event(void *arg, struct ifnet *ifp); 163 164 static void 165 ipsec_accel_init(void *arg) 166 { 167 mtx_init(&ipsec_accel_sav_tmp, "ipasat", MTX_DEF, 0); 168 mtx_init(&ipsec_accel_cnt_lock, "ipascn", MTX_DEF, 0); 169 drv_spi_unr = new_unrhdr(IPSEC_ACCEL_DRV_SPI_MIN, 170 IPSEC_ACCEL_DRV_SPI_MAX, &ipsec_accel_sav_tmp); 171 ipsec_accel_sa_newkey_p = ipsec_accel_sa_newkey_impl; 172 ipsec_accel_forget_sav_p = ipsec_accel_forget_sav_impl; 173 ipsec_accel_spdadd_p = ipsec_accel_spdadd_impl; 174 ipsec_accel_spddel_p = ipsec_accel_spddel_impl; 175 ipsec_accel_sa_lifetime_op_p = ipsec_accel_sa_lifetime_op_impl; 176 ipsec_accel_sync_p = ipsec_accel_sync_imp; 177 ipsec_accel_is_accel_sav_p = ipsec_accel_is_accel_sav_impl; 178 ipsec_accel_key_setaccelif_p = ipsec_accel_key_setaccelif_impl; 179 ipsec_accel_on_ifdown_p = ipsec_accel_on_ifdown_impl; 180 ipsec_accel_drv_sa_lifetime_update_p = 181 ipsec_accel_drv_sa_lifetime_update_impl; 182 ipsec_accel_drv_sa_lifetime_fetch_p = 183 ipsec_accel_drv_sa_lifetime_fetch_impl; 184 pctrie_init(&drv_spi_pctrie); 185 ipsec_accel_ifdetach_event_tag = EVENTHANDLER_REGISTER( 186 ifnet_departure_event, ipsec_accel_ifdetach_event, NULL, 187 EVENTHANDLER_PRI_ANY); 188 } 189 SYSINIT(ipsec_accel_init, SI_SUB_VNET_DONE, SI_ORDER_ANY, 190 ipsec_accel_init, NULL); 191 192 static void 193 ipsec_accel_fini(void *arg) 194 { 195 EVENTHANDLER_DEREGISTER(ifnet_departure_event, 196 ipsec_accel_ifdetach_event_tag); 197 ipsec_accel_sa_newkey_p = NULL; 198 ipsec_accel_forget_sav_p = NULL; 199 ipsec_accel_spdadd_p = NULL; 200 ipsec_accel_spddel_p = NULL; 201 ipsec_accel_sa_lifetime_op_p = NULL; 202 ipsec_accel_sync_p = NULL; 203 ipsec_accel_is_accel_sav_p = NULL; 204 ipsec_accel_key_setaccelif_p = NULL; 205 ipsec_accel_on_ifdown_p = NULL; 206 ipsec_accel_drv_sa_lifetime_update_p = NULL; 207 ipsec_accel_drv_sa_lifetime_fetch_p = NULL; 208 ipsec_accel_sync_imp(); 209 clean_unrhdr(drv_spi_unr); /* avoid panic, should go later */ 210 clear_unrhdr(drv_spi_unr); 211 delete_unrhdr(drv_spi_unr); 212 mtx_destroy(&ipsec_accel_sav_tmp); 213 mtx_destroy(&ipsec_accel_cnt_lock); 214 } 215 SYSUNINIT(ipsec_accel_fini, SI_SUB_VNET_DONE, SI_ORDER_ANY, 216 ipsec_accel_fini, NULL); 217 218 SYSCTL_NODE(_net_inet_ipsec, OID_AUTO, offload, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 219 ""); 220 221 static bool ipsec_offload_verbose = false; 222 SYSCTL_BOOL(_net_inet_ipsec_offload, OID_AUTO, verbose, CTLFLAG_RW, 223 &ipsec_offload_verbose, 0, 224 "Verbose SA/SP offload install and deinstall"); 225 226 static void 227 dprintf(const char *fmt, ...) 228 { 229 va_list ap; 230 231 if (!ipsec_offload_verbose) 232 return; 233 234 va_start(ap, fmt); 235 vprintf(fmt, ap); 236 va_end(ap); 237 } 238 239 static void 240 ipsec_accel_alloc_forget_tq(struct secasvar *sav) 241 { 242 void *ftq; 243 244 if (sav->accel_forget_tq != 0) 245 return; 246 247 ftq = malloc(sizeof(struct ipsec_accel_forget_tq), M_TEMP, M_WAITOK); 248 if (!atomic_cmpset_ptr(&sav->accel_forget_tq, 0, (uintptr_t)ftq)) 249 free(ftq, M_TEMP); 250 } 251 252 static bool 253 ipsec_accel_sa_install_match(if_t ifp, void *arg) 254 { 255 if ((ifp->if_capenable2 & IFCAP2_BIT(IFCAP2_IPSEC_OFFLOAD)) == 0) 256 return (false); 257 if (ifp->if_ipsec_accel_m->if_sa_newkey == NULL) { 258 dprintf("driver bug ifp %s if_sa_newkey NULL\n", 259 if_name(ifp)); 260 return (false); 261 } 262 return (true); 263 } 264 265 static int 266 ipsec_accel_sa_newkey_cb(if_t ifp, void *arg) 267 { 268 struct ipsec_accel_install_newkey_tq *tq; 269 void *priv; 270 u_int drv_spi; 271 int error; 272 273 tq = arg; 274 275 dprintf("ipsec_accel_sa_newkey_act: ifp %s h %p spi %#x " 276 "flags %#x seq %d\n", 277 if_name(ifp), ifp->if_ipsec_accel_m->if_sa_newkey, 278 be32toh(tq->sav->spi), tq->sav->flags, tq->sav->seq); 279 priv = NULL; 280 drv_spi = alloc_unr(drv_spi_unr); 281 if (tq->sav->accel_ifname != NULL && 282 strcmp(tq->sav->accel_ifname, if_name(ifp)) != 0) { 283 error = ipsec_accel_handle_sav(tq->sav, 284 ifp, drv_spi, priv, IFP_HS_REJECTED, NULL); 285 goto out; 286 } 287 if (drv_spi == -1) { 288 /* XXXKIB */ 289 dprintf("ipsec_accel_sa_install_newkey: cannot alloc " 290 "drv_spi if %s spi %#x\n", if_name(ifp), 291 be32toh(tq->sav->spi)); 292 return (ENOMEM); 293 } 294 error = ifp->if_ipsec_accel_m->if_sa_newkey(ifp, tq->sav, 295 drv_spi, &priv); 296 if (error != 0) { 297 if (error == EOPNOTSUPP) { 298 dprintf("ipsec_accel_sa_newkey: driver " 299 "refused sa if %s spi %#x\n", 300 if_name(ifp), be32toh(tq->sav->spi)); 301 error = ipsec_accel_handle_sav(tq->sav, 302 ifp, drv_spi, priv, IFP_HS_REJECTED, NULL); 303 /* XXXKIB */ 304 } else { 305 dprintf("ipsec_accel_sa_newkey: driver " 306 "error %d if %s spi %#x\n", 307 error, if_name(ifp), be32toh(tq->sav->spi)); 308 /* XXXKIB */ 309 } 310 } else { 311 error = ipsec_accel_handle_sav(tq->sav, ifp, 312 drv_spi, priv, IFP_HS_HANDLED, NULL); 313 if (error != 0) { 314 /* XXXKIB */ 315 dprintf("ipsec_accel_sa_newkey: handle_sav " 316 "err %d if %s spi %#x\n", error, 317 if_name(ifp), be32toh(tq->sav->spi)); 318 } 319 } 320 out: 321 return (error); 322 } 323 324 static void 325 ipsec_accel_sa_newkey_act(void *context, int pending) 326 { 327 struct ipsec_accel_install_newkey_tq *tq; 328 void *tqf; 329 struct secasvar *sav; 330 331 tq = context; 332 tqf = NULL; 333 sav = tq->sav; 334 CURVNET_SET(tq->install_vnet); 335 mtx_lock(&ipsec_accel_sav_tmp); 336 if ((sav->accel_flags & (SADB_KEY_ACCEL_INST | 337 SADB_KEY_ACCEL_DEINST)) == 0 && 338 sav->state == SADB_SASTATE_MATURE) { 339 sav->accel_flags |= SADB_KEY_ACCEL_INST; 340 mtx_unlock(&ipsec_accel_sav_tmp); 341 if_foreach_sleep(ipsec_accel_sa_install_match, context, 342 ipsec_accel_sa_newkey_cb, context); 343 ipsec_accel_alloc_forget_tq(sav); 344 mtx_lock(&ipsec_accel_sav_tmp); 345 346 /* 347 * If ipsec_accel_forget_sav() raced with us and set 348 * the flag, do its work. Its task cannot execute in 349 * parallel since taskqueue_thread is single-threaded. 350 */ 351 if ((sav->accel_flags & SADB_KEY_ACCEL_DEINST) != 0) { 352 tqf = (void *)sav->accel_forget_tq; 353 sav->accel_forget_tq = 0; 354 ipsec_accel_forget_sav_clear(sav); 355 } 356 } 357 mtx_unlock(&ipsec_accel_sav_tmp); 358 key_freesav(&tq->sav); 359 CURVNET_RESTORE(); 360 free(tq, M_TEMP); 361 free(tqf, M_TEMP); 362 } 363 364 static void 365 ipsec_accel_sa_newkey_impl(struct secasvar *sav) 366 { 367 struct ipsec_accel_install_newkey_tq *tq; 368 369 if ((sav->accel_flags & (SADB_KEY_ACCEL_INST | 370 SADB_KEY_ACCEL_DEINST)) != 0) 371 return; 372 373 dprintf( 374 "ipsec_accel_sa_install_newkey: spi %#x flags %#x seq %d\n", 375 be32toh(sav->spi), sav->flags, sav->seq); 376 377 tq = malloc(sizeof(*tq), M_TEMP, M_NOWAIT); 378 if (tq == NULL) { 379 dprintf("ipsec_accel_sa_install_newkey: no memory for tq, " 380 "spi %#x\n", be32toh(sav->spi)); 381 /* XXXKIB */ 382 return; 383 } 384 385 refcount_acquire(&sav->refcnt); 386 387 TASK_INIT(&tq->install_task, 0, ipsec_accel_sa_newkey_act, tq); 388 tq->sav = sav; 389 tq->install_vnet = curthread->td_vnet; /* XXXKIB liveness */ 390 taskqueue_enqueue(taskqueue_thread, &tq->install_task); 391 } 392 393 static int 394 ipsec_accel_handle_sav(struct secasvar *sav, struct ifnet *ifp, 395 u_int drv_spi, void *priv, uint32_t flags, struct ifp_handle_sav **ires) 396 { 397 struct ifp_handle_sav *ihs, *i; 398 int error; 399 400 MPASS(__bitcount(flags & (IFP_HS_HANDLED | IFP_HS_REJECTED)) == 1); 401 402 ihs = malloc(sizeof(*ihs), M_IPSEC_MISC, M_WAITOK | M_ZERO); 403 ihs->ifp = ifp; 404 ihs->sav = sav; 405 ihs->drv_spi = drv_spi; 406 ihs->ifdata = priv; 407 ihs->flags = flags; 408 if ((flags & IFP_HS_OUTPUT) != 0) 409 ihs->hdr_ext_size = esp_hdrsiz(sav); 410 mtx_lock(&ipsec_accel_sav_tmp); 411 CK_LIST_FOREACH(i, &sav->accel_ifps, sav_link) { 412 if (i->ifp == ifp) { 413 error = EALREADY; 414 goto errout; 415 } 416 } 417 error = DRVSPI_SA_PCTRIE_INSERT(&drv_spi_pctrie, ihs); 418 if (error != 0) 419 goto errout; 420 if_ref(ihs->ifp); 421 CK_LIST_INSERT_HEAD(&sav->accel_ifps, ihs, sav_link); 422 CK_LIST_INSERT_HEAD(&ipsec_accel_all_sav_handles, ihs, sav_allh_link); 423 mtx_unlock(&ipsec_accel_sav_tmp); 424 if (ires != NULL) 425 *ires = ihs; 426 return (0); 427 errout: 428 mtx_unlock(&ipsec_accel_sav_tmp); 429 free(ihs, M_IPSEC_MISC); 430 if (ires != NULL) 431 *ires = NULL; 432 return (error); 433 } 434 435 static void 436 ipsec_accel_forget_handle_sav(struct ifp_handle_sav *i, bool freesav) 437 { 438 struct ifnet *ifp; 439 struct secasvar *sav; 440 441 mtx_assert(&ipsec_accel_sav_tmp, MA_OWNED); 442 443 CK_LIST_REMOVE(i, sav_link); 444 CK_LIST_REMOVE(i, sav_allh_link); 445 DRVSPI_SA_PCTRIE_REMOVE(&drv_spi_pctrie, i->drv_spi); 446 mtx_unlock(&ipsec_accel_sav_tmp); 447 NET_EPOCH_WAIT(); 448 ifp = i->ifp; 449 sav = i->sav; 450 if ((i->flags & (IFP_HS_HANDLED | IFP_HS_REJECTED)) == 451 IFP_HS_HANDLED) { 452 dprintf("sa deinstall %s %p spi %#x ifl %#x\n", 453 if_name(ifp), sav, be32toh(sav->spi), i->flags); 454 ifp->if_ipsec_accel_m->if_sa_deinstall(ifp, 455 i->drv_spi, i->ifdata); 456 } 457 if_rele(ifp); 458 free_unr(drv_spi_unr, i->drv_spi); 459 free(i, M_IPSEC_MISC); 460 if (freesav) 461 key_freesav(&sav); 462 mtx_lock(&ipsec_accel_sav_tmp); 463 } 464 465 static void 466 ipsec_accel_forget_sav_clear(struct secasvar *sav) 467 { 468 struct ifp_handle_sav *i; 469 470 for (;;) { 471 i = CK_LIST_FIRST(&sav->accel_ifps); 472 if (i == NULL) 473 break; 474 ipsec_accel_forget_handle_sav(i, false); 475 } 476 } 477 478 static void 479 ipsec_accel_forget_sav_act(void *arg, int pending) 480 { 481 struct ipsec_accel_forget_tq *tq; 482 struct secasvar *sav; 483 484 tq = arg; 485 sav = tq->sav; 486 CURVNET_SET(tq->forget_vnet); 487 mtx_lock(&ipsec_accel_sav_tmp); 488 ipsec_accel_forget_sav_clear(sav); 489 mtx_unlock(&ipsec_accel_sav_tmp); 490 key_freesav(&sav); 491 CURVNET_RESTORE(); 492 free(tq, M_TEMP); 493 } 494 495 void 496 ipsec_accel_forget_sav_impl(struct secasvar *sav) 497 { 498 struct ipsec_accel_forget_tq *tq; 499 500 mtx_lock(&ipsec_accel_sav_tmp); 501 sav->accel_flags |= SADB_KEY_ACCEL_DEINST; 502 tq = (void *)atomic_load_ptr(&sav->accel_forget_tq); 503 if (tq == NULL || !atomic_cmpset_ptr(&sav->accel_forget_tq, 504 (uintptr_t)tq, 0)) { 505 mtx_unlock(&ipsec_accel_sav_tmp); 506 return; 507 } 508 mtx_unlock(&ipsec_accel_sav_tmp); 509 510 refcount_acquire(&sav->refcnt); 511 TASK_INIT(&tq->forget_task, 0, ipsec_accel_forget_sav_act, tq); 512 tq->forget_vnet = curthread->td_vnet; 513 tq->sav = sav; 514 taskqueue_enqueue(taskqueue_thread, &tq->forget_task); 515 } 516 517 static void 518 ipsec_accel_on_ifdown_sav(struct ifnet *ifp) 519 { 520 struct ifp_handle_sav *i, *marker; 521 522 marker = malloc(sizeof(*marker), M_IPSEC_MISC, M_WAITOK | M_ZERO); 523 marker->flags = IFP_HS_MARKER; 524 525 mtx_lock(&ipsec_accel_sav_tmp); 526 CK_LIST_INSERT_HEAD(&ipsec_accel_all_sav_handles, marker, 527 sav_allh_link); 528 for (;;) { 529 i = CK_LIST_NEXT(marker, sav_allh_link); 530 if (i == NULL) 531 break; 532 CK_LIST_REMOVE(marker, sav_allh_link); 533 CK_LIST_INSERT_AFTER(i, marker, sav_allh_link); 534 if (i->ifp == ifp) { 535 refcount_acquire(&i->sav->refcnt); /* XXXKIB wrap ? */ 536 ipsec_accel_forget_handle_sav(i, true); 537 } 538 } 539 CK_LIST_REMOVE(marker, sav_allh_link); 540 mtx_unlock(&ipsec_accel_sav_tmp); 541 free(marker, M_IPSEC_MISC); 542 } 543 544 static struct ifp_handle_sav * 545 ipsec_accel_is_accel_sav_ptr_raw(struct secasvar *sav, struct ifnet *ifp) 546 { 547 struct ifp_handle_sav *i; 548 549 if ((ifp->if_capenable2 & IFCAP2_BIT(IFCAP2_IPSEC_OFFLOAD)) == 0) 550 return (NULL); 551 CK_LIST_FOREACH(i, &sav->accel_ifps, sav_link) { 552 if (i->ifp == ifp) 553 return (i); 554 } 555 return (NULL); 556 } 557 558 static struct ifp_handle_sav * 559 ipsec_accel_is_accel_sav_ptr(struct secasvar *sav, struct ifnet *ifp) 560 { 561 NET_EPOCH_ASSERT(); 562 return (ipsec_accel_is_accel_sav_ptr_raw(sav, ifp)); 563 } 564 565 static bool 566 ipsec_accel_is_accel_sav_impl(struct secasvar *sav) 567 { 568 return (!CK_LIST_EMPTY(&sav->accel_ifps)); 569 } 570 571 static struct secasvar * 572 ipsec_accel_drvspi_to_sa(u_int drv_spi) 573 { 574 struct ifp_handle_sav *i; 575 576 i = DRVSPI_SA_PCTRIE_LOOKUP(&drv_spi_pctrie, drv_spi); 577 if (i == NULL) 578 return (NULL); 579 return (i->sav); 580 } 581 582 static struct ifp_handle_sp * 583 ipsec_accel_find_accel_sp(struct secpolicy *sp, if_t ifp) 584 { 585 struct ifp_handle_sp *i; 586 587 CK_LIST_FOREACH(i, &sp->accel_ifps, sp_link) { 588 if (i->ifp == ifp) 589 return (i); 590 } 591 return (NULL); 592 } 593 594 static bool 595 ipsec_accel_is_accel_sp(struct secpolicy *sp, if_t ifp) 596 { 597 return (ipsec_accel_find_accel_sp(sp, ifp) != NULL); 598 } 599 600 static int 601 ipsec_accel_remember_sp(struct secpolicy *sp, if_t ifp, 602 struct ifp_handle_sp **ip) 603 { 604 struct ifp_handle_sp *i; 605 606 i = malloc(sizeof(*i), M_IPSEC_MISC, M_WAITOK | M_ZERO); 607 i->sp = sp; 608 i->ifp = ifp; 609 if_ref(ifp); 610 i->flags = IFP_HP_HANDLED; 611 mtx_lock(&ipsec_accel_sav_tmp); 612 CK_LIST_INSERT_HEAD(&sp->accel_ifps, i, sp_link); 613 CK_LIST_INSERT_HEAD(&ipsec_accel_all_sp_handles, i, sp_allh_link); 614 mtx_unlock(&ipsec_accel_sav_tmp); 615 *ip = i; 616 return (0); 617 } 618 619 static bool 620 ipsec_accel_spdadd_match(if_t ifp, void *arg) 621 { 622 struct secpolicy *sp; 623 624 if ((ifp->if_capenable2 & IFCAP2_BIT(IFCAP2_IPSEC_OFFLOAD)) == 0 || 625 ifp->if_ipsec_accel_m->if_spdadd == NULL) 626 return (false); 627 sp = arg; 628 if (sp->accel_ifname != NULL && 629 strcmp(sp->accel_ifname, if_name(ifp)) != 0) 630 return (false); 631 if (ipsec_accel_is_accel_sp(sp, ifp)) 632 return (false); 633 return (true); 634 } 635 636 static int 637 ipsec_accel_spdadd_cb(if_t ifp, void *arg) 638 { 639 struct secpolicy *sp; 640 struct inpcb *inp; 641 struct ifp_handle_sp *i; 642 int error; 643 644 sp = arg; 645 inp = sp->ipsec_accel_add_sp_inp; 646 dprintf("ipsec_accel_spdadd_cb: ifp %s m %p sp %p inp %p\n", 647 if_name(ifp), ifp->if_ipsec_accel_m->if_spdadd, sp, inp); 648 error = ipsec_accel_remember_sp(sp, ifp, &i); 649 if (error != 0) { 650 dprintf("ipsec_accel_spdadd: %s if_spdadd %p remember res %d\n", 651 if_name(ifp), sp, error); 652 return (error); 653 } 654 error = ifp->if_ipsec_accel_m->if_spdadd(ifp, sp, inp, &i->ifdata); 655 if (error != 0) { 656 i->flags |= IFP_HP_REJECTED; 657 dprintf("ipsec_accel_spdadd: %s if_spdadd %p res %d\n", 658 if_name(ifp), sp, error); 659 } 660 return (error); 661 } 662 663 static void 664 ipsec_accel_spdadd_act(void *arg, int pending) 665 { 666 struct secpolicy *sp; 667 struct inpcb *inp; 668 669 sp = arg; 670 CURVNET_SET(sp->accel_add_tq.adddel_vnet); 671 if_foreach_sleep(ipsec_accel_spdadd_match, arg, 672 ipsec_accel_spdadd_cb, arg); 673 inp = sp->ipsec_accel_add_sp_inp; 674 if (inp != NULL) { 675 INP_WLOCK(inp); 676 if (!in_pcbrele_wlocked(inp)) 677 INP_WUNLOCK(inp); 678 sp->ipsec_accel_add_sp_inp = NULL; 679 } 680 CURVNET_RESTORE(); 681 key_freesp(&sp); 682 } 683 684 void 685 ipsec_accel_spdadd_impl(struct secpolicy *sp, struct inpcb *inp) 686 { 687 struct ipsec_accel_adddel_sp_tq *tq; 688 689 if (sp == NULL) 690 return; 691 if (sp->tcount == 0 && inp == NULL) 692 return; 693 tq = &sp->accel_add_tq; 694 if (atomic_cmpset_int(&tq->adddel_scheduled, 0, 1) == 0) 695 return; 696 tq->adddel_vnet = curthread->td_vnet; 697 sp->ipsec_accel_add_sp_inp = inp; 698 if (inp != NULL) 699 in_pcbref(inp); 700 TASK_INIT(&tq->adddel_task, 0, ipsec_accel_spdadd_act, sp); 701 key_addref(sp); 702 taskqueue_enqueue(taskqueue_thread, &tq->adddel_task); 703 } 704 705 static void 706 ipsec_accel_spddel_act(void *arg, int pending) 707 { 708 struct ifp_handle_sp *i; 709 struct secpolicy *sp; 710 int error; 711 712 sp = arg; 713 CURVNET_SET(sp->accel_del_tq.adddel_vnet); 714 mtx_lock(&ipsec_accel_sav_tmp); 715 for (;;) { 716 i = CK_LIST_FIRST(&sp->accel_ifps); 717 if (i == NULL) 718 break; 719 CK_LIST_REMOVE(i, sp_link); 720 CK_LIST_REMOVE(i, sp_allh_link); 721 mtx_unlock(&ipsec_accel_sav_tmp); 722 NET_EPOCH_WAIT(); 723 if ((i->flags & (IFP_HP_HANDLED | IFP_HP_REJECTED)) == 724 IFP_HP_HANDLED) { 725 dprintf("spd deinstall %s %p\n", if_name(i->ifp), sp); 726 error = i->ifp->if_ipsec_accel_m->if_spddel(i->ifp, 727 sp, i->ifdata); 728 if (error != 0) { 729 dprintf( 730 "ipsec_accel_spddel: %s if_spddel %p res %d\n", 731 if_name(i->ifp), sp, error); 732 } 733 } 734 if_rele(i->ifp); 735 free(i, M_IPSEC_MISC); 736 mtx_lock(&ipsec_accel_sav_tmp); 737 } 738 mtx_unlock(&ipsec_accel_sav_tmp); 739 key_freesp(&sp); 740 CURVNET_RESTORE(); 741 } 742 743 void 744 ipsec_accel_spddel_impl(struct secpolicy *sp) 745 { 746 struct ipsec_accel_adddel_sp_tq *tq; 747 748 if (sp == NULL) 749 return; 750 751 tq = &sp->accel_del_tq; 752 if (atomic_cmpset_int(&tq->adddel_scheduled, 0, 1) == 0) 753 return; 754 tq->adddel_vnet = curthread->td_vnet; 755 TASK_INIT(&tq->adddel_task, 0, ipsec_accel_spddel_act, sp); 756 key_addref(sp); 757 taskqueue_enqueue(taskqueue_thread, &tq->adddel_task); 758 } 759 760 static void 761 ipsec_accel_on_ifdown_sp(struct ifnet *ifp) 762 { 763 struct ifp_handle_sp *i, *marker; 764 struct secpolicy *sp; 765 int error; 766 767 marker = malloc(sizeof(*marker), M_IPSEC_MISC, M_WAITOK | M_ZERO); 768 marker->flags = IFP_HS_MARKER; 769 770 mtx_lock(&ipsec_accel_sav_tmp); 771 CK_LIST_INSERT_HEAD(&ipsec_accel_all_sp_handles, marker, 772 sp_allh_link); 773 for (;;) { 774 i = CK_LIST_NEXT(marker, sp_allh_link); 775 if (i == NULL) 776 break; 777 CK_LIST_REMOVE(marker, sp_allh_link); 778 CK_LIST_INSERT_AFTER(i, marker, sp_allh_link); 779 if (i->ifp != ifp) 780 continue; 781 782 sp = i->sp; 783 key_addref(sp); 784 CK_LIST_REMOVE(i, sp_link); 785 CK_LIST_REMOVE(i, sp_allh_link); 786 mtx_unlock(&ipsec_accel_sav_tmp); 787 NET_EPOCH_WAIT(); 788 if ((i->flags & (IFP_HP_HANDLED | IFP_HP_REJECTED)) == 789 IFP_HP_HANDLED) { 790 dprintf("spd deinstall %s %p\n", if_name(ifp), sp); 791 error = ifp->if_ipsec_accel_m->if_spddel(ifp, 792 sp, i->ifdata); 793 } 794 if (error != 0) { 795 dprintf( 796 "ipsec_accel_on_ifdown_sp: %s if_spddel %p res %d\n", 797 if_name(ifp), sp, error); 798 } 799 key_freesp(&sp); 800 if_rele(ifp); 801 free(i, M_IPSEC_MISC); 802 mtx_lock(&ipsec_accel_sav_tmp); 803 } 804 CK_LIST_REMOVE(marker, sp_allh_link); 805 mtx_unlock(&ipsec_accel_sav_tmp); 806 free(marker, M_IPSEC_MISC); 807 } 808 809 static void 810 ipsec_accel_on_ifdown_impl(struct ifnet *ifp) 811 { 812 ipsec_accel_on_ifdown_sp(ifp); 813 ipsec_accel_on_ifdown_sav(ifp); 814 } 815 816 static void 817 ipsec_accel_ifdetach_event(void *arg __unused, struct ifnet *ifp) 818 { 819 if ((ifp->if_flags & IFF_RENAMING) != 0) 820 return; 821 ipsec_accel_on_ifdown_impl(ifp); 822 } 823 824 static bool 825 ipsec_accel_output_pad(struct mbuf *m, struct secasvar *sav, int skip, int mtu) 826 { 827 int alen, blks, hlen, padding, rlen; 828 829 rlen = m->m_pkthdr.len - skip; 830 hlen = ((sav->flags & SADB_X_EXT_OLD) != 0 ? sizeof(struct esp) : 831 sizeof(struct newesp)) + sav->ivlen; 832 blks = MAX(4, SAV_ISCTR(sav) && VNET(esp_ctr_compatibility) ? 833 sav->tdb_encalgxform->native_blocksize : 834 sav->tdb_encalgxform->blocksize); 835 padding = ((blks - ((rlen + 2) % blks)) % blks) + 2; 836 alen = xform_ah_authsize(sav->tdb_authalgxform); 837 838 return (skip + hlen + rlen + padding + alen <= mtu); 839 } 840 841 static bool 842 ipsec_accel_output_tag(struct mbuf *m, u_int drv_spi) 843 { 844 struct ipsec_accel_out_tag *tag; 845 846 tag = (struct ipsec_accel_out_tag *)m_tag_get( 847 PACKET_TAG_IPSEC_ACCEL_OUT, sizeof(*tag), M_NOWAIT); 848 if (tag == NULL) 849 return (false); 850 tag->drv_spi = drv_spi; 851 m_tag_prepend(m, &tag->tag); 852 return (true); 853 } 854 855 bool 856 ipsec_accel_output(struct ifnet *ifp, struct mbuf *m, struct inpcb *inp, 857 struct secpolicy *sp, struct secasvar *sav, int af, int mtu, int *hwassist) 858 { 859 struct ifp_handle_sav *i; 860 struct ip *ip; 861 struct tcpcb *tp; 862 u_long ip_len, skip; 863 bool res; 864 865 *hwassist = 0; 866 res = false; 867 if (ifp == NULL) 868 return (res); 869 870 M_ASSERTPKTHDR(m); 871 NET_EPOCH_ASSERT(); 872 873 if (sav == NULL) { 874 res = ipsec_accel_output_tag(m, IPSEC_ACCEL_DRV_SPI_BYPASS); 875 goto out; 876 } 877 878 i = ipsec_accel_is_accel_sav_ptr(sav, ifp); 879 if (i == NULL) 880 goto out; 881 882 if ((m->m_pkthdr.csum_flags & CSUM_TSO) == 0) { 883 ip_len = m->m_pkthdr.len; 884 if (ip_len + i->hdr_ext_size > mtu) 885 goto out; 886 switch (af) { 887 case AF_INET: 888 ip = mtod(m, struct ip *); 889 skip = ip->ip_hl << 2; 890 break; 891 case AF_INET6: 892 skip = sizeof(struct ip6_hdr); 893 break; 894 default: 895 __unreachable(); 896 } 897 if (!ipsec_accel_output_pad(m, sav, skip, mtu)) 898 goto out; 899 } 900 901 if (!ipsec_accel_output_tag(m, i->drv_spi)) 902 goto out; 903 904 ipsec_accel_sa_recordxfer(sav, m); 905 key_freesav(&sav); 906 if (sp != NULL) 907 key_freesp(&sp); 908 909 *hwassist = ifp->if_ipsec_accel_m->if_hwassist(ifp, sav, 910 i->drv_spi, i->ifdata); 911 res = true; 912 out: 913 if (inp != NULL && inp->inp_pcbinfo == &V_tcbinfo) { 914 INP_WLOCK_ASSERT(inp); 915 tp = (struct tcpcb *)inp; 916 if (res && (*hwassist & (CSUM_TSO | CSUM_IP6_TSO)) != 0) { 917 tp->t_flags2 |= TF2_IPSEC_TSO; 918 } else { 919 tp->t_flags2 &= ~TF2_IPSEC_TSO; 920 } 921 } 922 return (res); 923 } 924 925 struct ipsec_accel_in_tag * 926 ipsec_accel_input_tag_lookup(const struct mbuf *m) 927 { 928 struct ipsec_accel_in_tag *tag; 929 struct m_tag *xtag; 930 931 xtag = m_tag_find(__DECONST(struct mbuf *, m), 932 PACKET_TAG_IPSEC_ACCEL_IN, NULL); 933 if (xtag == NULL) 934 return (NULL); 935 tag = __containerof(xtag, struct ipsec_accel_in_tag, tag); 936 return (tag); 937 } 938 939 int 940 ipsec_accel_input(struct mbuf *m, int offset, int proto) 941 { 942 struct secasvar *sav; 943 struct ipsec_accel_in_tag *tag; 944 945 tag = ipsec_accel_input_tag_lookup(m); 946 if (tag == NULL) 947 return (ENXIO); 948 949 if (tag->drv_spi < IPSEC_ACCEL_DRV_SPI_MIN || 950 tag->drv_spi > IPSEC_ACCEL_DRV_SPI_MAX) { 951 dprintf("if %s mbuf %p drv_spi %d invalid, packet dropped\n", 952 (m->m_flags & M_PKTHDR) != 0 ? if_name(m->m_pkthdr.rcvif) : 953 "<unknwn>", m, tag->drv_spi); 954 m_freem(m); 955 return (EINPROGRESS); 956 } 957 958 sav = ipsec_accel_drvspi_to_sa(tag->drv_spi); 959 if (sav != NULL) 960 ipsec_accel_sa_recordxfer(sav, m); 961 return (0); 962 } 963 964 static void 965 ipsec_accel_sa_recordxfer(struct secasvar *sav, struct mbuf *m) 966 { 967 counter_u64_add(sav->accel_lft_sw, 1); 968 counter_u64_add(sav->accel_lft_sw + 1, m->m_pkthdr.len); 969 if (sav->accel_firstused == 0) 970 sav->accel_firstused = time_second; 971 } 972 973 static void 974 ipsec_accel_sa_lifetime_update(struct seclifetime *lft_c, 975 const struct seclifetime *lft_l) 976 { 977 lft_c->allocations += lft_l->allocations; 978 lft_c->bytes += lft_l->bytes; 979 lft_c->usetime = min(lft_c->usetime, lft_l->usetime); 980 } 981 982 static void 983 ipsec_accel_drv_sa_lifetime_update_impl(struct secasvar *sav, if_t ifp, 984 u_int drv_spi, uint64_t octets, uint64_t allocs) 985 { 986 struct epoch_tracker et; 987 struct ifp_handle_sav *i; 988 uint64_t odiff, adiff; 989 990 NET_EPOCH_ENTER(et); 991 mtx_lock(&ipsec_accel_cnt_lock); 992 993 if (allocs != 0) { 994 if (sav->firstused == 0) 995 sav->firstused = time_second; 996 if (sav->accel_firstused == 0) 997 sav->accel_firstused = time_second; 998 } 999 1000 CK_LIST_FOREACH(i, &sav->accel_ifps, sav_link) { 1001 if (i->ifp == ifp && i->drv_spi == drv_spi) 1002 break; 1003 } 1004 if (i == NULL) 1005 goto out; 1006 1007 odiff = octets - i->cnt_octets; 1008 adiff = allocs - i->cnt_allocs; 1009 1010 if (sav->lft_c != NULL) { 1011 counter_u64_add(sav->lft_c_bytes, odiff); 1012 counter_u64_add(sav->lft_c_allocations, adiff); 1013 } 1014 1015 i->cnt_octets = octets; 1016 i->cnt_allocs = allocs; 1017 sav->accel_hw_octets += odiff; 1018 sav->accel_hw_allocs += adiff; 1019 1020 out: 1021 mtx_unlock(&ipsec_accel_cnt_lock); 1022 NET_EPOCH_EXIT(et); 1023 } 1024 1025 static int 1026 ipsec_accel_drv_sa_lifetime_fetch_impl(struct secasvar *sav, 1027 if_t ifp, u_int drv_spi, uint64_t *octets, uint64_t *allocs) 1028 { 1029 struct ifp_handle_sav *i; 1030 int error; 1031 1032 NET_EPOCH_ASSERT(); 1033 error = 0; 1034 1035 mtx_lock(&ipsec_accel_cnt_lock); 1036 CK_LIST_FOREACH(i, &sav->accel_ifps, sav_link) { 1037 if (i->ifp == ifp && i->drv_spi == drv_spi) { 1038 *octets = i->cnt_octets; 1039 *allocs = i->cnt_allocs; 1040 break; 1041 } 1042 } 1043 if (i == NULL) 1044 error = ENOENT; 1045 mtx_unlock(&ipsec_accel_cnt_lock); 1046 return (error); 1047 } 1048 1049 static void 1050 ipsec_accel_sa_lifetime_hw(struct secasvar *sav, if_t ifp, 1051 struct seclifetime *lft) 1052 { 1053 struct ifp_handle_sav *i; 1054 if_sa_cnt_fn_t p; 1055 1056 IFNET_RLOCK_ASSERT(); 1057 1058 i = ipsec_accel_is_accel_sav_ptr(sav, ifp); 1059 if (i != NULL && (i->flags & (IFP_HS_HANDLED | IFP_HS_REJECTED)) == 1060 IFP_HS_HANDLED) { 1061 p = ifp->if_ipsec_accel_m->if_sa_cnt; 1062 if (p != NULL) 1063 p(ifp, sav, i->drv_spi, i->ifdata, lft); 1064 } 1065 } 1066 1067 static int 1068 ipsec_accel_sa_lifetime_op_impl(struct secasvar *sav, 1069 struct seclifetime *lft_c, if_t ifp, enum IF_SA_CNT_WHICH op, 1070 struct rm_priotracker *sahtree_trackerp) 1071 { 1072 struct seclifetime lft_l, lft_s; 1073 struct ifp_handle_sav *i; 1074 if_t ifp1; 1075 if_sa_cnt_fn_t p; 1076 int error; 1077 1078 error = 0; 1079 memset(&lft_l, 0, sizeof(lft_l)); 1080 memset(&lft_s, 0, sizeof(lft_s)); 1081 1082 switch (op & ~IF_SA_CNT_UPD) { 1083 case IF_SA_CNT_IFP_HW_VAL: 1084 ipsec_accel_sa_lifetime_hw(sav, ifp, &lft_l); 1085 ipsec_accel_sa_lifetime_update(&lft_l, &lft_s); 1086 break; 1087 1088 case IF_SA_CNT_TOTAL_SW_VAL: 1089 lft_l.allocations = (uint32_t)counter_u64_fetch( 1090 sav->accel_lft_sw); 1091 lft_l.bytes = counter_u64_fetch(sav->accel_lft_sw + 1); 1092 lft_l.usetime = sav->accel_firstused; 1093 break; 1094 1095 case IF_SA_CNT_TOTAL_HW_VAL: 1096 IFNET_RLOCK_ASSERT(); 1097 CK_LIST_FOREACH(i, &sav->accel_ifps, sav_link) { 1098 if ((i->flags & (IFP_HS_HANDLED | IFP_HS_REJECTED)) != 1099 IFP_HS_HANDLED) 1100 continue; 1101 ifp1 = i->ifp; 1102 p = ifp1->if_ipsec_accel_m->if_sa_cnt; 1103 if (p == NULL) 1104 continue; 1105 memset(&lft_s, 0, sizeof(lft_s)); 1106 if (sahtree_trackerp != NULL) 1107 ipsec_sahtree_runlock(sahtree_trackerp); 1108 error = p(ifp1, sav, i->drv_spi, i->ifdata, &lft_s); 1109 if (sahtree_trackerp != NULL) 1110 ipsec_sahtree_rlock(sahtree_trackerp); 1111 if (error == 0) 1112 ipsec_accel_sa_lifetime_update(&lft_l, &lft_s); 1113 } 1114 break; 1115 } 1116 1117 if (error == 0) { 1118 if ((op & IF_SA_CNT_UPD) == 0) 1119 memset(lft_c, 0, sizeof(*lft_c)); 1120 ipsec_accel_sa_lifetime_update(lft_c, &lft_l); 1121 } 1122 1123 return (error); 1124 } 1125 1126 static void 1127 ipsec_accel_sync_imp(void) 1128 { 1129 taskqueue_drain_all(taskqueue_thread); 1130 } 1131 1132 static struct mbuf * 1133 ipsec_accel_key_setaccelif_impl(struct secasvar *sav) 1134 { 1135 struct mbuf *m, *m1; 1136 struct ifp_handle_sav *i; 1137 struct epoch_tracker et; 1138 1139 if (sav->accel_ifname != NULL) 1140 return (key_setaccelif(sav->accel_ifname)); 1141 1142 m = m1 = NULL; 1143 1144 NET_EPOCH_ENTER(et); 1145 CK_LIST_FOREACH(i, &sav->accel_ifps, sav_link) { 1146 if ((i->flags & (IFP_HS_HANDLED | IFP_HS_REJECTED)) == 1147 IFP_HS_HANDLED) { 1148 m1 = key_setaccelif(if_name(i->ifp)); 1149 if (m == NULL) 1150 m = m1; 1151 else if (m1 != NULL) 1152 m_cat(m, m1); 1153 } 1154 } 1155 NET_EPOCH_EXIT(et); 1156 return (m); 1157 } 1158 1159 #endif /* IPSEC_OFFLOAD */ 1160