1 /*- 2 * Copyright (c) 2021,2022 NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 */ 25 26 #include "opt_inet.h" 27 #include "opt_inet6.h" 28 #include "opt_ipsec.h" 29 30 #include <sys/param.h> 31 #include <sys/systm.h> 32 #include <sys/ck.h> 33 #include <sys/kernel.h> 34 #include <sys/mbuf.h> 35 #include <sys/pctrie.h> 36 #include <sys/proc.h> 37 #include <sys/socket.h> 38 #include <sys/sysctl.h> 39 #include <sys/protosw.h> 40 #include <sys/taskqueue.h> 41 42 #include <machine/stdarg.h> 43 44 #include <net/if.h> 45 #include <net/if_var.h> 46 #include <net/vnet.h> 47 #include <netinet/in.h> 48 #include <netinet/ip.h> 49 #include <netinet/ip_var.h> 50 #include <netinet/ip6.h> 51 #include <netinet6/ip6_var.h> 52 #include <netinet/in_pcb.h> 53 #include <netinet/tcp_var.h> 54 55 #include <netipsec/key.h> 56 #include <netipsec/keydb.h> 57 #include <netipsec/key_debug.h> 58 #include <netipsec/xform.h> 59 #include <netipsec/ipsec.h> 60 #include <netipsec/ipsec_offload.h> 61 #include <netipsec/ah_var.h> 62 #include <netipsec/esp.h> 63 #include <netipsec/esp_var.h> 64 #include <netipsec/ipcomp_var.h> 65 66 #ifdef IPSEC_OFFLOAD 67 68 static struct mtx ipsec_accel_sav_tmp; 69 static struct unrhdr *drv_spi_unr; 70 static struct mtx ipsec_accel_cnt_lock; 71 72 struct ipsec_accel_install_newkey_tq { 73 struct secasvar *sav; 74 struct vnet *install_vnet; 75 struct task install_task; 76 }; 77 78 struct ipsec_accel_forget_tq { 79 struct vnet *forget_vnet; 80 struct task forget_task; 81 struct secasvar *sav; 82 }; 83 84 struct ifp_handle_sav { 85 CK_LIST_ENTRY(ifp_handle_sav) sav_link; 86 CK_LIST_ENTRY(ifp_handle_sav) sav_allh_link; 87 struct secasvar *sav; 88 struct ifnet *ifp; 89 void *ifdata; 90 uint64_t drv_spi; 91 uint32_t flags; 92 size_t hdr_ext_size; 93 uint64_t cnt_octets; 94 uint64_t cnt_allocs; 95 }; 96 97 #define IFP_HS_HANDLED 0x00000001 98 #define IFP_HS_REJECTED 0x00000002 99 #define IFP_HS_INPUT 0x00000004 100 #define IFP_HS_OUTPUT 0x00000008 101 #define IFP_HS_MARKER 0x00000010 102 103 static CK_LIST_HEAD(, ifp_handle_sav) ipsec_accel_all_sav_handles; 104 105 struct ifp_handle_sp { 106 CK_LIST_ENTRY(ifp_handle_sp) sp_link; 107 CK_LIST_ENTRY(ifp_handle_sp) sp_allh_link; 108 struct secpolicy *sp; 109 struct ifnet *ifp; 110 void *ifdata; 111 uint32_t flags; 112 }; 113 114 #define IFP_HP_HANDLED 0x00000001 115 #define IFP_HP_REJECTED 0x00000002 116 #define IFP_HP_MARKER 0x00000004 117 118 static CK_LIST_HEAD(, ifp_handle_sp) ipsec_accel_all_sp_handles; 119 120 static void * 121 drvspi_sa_trie_alloc(struct pctrie *ptree) 122 { 123 void *res; 124 125 res = malloc(pctrie_node_size(), M_IPSEC_MISC, M_ZERO | M_NOWAIT); 126 if (res != NULL) 127 pctrie_zone_init(res, 0, 0); 128 return (res); 129 } 130 131 static void 132 drvspi_sa_trie_free(struct pctrie *ptree, void *node) 133 { 134 free(node, M_IPSEC_MISC); 135 } 136 137 PCTRIE_DEFINE(DRVSPI_SA, ifp_handle_sav, drv_spi, 138 drvspi_sa_trie_alloc, drvspi_sa_trie_free); 139 static struct pctrie drv_spi_pctrie; 140 141 static void ipsec_accel_sa_newkey_impl(struct secasvar *sav); 142 static int ipsec_accel_handle_sav(struct secasvar *sav, struct ifnet *ifp, 143 u_int drv_spi, void *priv, uint32_t flags, struct ifp_handle_sav **ires); 144 static void ipsec_accel_forget_sav_clear(struct secasvar *sav); 145 static struct ifp_handle_sav *ipsec_accel_is_accel_sav_ptr(struct secasvar *sav, 146 struct ifnet *ifp); 147 static int ipsec_accel_sa_lifetime_op_impl(struct secasvar *sav, 148 struct seclifetime *lft_c, if_t ifp, enum IF_SA_CNT_WHICH op, 149 struct rm_priotracker *sahtree_trackerp); 150 static void ipsec_accel_sa_recordxfer(struct secasvar *sav, struct mbuf *m); 151 static void ipsec_accel_sync_imp(void); 152 static bool ipsec_accel_is_accel_sav_impl(struct secasvar *sav); 153 static struct mbuf *ipsec_accel_key_setaccelif_impl(struct secasvar *sav); 154 static void ipsec_accel_on_ifdown_impl(struct ifnet *ifp); 155 static void ipsec_accel_drv_sa_lifetime_update_impl(struct secasvar *sav, 156 if_t ifp, u_int drv_spi, uint64_t octets, uint64_t allocs); 157 158 static void 159 ipsec_accel_init(void *arg) 160 { 161 mtx_init(&ipsec_accel_sav_tmp, "ipasat", MTX_DEF, 0); 162 mtx_init(&ipsec_accel_cnt_lock, "ipascn", MTX_DEF, 0); 163 drv_spi_unr = new_unrhdr(IPSEC_ACCEL_DRV_SPI_MIN, 164 IPSEC_ACCEL_DRV_SPI_MAX, &ipsec_accel_sav_tmp); 165 ipsec_accel_sa_newkey_p = ipsec_accel_sa_newkey_impl; 166 ipsec_accel_forget_sav_p = ipsec_accel_forget_sav_impl; 167 ipsec_accel_spdadd_p = ipsec_accel_spdadd_impl; 168 ipsec_accel_spddel_p = ipsec_accel_spddel_impl; 169 ipsec_accel_sa_lifetime_op_p = ipsec_accel_sa_lifetime_op_impl; 170 ipsec_accel_sync_p = ipsec_accel_sync_imp; 171 ipsec_accel_is_accel_sav_p = ipsec_accel_is_accel_sav_impl; 172 ipsec_accel_key_setaccelif_p = ipsec_accel_key_setaccelif_impl; 173 ipsec_accel_on_ifdown_p = ipsec_accel_on_ifdown_impl; 174 ipsec_accel_drv_sa_lifetime_update_p = 175 ipsec_accel_drv_sa_lifetime_update_impl; 176 pctrie_init(&drv_spi_pctrie); 177 } 178 SYSINIT(ipsec_accel_init, SI_SUB_VNET_DONE, SI_ORDER_ANY, 179 ipsec_accel_init, NULL); 180 181 static void 182 ipsec_accel_fini(void *arg) 183 { 184 ipsec_accel_sa_newkey_p = NULL; 185 ipsec_accel_forget_sav_p = NULL; 186 ipsec_accel_spdadd_p = NULL; 187 ipsec_accel_spddel_p = NULL; 188 ipsec_accel_sa_lifetime_op_p = NULL; 189 ipsec_accel_sync_p = NULL; 190 ipsec_accel_is_accel_sav_p = NULL; 191 ipsec_accel_key_setaccelif_p = NULL; 192 ipsec_accel_on_ifdown_p = NULL; 193 ipsec_accel_drv_sa_lifetime_update_p = NULL; 194 ipsec_accel_sync_imp(); 195 clean_unrhdr(drv_spi_unr); /* avoid panic, should go later */ 196 clear_unrhdr(drv_spi_unr); 197 delete_unrhdr(drv_spi_unr); 198 mtx_destroy(&ipsec_accel_sav_tmp); 199 mtx_destroy(&ipsec_accel_cnt_lock); 200 } 201 SYSUNINIT(ipsec_accel_fini, SI_SUB_VNET_DONE, SI_ORDER_ANY, 202 ipsec_accel_fini, NULL); 203 204 SYSCTL_NODE(_net_inet_ipsec, OID_AUTO, offload, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 205 ""); 206 207 static bool ipsec_offload_verbose = false; 208 SYSCTL_BOOL(_net_inet_ipsec_offload, OID_AUTO, verbose, CTLFLAG_RW, 209 &ipsec_offload_verbose, 0, 210 "Verbose SA/SP offload install and deinstall"); 211 212 static void 213 dprintf(const char *fmt, ...) 214 { 215 va_list ap; 216 217 if (!ipsec_offload_verbose) 218 return; 219 220 va_start(ap, fmt); 221 vprintf(fmt, ap); 222 va_end(ap); 223 } 224 225 static void 226 ipsec_accel_alloc_forget_tq(struct secasvar *sav) 227 { 228 void *ftq; 229 230 if (sav->accel_forget_tq != 0) 231 return; 232 233 ftq = malloc(sizeof(struct ipsec_accel_forget_tq), M_TEMP, M_WAITOK); 234 if (!atomic_cmpset_ptr(&sav->accel_forget_tq, 0, (uintptr_t)ftq)) 235 free(ftq, M_TEMP); 236 } 237 238 static bool 239 ipsec_accel_sa_install_match(if_t ifp, void *arg) 240 { 241 if ((ifp->if_capenable2 & IFCAP2_BIT(IFCAP2_IPSEC_OFFLOAD)) == 0) 242 return (false); 243 if (ifp->if_ipsec_accel_m->if_sa_newkey == NULL) { 244 dprintf("driver bug ifp %s if_sa_newkey NULL\n", 245 if_name(ifp)); 246 return (false); 247 } 248 return (true); 249 } 250 251 static int 252 ipsec_accel_sa_newkey_cb(if_t ifp, void *arg) 253 { 254 struct ipsec_accel_install_newkey_tq *tq; 255 void *priv; 256 u_int drv_spi; 257 int error; 258 259 tq = arg; 260 261 dprintf("ipsec_accel_sa_newkey_act: ifp %s h %p spi %#x " 262 "flags %#x seq %d\n", 263 if_name(ifp), ifp->if_ipsec_accel_m->if_sa_newkey, 264 be32toh(tq->sav->spi), tq->sav->flags, tq->sav->seq); 265 priv = NULL; 266 drv_spi = alloc_unr(drv_spi_unr); 267 if (tq->sav->accel_ifname != NULL && 268 strcmp(tq->sav->accel_ifname, if_name(ifp)) != 0) { 269 error = ipsec_accel_handle_sav(tq->sav, 270 ifp, drv_spi, priv, IFP_HS_REJECTED, NULL); 271 goto out; 272 } 273 if (drv_spi == -1) { 274 /* XXXKIB */ 275 dprintf("ipsec_accel_sa_install_newkey: cannot alloc " 276 "drv_spi if %s spi %#x\n", if_name(ifp), 277 be32toh(tq->sav->spi)); 278 return (ENOMEM); 279 } 280 error = ifp->if_ipsec_accel_m->if_sa_newkey(ifp, tq->sav, 281 drv_spi, &priv); 282 if (error != 0) { 283 if (error == EOPNOTSUPP) { 284 dprintf("ipsec_accel_sa_newkey: driver " 285 "refused sa if %s spi %#x\n", 286 if_name(ifp), be32toh(tq->sav->spi)); 287 error = ipsec_accel_handle_sav(tq->sav, 288 ifp, drv_spi, priv, IFP_HS_REJECTED, NULL); 289 /* XXXKIB */ 290 } else { 291 dprintf("ipsec_accel_sa_newkey: driver " 292 "error %d if %s spi %#x\n", 293 error, if_name(ifp), be32toh(tq->sav->spi)); 294 /* XXXKIB */ 295 } 296 } else { 297 error = ipsec_accel_handle_sav(tq->sav, ifp, 298 drv_spi, priv, IFP_HS_HANDLED, NULL); 299 if (error != 0) { 300 /* XXXKIB */ 301 dprintf("ipsec_accel_sa_newkey: handle_sav " 302 "err %d if %s spi %#x\n", error, 303 if_name(ifp), be32toh(tq->sav->spi)); 304 } 305 } 306 out: 307 return (error); 308 } 309 310 static void 311 ipsec_accel_sa_newkey_act(void *context, int pending) 312 { 313 struct ipsec_accel_install_newkey_tq *tq; 314 void *tqf; 315 struct secasvar *sav; 316 317 tq = context; 318 tqf = NULL; 319 sav = tq->sav; 320 CURVNET_SET(tq->install_vnet); 321 mtx_lock(&ipsec_accel_sav_tmp); 322 if ((sav->accel_flags & (SADB_KEY_ACCEL_INST | 323 SADB_KEY_ACCEL_DEINST)) == 0 && 324 sav->state == SADB_SASTATE_MATURE) { 325 sav->accel_flags |= SADB_KEY_ACCEL_INST; 326 mtx_unlock(&ipsec_accel_sav_tmp); 327 if_foreach_sleep(ipsec_accel_sa_install_match, context, 328 ipsec_accel_sa_newkey_cb, context); 329 ipsec_accel_alloc_forget_tq(sav); 330 mtx_lock(&ipsec_accel_sav_tmp); 331 332 /* 333 * If ipsec_accel_forget_sav() raced with us and set 334 * the flag, do its work. Its task cannot execute in 335 * parallel since taskqueue_thread is single-threaded. 336 */ 337 if ((sav->accel_flags & SADB_KEY_ACCEL_DEINST) != 0) { 338 tqf = (void *)sav->accel_forget_tq; 339 sav->accel_forget_tq = 0; 340 ipsec_accel_forget_sav_clear(sav); 341 } 342 } 343 mtx_unlock(&ipsec_accel_sav_tmp); 344 key_freesav(&tq->sav); 345 CURVNET_RESTORE(); 346 free(tq, M_TEMP); 347 free(tqf, M_TEMP); 348 } 349 350 static void 351 ipsec_accel_sa_newkey_impl(struct secasvar *sav) 352 { 353 struct ipsec_accel_install_newkey_tq *tq; 354 355 if ((sav->accel_flags & (SADB_KEY_ACCEL_INST | 356 SADB_KEY_ACCEL_DEINST)) != 0) 357 return; 358 359 dprintf( 360 "ipsec_accel_sa_install_newkey: spi %#x flags %#x seq %d\n", 361 be32toh(sav->spi), sav->flags, sav->seq); 362 363 tq = malloc(sizeof(*tq), M_TEMP, M_NOWAIT); 364 if (tq == NULL) { 365 dprintf("ipsec_accel_sa_install_newkey: no memory for tq, " 366 "spi %#x\n", be32toh(sav->spi)); 367 /* XXXKIB */ 368 return; 369 } 370 371 refcount_acquire(&sav->refcnt); 372 373 TASK_INIT(&tq->install_task, 0, ipsec_accel_sa_newkey_act, tq); 374 tq->sav = sav; 375 tq->install_vnet = curthread->td_vnet; /* XXXKIB liveness */ 376 taskqueue_enqueue(taskqueue_thread, &tq->install_task); 377 } 378 379 static int 380 ipsec_accel_handle_sav(struct secasvar *sav, struct ifnet *ifp, 381 u_int drv_spi, void *priv, uint32_t flags, struct ifp_handle_sav **ires) 382 { 383 struct ifp_handle_sav *ihs, *i; 384 int error; 385 386 MPASS(__bitcount(flags & (IFP_HS_HANDLED | IFP_HS_REJECTED)) == 1); 387 388 ihs = malloc(sizeof(*ihs), M_IPSEC_MISC, M_WAITOK | M_ZERO); 389 ihs->ifp = ifp; 390 ihs->sav = sav; 391 ihs->drv_spi = drv_spi; 392 ihs->ifdata = priv; 393 ihs->flags = flags; 394 if ((flags & IFP_HS_OUTPUT) != 0) 395 ihs->hdr_ext_size = esp_hdrsiz(sav); 396 mtx_lock(&ipsec_accel_sav_tmp); 397 CK_LIST_FOREACH(i, &sav->accel_ifps, sav_link) { 398 if (i->ifp == ifp) { 399 error = EALREADY; 400 goto errout; 401 } 402 } 403 error = DRVSPI_SA_PCTRIE_INSERT(&drv_spi_pctrie, ihs); 404 if (error != 0) 405 goto errout; 406 if_ref(ihs->ifp); 407 CK_LIST_INSERT_HEAD(&sav->accel_ifps, ihs, sav_link); 408 CK_LIST_INSERT_HEAD(&ipsec_accel_all_sav_handles, ihs, sav_allh_link); 409 mtx_unlock(&ipsec_accel_sav_tmp); 410 if (ires != NULL) 411 *ires = ihs; 412 return (0); 413 errout: 414 mtx_unlock(&ipsec_accel_sav_tmp); 415 free(ihs, M_IPSEC_MISC); 416 if (ires != NULL) 417 *ires = NULL; 418 return (error); 419 } 420 421 static void 422 ipsec_accel_forget_handle_sav(struct ifp_handle_sav *i, bool freesav) 423 { 424 struct ifnet *ifp; 425 struct secasvar *sav; 426 427 mtx_assert(&ipsec_accel_sav_tmp, MA_OWNED); 428 429 CK_LIST_REMOVE(i, sav_link); 430 CK_LIST_REMOVE(i, sav_allh_link); 431 DRVSPI_SA_PCTRIE_REMOVE(&drv_spi_pctrie, i->drv_spi); 432 mtx_unlock(&ipsec_accel_sav_tmp); 433 NET_EPOCH_WAIT(); 434 ifp = i->ifp; 435 sav = i->sav; 436 if ((i->flags & (IFP_HS_HANDLED | IFP_HS_REJECTED)) == 437 IFP_HS_HANDLED) { 438 dprintf("sa deinstall %s %p spi %#x ifl %#x\n", 439 if_name(ifp), sav, be32toh(sav->spi), i->flags); 440 ifp->if_ipsec_accel_m->if_sa_deinstall(ifp, 441 i->drv_spi, i->ifdata); 442 } 443 if_rele(ifp); 444 free_unr(drv_spi_unr, i->drv_spi); 445 free(i, M_IPSEC_MISC); 446 if (freesav) 447 key_freesav(&sav); 448 mtx_lock(&ipsec_accel_sav_tmp); 449 } 450 451 static void 452 ipsec_accel_forget_sav_clear(struct secasvar *sav) 453 { 454 struct ifp_handle_sav *i; 455 456 for (;;) { 457 i = CK_LIST_FIRST(&sav->accel_ifps); 458 if (i == NULL) 459 break; 460 ipsec_accel_forget_handle_sav(i, false); 461 } 462 } 463 464 static void 465 ipsec_accel_forget_sav_act(void *arg, int pending) 466 { 467 struct ipsec_accel_forget_tq *tq; 468 struct secasvar *sav; 469 470 tq = arg; 471 sav = tq->sav; 472 CURVNET_SET(tq->forget_vnet); 473 mtx_lock(&ipsec_accel_sav_tmp); 474 ipsec_accel_forget_sav_clear(sav); 475 mtx_unlock(&ipsec_accel_sav_tmp); 476 key_freesav(&sav); 477 CURVNET_RESTORE(); 478 free(tq, M_TEMP); 479 } 480 481 void 482 ipsec_accel_forget_sav_impl(struct secasvar *sav) 483 { 484 struct ipsec_accel_forget_tq *tq; 485 486 mtx_lock(&ipsec_accel_sav_tmp); 487 sav->accel_flags |= SADB_KEY_ACCEL_DEINST; 488 tq = (void *)atomic_load_ptr(&sav->accel_forget_tq); 489 if (tq == NULL || !atomic_cmpset_ptr(&sav->accel_forget_tq, 490 (uintptr_t)tq, 0)) { 491 mtx_unlock(&ipsec_accel_sav_tmp); 492 return; 493 } 494 mtx_unlock(&ipsec_accel_sav_tmp); 495 496 refcount_acquire(&sav->refcnt); 497 TASK_INIT(&tq->forget_task, 0, ipsec_accel_forget_sav_act, tq); 498 tq->forget_vnet = curthread->td_vnet; 499 tq->sav = sav; 500 taskqueue_enqueue(taskqueue_thread, &tq->forget_task); 501 } 502 503 static void 504 ipsec_accel_on_ifdown_sav(struct ifnet *ifp) 505 { 506 struct ifp_handle_sav *i, *marker; 507 508 marker = malloc(sizeof(*marker), M_IPSEC_MISC, M_WAITOK | M_ZERO); 509 marker->flags = IFP_HS_MARKER; 510 511 mtx_lock(&ipsec_accel_sav_tmp); 512 CK_LIST_INSERT_HEAD(&ipsec_accel_all_sav_handles, marker, 513 sav_allh_link); 514 for (;;) { 515 i = CK_LIST_NEXT(marker, sav_allh_link); 516 if (i == NULL) 517 break; 518 CK_LIST_REMOVE(marker, sav_allh_link); 519 CK_LIST_INSERT_AFTER(i, marker, sav_allh_link); 520 if (i->ifp == ifp) { 521 refcount_acquire(&i->sav->refcnt); /* XXXKIB wrap ? */ 522 ipsec_accel_forget_handle_sav(i, true); 523 } 524 } 525 CK_LIST_REMOVE(marker, sav_allh_link); 526 mtx_unlock(&ipsec_accel_sav_tmp); 527 free(marker, M_IPSEC_MISC); 528 } 529 530 static struct ifp_handle_sav * 531 ipsec_accel_is_accel_sav_ptr_raw(struct secasvar *sav, struct ifnet *ifp) 532 { 533 struct ifp_handle_sav *i; 534 535 if ((ifp->if_capenable2 & IFCAP2_BIT(IFCAP2_IPSEC_OFFLOAD)) == 0) 536 return (NULL); 537 CK_LIST_FOREACH(i, &sav->accel_ifps, sav_link) { 538 if (i->ifp == ifp) 539 return (i); 540 } 541 return (NULL); 542 } 543 544 static struct ifp_handle_sav * 545 ipsec_accel_is_accel_sav_ptr(struct secasvar *sav, struct ifnet *ifp) 546 { 547 NET_EPOCH_ASSERT(); 548 return (ipsec_accel_is_accel_sav_ptr_raw(sav, ifp)); 549 } 550 551 static bool 552 ipsec_accel_is_accel_sav_impl(struct secasvar *sav) 553 { 554 return (!CK_LIST_EMPTY(&sav->accel_ifps)); 555 } 556 557 static struct secasvar * 558 ipsec_accel_drvspi_to_sa(u_int drv_spi) 559 { 560 struct ifp_handle_sav *i; 561 562 i = DRVSPI_SA_PCTRIE_LOOKUP(&drv_spi_pctrie, drv_spi); 563 if (i == NULL) 564 return (NULL); 565 return (i->sav); 566 } 567 568 static struct ifp_handle_sp * 569 ipsec_accel_find_accel_sp(struct secpolicy *sp, if_t ifp) 570 { 571 struct ifp_handle_sp *i; 572 573 CK_LIST_FOREACH(i, &sp->accel_ifps, sp_link) { 574 if (i->ifp == ifp) 575 return (i); 576 } 577 return (NULL); 578 } 579 580 static bool 581 ipsec_accel_is_accel_sp(struct secpolicy *sp, if_t ifp) 582 { 583 return (ipsec_accel_find_accel_sp(sp, ifp) != NULL); 584 } 585 586 static int 587 ipsec_accel_remember_sp(struct secpolicy *sp, if_t ifp, 588 struct ifp_handle_sp **ip) 589 { 590 struct ifp_handle_sp *i; 591 592 i = malloc(sizeof(*i), M_IPSEC_MISC, M_WAITOK | M_ZERO); 593 i->sp = sp; 594 i->ifp = ifp; 595 if_ref(ifp); 596 i->flags = IFP_HP_HANDLED; 597 mtx_lock(&ipsec_accel_sav_tmp); 598 CK_LIST_INSERT_HEAD(&sp->accel_ifps, i, sp_link); 599 CK_LIST_INSERT_HEAD(&ipsec_accel_all_sp_handles, i, sp_allh_link); 600 mtx_unlock(&ipsec_accel_sav_tmp); 601 *ip = i; 602 return (0); 603 } 604 605 static bool 606 ipsec_accel_spdadd_match(if_t ifp, void *arg) 607 { 608 struct secpolicy *sp; 609 610 if ((ifp->if_capenable2 & IFCAP2_BIT(IFCAP2_IPSEC_OFFLOAD)) == 0 || 611 ifp->if_ipsec_accel_m->if_spdadd == NULL) 612 return (false); 613 sp = arg; 614 if (sp->accel_ifname != NULL && 615 strcmp(sp->accel_ifname, if_name(ifp)) != 0) 616 return (false); 617 if (ipsec_accel_is_accel_sp(sp, ifp)) 618 return (false); 619 return (true); 620 } 621 622 static int 623 ipsec_accel_spdadd_cb(if_t ifp, void *arg) 624 { 625 struct secpolicy *sp; 626 struct inpcb *inp; 627 struct ifp_handle_sp *i; 628 int error; 629 630 sp = arg; 631 inp = sp->ipsec_accel_add_sp_inp; 632 dprintf("ipsec_accel_spdadd_cb: ifp %s m %p sp %p inp %p\n", 633 if_name(ifp), ifp->if_ipsec_accel_m->if_spdadd, sp, inp); 634 error = ipsec_accel_remember_sp(sp, ifp, &i); 635 if (error != 0) { 636 dprintf("ipsec_accel_spdadd: %s if_spdadd %p remember res %d\n", 637 if_name(ifp), sp, error); 638 return (error); 639 } 640 error = ifp->if_ipsec_accel_m->if_spdadd(ifp, sp, inp, &i->ifdata); 641 if (error != 0) { 642 i->flags |= IFP_HP_REJECTED; 643 dprintf("ipsec_accel_spdadd: %s if_spdadd %p res %d\n", 644 if_name(ifp), sp, error); 645 } 646 return (error); 647 } 648 649 static void 650 ipsec_accel_spdadd_act(void *arg, int pending) 651 { 652 struct secpolicy *sp; 653 struct inpcb *inp; 654 655 sp = arg; 656 CURVNET_SET(sp->accel_add_tq.adddel_vnet); 657 if_foreach_sleep(ipsec_accel_spdadd_match, arg, 658 ipsec_accel_spdadd_cb, arg); 659 inp = sp->ipsec_accel_add_sp_inp; 660 if (inp != NULL) { 661 INP_WLOCK(inp); 662 if (!in_pcbrele_wlocked(inp)) 663 INP_WUNLOCK(inp); 664 sp->ipsec_accel_add_sp_inp = NULL; 665 } 666 CURVNET_RESTORE(); 667 key_freesp(&sp); 668 } 669 670 void 671 ipsec_accel_spdadd_impl(struct secpolicy *sp, struct inpcb *inp) 672 { 673 struct ipsec_accel_adddel_sp_tq *tq; 674 675 if (sp == NULL) 676 return; 677 if (sp->tcount == 0 && inp == NULL) 678 return; 679 tq = &sp->accel_add_tq; 680 if (atomic_cmpset_int(&tq->adddel_scheduled, 0, 1) == 0) 681 return; 682 tq->adddel_vnet = curthread->td_vnet; 683 sp->ipsec_accel_add_sp_inp = inp; 684 if (inp != NULL) 685 in_pcbref(inp); 686 TASK_INIT(&tq->adddel_task, 0, ipsec_accel_spdadd_act, sp); 687 key_addref(sp); 688 taskqueue_enqueue(taskqueue_thread, &tq->adddel_task); 689 } 690 691 static void 692 ipsec_accel_spddel_act(void *arg, int pending) 693 { 694 struct ifp_handle_sp *i; 695 struct secpolicy *sp; 696 int error; 697 698 sp = arg; 699 CURVNET_SET(sp->accel_del_tq.adddel_vnet); 700 mtx_lock(&ipsec_accel_sav_tmp); 701 for (;;) { 702 i = CK_LIST_FIRST(&sp->accel_ifps); 703 if (i == NULL) 704 break; 705 CK_LIST_REMOVE(i, sp_link); 706 CK_LIST_REMOVE(i, sp_allh_link); 707 mtx_unlock(&ipsec_accel_sav_tmp); 708 NET_EPOCH_WAIT(); 709 if ((i->flags & (IFP_HP_HANDLED | IFP_HP_REJECTED)) == 710 IFP_HP_HANDLED) { 711 dprintf("spd deinstall %s %p\n", if_name(i->ifp), sp); 712 error = i->ifp->if_ipsec_accel_m->if_spddel(i->ifp, 713 sp, i->ifdata); 714 if (error != 0) { 715 dprintf( 716 "ipsec_accel_spddel: %s if_spddel %p res %d\n", 717 if_name(i->ifp), sp, error); 718 } 719 } 720 if_rele(i->ifp); 721 free(i, M_IPSEC_MISC); 722 mtx_lock(&ipsec_accel_sav_tmp); 723 } 724 mtx_unlock(&ipsec_accel_sav_tmp); 725 key_freesp(&sp); 726 CURVNET_RESTORE(); 727 } 728 729 void 730 ipsec_accel_spddel_impl(struct secpolicy *sp) 731 { 732 struct ipsec_accel_adddel_sp_tq *tq; 733 734 if (sp == NULL) 735 return; 736 737 tq = &sp->accel_del_tq; 738 if (atomic_cmpset_int(&tq->adddel_scheduled, 0, 1) == 0) 739 return; 740 tq->adddel_vnet = curthread->td_vnet; 741 TASK_INIT(&tq->adddel_task, 0, ipsec_accel_spddel_act, sp); 742 key_addref(sp); 743 taskqueue_enqueue(taskqueue_thread, &tq->adddel_task); 744 } 745 746 static void 747 ipsec_accel_on_ifdown_sp(struct ifnet *ifp) 748 { 749 struct ifp_handle_sp *i, *marker; 750 struct secpolicy *sp; 751 int error; 752 753 marker = malloc(sizeof(*marker), M_IPSEC_MISC, M_WAITOK | M_ZERO); 754 marker->flags = IFP_HS_MARKER; 755 756 mtx_lock(&ipsec_accel_sav_tmp); 757 CK_LIST_INSERT_HEAD(&ipsec_accel_all_sp_handles, marker, 758 sp_allh_link); 759 for (;;) { 760 i = CK_LIST_NEXT(marker, sp_allh_link); 761 if (i == NULL) 762 break; 763 CK_LIST_REMOVE(marker, sp_allh_link); 764 CK_LIST_INSERT_AFTER(i, marker, sp_allh_link); 765 if (i->ifp != ifp) 766 continue; 767 768 sp = i->sp; 769 key_addref(sp); 770 CK_LIST_REMOVE(i, sp_link); 771 CK_LIST_REMOVE(i, sp_allh_link); 772 mtx_unlock(&ipsec_accel_sav_tmp); 773 NET_EPOCH_WAIT(); 774 if ((i->flags & (IFP_HP_HANDLED | IFP_HP_REJECTED)) == 775 IFP_HP_HANDLED) { 776 dprintf("spd deinstall %s %p\n", if_name(ifp), sp); 777 error = ifp->if_ipsec_accel_m->if_spddel(ifp, 778 sp, i->ifdata); 779 } 780 if (error != 0) { 781 dprintf( 782 "ipsec_accel_on_ifdown_sp: %s if_spddel %p res %d\n", 783 if_name(ifp), sp, error); 784 } 785 key_freesp(&sp); 786 if_rele(ifp); 787 free(i, M_IPSEC_MISC); 788 mtx_lock(&ipsec_accel_sav_tmp); 789 } 790 CK_LIST_REMOVE(marker, sp_allh_link); 791 mtx_unlock(&ipsec_accel_sav_tmp); 792 free(marker, M_IPSEC_MISC); 793 } 794 795 static void 796 ipsec_accel_on_ifdown_impl(struct ifnet *ifp) 797 { 798 ipsec_accel_on_ifdown_sp(ifp); 799 ipsec_accel_on_ifdown_sav(ifp); 800 } 801 802 static bool 803 ipsec_accel_output_pad(struct mbuf *m, struct secasvar *sav, int skip, int mtu) 804 { 805 int alen, blks, hlen, padding, rlen; 806 807 rlen = m->m_pkthdr.len - skip; 808 hlen = ((sav->flags & SADB_X_EXT_OLD) != 0 ? sizeof(struct esp) : 809 sizeof(struct newesp)) + sav->ivlen; 810 blks = MAX(4, SAV_ISCTR(sav) && VNET(esp_ctr_compatibility) ? 811 sav->tdb_encalgxform->native_blocksize : 812 sav->tdb_encalgxform->blocksize); 813 padding = ((blks - ((rlen + 2) % blks)) % blks) + 2; 814 alen = xform_ah_authsize(sav->tdb_authalgxform); 815 816 return (skip + hlen + rlen + padding + alen <= mtu); 817 } 818 819 static bool 820 ipsec_accel_output_tag(struct mbuf *m, u_int drv_spi) 821 { 822 struct ipsec_accel_out_tag *tag; 823 824 tag = (struct ipsec_accel_out_tag *)m_tag_get( 825 PACKET_TAG_IPSEC_ACCEL_OUT, sizeof(*tag), M_NOWAIT); 826 if (tag == NULL) 827 return (false); 828 tag->drv_spi = drv_spi; 829 m_tag_prepend(m, &tag->tag); 830 return (true); 831 } 832 833 bool 834 ipsec_accel_output(struct ifnet *ifp, struct mbuf *m, struct inpcb *inp, 835 struct secpolicy *sp, struct secasvar *sav, int af, int mtu, int *hwassist) 836 { 837 struct ifp_handle_sav *i; 838 struct ip *ip; 839 struct tcpcb *tp; 840 u_long ip_len, skip; 841 bool res; 842 843 *hwassist = 0; 844 res = false; 845 if (ifp == NULL) 846 return (res); 847 848 M_ASSERTPKTHDR(m); 849 NET_EPOCH_ASSERT(); 850 851 if (sav == NULL) { 852 res = ipsec_accel_output_tag(m, IPSEC_ACCEL_DRV_SPI_BYPASS); 853 goto out; 854 } 855 856 i = ipsec_accel_is_accel_sav_ptr(sav, ifp); 857 if (i == NULL) 858 goto out; 859 860 if ((m->m_pkthdr.csum_flags & CSUM_TSO) == 0) { 861 ip_len = m->m_pkthdr.len; 862 if (ip_len + i->hdr_ext_size > mtu) 863 goto out; 864 switch (af) { 865 case AF_INET: 866 ip = mtod(m, struct ip *); 867 skip = ip->ip_hl << 2; 868 break; 869 case AF_INET6: 870 skip = sizeof(struct ip6_hdr); 871 break; 872 default: 873 __unreachable(); 874 } 875 if (!ipsec_accel_output_pad(m, sav, skip, mtu)) 876 goto out; 877 } 878 879 if (!ipsec_accel_output_tag(m, i->drv_spi)) 880 goto out; 881 882 ipsec_accel_sa_recordxfer(sav, m); 883 key_freesav(&sav); 884 if (sp != NULL) 885 key_freesp(&sp); 886 887 *hwassist = ifp->if_ipsec_accel_m->if_hwassist(ifp, sav, 888 i->drv_spi, i->ifdata); 889 res = true; 890 out: 891 if (inp != NULL && inp->inp_pcbinfo == &V_tcbinfo) { 892 INP_WLOCK_ASSERT(inp); 893 tp = (struct tcpcb *)inp; 894 if (res && (*hwassist & (CSUM_TSO | CSUM_IP6_TSO)) != 0) { 895 tp->t_flags2 |= TF2_IPSEC_TSO; 896 } else { 897 tp->t_flags2 &= ~TF2_IPSEC_TSO; 898 } 899 } 900 return (res); 901 } 902 903 struct ipsec_accel_in_tag * 904 ipsec_accel_input_tag_lookup(const struct mbuf *m) 905 { 906 struct ipsec_accel_in_tag *tag; 907 struct m_tag *xtag; 908 909 xtag = m_tag_find(__DECONST(struct mbuf *, m), 910 PACKET_TAG_IPSEC_ACCEL_IN, NULL); 911 if (xtag == NULL) 912 return (NULL); 913 tag = __containerof(xtag, struct ipsec_accel_in_tag, tag); 914 return (tag); 915 } 916 917 int 918 ipsec_accel_input(struct mbuf *m, int offset, int proto) 919 { 920 struct secasvar *sav; 921 struct ipsec_accel_in_tag *tag; 922 923 tag = ipsec_accel_input_tag_lookup(m); 924 if (tag == NULL) 925 return (ENXIO); 926 927 if (tag->drv_spi < IPSEC_ACCEL_DRV_SPI_MIN || 928 tag->drv_spi > IPSEC_ACCEL_DRV_SPI_MAX) { 929 dprintf("if %s mbuf %p drv_spi %d invalid, packet dropped\n", 930 (m->m_flags & M_PKTHDR) != 0 ? if_name(m->m_pkthdr.rcvif) : 931 "<unknwn>", m, tag->drv_spi); 932 m_freem(m); 933 return (EINPROGRESS); 934 } 935 936 sav = ipsec_accel_drvspi_to_sa(tag->drv_spi); 937 if (sav != NULL) 938 ipsec_accel_sa_recordxfer(sav, m); 939 return (0); 940 } 941 942 static void 943 ipsec_accel_sa_recordxfer(struct secasvar *sav, struct mbuf *m) 944 { 945 counter_u64_add(sav->accel_lft_sw, 1); 946 counter_u64_add(sav->accel_lft_sw + 1, m->m_pkthdr.len); 947 if (sav->accel_firstused == 0) 948 sav->accel_firstused = time_second; 949 } 950 951 static void 952 ipsec_accel_sa_lifetime_update(struct seclifetime *lft_c, 953 const struct seclifetime *lft_l) 954 { 955 lft_c->allocations += lft_l->allocations; 956 lft_c->bytes += lft_l->bytes; 957 lft_c->usetime = min(lft_c->usetime, lft_l->usetime); 958 } 959 960 static void 961 ipsec_accel_drv_sa_lifetime_update_impl(struct secasvar *sav, if_t ifp, 962 u_int drv_spi, uint64_t octets, uint64_t allocs) 963 { 964 struct epoch_tracker et; 965 struct ifp_handle_sav *i; 966 uint64_t odiff, adiff; 967 968 NET_EPOCH_ENTER(et); 969 mtx_lock(&ipsec_accel_cnt_lock); 970 971 if (allocs != 0) { 972 if (sav->firstused == 0) 973 sav->firstused = time_second; 974 if (sav->accel_firstused == 0) 975 sav->accel_firstused = time_second; 976 } 977 978 CK_LIST_FOREACH(i, &sav->accel_ifps, sav_link) { 979 if (i->ifp == ifp && i->drv_spi == drv_spi) 980 break; 981 } 982 if (i == NULL) 983 goto out; 984 985 odiff = octets - i->cnt_octets; 986 adiff = allocs - i->cnt_allocs; 987 988 if (sav->lft_c != NULL) { 989 counter_u64_add(sav->lft_c_bytes, odiff); 990 counter_u64_add(sav->lft_c_allocations, adiff); 991 } 992 993 i->cnt_octets = octets; 994 i->cnt_allocs = allocs; 995 sav->accel_hw_octets += odiff; 996 sav->accel_hw_allocs += adiff; 997 998 out: 999 mtx_unlock(&ipsec_accel_cnt_lock); 1000 NET_EPOCH_EXIT(et); 1001 } 1002 1003 static void 1004 ipsec_accel_sa_lifetime_hw(struct secasvar *sav, if_t ifp, 1005 struct seclifetime *lft) 1006 { 1007 struct ifp_handle_sav *i; 1008 if_sa_cnt_fn_t p; 1009 1010 IFNET_RLOCK_ASSERT(); 1011 1012 i = ipsec_accel_is_accel_sav_ptr(sav, ifp); 1013 if (i != NULL && (i->flags & (IFP_HS_HANDLED | IFP_HS_REJECTED)) == 1014 IFP_HS_HANDLED) { 1015 p = ifp->if_ipsec_accel_m->if_sa_cnt; 1016 if (p != NULL) 1017 p(ifp, sav, i->drv_spi, i->ifdata, lft); 1018 } 1019 } 1020 1021 static int 1022 ipsec_accel_sa_lifetime_op_impl(struct secasvar *sav, 1023 struct seclifetime *lft_c, if_t ifp, enum IF_SA_CNT_WHICH op, 1024 struct rm_priotracker *sahtree_trackerp) 1025 { 1026 struct seclifetime lft_l, lft_s; 1027 struct ifp_handle_sav *i; 1028 if_t ifp1; 1029 if_sa_cnt_fn_t p; 1030 int error; 1031 1032 error = 0; 1033 memset(&lft_l, 0, sizeof(lft_l)); 1034 memset(&lft_s, 0, sizeof(lft_s)); 1035 1036 switch (op & ~IF_SA_CNT_UPD) { 1037 case IF_SA_CNT_IFP_HW_VAL: 1038 ipsec_accel_sa_lifetime_hw(sav, ifp, &lft_l); 1039 ipsec_accel_sa_lifetime_update(&lft_l, &lft_s); 1040 break; 1041 1042 case IF_SA_CNT_TOTAL_SW_VAL: 1043 lft_l.allocations = (uint32_t)counter_u64_fetch( 1044 sav->accel_lft_sw); 1045 lft_l.bytes = counter_u64_fetch(sav->accel_lft_sw + 1); 1046 lft_l.usetime = sav->accel_firstused; 1047 break; 1048 1049 case IF_SA_CNT_TOTAL_HW_VAL: 1050 IFNET_RLOCK_ASSERT(); 1051 CK_LIST_FOREACH(i, &sav->accel_ifps, sav_link) { 1052 if ((i->flags & (IFP_HS_HANDLED | IFP_HS_REJECTED)) != 1053 IFP_HS_HANDLED) 1054 continue; 1055 ifp1 = i->ifp; 1056 p = ifp1->if_ipsec_accel_m->if_sa_cnt; 1057 if (p == NULL) 1058 continue; 1059 memset(&lft_s, 0, sizeof(lft_s)); 1060 if (sahtree_trackerp != NULL) 1061 ipsec_sahtree_runlock(sahtree_trackerp); 1062 error = p(ifp1, sav, i->drv_spi, i->ifdata, &lft_s); 1063 if (sahtree_trackerp != NULL) 1064 ipsec_sahtree_rlock(sahtree_trackerp); 1065 if (error == 0) 1066 ipsec_accel_sa_lifetime_update(&lft_l, &lft_s); 1067 } 1068 break; 1069 } 1070 1071 if (error == 0) { 1072 if ((op & IF_SA_CNT_UPD) == 0) 1073 memset(lft_c, 0, sizeof(*lft_c)); 1074 ipsec_accel_sa_lifetime_update(lft_c, &lft_l); 1075 } 1076 1077 return (error); 1078 } 1079 1080 static void 1081 ipsec_accel_sync_imp(void) 1082 { 1083 taskqueue_drain_all(taskqueue_thread); 1084 } 1085 1086 static struct mbuf * 1087 ipsec_accel_key_setaccelif_impl(struct secasvar *sav) 1088 { 1089 struct mbuf *m, *m1; 1090 struct ifp_handle_sav *i; 1091 struct epoch_tracker et; 1092 1093 if (sav->accel_ifname != NULL) 1094 return (key_setaccelif(sav->accel_ifname)); 1095 1096 m = m1 = NULL; 1097 1098 NET_EPOCH_ENTER(et); 1099 CK_LIST_FOREACH(i, &sav->accel_ifps, sav_link) { 1100 if ((i->flags & (IFP_HS_HANDLED | IFP_HS_REJECTED)) == 1101 IFP_HS_HANDLED) { 1102 m1 = key_setaccelif(if_name(i->ifp)); 1103 if (m == NULL) 1104 m = m1; 1105 else if (m1 != NULL) 1106 m_cat(m, m1); 1107 } 1108 } 1109 NET_EPOCH_EXIT(et); 1110 return (m); 1111 } 1112 1113 #endif /* IPSEC_OFFLOAD */ 1114