1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2012 Chelsio Communications, Inc. 5 * All rights reserved. 6 * Written by: Navdeep Parhar <np@FreeBSD.org> 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include <sys/cdefs.h> 31 __FBSDID("$FreeBSD$"); 32 33 #include "opt_inet.h" 34 #include "opt_inet6.h" 35 36 #include <sys/param.h> 37 #include <sys/eventhandler.h> 38 #include <sys/kernel.h> 39 #include <sys/systm.h> 40 #include <sys/malloc.h> 41 #include <sys/mbuf.h> 42 #include <sys/module.h> 43 #include <sys/types.h> 44 #include <sys/sockopt.h> 45 #include <sys/sysctl.h> 46 #include <sys/socket.h> 47 48 #include <net/ethernet.h> 49 #include <net/if.h> 50 #include <net/if_var.h> 51 #include <net/if_private.h> 52 #include <net/if_types.h> 53 #include <net/if_vlan_var.h> 54 #include <net/if_llatbl.h> 55 #include <net/route.h> 56 57 #include <netinet/if_ether.h> 58 #include <netinet/in.h> 59 #include <netinet/in_pcb.h> 60 #include <netinet/in_var.h> 61 #include <netinet6/in6_var.h> 62 #include <netinet6/in6_pcb.h> 63 #include <netinet6/nd6.h> 64 #define TCPSTATES 65 #include <netinet/tcp.h> 66 #include <netinet/tcp_fsm.h> 67 #include <netinet/tcp_timer.h> 68 #include <netinet/tcp_var.h> 69 #include <netinet/tcp_syncache.h> 70 #include <netinet/tcp_offload.h> 71 #include <netinet/toecore.h> 72 73 static struct mtx toedev_lock; 74 static TAILQ_HEAD(, toedev) toedev_list; 75 static eventhandler_tag listen_start_eh; 76 static eventhandler_tag listen_stop_eh; 77 static eventhandler_tag lle_event_eh; 78 79 static int 80 toedev_connect(struct toedev *tod __unused, struct socket *so __unused, 81 struct nhop_object *nh __unused, struct sockaddr *nam __unused) 82 { 83 84 return (ENOTSUP); 85 } 86 87 static int 88 toedev_listen_start(struct toedev *tod __unused, struct tcpcb *tp __unused) 89 { 90 91 return (ENOTSUP); 92 } 93 94 static int 95 toedev_listen_stop(struct toedev *tod __unused, struct tcpcb *tp __unused) 96 { 97 98 return (ENOTSUP); 99 } 100 101 static void 102 toedev_input(struct toedev *tod __unused, struct tcpcb *tp __unused, 103 struct mbuf *m) 104 { 105 106 m_freem(m); 107 return; 108 } 109 110 static void 111 toedev_rcvd(struct toedev *tod __unused, struct tcpcb *tp __unused) 112 { 113 114 return; 115 } 116 117 static int 118 toedev_output(struct toedev *tod __unused, struct tcpcb *tp __unused) 119 { 120 121 return (ENOTSUP); 122 } 123 124 static void 125 toedev_pcb_detach(struct toedev *tod __unused, struct tcpcb *tp __unused) 126 { 127 128 return; 129 } 130 131 static void 132 toedev_l2_update(struct toedev *tod __unused, struct ifnet *ifp __unused, 133 struct sockaddr *sa __unused, uint8_t *lladdr __unused, 134 uint16_t vtag __unused) 135 { 136 137 return; 138 } 139 140 static void 141 toedev_route_redirect(struct toedev *tod __unused, struct ifnet *ifp __unused, 142 struct nhop_object *nh0 __unused, struct nhop_object *nh1 __unused) 143 { 144 145 return; 146 } 147 148 static void 149 toedev_syncache_added(struct toedev *tod __unused, void *ctx __unused) 150 { 151 152 return; 153 } 154 155 static void 156 toedev_syncache_removed(struct toedev *tod __unused, void *ctx __unused) 157 { 158 159 return; 160 } 161 162 static int 163 toedev_syncache_respond(struct toedev *tod __unused, void *ctx __unused, 164 struct mbuf *m) 165 { 166 167 m_freem(m); 168 return (0); 169 } 170 171 static void 172 toedev_offload_socket(struct toedev *tod __unused, void *ctx __unused, 173 struct socket *so __unused) 174 { 175 176 return; 177 } 178 179 static void 180 toedev_ctloutput(struct toedev *tod __unused, struct tcpcb *tp __unused, 181 int sopt_dir __unused, int sopt_name __unused) 182 { 183 184 return; 185 } 186 187 static void 188 toedev_tcp_info(struct toedev *tod __unused, struct tcpcb *tp __unused, 189 struct tcp_info *ti __unused) 190 { 191 192 return; 193 } 194 195 static int 196 toedev_alloc_tls_session(struct toedev *tod __unused, struct tcpcb *tp __unused, 197 struct ktls_session *tls __unused, int direction __unused) 198 { 199 200 return (EINVAL); 201 } 202 203 static void 204 toedev_pmtu_update(struct toedev *tod __unused, struct tcpcb *tp __unused, 205 tcp_seq seq __unused, int mtu __unused) 206 { 207 208 return; 209 } 210 211 /* 212 * Inform one or more TOE devices about a listening socket. 213 */ 214 static void 215 toe_listen_start(struct inpcb *inp, void *arg) 216 { 217 struct toedev *t, *tod; 218 struct tcpcb *tp; 219 220 INP_WLOCK_ASSERT(inp); 221 KASSERT(inp->inp_pcbinfo == &V_tcbinfo, 222 ("%s: inp is not a TCP inp", __func__)); 223 224 if (inp->inp_flags & INP_DROPPED) 225 return; 226 227 tp = intotcpcb(inp); 228 if (tp->t_state != TCPS_LISTEN) 229 return; 230 231 t = arg; 232 mtx_lock(&toedev_lock); 233 TAILQ_FOREACH(tod, &toedev_list, link) { 234 if (t == NULL || t == tod) 235 tod->tod_listen_start(tod, tp); 236 } 237 mtx_unlock(&toedev_lock); 238 } 239 240 static void 241 toe_listen_start_event(void *arg __unused, struct tcpcb *tp) 242 { 243 struct inpcb *inp = tptoinpcb(tp); 244 245 INP_WLOCK_ASSERT(inp); 246 KASSERT(tp->t_state == TCPS_LISTEN, 247 ("%s: t_state %s", __func__, tcpstates[tp->t_state])); 248 249 toe_listen_start(inp, NULL); 250 } 251 252 static void 253 toe_listen_stop_event(void *arg __unused, struct tcpcb *tp) 254 { 255 struct toedev *tod; 256 #ifdef INVARIANTS 257 struct inpcb *inp = tptoinpcb(tp); 258 #endif 259 260 INP_WLOCK_ASSERT(inp); 261 KASSERT(tp->t_state == TCPS_LISTEN, 262 ("%s: t_state %s", __func__, tcpstates[tp->t_state])); 263 264 mtx_lock(&toedev_lock); 265 TAILQ_FOREACH(tod, &toedev_list, link) 266 tod->tod_listen_stop(tod, tp); 267 mtx_unlock(&toedev_lock); 268 } 269 270 /* 271 * Fill up a freshly allocated toedev struct with reasonable defaults. 272 */ 273 void 274 init_toedev(struct toedev *tod) 275 { 276 277 tod->tod_softc = NULL; 278 279 /* 280 * Provide no-op defaults so that the kernel can call any toedev 281 * function without having to check whether the TOE driver supplied one 282 * or not. 283 */ 284 tod->tod_connect = toedev_connect; 285 tod->tod_listen_start = toedev_listen_start; 286 tod->tod_listen_stop = toedev_listen_stop; 287 tod->tod_input = toedev_input; 288 tod->tod_rcvd = toedev_rcvd; 289 tod->tod_output = toedev_output; 290 tod->tod_send_rst = toedev_output; 291 tod->tod_send_fin = toedev_output; 292 tod->tod_pcb_detach = toedev_pcb_detach; 293 tod->tod_l2_update = toedev_l2_update; 294 tod->tod_route_redirect = toedev_route_redirect; 295 tod->tod_syncache_added = toedev_syncache_added; 296 tod->tod_syncache_removed = toedev_syncache_removed; 297 tod->tod_syncache_respond = toedev_syncache_respond; 298 tod->tod_offload_socket = toedev_offload_socket; 299 tod->tod_ctloutput = toedev_ctloutput; 300 tod->tod_tcp_info = toedev_tcp_info; 301 tod->tod_alloc_tls_session = toedev_alloc_tls_session; 302 tod->tod_pmtu_update = toedev_pmtu_update; 303 } 304 305 /* 306 * Register an active TOE device with the system. This allows it to receive 307 * notifications from the kernel. 308 */ 309 int 310 register_toedev(struct toedev *tod) 311 { 312 struct toedev *t; 313 314 mtx_lock(&toedev_lock); 315 TAILQ_FOREACH(t, &toedev_list, link) { 316 if (t == tod) { 317 mtx_unlock(&toedev_lock); 318 return (EEXIST); 319 } 320 } 321 322 TAILQ_INSERT_TAIL(&toedev_list, tod, link); 323 registered_toedevs++; 324 mtx_unlock(&toedev_lock); 325 326 inp_apply_all(&V_tcbinfo, toe_listen_start, tod); 327 328 return (0); 329 } 330 331 /* 332 * Remove the TOE device from the global list of active TOE devices. It is the 333 * caller's responsibility to ensure that the TOE device is quiesced prior to 334 * this call. 335 */ 336 int 337 unregister_toedev(struct toedev *tod) 338 { 339 struct toedev *t, *t2; 340 int rc = ENODEV; 341 342 mtx_lock(&toedev_lock); 343 TAILQ_FOREACH_SAFE(t, &toedev_list, link, t2) { 344 if (t == tod) { 345 TAILQ_REMOVE(&toedev_list, tod, link); 346 registered_toedevs--; 347 rc = 0; 348 break; 349 } 350 } 351 KASSERT(registered_toedevs >= 0, 352 ("%s: registered_toedevs (%d) < 0", __func__, registered_toedevs)); 353 mtx_unlock(&toedev_lock); 354 return (rc); 355 } 356 357 void 358 toe_syncache_add(struct in_conninfo *inc, struct tcpopt *to, struct tcphdr *th, 359 struct inpcb *inp, void *tod, void *todctx, uint8_t iptos) 360 { 361 362 INP_RLOCK_ASSERT(inp); 363 364 (void )syncache_add(inc, to, th, inp, inp->inp_socket, NULL, tod, 365 todctx, iptos, htons(0)); 366 } 367 368 int 369 toe_syncache_expand(struct in_conninfo *inc, struct tcpopt *to, 370 struct tcphdr *th, struct socket **lsop) 371 { 372 373 NET_EPOCH_ASSERT(); 374 375 return (syncache_expand(inc, to, th, lsop, NULL, htons(0))); 376 } 377 378 /* 379 * General purpose check to see if a 4-tuple is in use by the kernel. If a TCP 380 * header (presumably for an incoming SYN) is also provided, an existing 4-tuple 381 * in TIME_WAIT may be assassinated freeing it up for re-use. 382 * 383 * Note that the TCP header must have been run through tcp_fields_to_host() or 384 * equivalent. 385 */ 386 int 387 toe_4tuple_check(struct in_conninfo *inc, struct tcphdr *th, struct ifnet *ifp) 388 { 389 struct inpcb *inp; 390 struct tcpcb *tp; 391 392 if (inc->inc_flags & INC_ISIPV6) { 393 inp = in6_pcblookup(&V_tcbinfo, &inc->inc6_faddr, 394 inc->inc_fport, &inc->inc6_laddr, inc->inc_lport, 395 INPLOOKUP_RLOCKPCB, ifp); 396 } else { 397 inp = in_pcblookup(&V_tcbinfo, inc->inc_faddr, inc->inc_fport, 398 inc->inc_laddr, inc->inc_lport, INPLOOKUP_RLOCKPCB, ifp); 399 } 400 if (inp != NULL) { 401 INP_RLOCK_ASSERT(inp); 402 403 tp = intotcpcb(inp); 404 if (tp->t_state == TCPS_TIME_WAIT && th != NULL) { 405 if (!tcp_twcheck(inp, NULL, th, NULL, 0)) 406 return (EADDRINUSE); 407 } else { 408 INP_RUNLOCK(inp); 409 return (EADDRINUSE); 410 } 411 } 412 413 return (0); 414 } 415 416 static void 417 toe_lle_event(void *arg __unused, struct llentry *lle, int evt) 418 { 419 struct toedev *tod; 420 struct ifnet *ifp; 421 struct sockaddr *sa; 422 uint8_t *lladdr; 423 uint16_t vid, pcp; 424 int family; 425 struct sockaddr_in6 sin6; 426 427 LLE_WLOCK_ASSERT(lle); 428 429 ifp = lltable_get_ifp(lle->lle_tbl); 430 family = lltable_get_af(lle->lle_tbl); 431 432 if (family != AF_INET && family != AF_INET6) 433 return; 434 /* 435 * Not interested if the interface's TOE capability is not enabled. 436 */ 437 if ((family == AF_INET && !(ifp->if_capenable & IFCAP_TOE4)) || 438 (family == AF_INET6 && !(ifp->if_capenable & IFCAP_TOE6))) 439 return; 440 441 tod = TOEDEV(ifp); 442 if (tod == NULL) 443 return; 444 445 sa = (struct sockaddr *)&sin6; 446 lltable_fill_sa_entry(lle, sa); 447 448 vid = 0xfff; 449 pcp = 0; 450 if (evt != LLENTRY_RESOLVED) { 451 /* 452 * LLENTRY_TIMEDOUT, LLENTRY_DELETED, LLENTRY_EXPIRED all mean 453 * this entry is going to be deleted. 454 */ 455 456 lladdr = NULL; 457 } else { 458 KASSERT(lle->la_flags & LLE_VALID, 459 ("%s: %p resolved but not valid?", __func__, lle)); 460 461 lladdr = (uint8_t *)lle->ll_addr; 462 VLAN_TAG(ifp, &vid); 463 VLAN_PCP(ifp, &pcp); 464 } 465 466 tod->tod_l2_update(tod, ifp, sa, lladdr, EVL_MAKETAG(vid, pcp, 0)); 467 } 468 469 /* 470 * Returns 0 or EWOULDBLOCK on success (any other value is an error). 0 means 471 * lladdr and vtag are valid on return, EWOULDBLOCK means the TOE driver's 472 * tod_l2_update will be called later, when the entry is resolved or times out. 473 */ 474 int 475 toe_l2_resolve(struct toedev *tod, struct ifnet *ifp, struct sockaddr *sa, 476 uint8_t *lladdr, uint16_t *vtag) 477 { 478 int rc; 479 uint16_t vid, pcp; 480 481 switch (sa->sa_family) { 482 #ifdef INET 483 case AF_INET: 484 rc = arpresolve(ifp, 0, NULL, sa, lladdr, NULL, NULL); 485 break; 486 #endif 487 #ifdef INET6 488 case AF_INET6: 489 rc = nd6_resolve(ifp, LLE_SF(AF_INET6, 0), NULL, sa, lladdr, 490 NULL, NULL); 491 break; 492 #endif 493 default: 494 return (EPROTONOSUPPORT); 495 } 496 497 if (rc == 0) { 498 vid = 0xfff; 499 pcp = 0; 500 if (ifp->if_type == IFT_L2VLAN) { 501 VLAN_TAG(ifp, &vid); 502 VLAN_PCP(ifp, &pcp); 503 } else if (ifp->if_pcp != IFNET_PCP_NONE) { 504 vid = 0; 505 pcp = ifp->if_pcp; 506 } 507 *vtag = EVL_MAKETAG(vid, pcp, 0); 508 } 509 510 return (rc); 511 } 512 513 void 514 toe_connect_failed(struct toedev *tod, struct inpcb *inp, int err) 515 { 516 517 NET_EPOCH_ASSERT(); 518 INP_WLOCK_ASSERT(inp); 519 520 if (!(inp->inp_flags & INP_DROPPED)) { 521 struct tcpcb *tp = intotcpcb(inp); 522 523 KASSERT(tp->t_flags & TF_TOE, 524 ("%s: tp %p not offloaded.", __func__, tp)); 525 526 if (err == EAGAIN) { 527 /* 528 * Temporary failure during offload, take this PCB back. 529 * Detach from the TOE driver and do the rest of what 530 * TCP's pru_connect would have done if the connection 531 * wasn't offloaded. 532 */ 533 534 tod->tod_pcb_detach(tod, tp); 535 KASSERT(!(tp->t_flags & TF_TOE), 536 ("%s: tp %p still offloaded.", __func__, tp)); 537 tcp_timer_activate(tp, TT_KEEP, TP_KEEPINIT(tp)); 538 if (tcp_output(tp) < 0) 539 INP_WLOCK(inp); /* re-acquire */ 540 } else { 541 tp = tcp_drop(tp, err); 542 if (tp == NULL) 543 INP_WLOCK(inp); /* re-acquire */ 544 } 545 } 546 INP_WLOCK_ASSERT(inp); 547 } 548 549 static int 550 toecore_load(void) 551 { 552 553 mtx_init(&toedev_lock, "toedev lock", NULL, MTX_DEF); 554 TAILQ_INIT(&toedev_list); 555 556 listen_start_eh = EVENTHANDLER_REGISTER(tcp_offload_listen_start, 557 toe_listen_start_event, NULL, EVENTHANDLER_PRI_ANY); 558 listen_stop_eh = EVENTHANDLER_REGISTER(tcp_offload_listen_stop, 559 toe_listen_stop_event, NULL, EVENTHANDLER_PRI_ANY); 560 lle_event_eh = EVENTHANDLER_REGISTER(lle_event, toe_lle_event, NULL, 561 EVENTHANDLER_PRI_ANY); 562 563 return (0); 564 } 565 566 static int 567 toecore_unload(void) 568 { 569 570 mtx_lock(&toedev_lock); 571 if (!TAILQ_EMPTY(&toedev_list)) { 572 mtx_unlock(&toedev_lock); 573 return (EBUSY); 574 } 575 576 EVENTHANDLER_DEREGISTER(tcp_offload_listen_start, listen_start_eh); 577 EVENTHANDLER_DEREGISTER(tcp_offload_listen_stop, listen_stop_eh); 578 EVENTHANDLER_DEREGISTER(lle_event, lle_event_eh); 579 580 mtx_unlock(&toedev_lock); 581 mtx_destroy(&toedev_lock); 582 583 return (0); 584 } 585 586 static int 587 toecore_mod_handler(module_t mod, int cmd, void *arg) 588 { 589 590 if (cmd == MOD_LOAD) 591 return (toecore_load()); 592 593 if (cmd == MOD_UNLOAD) 594 return (toecore_unload()); 595 596 return (EOPNOTSUPP); 597 } 598 599 static moduledata_t mod_data= { 600 "toecore", 601 toecore_mod_handler, 602 0 603 }; 604 605 MODULE_VERSION(toecore, 1); 606 DECLARE_MODULE(toecore, mod_data, SI_SUB_EXEC, SI_ORDER_ANY); 607