1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2012 Chelsio Communications, Inc. 5 * All rights reserved. 6 * Written by: Navdeep Parhar <np@FreeBSD.org> 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include <sys/cdefs.h> 31 __FBSDID("$FreeBSD$"); 32 33 #include "opt_inet.h" 34 #include "opt_inet6.h" 35 36 #include <sys/param.h> 37 #include <sys/eventhandler.h> 38 #include <sys/kernel.h> 39 #include <sys/systm.h> 40 #include <sys/malloc.h> 41 #include <sys/mbuf.h> 42 #include <sys/module.h> 43 #include <sys/types.h> 44 #include <sys/sockopt.h> 45 #include <sys/sysctl.h> 46 #include <sys/socket.h> 47 48 #include <net/ethernet.h> 49 #include <net/if.h> 50 #include <net/if_var.h> 51 #include <net/if_types.h> 52 #include <net/if_vlan_var.h> 53 #include <net/if_llatbl.h> 54 #include <net/route.h> 55 56 #include <netinet/if_ether.h> 57 #include <netinet/in.h> 58 #include <netinet/in_pcb.h> 59 #include <netinet/in_var.h> 60 #include <netinet6/in6_var.h> 61 #include <netinet6/in6_pcb.h> 62 #include <netinet6/nd6.h> 63 #define TCPSTATES 64 #include <netinet/tcp.h> 65 #include <netinet/tcp_fsm.h> 66 #include <netinet/tcp_timer.h> 67 #include <netinet/tcp_var.h> 68 #include <netinet/tcp_syncache.h> 69 #include <netinet/tcp_offload.h> 70 #include <netinet/toecore.h> 71 72 static struct mtx toedev_lock; 73 static TAILQ_HEAD(, toedev) toedev_list; 74 static eventhandler_tag listen_start_eh; 75 static eventhandler_tag listen_stop_eh; 76 static eventhandler_tag lle_event_eh; 77 78 static int 79 toedev_connect(struct toedev *tod __unused, struct socket *so __unused, 80 struct nhop_object *nh __unused, struct sockaddr *nam __unused) 81 { 82 83 return (ENOTSUP); 84 } 85 86 static int 87 toedev_listen_start(struct toedev *tod __unused, struct tcpcb *tp __unused) 88 { 89 90 return (ENOTSUP); 91 } 92 93 static int 94 toedev_listen_stop(struct toedev *tod __unused, struct tcpcb *tp __unused) 95 { 96 97 return (ENOTSUP); 98 } 99 100 static void 101 toedev_input(struct toedev *tod __unused, struct tcpcb *tp __unused, 102 struct mbuf *m) 103 { 104 105 m_freem(m); 106 return; 107 } 108 109 static void 110 toedev_rcvd(struct toedev *tod __unused, struct tcpcb *tp __unused) 111 { 112 113 return; 114 } 115 116 static int 117 toedev_output(struct toedev *tod __unused, struct tcpcb *tp __unused) 118 { 119 120 return (ENOTSUP); 121 } 122 123 static void 124 toedev_pcb_detach(struct toedev *tod __unused, struct tcpcb *tp __unused) 125 { 126 127 return; 128 } 129 130 static void 131 toedev_l2_update(struct toedev *tod __unused, struct ifnet *ifp __unused, 132 struct sockaddr *sa __unused, uint8_t *lladdr __unused, 133 uint16_t vtag __unused) 134 { 135 136 return; 137 } 138 139 static void 140 toedev_route_redirect(struct toedev *tod __unused, struct ifnet *ifp __unused, 141 struct nhop_object *nh0 __unused, struct nhop_object *nh1 __unused) 142 { 143 144 return; 145 } 146 147 static void 148 toedev_syncache_added(struct toedev *tod __unused, void *ctx __unused) 149 { 150 151 return; 152 } 153 154 static void 155 toedev_syncache_removed(struct toedev *tod __unused, void *ctx __unused) 156 { 157 158 return; 159 } 160 161 static int 162 toedev_syncache_respond(struct toedev *tod __unused, void *ctx __unused, 163 struct mbuf *m) 164 { 165 166 m_freem(m); 167 return (0); 168 } 169 170 static void 171 toedev_offload_socket(struct toedev *tod __unused, void *ctx __unused, 172 struct socket *so __unused) 173 { 174 175 return; 176 } 177 178 static void 179 toedev_ctloutput(struct toedev *tod __unused, struct tcpcb *tp __unused, 180 int sopt_dir __unused, int sopt_name __unused) 181 { 182 183 return; 184 } 185 186 static void 187 toedev_tcp_info(struct toedev *tod __unused, struct tcpcb *tp __unused, 188 struct tcp_info *ti __unused) 189 { 190 191 return; 192 } 193 194 static int 195 toedev_alloc_tls_session(struct toedev *tod __unused, struct tcpcb *tp __unused, 196 struct ktls_session *tls __unused, int direction __unused) 197 { 198 199 return (EINVAL); 200 } 201 202 static void 203 toedev_pmtu_update(struct toedev *tod __unused, struct tcpcb *tp __unused, 204 tcp_seq seq __unused, int mtu __unused) 205 { 206 207 return; 208 } 209 210 /* 211 * Inform one or more TOE devices about a listening socket. 212 */ 213 static void 214 toe_listen_start(struct inpcb *inp, void *arg) 215 { 216 struct toedev *t, *tod; 217 struct tcpcb *tp; 218 219 INP_WLOCK_ASSERT(inp); 220 KASSERT(inp->inp_pcbinfo == &V_tcbinfo, 221 ("%s: inp is not a TCP inp", __func__)); 222 223 if (inp->inp_flags & INP_DROPPED) 224 return; 225 226 tp = intotcpcb(inp); 227 if (tp->t_state != TCPS_LISTEN) 228 return; 229 230 t = arg; 231 mtx_lock(&toedev_lock); 232 TAILQ_FOREACH(tod, &toedev_list, link) { 233 if (t == NULL || t == tod) 234 tod->tod_listen_start(tod, tp); 235 } 236 mtx_unlock(&toedev_lock); 237 } 238 239 static void 240 toe_listen_start_event(void *arg __unused, struct tcpcb *tp) 241 { 242 struct inpcb *inp = tptoinpcb(tp); 243 244 INP_WLOCK_ASSERT(inp); 245 KASSERT(tp->t_state == TCPS_LISTEN, 246 ("%s: t_state %s", __func__, tcpstates[tp->t_state])); 247 248 toe_listen_start(inp, NULL); 249 } 250 251 static void 252 toe_listen_stop_event(void *arg __unused, struct tcpcb *tp) 253 { 254 struct toedev *tod; 255 #ifdef INVARIANTS 256 struct inpcb *inp = tptoinpcb(tp); 257 #endif 258 259 INP_WLOCK_ASSERT(inp); 260 KASSERT(tp->t_state == TCPS_LISTEN, 261 ("%s: t_state %s", __func__, tcpstates[tp->t_state])); 262 263 mtx_lock(&toedev_lock); 264 TAILQ_FOREACH(tod, &toedev_list, link) 265 tod->tod_listen_stop(tod, tp); 266 mtx_unlock(&toedev_lock); 267 } 268 269 /* 270 * Fill up a freshly allocated toedev struct with reasonable defaults. 271 */ 272 void 273 init_toedev(struct toedev *tod) 274 { 275 276 tod->tod_softc = NULL; 277 278 /* 279 * Provide no-op defaults so that the kernel can call any toedev 280 * function without having to check whether the TOE driver supplied one 281 * or not. 282 */ 283 tod->tod_connect = toedev_connect; 284 tod->tod_listen_start = toedev_listen_start; 285 tod->tod_listen_stop = toedev_listen_stop; 286 tod->tod_input = toedev_input; 287 tod->tod_rcvd = toedev_rcvd; 288 tod->tod_output = toedev_output; 289 tod->tod_send_rst = toedev_output; 290 tod->tod_send_fin = toedev_output; 291 tod->tod_pcb_detach = toedev_pcb_detach; 292 tod->tod_l2_update = toedev_l2_update; 293 tod->tod_route_redirect = toedev_route_redirect; 294 tod->tod_syncache_added = toedev_syncache_added; 295 tod->tod_syncache_removed = toedev_syncache_removed; 296 tod->tod_syncache_respond = toedev_syncache_respond; 297 tod->tod_offload_socket = toedev_offload_socket; 298 tod->tod_ctloutput = toedev_ctloutput; 299 tod->tod_tcp_info = toedev_tcp_info; 300 tod->tod_alloc_tls_session = toedev_alloc_tls_session; 301 tod->tod_pmtu_update = toedev_pmtu_update; 302 } 303 304 /* 305 * Register an active TOE device with the system. This allows it to receive 306 * notifications from the kernel. 307 */ 308 int 309 register_toedev(struct toedev *tod) 310 { 311 struct toedev *t; 312 313 mtx_lock(&toedev_lock); 314 TAILQ_FOREACH(t, &toedev_list, link) { 315 if (t == tod) { 316 mtx_unlock(&toedev_lock); 317 return (EEXIST); 318 } 319 } 320 321 TAILQ_INSERT_TAIL(&toedev_list, tod, link); 322 registered_toedevs++; 323 mtx_unlock(&toedev_lock); 324 325 inp_apply_all(&V_tcbinfo, toe_listen_start, tod); 326 327 return (0); 328 } 329 330 /* 331 * Remove the TOE device from the global list of active TOE devices. It is the 332 * caller's responsibility to ensure that the TOE device is quiesced prior to 333 * this call. 334 */ 335 int 336 unregister_toedev(struct toedev *tod) 337 { 338 struct toedev *t, *t2; 339 int rc = ENODEV; 340 341 mtx_lock(&toedev_lock); 342 TAILQ_FOREACH_SAFE(t, &toedev_list, link, t2) { 343 if (t == tod) { 344 TAILQ_REMOVE(&toedev_list, tod, link); 345 registered_toedevs--; 346 rc = 0; 347 break; 348 } 349 } 350 KASSERT(registered_toedevs >= 0, 351 ("%s: registered_toedevs (%d) < 0", __func__, registered_toedevs)); 352 mtx_unlock(&toedev_lock); 353 return (rc); 354 } 355 356 void 357 toe_syncache_add(struct in_conninfo *inc, struct tcpopt *to, struct tcphdr *th, 358 struct inpcb *inp, void *tod, void *todctx, uint8_t iptos) 359 { 360 361 INP_RLOCK_ASSERT(inp); 362 363 (void )syncache_add(inc, to, th, inp, inp->inp_socket, NULL, tod, 364 todctx, iptos, htons(0)); 365 } 366 367 int 368 toe_syncache_expand(struct in_conninfo *inc, struct tcpopt *to, 369 struct tcphdr *th, struct socket **lsop) 370 { 371 372 NET_EPOCH_ASSERT(); 373 374 return (syncache_expand(inc, to, th, lsop, NULL, htons(0))); 375 } 376 377 /* 378 * General purpose check to see if a 4-tuple is in use by the kernel. If a TCP 379 * header (presumably for an incoming SYN) is also provided, an existing 4-tuple 380 * in TIME_WAIT may be assassinated freeing it up for re-use. 381 * 382 * Note that the TCP header must have been run through tcp_fields_to_host() or 383 * equivalent. 384 */ 385 int 386 toe_4tuple_check(struct in_conninfo *inc, struct tcphdr *th, struct ifnet *ifp) 387 { 388 struct inpcb *inp; 389 struct tcpcb *tp; 390 391 if (inc->inc_flags & INC_ISIPV6) { 392 inp = in6_pcblookup(&V_tcbinfo, &inc->inc6_faddr, 393 inc->inc_fport, &inc->inc6_laddr, inc->inc_lport, 394 INPLOOKUP_RLOCKPCB, ifp); 395 } else { 396 inp = in_pcblookup(&V_tcbinfo, inc->inc_faddr, inc->inc_fport, 397 inc->inc_laddr, inc->inc_lport, INPLOOKUP_RLOCKPCB, ifp); 398 } 399 if (inp != NULL) { 400 INP_RLOCK_ASSERT(inp); 401 402 tp = intotcpcb(inp); 403 if (tp->t_state == TCPS_TIME_WAIT && th != NULL) { 404 if (!tcp_twcheck(inp, NULL, th, NULL, 0)) 405 return (EADDRINUSE); 406 } else { 407 INP_RUNLOCK(inp); 408 return (EADDRINUSE); 409 } 410 } 411 412 return (0); 413 } 414 415 static void 416 toe_lle_event(void *arg __unused, struct llentry *lle, int evt) 417 { 418 struct toedev *tod; 419 struct ifnet *ifp; 420 struct sockaddr *sa; 421 uint8_t *lladdr; 422 uint16_t vid, pcp; 423 int family; 424 struct sockaddr_in6 sin6; 425 426 LLE_WLOCK_ASSERT(lle); 427 428 ifp = lltable_get_ifp(lle->lle_tbl); 429 family = lltable_get_af(lle->lle_tbl); 430 431 if (family != AF_INET && family != AF_INET6) 432 return; 433 /* 434 * Not interested if the interface's TOE capability is not enabled. 435 */ 436 if ((family == AF_INET && !(ifp->if_capenable & IFCAP_TOE4)) || 437 (family == AF_INET6 && !(ifp->if_capenable & IFCAP_TOE6))) 438 return; 439 440 tod = TOEDEV(ifp); 441 if (tod == NULL) 442 return; 443 444 sa = (struct sockaddr *)&sin6; 445 lltable_fill_sa_entry(lle, sa); 446 447 vid = 0xfff; 448 pcp = 0; 449 if (evt != LLENTRY_RESOLVED) { 450 /* 451 * LLENTRY_TIMEDOUT, LLENTRY_DELETED, LLENTRY_EXPIRED all mean 452 * this entry is going to be deleted. 453 */ 454 455 lladdr = NULL; 456 } else { 457 KASSERT(lle->la_flags & LLE_VALID, 458 ("%s: %p resolved but not valid?", __func__, lle)); 459 460 lladdr = (uint8_t *)lle->ll_addr; 461 VLAN_TAG(ifp, &vid); 462 VLAN_PCP(ifp, &pcp); 463 } 464 465 tod->tod_l2_update(tod, ifp, sa, lladdr, EVL_MAKETAG(vid, pcp, 0)); 466 } 467 468 /* 469 * Returns 0 or EWOULDBLOCK on success (any other value is an error). 0 means 470 * lladdr and vtag are valid on return, EWOULDBLOCK means the TOE driver's 471 * tod_l2_update will be called later, when the entry is resolved or times out. 472 */ 473 int 474 toe_l2_resolve(struct toedev *tod, struct ifnet *ifp, struct sockaddr *sa, 475 uint8_t *lladdr, uint16_t *vtag) 476 { 477 int rc; 478 uint16_t vid, pcp; 479 480 switch (sa->sa_family) { 481 #ifdef INET 482 case AF_INET: 483 rc = arpresolve(ifp, 0, NULL, sa, lladdr, NULL, NULL); 484 break; 485 #endif 486 #ifdef INET6 487 case AF_INET6: 488 rc = nd6_resolve(ifp, LLE_SF(AF_INET6, 0), NULL, sa, lladdr, 489 NULL, NULL); 490 break; 491 #endif 492 default: 493 return (EPROTONOSUPPORT); 494 } 495 496 if (rc == 0) { 497 vid = 0xfff; 498 pcp = 0; 499 if (ifp->if_type == IFT_L2VLAN) { 500 VLAN_TAG(ifp, &vid); 501 VLAN_PCP(ifp, &pcp); 502 } else if (ifp->if_pcp != IFNET_PCP_NONE) { 503 vid = 0; 504 pcp = ifp->if_pcp; 505 } 506 *vtag = EVL_MAKETAG(vid, pcp, 0); 507 } 508 509 return (rc); 510 } 511 512 void 513 toe_connect_failed(struct toedev *tod, struct inpcb *inp, int err) 514 { 515 516 NET_EPOCH_ASSERT(); 517 INP_WLOCK_ASSERT(inp); 518 519 if (!(inp->inp_flags & INP_DROPPED)) { 520 struct tcpcb *tp = intotcpcb(inp); 521 522 KASSERT(tp->t_flags & TF_TOE, 523 ("%s: tp %p not offloaded.", __func__, tp)); 524 525 if (err == EAGAIN) { 526 /* 527 * Temporary failure during offload, take this PCB back. 528 * Detach from the TOE driver and do the rest of what 529 * TCP's pru_connect would have done if the connection 530 * wasn't offloaded. 531 */ 532 533 tod->tod_pcb_detach(tod, tp); 534 KASSERT(!(tp->t_flags & TF_TOE), 535 ("%s: tp %p still offloaded.", __func__, tp)); 536 tcp_timer_activate(tp, TT_KEEP, TP_KEEPINIT(tp)); 537 if (tcp_output(tp) < 0) 538 INP_WLOCK(inp); /* re-acquire */ 539 } else { 540 tp = tcp_drop(tp, err); 541 if (tp == NULL) 542 INP_WLOCK(inp); /* re-acquire */ 543 } 544 } 545 INP_WLOCK_ASSERT(inp); 546 } 547 548 static int 549 toecore_load(void) 550 { 551 552 mtx_init(&toedev_lock, "toedev lock", NULL, MTX_DEF); 553 TAILQ_INIT(&toedev_list); 554 555 listen_start_eh = EVENTHANDLER_REGISTER(tcp_offload_listen_start, 556 toe_listen_start_event, NULL, EVENTHANDLER_PRI_ANY); 557 listen_stop_eh = EVENTHANDLER_REGISTER(tcp_offload_listen_stop, 558 toe_listen_stop_event, NULL, EVENTHANDLER_PRI_ANY); 559 lle_event_eh = EVENTHANDLER_REGISTER(lle_event, toe_lle_event, NULL, 560 EVENTHANDLER_PRI_ANY); 561 562 return (0); 563 } 564 565 static int 566 toecore_unload(void) 567 { 568 569 mtx_lock(&toedev_lock); 570 if (!TAILQ_EMPTY(&toedev_list)) { 571 mtx_unlock(&toedev_lock); 572 return (EBUSY); 573 } 574 575 EVENTHANDLER_DEREGISTER(tcp_offload_listen_start, listen_start_eh); 576 EVENTHANDLER_DEREGISTER(tcp_offload_listen_stop, listen_stop_eh); 577 EVENTHANDLER_DEREGISTER(lle_event, lle_event_eh); 578 579 mtx_unlock(&toedev_lock); 580 mtx_destroy(&toedev_lock); 581 582 return (0); 583 } 584 585 static int 586 toecore_mod_handler(module_t mod, int cmd, void *arg) 587 { 588 589 if (cmd == MOD_LOAD) 590 return (toecore_load()); 591 592 if (cmd == MOD_UNLOAD) 593 return (toecore_unload()); 594 595 return (EOPNOTSUPP); 596 } 597 598 static moduledata_t mod_data= { 599 "toecore", 600 toecore_mod_handler, 601 0 602 }; 603 604 MODULE_VERSION(toecore, 1); 605 DECLARE_MODULE(toecore, mod_data, SI_SUB_EXEC, SI_ORDER_ANY); 606