1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2012 Chelsio Communications, Inc. 5 * All rights reserved. 6 * Written by: Navdeep Parhar <np@FreeBSD.org> 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include <sys/cdefs.h> 31 __FBSDID("$FreeBSD$"); 32 33 #include "opt_inet.h" 34 #include "opt_inet6.h" 35 36 #include <sys/param.h> 37 #include <sys/kernel.h> 38 #include <sys/systm.h> 39 #include <sys/malloc.h> 40 #include <sys/mbuf.h> 41 #include <sys/module.h> 42 #include <sys/types.h> 43 #include <sys/sockopt.h> 44 #include <sys/sysctl.h> 45 #include <sys/socket.h> 46 47 #include <net/ethernet.h> 48 #include <net/if.h> 49 #include <net/if_var.h> 50 #include <net/if_types.h> 51 #include <net/if_vlan_var.h> 52 #include <net/if_llatbl.h> 53 #include <net/route.h> 54 55 #include <netinet/if_ether.h> 56 #include <netinet/in.h> 57 #include <netinet/in_pcb.h> 58 #include <netinet/in_var.h> 59 #include <netinet6/in6_var.h> 60 #include <netinet6/in6_pcb.h> 61 #include <netinet6/nd6.h> 62 #define TCPSTATES 63 #include <netinet/tcp.h> 64 #include <netinet/tcp_fsm.h> 65 #include <netinet/tcp_timer.h> 66 #include <netinet/tcp_var.h> 67 #include <netinet/tcp_syncache.h> 68 #include <netinet/tcp_offload.h> 69 #include <netinet/toecore.h> 70 71 static struct mtx toedev_lock; 72 static TAILQ_HEAD(, toedev) toedev_list; 73 static eventhandler_tag listen_start_eh; 74 static eventhandler_tag listen_stop_eh; 75 static eventhandler_tag lle_event_eh; 76 77 static int 78 toedev_connect(struct toedev *tod __unused, struct socket *so __unused, 79 struct rtentry *rt __unused, struct sockaddr *nam __unused) 80 { 81 82 return (ENOTSUP); 83 } 84 85 static int 86 toedev_listen_start(struct toedev *tod __unused, struct tcpcb *tp __unused) 87 { 88 89 return (ENOTSUP); 90 } 91 92 static int 93 toedev_listen_stop(struct toedev *tod __unused, struct tcpcb *tp __unused) 94 { 95 96 return (ENOTSUP); 97 } 98 99 static void 100 toedev_input(struct toedev *tod __unused, struct tcpcb *tp __unused, 101 struct mbuf *m) 102 { 103 104 m_freem(m); 105 return; 106 } 107 108 static void 109 toedev_rcvd(struct toedev *tod __unused, struct tcpcb *tp __unused) 110 { 111 112 return; 113 } 114 115 static int 116 toedev_output(struct toedev *tod __unused, struct tcpcb *tp __unused) 117 { 118 119 return (ENOTSUP); 120 } 121 122 static void 123 toedev_pcb_detach(struct toedev *tod __unused, struct tcpcb *tp __unused) 124 { 125 126 return; 127 } 128 129 static void 130 toedev_l2_update(struct toedev *tod __unused, struct ifnet *ifp __unused, 131 struct sockaddr *sa __unused, uint8_t *lladdr __unused, 132 uint16_t vtag __unused) 133 { 134 135 return; 136 } 137 138 static void 139 toedev_route_redirect(struct toedev *tod __unused, struct ifnet *ifp __unused, 140 struct rtentry *rt0 __unused, struct rtentry *rt1 __unused) 141 { 142 143 return; 144 } 145 146 static void 147 toedev_syncache_added(struct toedev *tod __unused, void *ctx __unused) 148 { 149 150 return; 151 } 152 153 static void 154 toedev_syncache_removed(struct toedev *tod __unused, void *ctx __unused) 155 { 156 157 return; 158 } 159 160 static int 161 toedev_syncache_respond(struct toedev *tod __unused, void *ctx __unused, 162 struct mbuf *m) 163 { 164 165 m_freem(m); 166 return (0); 167 } 168 169 static void 170 toedev_offload_socket(struct toedev *tod __unused, void *ctx __unused, 171 struct socket *so __unused) 172 { 173 174 return; 175 } 176 177 static void 178 toedev_ctloutput(struct toedev *tod __unused, struct tcpcb *tp __unused, 179 int sopt_dir __unused, int sopt_name __unused) 180 { 181 182 return; 183 } 184 185 /* 186 * Inform one or more TOE devices about a listening socket. 187 */ 188 static void 189 toe_listen_start(struct inpcb *inp, void *arg) 190 { 191 struct toedev *t, *tod; 192 struct tcpcb *tp; 193 194 INP_WLOCK_ASSERT(inp); 195 KASSERT(inp->inp_pcbinfo == &V_tcbinfo, 196 ("%s: inp is not a TCP inp", __func__)); 197 198 if (inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) 199 return; 200 201 tp = intotcpcb(inp); 202 if (tp->t_state != TCPS_LISTEN) 203 return; 204 205 t = arg; 206 mtx_lock(&toedev_lock); 207 TAILQ_FOREACH(tod, &toedev_list, link) { 208 if (t == NULL || t == tod) 209 tod->tod_listen_start(tod, tp); 210 } 211 mtx_unlock(&toedev_lock); 212 } 213 214 static void 215 toe_listen_start_event(void *arg __unused, struct tcpcb *tp) 216 { 217 struct inpcb *inp = tp->t_inpcb; 218 219 INP_WLOCK_ASSERT(inp); 220 KASSERT(tp->t_state == TCPS_LISTEN, 221 ("%s: t_state %s", __func__, tcpstates[tp->t_state])); 222 223 toe_listen_start(inp, NULL); 224 } 225 226 static void 227 toe_listen_stop_event(void *arg __unused, struct tcpcb *tp) 228 { 229 struct toedev *tod; 230 #ifdef INVARIANTS 231 struct inpcb *inp = tp->t_inpcb; 232 #endif 233 234 INP_WLOCK_ASSERT(inp); 235 KASSERT(tp->t_state == TCPS_LISTEN, 236 ("%s: t_state %s", __func__, tcpstates[tp->t_state])); 237 238 mtx_lock(&toedev_lock); 239 TAILQ_FOREACH(tod, &toedev_list, link) 240 tod->tod_listen_stop(tod, tp); 241 mtx_unlock(&toedev_lock); 242 } 243 244 /* 245 * Fill up a freshly allocated toedev struct with reasonable defaults. 246 */ 247 void 248 init_toedev(struct toedev *tod) 249 { 250 251 tod->tod_softc = NULL; 252 253 /* 254 * Provide no-op defaults so that the kernel can call any toedev 255 * function without having to check whether the TOE driver supplied one 256 * or not. 257 */ 258 tod->tod_connect = toedev_connect; 259 tod->tod_listen_start = toedev_listen_start; 260 tod->tod_listen_stop = toedev_listen_stop; 261 tod->tod_input = toedev_input; 262 tod->tod_rcvd = toedev_rcvd; 263 tod->tod_output = toedev_output; 264 tod->tod_send_rst = toedev_output; 265 tod->tod_send_fin = toedev_output; 266 tod->tod_pcb_detach = toedev_pcb_detach; 267 tod->tod_l2_update = toedev_l2_update; 268 tod->tod_route_redirect = toedev_route_redirect; 269 tod->tod_syncache_added = toedev_syncache_added; 270 tod->tod_syncache_removed = toedev_syncache_removed; 271 tod->tod_syncache_respond = toedev_syncache_respond; 272 tod->tod_offload_socket = toedev_offload_socket; 273 tod->tod_ctloutput = toedev_ctloutput; 274 } 275 276 /* 277 * Register an active TOE device with the system. This allows it to receive 278 * notifications from the kernel. 279 */ 280 int 281 register_toedev(struct toedev *tod) 282 { 283 struct toedev *t; 284 285 mtx_lock(&toedev_lock); 286 TAILQ_FOREACH(t, &toedev_list, link) { 287 if (t == tod) { 288 mtx_unlock(&toedev_lock); 289 return (EEXIST); 290 } 291 } 292 293 TAILQ_INSERT_TAIL(&toedev_list, tod, link); 294 registered_toedevs++; 295 mtx_unlock(&toedev_lock); 296 297 inp_apply_all(toe_listen_start, tod); 298 299 return (0); 300 } 301 302 /* 303 * Remove the TOE device from the global list of active TOE devices. It is the 304 * caller's responsibility to ensure that the TOE device is quiesced prior to 305 * this call. 306 */ 307 int 308 unregister_toedev(struct toedev *tod) 309 { 310 struct toedev *t, *t2; 311 int rc = ENODEV; 312 313 mtx_lock(&toedev_lock); 314 TAILQ_FOREACH_SAFE(t, &toedev_list, link, t2) { 315 if (t == tod) { 316 TAILQ_REMOVE(&toedev_list, tod, link); 317 registered_toedevs--; 318 rc = 0; 319 break; 320 } 321 } 322 KASSERT(registered_toedevs >= 0, 323 ("%s: registered_toedevs (%d) < 0", __func__, registered_toedevs)); 324 mtx_unlock(&toedev_lock); 325 return (rc); 326 } 327 328 void 329 toe_syncache_add(struct in_conninfo *inc, struct tcpopt *to, struct tcphdr *th, 330 struct inpcb *inp, void *tod, void *todctx) 331 { 332 struct socket *lso = inp->inp_socket; 333 334 INP_WLOCK_ASSERT(inp); 335 336 syncache_add(inc, to, th, inp, &lso, NULL, tod, todctx); 337 } 338 339 int 340 toe_syncache_expand(struct in_conninfo *inc, struct tcpopt *to, 341 struct tcphdr *th, struct socket **lsop) 342 { 343 344 INP_INFO_RLOCK_ASSERT(&V_tcbinfo); 345 346 return (syncache_expand(inc, to, th, lsop, NULL)); 347 } 348 349 /* 350 * General purpose check to see if a 4-tuple is in use by the kernel. If a TCP 351 * header (presumably for an incoming SYN) is also provided, an existing 4-tuple 352 * in TIME_WAIT may be assassinated freeing it up for re-use. 353 * 354 * Note that the TCP header must have been run through tcp_fields_to_host() or 355 * equivalent. 356 */ 357 int 358 toe_4tuple_check(struct in_conninfo *inc, struct tcphdr *th, struct ifnet *ifp) 359 { 360 struct inpcb *inp; 361 362 if (inc->inc_flags & INC_ISIPV6) { 363 inp = in6_pcblookup(&V_tcbinfo, &inc->inc6_faddr, 364 inc->inc_fport, &inc->inc6_laddr, inc->inc_lport, 365 INPLOOKUP_WLOCKPCB, ifp); 366 } else { 367 inp = in_pcblookup(&V_tcbinfo, inc->inc_faddr, inc->inc_fport, 368 inc->inc_laddr, inc->inc_lport, INPLOOKUP_WLOCKPCB, ifp); 369 } 370 if (inp != NULL) { 371 INP_WLOCK_ASSERT(inp); 372 373 if ((inp->inp_flags & INP_TIMEWAIT) && th != NULL) { 374 375 INP_INFO_RLOCK_ASSERT(&V_tcbinfo); /* for twcheck */ 376 if (!tcp_twcheck(inp, NULL, th, NULL, 0)) 377 return (EADDRINUSE); 378 } else { 379 INP_WUNLOCK(inp); 380 return (EADDRINUSE); 381 } 382 } 383 384 return (0); 385 } 386 387 static void 388 toe_lle_event(void *arg __unused, struct llentry *lle, int evt) 389 { 390 struct toedev *tod; 391 struct ifnet *ifp; 392 struct sockaddr *sa; 393 uint8_t *lladdr; 394 uint16_t vtag; 395 int family; 396 struct sockaddr_in6 sin6; 397 398 LLE_WLOCK_ASSERT(lle); 399 400 ifp = lltable_get_ifp(lle->lle_tbl); 401 family = lltable_get_af(lle->lle_tbl); 402 403 if (family != AF_INET && family != AF_INET6) 404 return; 405 /* 406 * Not interested if the interface's TOE capability is not enabled. 407 */ 408 if ((family == AF_INET && !(ifp->if_capenable & IFCAP_TOE4)) || 409 (family == AF_INET6 && !(ifp->if_capenable & IFCAP_TOE6))) 410 return; 411 412 tod = TOEDEV(ifp); 413 if (tod == NULL) 414 return; 415 416 sa = (struct sockaddr *)&sin6; 417 lltable_fill_sa_entry(lle, sa); 418 419 vtag = 0xfff; 420 if (evt != LLENTRY_RESOLVED) { 421 422 /* 423 * LLENTRY_TIMEDOUT, LLENTRY_DELETED, LLENTRY_EXPIRED all mean 424 * this entry is going to be deleted. 425 */ 426 427 lladdr = NULL; 428 } else { 429 430 KASSERT(lle->la_flags & LLE_VALID, 431 ("%s: %p resolved but not valid?", __func__, lle)); 432 433 lladdr = (uint8_t *)lle->ll_addr; 434 #ifdef VLAN_TAG 435 VLAN_TAG(ifp, &vtag); 436 #endif 437 } 438 439 tod->tod_l2_update(tod, ifp, sa, lladdr, vtag); 440 } 441 442 /* 443 * Returns 0 or EWOULDBLOCK on success (any other value is an error). 0 means 444 * lladdr and vtag are valid on return, EWOULDBLOCK means the TOE driver's 445 * tod_l2_update will be called later, when the entry is resolved or times out. 446 */ 447 int 448 toe_l2_resolve(struct toedev *tod, struct ifnet *ifp, struct sockaddr *sa, 449 uint8_t *lladdr, uint16_t *vtag) 450 { 451 int rc; 452 453 switch (sa->sa_family) { 454 #ifdef INET 455 case AF_INET: 456 rc = arpresolve(ifp, 0, NULL, sa, lladdr, NULL, NULL); 457 break; 458 #endif 459 #ifdef INET6 460 case AF_INET6: 461 rc = nd6_resolve(ifp, 0, NULL, sa, lladdr, NULL, NULL); 462 break; 463 #endif 464 default: 465 return (EPROTONOSUPPORT); 466 } 467 468 if (rc == 0) { 469 #ifdef VLAN_TAG 470 if (VLAN_TAG(ifp, vtag) != 0) 471 #endif 472 *vtag = 0xfff; 473 } 474 475 return (rc); 476 } 477 478 void 479 toe_connect_failed(struct toedev *tod, struct inpcb *inp, int err) 480 { 481 482 INP_WLOCK_ASSERT(inp); 483 484 if (!(inp->inp_flags & INP_DROPPED)) { 485 struct tcpcb *tp = intotcpcb(inp); 486 487 KASSERT(tp->t_flags & TF_TOE, 488 ("%s: tp %p not offloaded.", __func__, tp)); 489 490 if (err == EAGAIN) { 491 492 /* 493 * Temporary failure during offload, take this PCB back. 494 * Detach from the TOE driver and do the rest of what 495 * TCP's pru_connect would have done if the connection 496 * wasn't offloaded. 497 */ 498 499 tod->tod_pcb_detach(tod, tp); 500 KASSERT(!(tp->t_flags & TF_TOE), 501 ("%s: tp %p still offloaded.", __func__, tp)); 502 tcp_timer_activate(tp, TT_KEEP, TP_KEEPINIT(tp)); 503 (void) tp->t_fb->tfb_tcp_output(tp); 504 } else { 505 506 INP_INFO_RLOCK_ASSERT(&V_tcbinfo); 507 tp = tcp_drop(tp, err); 508 if (tp == NULL) 509 INP_WLOCK(inp); /* re-acquire */ 510 } 511 } 512 INP_WLOCK_ASSERT(inp); 513 } 514 515 static int 516 toecore_load(void) 517 { 518 519 mtx_init(&toedev_lock, "toedev lock", NULL, MTX_DEF); 520 TAILQ_INIT(&toedev_list); 521 522 listen_start_eh = EVENTHANDLER_REGISTER(tcp_offload_listen_start, 523 toe_listen_start_event, NULL, EVENTHANDLER_PRI_ANY); 524 listen_stop_eh = EVENTHANDLER_REGISTER(tcp_offload_listen_stop, 525 toe_listen_stop_event, NULL, EVENTHANDLER_PRI_ANY); 526 lle_event_eh = EVENTHANDLER_REGISTER(lle_event, toe_lle_event, NULL, 527 EVENTHANDLER_PRI_ANY); 528 529 return (0); 530 } 531 532 static int 533 toecore_unload(void) 534 { 535 536 mtx_lock(&toedev_lock); 537 if (!TAILQ_EMPTY(&toedev_list)) { 538 mtx_unlock(&toedev_lock); 539 return (EBUSY); 540 } 541 542 EVENTHANDLER_DEREGISTER(tcp_offload_listen_start, listen_start_eh); 543 EVENTHANDLER_DEREGISTER(tcp_offload_listen_stop, listen_stop_eh); 544 EVENTHANDLER_DEREGISTER(lle_event, lle_event_eh); 545 546 mtx_unlock(&toedev_lock); 547 mtx_destroy(&toedev_lock); 548 549 return (0); 550 } 551 552 static int 553 toecore_mod_handler(module_t mod, int cmd, void *arg) 554 { 555 556 if (cmd == MOD_LOAD) 557 return (toecore_load()); 558 559 if (cmd == MOD_UNLOAD) 560 return (toecore_unload()); 561 562 return (EOPNOTSUPP); 563 } 564 565 static moduledata_t mod_data= { 566 "toecore", 567 toecore_mod_handler, 568 0 569 }; 570 571 MODULE_VERSION(toecore, 1); 572 DECLARE_MODULE(toecore, mod_data, SI_SUB_EXEC, SI_ORDER_ANY); 573