1 /*- 2 * Copyright (c) 2012 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __FBSDID("$FreeBSD$"); 30 31 #include "opt_inet.h" 32 #include "opt_inet6.h" 33 34 #include <sys/param.h> 35 #include <sys/kernel.h> 36 #include <sys/systm.h> 37 #include <sys/mbuf.h> 38 #include <sys/module.h> 39 #include <sys/types.h> 40 #include <sys/sockopt.h> 41 #include <sys/sysctl.h> 42 #include <sys/socket.h> 43 44 #include <net/ethernet.h> 45 #include <net/if.h> 46 #include <net/if_var.h> 47 #include <net/if_types.h> 48 #include <net/if_vlan_var.h> 49 #include <net/if_llatbl.h> 50 #include <net/route.h> 51 52 #include <netinet/if_ether.h> 53 #include <netinet/in.h> 54 #include <netinet/in_pcb.h> 55 #include <netinet/in_var.h> 56 #include <netinet6/in6_var.h> 57 #include <netinet6/in6_pcb.h> 58 #include <netinet6/nd6.h> 59 #define TCPSTATES 60 #include <netinet/tcp.h> 61 #include <netinet/tcp_fsm.h> 62 #include <netinet/tcp_timer.h> 63 #include <netinet/tcp_var.h> 64 #include <netinet/tcp_syncache.h> 65 #include <netinet/tcp_offload.h> 66 #include <netinet/toecore.h> 67 68 static struct mtx toedev_lock; 69 static TAILQ_HEAD(, toedev) toedev_list; 70 static eventhandler_tag listen_start_eh; 71 static eventhandler_tag listen_stop_eh; 72 static eventhandler_tag lle_event_eh; 73 static eventhandler_tag route_redirect_eh; 74 75 static int 76 toedev_connect(struct toedev *tod __unused, struct socket *so __unused, 77 struct rtentry *rt __unused, struct sockaddr *nam __unused) 78 { 79 80 return (ENOTSUP); 81 } 82 83 static int 84 toedev_listen_start(struct toedev *tod __unused, struct tcpcb *tp __unused) 85 { 86 87 return (ENOTSUP); 88 } 89 90 static int 91 toedev_listen_stop(struct toedev *tod __unused, struct tcpcb *tp __unused) 92 { 93 94 return (ENOTSUP); 95 } 96 97 static void 98 toedev_input(struct toedev *tod __unused, struct tcpcb *tp __unused, 99 struct mbuf *m) 100 { 101 102 m_freem(m); 103 return; 104 } 105 106 static void 107 toedev_rcvd(struct toedev *tod __unused, struct tcpcb *tp __unused) 108 { 109 110 return; 111 } 112 113 static int 114 toedev_output(struct toedev *tod __unused, struct tcpcb *tp __unused) 115 { 116 117 return (ENOTSUP); 118 } 119 120 static void 121 toedev_pcb_detach(struct toedev *tod __unused, struct tcpcb *tp __unused) 122 { 123 124 return; 125 } 126 127 static void 128 toedev_l2_update(struct toedev *tod __unused, struct ifnet *ifp __unused, 129 struct sockaddr *sa __unused, uint8_t *lladdr __unused, 130 uint16_t vtag __unused) 131 { 132 133 return; 134 } 135 136 static void 137 toedev_route_redirect(struct toedev *tod __unused, struct ifnet *ifp __unused, 138 struct rtentry *rt0 __unused, struct rtentry *rt1 __unused) 139 { 140 141 return; 142 } 143 144 static void 145 toedev_syncache_added(struct toedev *tod __unused, void *ctx __unused) 146 { 147 148 return; 149 } 150 151 static void 152 toedev_syncache_removed(struct toedev *tod __unused, void *ctx __unused) 153 { 154 155 return; 156 } 157 158 static int 159 toedev_syncache_respond(struct toedev *tod __unused, void *ctx __unused, 160 struct mbuf *m) 161 { 162 163 m_freem(m); 164 return (0); 165 } 166 167 static void 168 toedev_offload_socket(struct toedev *tod __unused, void *ctx __unused, 169 struct socket *so __unused) 170 { 171 172 return; 173 } 174 175 static void 176 toedev_ctloutput(struct toedev *tod __unused, struct tcpcb *tp __unused, 177 int sopt_dir __unused, int sopt_name __unused) 178 { 179 180 return; 181 } 182 183 /* 184 * Inform one or more TOE devices about a listening socket. 185 */ 186 static void 187 toe_listen_start(struct inpcb *inp, void *arg) 188 { 189 struct toedev *t, *tod; 190 struct tcpcb *tp; 191 192 INP_WLOCK_ASSERT(inp); 193 KASSERT(inp->inp_pcbinfo == &V_tcbinfo, 194 ("%s: inp is not a TCP inp", __func__)); 195 196 if (inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) 197 return; 198 199 tp = intotcpcb(inp); 200 if (tp->t_state != TCPS_LISTEN) 201 return; 202 203 t = arg; 204 mtx_lock(&toedev_lock); 205 TAILQ_FOREACH(tod, &toedev_list, link) { 206 if (t == NULL || t == tod) 207 tod->tod_listen_start(tod, tp); 208 } 209 mtx_unlock(&toedev_lock); 210 } 211 212 static void 213 toe_listen_start_event(void *arg __unused, struct tcpcb *tp) 214 { 215 struct inpcb *inp = tp->t_inpcb; 216 217 INP_WLOCK_ASSERT(inp); 218 KASSERT(tp->t_state == TCPS_LISTEN, 219 ("%s: t_state %s", __func__, tcpstates[tp->t_state])); 220 221 toe_listen_start(inp, NULL); 222 } 223 224 static void 225 toe_listen_stop_event(void *arg __unused, struct tcpcb *tp) 226 { 227 struct toedev *tod; 228 #ifdef INVARIANTS 229 struct inpcb *inp = tp->t_inpcb; 230 #endif 231 232 INP_WLOCK_ASSERT(inp); 233 KASSERT(tp->t_state == TCPS_LISTEN, 234 ("%s: t_state %s", __func__, tcpstates[tp->t_state])); 235 236 mtx_lock(&toedev_lock); 237 TAILQ_FOREACH(tod, &toedev_list, link) 238 tod->tod_listen_stop(tod, tp); 239 mtx_unlock(&toedev_lock); 240 } 241 242 /* 243 * Fill up a freshly allocated toedev struct with reasonable defaults. 244 */ 245 void 246 init_toedev(struct toedev *tod) 247 { 248 249 tod->tod_softc = NULL; 250 251 /* 252 * Provide no-op defaults so that the kernel can call any toedev 253 * function without having to check whether the TOE driver supplied one 254 * or not. 255 */ 256 tod->tod_connect = toedev_connect; 257 tod->tod_listen_start = toedev_listen_start; 258 tod->tod_listen_stop = toedev_listen_stop; 259 tod->tod_input = toedev_input; 260 tod->tod_rcvd = toedev_rcvd; 261 tod->tod_output = toedev_output; 262 tod->tod_send_rst = toedev_output; 263 tod->tod_send_fin = toedev_output; 264 tod->tod_pcb_detach = toedev_pcb_detach; 265 tod->tod_l2_update = toedev_l2_update; 266 tod->tod_route_redirect = toedev_route_redirect; 267 tod->tod_syncache_added = toedev_syncache_added; 268 tod->tod_syncache_removed = toedev_syncache_removed; 269 tod->tod_syncache_respond = toedev_syncache_respond; 270 tod->tod_offload_socket = toedev_offload_socket; 271 tod->tod_ctloutput = toedev_ctloutput; 272 } 273 274 /* 275 * Register an active TOE device with the system. This allows it to receive 276 * notifications from the kernel. 277 */ 278 int 279 register_toedev(struct toedev *tod) 280 { 281 struct toedev *t; 282 283 mtx_lock(&toedev_lock); 284 TAILQ_FOREACH(t, &toedev_list, link) { 285 if (t == tod) { 286 mtx_unlock(&toedev_lock); 287 return (EEXIST); 288 } 289 } 290 291 TAILQ_INSERT_TAIL(&toedev_list, tod, link); 292 registered_toedevs++; 293 mtx_unlock(&toedev_lock); 294 295 inp_apply_all(toe_listen_start, tod); 296 297 return (0); 298 } 299 300 /* 301 * Remove the TOE device from the global list of active TOE devices. It is the 302 * caller's responsibility to ensure that the TOE device is quiesced prior to 303 * this call. 304 */ 305 int 306 unregister_toedev(struct toedev *tod) 307 { 308 struct toedev *t, *t2; 309 int rc = ENODEV; 310 311 mtx_lock(&toedev_lock); 312 TAILQ_FOREACH_SAFE(t, &toedev_list, link, t2) { 313 if (t == tod) { 314 TAILQ_REMOVE(&toedev_list, tod, link); 315 registered_toedevs--; 316 rc = 0; 317 break; 318 } 319 } 320 KASSERT(registered_toedevs >= 0, 321 ("%s: registered_toedevs (%d) < 0", __func__, registered_toedevs)); 322 mtx_unlock(&toedev_lock); 323 return (rc); 324 } 325 326 void 327 toe_syncache_add(struct in_conninfo *inc, struct tcpopt *to, struct tcphdr *th, 328 struct inpcb *inp, void *tod, void *todctx) 329 { 330 struct socket *lso = inp->inp_socket; 331 332 INP_INFO_WLOCK_ASSERT(&V_tcbinfo); 333 INP_WLOCK_ASSERT(inp); 334 335 syncache_add(inc, to, th, inp, &lso, NULL, tod, todctx); 336 } 337 338 int 339 toe_syncache_expand(struct in_conninfo *inc, struct tcpopt *to, 340 struct tcphdr *th, struct socket **lsop) 341 { 342 343 INP_INFO_WLOCK_ASSERT(&V_tcbinfo); 344 345 return (syncache_expand(inc, to, th, lsop, NULL)); 346 } 347 348 /* 349 * General purpose check to see if a 4-tuple is in use by the kernel. If a TCP 350 * header (presumably for an incoming SYN) is also provided, an existing 4-tuple 351 * in TIME_WAIT may be assassinated freeing it up for re-use. 352 * 353 * Note that the TCP header must have been run through tcp_fields_to_host() or 354 * equivalent. 355 */ 356 int 357 toe_4tuple_check(struct in_conninfo *inc, struct tcphdr *th, struct ifnet *ifp) 358 { 359 struct inpcb *inp; 360 361 if (inc->inc_flags & INC_ISIPV6) { 362 inp = in6_pcblookup(&V_tcbinfo, &inc->inc6_faddr, 363 inc->inc_fport, &inc->inc6_laddr, inc->inc_lport, 364 INPLOOKUP_WLOCKPCB, ifp); 365 } else { 366 inp = in_pcblookup(&V_tcbinfo, inc->inc_faddr, inc->inc_fport, 367 inc->inc_laddr, inc->inc_lport, INPLOOKUP_WLOCKPCB, ifp); 368 } 369 if (inp != NULL) { 370 INP_WLOCK_ASSERT(inp); 371 372 if ((inp->inp_flags & INP_TIMEWAIT) && th != NULL) { 373 374 INP_INFO_WLOCK_ASSERT(&V_tcbinfo); /* for twcheck */ 375 if (!tcp_twcheck(inp, NULL, th, NULL, 0)) 376 return (EADDRINUSE); 377 } else { 378 INP_WUNLOCK(inp); 379 return (EADDRINUSE); 380 } 381 } 382 383 return (0); 384 } 385 386 static void 387 toe_lle_event(void *arg __unused, struct llentry *lle, int evt) 388 { 389 struct toedev *tod; 390 struct ifnet *ifp; 391 struct sockaddr *sa; 392 uint8_t *lladdr; 393 uint16_t vtag; 394 395 LLE_WLOCK_ASSERT(lle); 396 397 ifp = lle->lle_tbl->llt_ifp; 398 sa = L3_ADDR(lle); 399 400 KASSERT(sa->sa_family == AF_INET || sa->sa_family == AF_INET6, 401 ("%s: lle_event %d for lle %p but sa %p !INET && !INET6", 402 __func__, evt, lle, sa)); 403 404 /* 405 * Not interested if the interface's TOE capability is not enabled. 406 */ 407 if ((sa->sa_family == AF_INET && !(ifp->if_capenable & IFCAP_TOE4)) || 408 (sa->sa_family == AF_INET6 && !(ifp->if_capenable & IFCAP_TOE6))) 409 return; 410 411 tod = TOEDEV(ifp); 412 if (tod == NULL) 413 return; 414 415 vtag = 0xfff; 416 if (evt != LLENTRY_RESOLVED) { 417 418 /* 419 * LLENTRY_TIMEDOUT, LLENTRY_DELETED, LLENTRY_EXPIRED all mean 420 * this entry is going to be deleted. 421 */ 422 423 lladdr = NULL; 424 } else { 425 426 KASSERT(lle->la_flags & LLE_VALID, 427 ("%s: %p resolved but not valid?", __func__, lle)); 428 429 lladdr = (uint8_t *)&lle->ll_addr; 430 #ifdef VLAN_TAG 431 VLAN_TAG(ifp, &vtag); 432 #endif 433 } 434 435 tod->tod_l2_update(tod, ifp, sa, lladdr, vtag); 436 } 437 438 /* 439 * XXX: implement. 440 */ 441 static void 442 toe_route_redirect_event(void *arg __unused, struct rtentry *rt0, 443 struct rtentry *rt1, struct sockaddr *sa) 444 { 445 446 return; 447 } 448 449 #ifdef INET6 450 /* 451 * XXX: no checks to verify that sa is really a neighbor because we assume it is 452 * the result of a route lookup and is on-link on the given ifp. 453 */ 454 static int 455 toe_nd6_resolve(struct ifnet *ifp, struct sockaddr *sa, uint8_t *lladdr) 456 { 457 struct llentry *lle; 458 struct sockaddr_in6 *sin6 = (void *)sa; 459 int rc, flags = 0; 460 461 restart: 462 IF_AFDATA_RLOCK(ifp); 463 lle = lla_lookup(LLTABLE6(ifp), flags, sa); 464 IF_AFDATA_RUNLOCK(ifp); 465 if (lle == NULL) { 466 IF_AFDATA_LOCK(ifp); 467 lle = nd6_lookup(&sin6->sin6_addr, ND6_CREATE | ND6_EXCLUSIVE, 468 ifp); 469 IF_AFDATA_UNLOCK(ifp); 470 if (lle == NULL) 471 return (ENOMEM); /* Couldn't create entry in cache. */ 472 lle->ln_state = ND6_LLINFO_INCOMPLETE; 473 nd6_llinfo_settimer_locked(lle, 474 (long)ND_IFINFO(ifp)->retrans * hz / 1000); 475 LLE_WUNLOCK(lle); 476 477 nd6_ns_output(ifp, NULL, &sin6->sin6_addr, NULL, 0); 478 479 return (EWOULDBLOCK); 480 } 481 482 if (lle->ln_state == ND6_LLINFO_STALE) { 483 if ((flags & LLE_EXCLUSIVE) == 0) { 484 LLE_RUNLOCK(lle); 485 flags |= LLE_EXCLUSIVE; 486 goto restart; 487 } 488 489 LLE_WLOCK_ASSERT(lle); 490 491 lle->la_asked = 0; 492 lle->ln_state = ND6_LLINFO_DELAY; 493 nd6_llinfo_settimer_locked(lle, (long)V_nd6_delay * hz); 494 } 495 496 if (lle->la_flags & LLE_VALID) { 497 memcpy(lladdr, &lle->ll_addr, ifp->if_addrlen); 498 rc = 0; 499 } else 500 rc = EWOULDBLOCK; 501 502 if (flags & LLE_EXCLUSIVE) 503 LLE_WUNLOCK(lle); 504 else 505 LLE_RUNLOCK(lle); 506 507 return (rc); 508 } 509 #endif 510 511 /* 512 * Returns 0 or EWOULDBLOCK on success (any other value is an error). 0 means 513 * lladdr and vtag are valid on return, EWOULDBLOCK means the TOE driver's 514 * tod_l2_update will be called later, when the entry is resolved or times out. 515 */ 516 int 517 toe_l2_resolve(struct toedev *tod, struct ifnet *ifp, struct sockaddr *sa, 518 uint8_t *lladdr, uint16_t *vtag) 519 { 520 #ifdef INET 521 struct llentry *lle; 522 #endif 523 int rc; 524 525 switch (sa->sa_family) { 526 #ifdef INET 527 case AF_INET: 528 rc = arpresolve(ifp, NULL, NULL, sa, lladdr, &lle); 529 break; 530 #endif 531 #ifdef INET6 532 case AF_INET6: 533 rc = toe_nd6_resolve(ifp, sa, lladdr); 534 break; 535 #endif 536 default: 537 return (EPROTONOSUPPORT); 538 } 539 540 if (rc == 0) { 541 #ifdef VLAN_TAG 542 if (VLAN_TAG(ifp, vtag) != 0) 543 #endif 544 *vtag = 0xfff; 545 } 546 547 return (rc); 548 } 549 550 void 551 toe_connect_failed(struct toedev *tod, struct inpcb *inp, int err) 552 { 553 554 INP_WLOCK_ASSERT(inp); 555 556 if (!(inp->inp_flags & INP_DROPPED)) { 557 struct tcpcb *tp = intotcpcb(inp); 558 559 KASSERT(tp->t_flags & TF_TOE, 560 ("%s: tp %p not offloaded.", __func__, tp)); 561 562 if (err == EAGAIN) { 563 564 /* 565 * Temporary failure during offload, take this PCB back. 566 * Detach from the TOE driver and do the rest of what 567 * TCP's pru_connect would have done if the connection 568 * wasn't offloaded. 569 */ 570 571 tod->tod_pcb_detach(tod, tp); 572 KASSERT(!(tp->t_flags & TF_TOE), 573 ("%s: tp %p still offloaded.", __func__, tp)); 574 tcp_timer_activate(tp, TT_KEEP, TP_KEEPINIT(tp)); 575 (void) tcp_output(tp); 576 } else { 577 578 INP_INFO_WLOCK_ASSERT(&V_tcbinfo); 579 tp = tcp_drop(tp, err); 580 if (tp == NULL) 581 INP_WLOCK(inp); /* re-acquire */ 582 } 583 } 584 INP_WLOCK_ASSERT(inp); 585 } 586 587 static int 588 toecore_load(void) 589 { 590 591 mtx_init(&toedev_lock, "toedev lock", NULL, MTX_DEF); 592 TAILQ_INIT(&toedev_list); 593 594 listen_start_eh = EVENTHANDLER_REGISTER(tcp_offload_listen_start, 595 toe_listen_start_event, NULL, EVENTHANDLER_PRI_ANY); 596 listen_stop_eh = EVENTHANDLER_REGISTER(tcp_offload_listen_stop, 597 toe_listen_stop_event, NULL, EVENTHANDLER_PRI_ANY); 598 lle_event_eh = EVENTHANDLER_REGISTER(lle_event, toe_lle_event, NULL, 599 EVENTHANDLER_PRI_ANY); 600 route_redirect_eh = EVENTHANDLER_REGISTER(route_redirect_event, 601 toe_route_redirect_event, NULL, EVENTHANDLER_PRI_ANY); 602 603 return (0); 604 } 605 606 static int 607 toecore_unload(void) 608 { 609 610 mtx_lock(&toedev_lock); 611 if (!TAILQ_EMPTY(&toedev_list)) { 612 mtx_unlock(&toedev_lock); 613 return (EBUSY); 614 } 615 616 EVENTHANDLER_DEREGISTER(tcp_offload_listen_start, listen_start_eh); 617 EVENTHANDLER_DEREGISTER(tcp_offload_listen_stop, listen_stop_eh); 618 EVENTHANDLER_DEREGISTER(lle_event, lle_event_eh); 619 EVENTHANDLER_DEREGISTER(route_redirect_event, route_redirect_eh); 620 621 mtx_unlock(&toedev_lock); 622 mtx_destroy(&toedev_lock); 623 624 return (0); 625 } 626 627 static int 628 toecore_mod_handler(module_t mod, int cmd, void *arg) 629 { 630 631 if (cmd == MOD_LOAD) 632 return (toecore_load()); 633 634 if (cmd == MOD_UNLOAD) 635 return (toecore_unload()); 636 637 return (EOPNOTSUPP); 638 } 639 640 static moduledata_t mod_data= { 641 "toecore", 642 toecore_mod_handler, 643 0 644 }; 645 646 MODULE_VERSION(toecore, 1); 647 DECLARE_MODULE(toecore, mod_data, SI_SUB_EXEC, SI_ORDER_ANY); 648