1 /*- 2 * Copyright (c) 2012 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __FBSDID("$FreeBSD$"); 30 31 #include "opt_inet.h" 32 #include "opt_inet6.h" 33 34 #include <sys/param.h> 35 #include <sys/kernel.h> 36 #include <sys/systm.h> 37 #include <sys/mbuf.h> 38 #include <sys/module.h> 39 #include <sys/types.h> 40 #include <sys/sockopt.h> 41 #include <sys/sysctl.h> 42 #include <sys/socket.h> 43 44 #include <net/ethernet.h> 45 #include <net/if.h> 46 #include <net/if_var.h> 47 #include <net/if_types.h> 48 #include <net/if_vlan_var.h> 49 #include <net/if_llatbl.h> 50 #include <net/route.h> 51 52 #include <netinet/if_ether.h> 53 #include <netinet/in.h> 54 #include <netinet/in_pcb.h> 55 #include <netinet/in_var.h> 56 #include <netinet6/in6_var.h> 57 #include <netinet6/in6_pcb.h> 58 #include <netinet6/nd6.h> 59 #define TCPSTATES 60 #include <netinet/tcp.h> 61 #include <netinet/tcp_fsm.h> 62 #include <netinet/tcp_timer.h> 63 #include <netinet/tcp_var.h> 64 #include <netinet/tcp_syncache.h> 65 #include <netinet/tcp_offload.h> 66 #include <netinet/toecore.h> 67 68 static struct mtx toedev_lock; 69 static TAILQ_HEAD(, toedev) toedev_list; 70 static eventhandler_tag listen_start_eh; 71 static eventhandler_tag listen_stop_eh; 72 static eventhandler_tag lle_event_eh; 73 74 static int 75 toedev_connect(struct toedev *tod __unused, struct socket *so __unused, 76 struct rtentry *rt __unused, struct sockaddr *nam __unused) 77 { 78 79 return (ENOTSUP); 80 } 81 82 static int 83 toedev_listen_start(struct toedev *tod __unused, struct tcpcb *tp __unused) 84 { 85 86 return (ENOTSUP); 87 } 88 89 static int 90 toedev_listen_stop(struct toedev *tod __unused, struct tcpcb *tp __unused) 91 { 92 93 return (ENOTSUP); 94 } 95 96 static void 97 toedev_input(struct toedev *tod __unused, struct tcpcb *tp __unused, 98 struct mbuf *m) 99 { 100 101 m_freem(m); 102 return; 103 } 104 105 static void 106 toedev_rcvd(struct toedev *tod __unused, struct tcpcb *tp __unused) 107 { 108 109 return; 110 } 111 112 static int 113 toedev_output(struct toedev *tod __unused, struct tcpcb *tp __unused) 114 { 115 116 return (ENOTSUP); 117 } 118 119 static void 120 toedev_pcb_detach(struct toedev *tod __unused, struct tcpcb *tp __unused) 121 { 122 123 return; 124 } 125 126 static void 127 toedev_l2_update(struct toedev *tod __unused, struct ifnet *ifp __unused, 128 struct sockaddr *sa __unused, uint8_t *lladdr __unused, 129 uint16_t vtag __unused) 130 { 131 132 return; 133 } 134 135 static void 136 toedev_route_redirect(struct toedev *tod __unused, struct ifnet *ifp __unused, 137 struct rtentry *rt0 __unused, struct rtentry *rt1 __unused) 138 { 139 140 return; 141 } 142 143 static void 144 toedev_syncache_added(struct toedev *tod __unused, void *ctx __unused) 145 { 146 147 return; 148 } 149 150 static void 151 toedev_syncache_removed(struct toedev *tod __unused, void *ctx __unused) 152 { 153 154 return; 155 } 156 157 static int 158 toedev_syncache_respond(struct toedev *tod __unused, void *ctx __unused, 159 struct mbuf *m) 160 { 161 162 m_freem(m); 163 return (0); 164 } 165 166 static void 167 toedev_offload_socket(struct toedev *tod __unused, void *ctx __unused, 168 struct socket *so __unused) 169 { 170 171 return; 172 } 173 174 static void 175 toedev_ctloutput(struct toedev *tod __unused, struct tcpcb *tp __unused, 176 int sopt_dir __unused, int sopt_name __unused) 177 { 178 179 return; 180 } 181 182 /* 183 * Inform one or more TOE devices about a listening socket. 184 */ 185 static void 186 toe_listen_start(struct inpcb *inp, void *arg) 187 { 188 struct toedev *t, *tod; 189 struct tcpcb *tp; 190 191 INP_WLOCK_ASSERT(inp); 192 KASSERT(inp->inp_pcbinfo == &V_tcbinfo, 193 ("%s: inp is not a TCP inp", __func__)); 194 195 if (inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) 196 return; 197 198 tp = intotcpcb(inp); 199 if (tp->t_state != TCPS_LISTEN) 200 return; 201 202 t = arg; 203 mtx_lock(&toedev_lock); 204 TAILQ_FOREACH(tod, &toedev_list, link) { 205 if (t == NULL || t == tod) 206 tod->tod_listen_start(tod, tp); 207 } 208 mtx_unlock(&toedev_lock); 209 } 210 211 static void 212 toe_listen_start_event(void *arg __unused, struct tcpcb *tp) 213 { 214 struct inpcb *inp = tp->t_inpcb; 215 216 INP_WLOCK_ASSERT(inp); 217 KASSERT(tp->t_state == TCPS_LISTEN, 218 ("%s: t_state %s", __func__, tcpstates[tp->t_state])); 219 220 toe_listen_start(inp, NULL); 221 } 222 223 static void 224 toe_listen_stop_event(void *arg __unused, struct tcpcb *tp) 225 { 226 struct toedev *tod; 227 #ifdef INVARIANTS 228 struct inpcb *inp = tp->t_inpcb; 229 #endif 230 231 INP_WLOCK_ASSERT(inp); 232 KASSERT(tp->t_state == TCPS_LISTEN, 233 ("%s: t_state %s", __func__, tcpstates[tp->t_state])); 234 235 mtx_lock(&toedev_lock); 236 TAILQ_FOREACH(tod, &toedev_list, link) 237 tod->tod_listen_stop(tod, tp); 238 mtx_unlock(&toedev_lock); 239 } 240 241 /* 242 * Fill up a freshly allocated toedev struct with reasonable defaults. 243 */ 244 void 245 init_toedev(struct toedev *tod) 246 { 247 248 tod->tod_softc = NULL; 249 250 /* 251 * Provide no-op defaults so that the kernel can call any toedev 252 * function without having to check whether the TOE driver supplied one 253 * or not. 254 */ 255 tod->tod_connect = toedev_connect; 256 tod->tod_listen_start = toedev_listen_start; 257 tod->tod_listen_stop = toedev_listen_stop; 258 tod->tod_input = toedev_input; 259 tod->tod_rcvd = toedev_rcvd; 260 tod->tod_output = toedev_output; 261 tod->tod_send_rst = toedev_output; 262 tod->tod_send_fin = toedev_output; 263 tod->tod_pcb_detach = toedev_pcb_detach; 264 tod->tod_l2_update = toedev_l2_update; 265 tod->tod_route_redirect = toedev_route_redirect; 266 tod->tod_syncache_added = toedev_syncache_added; 267 tod->tod_syncache_removed = toedev_syncache_removed; 268 tod->tod_syncache_respond = toedev_syncache_respond; 269 tod->tod_offload_socket = toedev_offload_socket; 270 tod->tod_ctloutput = toedev_ctloutput; 271 } 272 273 /* 274 * Register an active TOE device with the system. This allows it to receive 275 * notifications from the kernel. 276 */ 277 int 278 register_toedev(struct toedev *tod) 279 { 280 struct toedev *t; 281 282 mtx_lock(&toedev_lock); 283 TAILQ_FOREACH(t, &toedev_list, link) { 284 if (t == tod) { 285 mtx_unlock(&toedev_lock); 286 return (EEXIST); 287 } 288 } 289 290 TAILQ_INSERT_TAIL(&toedev_list, tod, link); 291 registered_toedevs++; 292 mtx_unlock(&toedev_lock); 293 294 inp_apply_all(toe_listen_start, tod); 295 296 return (0); 297 } 298 299 /* 300 * Remove the TOE device from the global list of active TOE devices. It is the 301 * caller's responsibility to ensure that the TOE device is quiesced prior to 302 * this call. 303 */ 304 int 305 unregister_toedev(struct toedev *tod) 306 { 307 struct toedev *t, *t2; 308 int rc = ENODEV; 309 310 mtx_lock(&toedev_lock); 311 TAILQ_FOREACH_SAFE(t, &toedev_list, link, t2) { 312 if (t == tod) { 313 TAILQ_REMOVE(&toedev_list, tod, link); 314 registered_toedevs--; 315 rc = 0; 316 break; 317 } 318 } 319 KASSERT(registered_toedevs >= 0, 320 ("%s: registered_toedevs (%d) < 0", __func__, registered_toedevs)); 321 mtx_unlock(&toedev_lock); 322 return (rc); 323 } 324 325 void 326 toe_syncache_add(struct in_conninfo *inc, struct tcpopt *to, struct tcphdr *th, 327 struct inpcb *inp, void *tod, void *todctx) 328 { 329 struct socket *lso = inp->inp_socket; 330 331 INP_WLOCK_ASSERT(inp); 332 333 syncache_add(inc, to, th, inp, &lso, NULL, tod, todctx); 334 } 335 336 int 337 toe_syncache_expand(struct in_conninfo *inc, struct tcpopt *to, 338 struct tcphdr *th, struct socket **lsop) 339 { 340 341 INP_INFO_RLOCK_ASSERT(&V_tcbinfo); 342 343 return (syncache_expand(inc, to, th, lsop, NULL)); 344 } 345 346 /* 347 * General purpose check to see if a 4-tuple is in use by the kernel. If a TCP 348 * header (presumably for an incoming SYN) is also provided, an existing 4-tuple 349 * in TIME_WAIT may be assassinated freeing it up for re-use. 350 * 351 * Note that the TCP header must have been run through tcp_fields_to_host() or 352 * equivalent. 353 */ 354 int 355 toe_4tuple_check(struct in_conninfo *inc, struct tcphdr *th, struct ifnet *ifp) 356 { 357 struct inpcb *inp; 358 359 if (inc->inc_flags & INC_ISIPV6) { 360 inp = in6_pcblookup(&V_tcbinfo, &inc->inc6_faddr, 361 inc->inc_fport, &inc->inc6_laddr, inc->inc_lport, 362 INPLOOKUP_WLOCKPCB, ifp); 363 } else { 364 inp = in_pcblookup(&V_tcbinfo, inc->inc_faddr, inc->inc_fport, 365 inc->inc_laddr, inc->inc_lport, INPLOOKUP_WLOCKPCB, ifp); 366 } 367 if (inp != NULL) { 368 INP_WLOCK_ASSERT(inp); 369 370 if ((inp->inp_flags & INP_TIMEWAIT) && th != NULL) { 371 372 INP_INFO_RLOCK_ASSERT(&V_tcbinfo); /* for twcheck */ 373 if (!tcp_twcheck(inp, NULL, th, NULL, 0)) 374 return (EADDRINUSE); 375 } else { 376 INP_WUNLOCK(inp); 377 return (EADDRINUSE); 378 } 379 } 380 381 return (0); 382 } 383 384 static void 385 toe_lle_event(void *arg __unused, struct llentry *lle, int evt) 386 { 387 struct toedev *tod; 388 struct ifnet *ifp; 389 struct sockaddr *sa; 390 uint8_t *lladdr; 391 uint16_t vtag; 392 int family; 393 struct sockaddr_in6 sin6; 394 395 LLE_WLOCK_ASSERT(lle); 396 397 ifp = lltable_get_ifp(lle->lle_tbl); 398 family = lltable_get_af(lle->lle_tbl); 399 400 if (family != AF_INET && family != AF_INET6) 401 return; 402 /* 403 * Not interested if the interface's TOE capability is not enabled. 404 */ 405 if ((family == AF_INET && !(ifp->if_capenable & IFCAP_TOE4)) || 406 (family == AF_INET6 && !(ifp->if_capenable & IFCAP_TOE6))) 407 return; 408 409 tod = TOEDEV(ifp); 410 if (tod == NULL) 411 return; 412 413 sa = (struct sockaddr *)&sin6; 414 lltable_fill_sa_entry(lle, sa); 415 416 vtag = 0xfff; 417 if (evt != LLENTRY_RESOLVED) { 418 419 /* 420 * LLENTRY_TIMEDOUT, LLENTRY_DELETED, LLENTRY_EXPIRED all mean 421 * this entry is going to be deleted. 422 */ 423 424 lladdr = NULL; 425 } else { 426 427 KASSERT(lle->la_flags & LLE_VALID, 428 ("%s: %p resolved but not valid?", __func__, lle)); 429 430 lladdr = (uint8_t *)lle->ll_addr; 431 #ifdef VLAN_TAG 432 VLAN_TAG(ifp, &vtag); 433 #endif 434 } 435 436 tod->tod_l2_update(tod, ifp, sa, lladdr, vtag); 437 } 438 439 /* 440 * Returns 0 or EWOULDBLOCK on success (any other value is an error). 0 means 441 * lladdr and vtag are valid on return, EWOULDBLOCK means the TOE driver's 442 * tod_l2_update will be called later, when the entry is resolved or times out. 443 */ 444 int 445 toe_l2_resolve(struct toedev *tod, struct ifnet *ifp, struct sockaddr *sa, 446 uint8_t *lladdr, uint16_t *vtag) 447 { 448 int rc; 449 450 switch (sa->sa_family) { 451 #ifdef INET 452 case AF_INET: 453 rc = arpresolve(ifp, 0, NULL, sa, lladdr, NULL); 454 break; 455 #endif 456 #ifdef INET6 457 case AF_INET6: 458 rc = nd6_resolve(ifp, 0, NULL, sa, lladdr, NULL); 459 break; 460 #endif 461 default: 462 return (EPROTONOSUPPORT); 463 } 464 465 if (rc == 0) { 466 #ifdef VLAN_TAG 467 if (VLAN_TAG(ifp, vtag) != 0) 468 #endif 469 *vtag = 0xfff; 470 } 471 472 return (rc); 473 } 474 475 void 476 toe_connect_failed(struct toedev *tod, struct inpcb *inp, int err) 477 { 478 479 INP_WLOCK_ASSERT(inp); 480 481 if (!(inp->inp_flags & INP_DROPPED)) { 482 struct tcpcb *tp = intotcpcb(inp); 483 484 KASSERT(tp->t_flags & TF_TOE, 485 ("%s: tp %p not offloaded.", __func__, tp)); 486 487 if (err == EAGAIN) { 488 489 /* 490 * Temporary failure during offload, take this PCB back. 491 * Detach from the TOE driver and do the rest of what 492 * TCP's pru_connect would have done if the connection 493 * wasn't offloaded. 494 */ 495 496 tod->tod_pcb_detach(tod, tp); 497 KASSERT(!(tp->t_flags & TF_TOE), 498 ("%s: tp %p still offloaded.", __func__, tp)); 499 tcp_timer_activate(tp, TT_KEEP, TP_KEEPINIT(tp)); 500 (void) tp->t_fb->tfb_tcp_output(tp); 501 } else { 502 503 INP_INFO_RLOCK_ASSERT(&V_tcbinfo); 504 tp = tcp_drop(tp, err); 505 if (tp == NULL) 506 INP_WLOCK(inp); /* re-acquire */ 507 } 508 } 509 INP_WLOCK_ASSERT(inp); 510 } 511 512 static int 513 toecore_load(void) 514 { 515 516 mtx_init(&toedev_lock, "toedev lock", NULL, MTX_DEF); 517 TAILQ_INIT(&toedev_list); 518 519 listen_start_eh = EVENTHANDLER_REGISTER(tcp_offload_listen_start, 520 toe_listen_start_event, NULL, EVENTHANDLER_PRI_ANY); 521 listen_stop_eh = EVENTHANDLER_REGISTER(tcp_offload_listen_stop, 522 toe_listen_stop_event, NULL, EVENTHANDLER_PRI_ANY); 523 lle_event_eh = EVENTHANDLER_REGISTER(lle_event, toe_lle_event, NULL, 524 EVENTHANDLER_PRI_ANY); 525 526 return (0); 527 } 528 529 static int 530 toecore_unload(void) 531 { 532 533 mtx_lock(&toedev_lock); 534 if (!TAILQ_EMPTY(&toedev_list)) { 535 mtx_unlock(&toedev_lock); 536 return (EBUSY); 537 } 538 539 EVENTHANDLER_DEREGISTER(tcp_offload_listen_start, listen_start_eh); 540 EVENTHANDLER_DEREGISTER(tcp_offload_listen_stop, listen_stop_eh); 541 EVENTHANDLER_DEREGISTER(lle_event, lle_event_eh); 542 543 mtx_unlock(&toedev_lock); 544 mtx_destroy(&toedev_lock); 545 546 return (0); 547 } 548 549 static int 550 toecore_mod_handler(module_t mod, int cmd, void *arg) 551 { 552 553 if (cmd == MOD_LOAD) 554 return (toecore_load()); 555 556 if (cmd == MOD_UNLOAD) 557 return (toecore_unload()); 558 559 return (EOPNOTSUPP); 560 } 561 562 static moduledata_t mod_data= { 563 "toecore", 564 toecore_mod_handler, 565 0 566 }; 567 568 MODULE_VERSION(toecore, 1); 569 DECLARE_MODULE(toecore, mod_data, SI_SUB_EXEC, SI_ORDER_ANY); 570