1 /*- 2 * Copyright (c) 2012 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __FBSDID("$FreeBSD$"); 30 31 #include "opt_inet.h" 32 33 #include <sys/param.h> 34 #include <sys/types.h> 35 #include <sys/systm.h> 36 #include <sys/kernel.h> 37 #include <sys/ktr.h> 38 #include <sys/module.h> 39 #include <sys/protosw.h> 40 #include <sys/domain.h> 41 #include <sys/socket.h> 42 #include <sys/socketvar.h> 43 #include <netinet/in.h> 44 #include <netinet/in_pcb.h> 45 #include <netinet/ip.h> 46 #include <netinet/tcp_var.h> 47 #define TCPSTATES 48 #include <netinet/tcp_fsm.h> 49 #include <netinet/toecore.h> 50 51 #ifdef TCP_OFFLOAD 52 #include "common/common.h" 53 #include "common/t4_msg.h" 54 #include "common/t4_regs.h" 55 #include "tom/t4_tom_l2t.h" 56 #include "tom/t4_tom.h" 57 58 static struct protosw ddp_protosw; 59 static struct pr_usrreqs ddp_usrreqs; 60 61 /* Module ops */ 62 static int t4_tom_mod_load(void); 63 static int t4_tom_mod_unload(void); 64 static int t4_tom_modevent(module_t, int, void *); 65 66 /* ULD ops and helpers */ 67 static int t4_tom_activate(struct adapter *); 68 static int t4_tom_deactivate(struct adapter *); 69 70 static struct uld_info tom_uld_info = { 71 .uld_id = ULD_TOM, 72 .activate = t4_tom_activate, 73 .deactivate = t4_tom_deactivate, 74 }; 75 76 static void queue_tid_release(struct adapter *, int); 77 static void release_offload_resources(struct toepcb *); 78 static int alloc_tid_tabs(struct tid_info *); 79 static void free_tid_tabs(struct tid_info *); 80 static void free_tom_data(struct adapter *, struct tom_data *); 81 82 struct toepcb * 83 alloc_toepcb(struct port_info *pi, int txqid, int rxqid, int flags) 84 { 85 struct adapter *sc = pi->adapter; 86 struct toepcb *toep; 87 int tx_credits, txsd_total, len; 88 89 /* 90 * The firmware counts tx work request credits in units of 16 bytes 91 * each. Reserve room for an ABORT_REQ so the driver never has to worry 92 * about tx credits if it wants to abort a connection. 93 */ 94 tx_credits = sc->params.ofldq_wr_cred; 95 tx_credits -= howmany(sizeof(struct cpl_abort_req), 16); 96 97 /* 98 * Shortest possible tx work request is a fw_ofld_tx_data_wr + 1 byte 99 * immediate payload, and firmware counts tx work request credits in 100 * units of 16 byte. Calculate the maximum work requests possible. 101 */ 102 txsd_total = tx_credits / 103 howmany((sizeof(struct fw_ofld_tx_data_wr) + 1), 16); 104 105 if (txqid < 0) 106 txqid = (arc4random() % pi->nofldtxq) + pi->first_ofld_txq; 107 KASSERT(txqid >= pi->first_ofld_txq && 108 txqid < pi->first_ofld_txq + pi->nofldtxq, 109 ("%s: txqid %d for port %p (first %d, n %d)", __func__, txqid, pi, 110 pi->first_ofld_txq, pi->nofldtxq)); 111 112 if (rxqid < 0) 113 rxqid = (arc4random() % pi->nofldrxq) + pi->first_ofld_rxq; 114 KASSERT(rxqid >= pi->first_ofld_rxq && 115 rxqid < pi->first_ofld_rxq + pi->nofldrxq, 116 ("%s: rxqid %d for port %p (first %d, n %d)", __func__, rxqid, pi, 117 pi->first_ofld_rxq, pi->nofldrxq)); 118 119 len = offsetof(struct toepcb, txsd) + 120 txsd_total * sizeof(struct ofld_tx_sdesc); 121 122 toep = malloc(len, M_CXGBE, M_ZERO | flags); 123 if (toep == NULL) 124 return (NULL); 125 126 toep->td = sc->tom_softc; 127 toep->port = pi; 128 toep->tx_credits = tx_credits; 129 toep->ofld_txq = &sc->sge.ofld_txq[txqid]; 130 toep->ofld_rxq = &sc->sge.ofld_rxq[rxqid]; 131 toep->ctrlq = &sc->sge.ctrlq[pi->port_id]; 132 toep->txsd_total = txsd_total; 133 toep->txsd_avail = txsd_total; 134 toep->txsd_pidx = 0; 135 toep->txsd_cidx = 0; 136 137 return (toep); 138 } 139 140 void 141 free_toepcb(struct toepcb *toep) 142 { 143 144 KASSERT(!(toep->flags & TPF_ATTACHED), 145 ("%s: attached to an inpcb", __func__)); 146 KASSERT(!(toep->flags & TPF_CPL_PENDING), 147 ("%s: CPL pending", __func__)); 148 149 free(toep, M_CXGBE); 150 } 151 152 /* 153 * Set up the socket for TCP offload. 154 */ 155 void 156 offload_socket(struct socket *so, struct toepcb *toep) 157 { 158 struct tom_data *td = toep->td; 159 struct inpcb *inp = sotoinpcb(so); 160 struct tcpcb *tp = intotcpcb(inp); 161 struct sockbuf *sb; 162 163 INP_WLOCK_ASSERT(inp); 164 165 /* Update socket */ 166 sb = &so->so_snd; 167 SOCKBUF_LOCK(sb); 168 sb->sb_flags |= SB_NOCOALESCE; 169 SOCKBUF_UNLOCK(sb); 170 sb = &so->so_rcv; 171 SOCKBUF_LOCK(sb); 172 sb->sb_flags |= SB_NOCOALESCE; 173 if (toep->ulp_mode == ULP_MODE_TCPDDP) 174 so->so_proto = &ddp_protosw; 175 SOCKBUF_UNLOCK(sb); 176 177 /* Update TCP PCB */ 178 tp->tod = &td->tod; 179 tp->t_toe = toep; 180 tp->t_flags |= TF_TOE; 181 182 /* Install an extra hold on inp */ 183 toep->inp = inp; 184 toep->flags |= TPF_ATTACHED; 185 in_pcbref(inp); 186 187 /* Add the TOE PCB to the active list */ 188 mtx_lock(&td->toep_list_lock); 189 TAILQ_INSERT_HEAD(&td->toep_list, toep, link); 190 mtx_unlock(&td->toep_list_lock); 191 } 192 193 /* This is _not_ the normal way to "unoffload" a socket. */ 194 void 195 undo_offload_socket(struct socket *so) 196 { 197 struct inpcb *inp = sotoinpcb(so); 198 struct tcpcb *tp = intotcpcb(inp); 199 struct toepcb *toep = tp->t_toe; 200 struct tom_data *td = toep->td; 201 struct sockbuf *sb; 202 203 INP_WLOCK_ASSERT(inp); 204 205 sb = &so->so_snd; 206 SOCKBUF_LOCK(sb); 207 sb->sb_flags &= ~SB_NOCOALESCE; 208 SOCKBUF_UNLOCK(sb); 209 sb = &so->so_rcv; 210 SOCKBUF_LOCK(sb); 211 sb->sb_flags &= ~SB_NOCOALESCE; 212 SOCKBUF_UNLOCK(sb); 213 214 tp->tod = NULL; 215 tp->t_toe = NULL; 216 tp->t_flags &= ~TF_TOE; 217 218 toep->inp = NULL; 219 toep->flags &= ~TPF_ATTACHED; 220 if (in_pcbrele_wlocked(inp)) 221 panic("%s: inp freed.", __func__); 222 223 mtx_lock(&td->toep_list_lock); 224 TAILQ_REMOVE(&td->toep_list, toep, link); 225 mtx_unlock(&td->toep_list_lock); 226 } 227 228 static void 229 release_offload_resources(struct toepcb *toep) 230 { 231 struct tom_data *td = toep->td; 232 struct adapter *sc = td_adapter(td); 233 int tid = toep->tid; 234 235 KASSERT(!(toep->flags & TPF_CPL_PENDING), 236 ("%s: %p has CPL pending.", __func__, toep)); 237 KASSERT(!(toep->flags & TPF_ATTACHED), 238 ("%s: %p is still attached.", __func__, toep)); 239 240 CTR4(KTR_CXGBE, "%s: toep %p (tid %d, l2te %p)", 241 __func__, toep, tid, toep->l2te); 242 243 if (toep->ulp_mode == ULP_MODE_TCPDDP) 244 release_ddp_resources(toep); 245 246 if (toep->l2te) 247 t4_l2t_release(toep->l2te); 248 249 if (tid >= 0) { 250 remove_tid(sc, tid); 251 release_tid(sc, tid, toep->ctrlq); 252 } 253 254 mtx_lock(&td->toep_list_lock); 255 TAILQ_REMOVE(&td->toep_list, toep, link); 256 mtx_unlock(&td->toep_list_lock); 257 258 free_toepcb(toep); 259 } 260 261 /* 262 * The kernel is done with the TCP PCB and this is our opportunity to unhook the 263 * toepcb hanging off of it. If the TOE driver is also done with the toepcb (no 264 * pending CPL) then it is time to release all resources tied to the toepcb. 265 * 266 * Also gets called when an offloaded active open fails and the TOM wants the 267 * kernel to take the TCP PCB back. 268 */ 269 static void 270 t4_pcb_detach(struct toedev *tod __unused, struct tcpcb *tp) 271 { 272 #if defined(KTR) || defined(INVARIANTS) 273 struct inpcb *inp = tp->t_inpcb; 274 #endif 275 struct toepcb *toep = tp->t_toe; 276 277 INP_WLOCK_ASSERT(inp); 278 279 KASSERT(toep != NULL, ("%s: toep is NULL", __func__)); 280 KASSERT(toep->flags & TPF_ATTACHED, 281 ("%s: not attached", __func__)); 282 283 #ifdef KTR 284 if (tp->t_state == TCPS_SYN_SENT) { 285 CTR6(KTR_CXGBE, "%s: atid %d, toep %p (0x%x), inp %p (0x%x)", 286 __func__, toep->tid, toep, toep->flags, inp, 287 inp->inp_flags); 288 } else { 289 CTR6(KTR_CXGBE, 290 "t4_pcb_detach: tid %d (%s), toep %p (0x%x), inp %p (0x%x)", 291 toep->tid, tcpstates[tp->t_state], toep, toep->flags, inp, 292 inp->inp_flags); 293 } 294 #endif 295 296 tp->t_toe = NULL; 297 tp->t_flags &= ~TF_TOE; 298 toep->flags &= ~TPF_ATTACHED; 299 300 if (!(toep->flags & TPF_CPL_PENDING)) 301 release_offload_resources(toep); 302 } 303 304 /* 305 * The TOE driver will not receive any more CPLs for the tid associated with the 306 * toepcb; release the hold on the inpcb. 307 */ 308 void 309 final_cpl_received(struct toepcb *toep) 310 { 311 struct inpcb *inp = toep->inp; 312 313 KASSERT(inp != NULL, ("%s: inp is NULL", __func__)); 314 INP_WLOCK_ASSERT(inp); 315 KASSERT(toep->flags & TPF_CPL_PENDING, 316 ("%s: CPL not pending already?", __func__)); 317 318 CTR6(KTR_CXGBE, "%s: tid %d, toep %p (0x%x), inp %p (0x%x)", 319 __func__, toep->tid, toep, toep->flags, inp, inp->inp_flags); 320 321 toep->inp = NULL; 322 toep->flags &= ~TPF_CPL_PENDING; 323 324 if (!(toep->flags & TPF_ATTACHED)) 325 release_offload_resources(toep); 326 327 if (!in_pcbrele_wlocked(inp)) 328 INP_WUNLOCK(inp); 329 } 330 331 void 332 insert_tid(struct adapter *sc, int tid, void *ctx) 333 { 334 struct tid_info *t = &sc->tids; 335 336 t->tid_tab[tid] = ctx; 337 atomic_add_int(&t->tids_in_use, 1); 338 } 339 340 void * 341 lookup_tid(struct adapter *sc, int tid) 342 { 343 struct tid_info *t = &sc->tids; 344 345 return (t->tid_tab[tid]); 346 } 347 348 void 349 update_tid(struct adapter *sc, int tid, void *ctx) 350 { 351 struct tid_info *t = &sc->tids; 352 353 t->tid_tab[tid] = ctx; 354 } 355 356 void 357 remove_tid(struct adapter *sc, int tid) 358 { 359 struct tid_info *t = &sc->tids; 360 361 t->tid_tab[tid] = NULL; 362 atomic_subtract_int(&t->tids_in_use, 1); 363 } 364 365 void 366 release_tid(struct adapter *sc, int tid, struct sge_wrq *ctrlq) 367 { 368 struct wrqe *wr; 369 struct cpl_tid_release *req; 370 371 wr = alloc_wrqe(sizeof(*req), ctrlq); 372 if (wr == NULL) { 373 queue_tid_release(sc, tid); /* defer */ 374 return; 375 } 376 req = wrtod(wr); 377 378 INIT_TP_WR_MIT_CPL(req, CPL_TID_RELEASE, tid); 379 380 t4_wrq_tx(sc, wr); 381 } 382 383 static void 384 queue_tid_release(struct adapter *sc, int tid) 385 { 386 387 CXGBE_UNIMPLEMENTED("deferred tid release"); 388 } 389 390 /* 391 * What mtu_idx to use, given a 4-tuple and/or an MSS cap 392 */ 393 int 394 find_best_mtu_idx(struct adapter *sc, struct in_conninfo *inc, int pmss) 395 { 396 unsigned short *mtus = &sc->params.mtus[0]; 397 int i = 0, mss; 398 399 KASSERT(inc != NULL || pmss > 0, 400 ("%s: at least one of inc/pmss must be specified", __func__)); 401 402 mss = inc ? tcp_mssopt(inc) : pmss; 403 if (pmss > 0 && mss > pmss) 404 mss = pmss; 405 406 while (i < NMTUS - 1 && mtus[i + 1] <= mss + 40) 407 ++i; 408 409 return (i); 410 } 411 412 /* 413 * Determine the receive window size for a socket. 414 */ 415 u_long 416 select_rcv_wnd(struct socket *so) 417 { 418 unsigned long wnd; 419 420 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 421 422 wnd = sbspace(&so->so_rcv); 423 if (wnd < MIN_RCV_WND) 424 wnd = MIN_RCV_WND; 425 426 return min(wnd, MAX_RCV_WND); 427 } 428 429 int 430 select_rcv_wscale(void) 431 { 432 int wscale = 0; 433 unsigned long space = sb_max; 434 435 if (space > MAX_RCV_WND) 436 space = MAX_RCV_WND; 437 438 while (wscale < TCP_MAX_WINSHIFT && (TCP_MAXWIN << wscale) < space) 439 wscale++; 440 441 return (wscale); 442 } 443 444 extern int always_keepalive; 445 #define VIID_SMACIDX(v) (((unsigned int)(v) & 0x7f) << 1) 446 447 /* 448 * socket so could be a listening socket too. 449 */ 450 uint64_t 451 calc_opt0(struct socket *so, struct port_info *pi, struct l2t_entry *e, 452 int mtu_idx, int rscale, int rx_credits, int ulp_mode) 453 { 454 uint64_t opt0; 455 456 KASSERT(rx_credits <= M_RCV_BUFSIZ, 457 ("%s: rcv_bufsiz too high", __func__)); 458 459 opt0 = F_TCAM_BYPASS | V_WND_SCALE(rscale) | V_MSS_IDX(mtu_idx) | 460 V_ULP_MODE(ulp_mode) | V_RCV_BUFSIZ(rx_credits); 461 462 if (so != NULL) { 463 struct inpcb *inp = sotoinpcb(so); 464 struct tcpcb *tp = intotcpcb(inp); 465 int keepalive = always_keepalive || 466 so_options_get(so) & SO_KEEPALIVE; 467 468 opt0 |= V_NAGLE((tp->t_flags & TF_NODELAY) == 0); 469 opt0 |= V_KEEP_ALIVE(keepalive != 0); 470 } 471 472 if (e != NULL) 473 opt0 |= V_L2T_IDX(e->idx); 474 475 if (pi != NULL) { 476 opt0 |= V_SMAC_SEL(VIID_SMACIDX(pi->viid)); 477 opt0 |= V_TX_CHAN(pi->tx_chan); 478 } 479 480 return htobe64(opt0); 481 } 482 483 #define FILTER_SEL_WIDTH_P_FC (3 + 1) 484 #define FILTER_SEL_WIDTH_VIN_P_FC (6 + 7 + FILTER_SEL_WIDTH_P_FC) 485 #define FILTER_SEL_WIDTH_TAG_P_FC (3 + FILTER_SEL_WIDTH_VIN_P_FC) 486 #define FILTER_SEL_WIDTH_VLD_TAG_P_FC (1 + FILTER_SEL_WIDTH_TAG_P_FC) 487 #define VLAN_NONE 0xfff 488 #define FILTER_SEL_VLAN_NONE 0xffff 489 490 uint32_t 491 select_ntuple(struct port_info *pi, struct l2t_entry *e, uint32_t filter_mode) 492 { 493 uint16_t viid = pi->viid; 494 uint32_t ntuple = 0; 495 496 if (filter_mode == HW_TPL_FR_MT_PR_IV_P_FC) { 497 if (e->vlan == VLAN_NONE) 498 ntuple |= FILTER_SEL_VLAN_NONE << FILTER_SEL_WIDTH_P_FC; 499 else { 500 ntuple |= e->vlan << FILTER_SEL_WIDTH_P_FC; 501 ntuple |= 1 << FILTER_SEL_WIDTH_VLD_TAG_P_FC; 502 } 503 ntuple |= e->lport << S_PORT; 504 ntuple |= IPPROTO_TCP << FILTER_SEL_WIDTH_VLD_TAG_P_FC; 505 } else if (filter_mode == HW_TPL_FR_MT_PR_OV_P_FC) { 506 ntuple |= G_FW_VIID_VIN(viid) << FILTER_SEL_WIDTH_P_FC; 507 ntuple |= G_FW_VIID_PFN(viid) << FILTER_SEL_WIDTH_VIN_P_FC; 508 ntuple |= G_FW_VIID_VIVLD(viid) << FILTER_SEL_WIDTH_TAG_P_FC; 509 ntuple |= e->lport << S_PORT; 510 ntuple |= IPPROTO_TCP << FILTER_SEL_WIDTH_VLD_TAG_P_FC; 511 } 512 513 return (htobe32(ntuple)); 514 } 515 516 static int 517 alloc_tid_tabs(struct tid_info *t) 518 { 519 size_t size; 520 unsigned int i; 521 522 size = t->ntids * sizeof(*t->tid_tab) + 523 t->natids * sizeof(*t->atid_tab) + 524 t->nstids * sizeof(*t->stid_tab); 525 526 t->tid_tab = malloc(size, M_CXGBE, M_ZERO | M_NOWAIT); 527 if (t->tid_tab == NULL) 528 return (ENOMEM); 529 530 mtx_init(&t->atid_lock, "atid lock", NULL, MTX_DEF); 531 t->atid_tab = (union aopen_entry *)&t->tid_tab[t->ntids]; 532 t->afree = t->atid_tab; 533 t->atids_in_use = 0; 534 for (i = 1; i < t->natids; i++) 535 t->atid_tab[i - 1].next = &t->atid_tab[i]; 536 t->atid_tab[t->natids - 1].next = NULL; 537 538 mtx_init(&t->stid_lock, "stid lock", NULL, MTX_DEF); 539 t->stid_tab = (union serv_entry *)&t->atid_tab[t->natids]; 540 t->sfree = t->stid_tab; 541 t->stids_in_use = 0; 542 for (i = 1; i < t->nstids; i++) 543 t->stid_tab[i - 1].next = &t->stid_tab[i]; 544 t->stid_tab[t->nstids - 1].next = NULL; 545 546 atomic_store_rel_int(&t->tids_in_use, 0); 547 548 return (0); 549 } 550 551 static void 552 free_tid_tabs(struct tid_info *t) 553 { 554 KASSERT(t->tids_in_use == 0, 555 ("%s: %d tids still in use.", __func__, t->tids_in_use)); 556 KASSERT(t->atids_in_use == 0, 557 ("%s: %d atids still in use.", __func__, t->atids_in_use)); 558 KASSERT(t->stids_in_use == 0, 559 ("%s: %d tids still in use.", __func__, t->stids_in_use)); 560 561 free(t->tid_tab, M_CXGBE); 562 t->tid_tab = NULL; 563 564 if (mtx_initialized(&t->atid_lock)) 565 mtx_destroy(&t->atid_lock); 566 if (mtx_initialized(&t->stid_lock)) 567 mtx_destroy(&t->stid_lock); 568 } 569 570 static void 571 free_tom_data(struct adapter *sc, struct tom_data *td) 572 { 573 KASSERT(TAILQ_EMPTY(&td->toep_list), 574 ("%s: TOE PCB list is not empty.", __func__)); 575 KASSERT(td->lctx_count == 0, 576 ("%s: lctx hash table is not empty.", __func__)); 577 578 t4_uninit_l2t_cpl_handlers(sc); 579 t4_uninit_cpl_io_handlers(sc); 580 t4_uninit_ddp(sc, td); 581 582 if (td->listen_mask != 0) 583 hashdestroy(td->listen_hash, M_CXGBE, td->listen_mask); 584 585 if (mtx_initialized(&td->lctx_hash_lock)) 586 mtx_destroy(&td->lctx_hash_lock); 587 if (mtx_initialized(&td->toep_list_lock)) 588 mtx_destroy(&td->toep_list_lock); 589 590 free_tid_tabs(&sc->tids); 591 free(td, M_CXGBE); 592 } 593 594 /* 595 * Ground control to Major TOM 596 * Commencing countdown, engines on 597 */ 598 static int 599 t4_tom_activate(struct adapter *sc) 600 { 601 struct tom_data *td; 602 struct toedev *tod; 603 int i, rc; 604 605 ADAPTER_LOCK_ASSERT_OWNED(sc); /* for sc->flags */ 606 607 /* per-adapter softc for TOM */ 608 td = malloc(sizeof(*td), M_CXGBE, M_ZERO | M_NOWAIT); 609 if (td == NULL) 610 return (ENOMEM); 611 612 /* List of TOE PCBs and associated lock */ 613 mtx_init(&td->toep_list_lock, "PCB list lock", NULL, MTX_DEF); 614 TAILQ_INIT(&td->toep_list); 615 616 /* Listen context */ 617 mtx_init(&td->lctx_hash_lock, "lctx hash lock", NULL, MTX_DEF); 618 td->listen_hash = hashinit_flags(LISTEN_HASH_SIZE, M_CXGBE, 619 &td->listen_mask, HASH_NOWAIT); 620 621 /* TID tables */ 622 rc = alloc_tid_tabs(&sc->tids); 623 if (rc != 0) 624 goto done; 625 626 t4_init_ddp(sc, td); 627 628 /* CPL handlers */ 629 t4_init_connect_cpl_handlers(sc); 630 t4_init_l2t_cpl_handlers(sc); 631 t4_init_listen_cpl_handlers(sc); 632 t4_init_cpl_io_handlers(sc); 633 634 /* toedev ops */ 635 tod = &td->tod; 636 init_toedev(tod); 637 tod->tod_softc = sc; 638 tod->tod_connect = t4_connect; 639 tod->tod_listen_start = t4_listen_start; 640 tod->tod_listen_stop = t4_listen_stop; 641 tod->tod_rcvd = t4_rcvd; 642 tod->tod_output = t4_tod_output; 643 tod->tod_send_rst = t4_send_rst; 644 tod->tod_send_fin = t4_send_fin; 645 tod->tod_pcb_detach = t4_pcb_detach; 646 tod->tod_l2_update = t4_l2_update; 647 tod->tod_syncache_added = t4_syncache_added; 648 tod->tod_syncache_removed = t4_syncache_removed; 649 tod->tod_syncache_respond = t4_syncache_respond; 650 tod->tod_offload_socket = t4_offload_socket; 651 652 for_each_port(sc, i) 653 TOEDEV(sc->port[i]->ifp) = &td->tod; 654 655 sc->tom_softc = td; 656 sc->flags |= TOM_INIT_DONE; 657 register_toedev(sc->tom_softc); 658 659 done: 660 if (rc != 0) 661 free_tom_data(sc, td); 662 return (rc); 663 } 664 665 static int 666 t4_tom_deactivate(struct adapter *sc) 667 { 668 int rc = 0; 669 struct tom_data *td = sc->tom_softc; 670 671 ADAPTER_LOCK_ASSERT_OWNED(sc); /* for sc->flags */ 672 673 if (td == NULL) 674 return (0); /* XXX. KASSERT? */ 675 676 if (sc->offload_map != 0) 677 return (EBUSY); /* at least one port has IFCAP_TOE enabled */ 678 679 mtx_lock(&td->toep_list_lock); 680 if (!TAILQ_EMPTY(&td->toep_list)) 681 rc = EBUSY; 682 mtx_unlock(&td->toep_list_lock); 683 684 mtx_lock(&td->lctx_hash_lock); 685 if (td->lctx_count > 0) 686 rc = EBUSY; 687 mtx_unlock(&td->lctx_hash_lock); 688 689 if (rc == 0) { 690 unregister_toedev(sc->tom_softc); 691 free_tom_data(sc, td); 692 sc->tom_softc = NULL; 693 sc->flags &= ~TOM_INIT_DONE; 694 } 695 696 return (rc); 697 } 698 699 static int 700 t4_tom_mod_load(void) 701 { 702 int rc; 703 struct protosw *tcp_protosw; 704 705 tcp_protosw = pffindproto(PF_INET, IPPROTO_TCP, SOCK_STREAM); 706 if (tcp_protosw == NULL) 707 return (ENOPROTOOPT); 708 709 bcopy(tcp_protosw, &ddp_protosw, sizeof(ddp_protosw)); 710 bcopy(tcp_protosw->pr_usrreqs, &ddp_usrreqs, sizeof(ddp_usrreqs)); 711 ddp_usrreqs.pru_soreceive = t4_soreceive_ddp; 712 ddp_protosw.pr_usrreqs = &ddp_usrreqs; 713 714 rc = t4_register_uld(&tom_uld_info); 715 if (rc != 0) 716 t4_tom_mod_unload(); 717 718 return (rc); 719 } 720 721 static void 722 tom_uninit(struct adapter *sc, void *arg __unused) 723 { 724 /* Try to free resources (works only if no port has IFCAP_TOE) */ 725 ADAPTER_LOCK(sc); 726 if (sc->flags & TOM_INIT_DONE) 727 t4_deactivate_uld(sc, ULD_TOM); 728 ADAPTER_UNLOCK(sc); 729 } 730 731 static int 732 t4_tom_mod_unload(void) 733 { 734 t4_iterate(tom_uninit, NULL); 735 736 if (t4_unregister_uld(&tom_uld_info) == EBUSY) 737 return (EBUSY); 738 739 return (0); 740 } 741 #endif /* TCP_OFFLOAD */ 742 743 static int 744 t4_tom_modevent(module_t mod, int cmd, void *arg) 745 { 746 int rc = 0; 747 748 #ifdef TCP_OFFLOAD 749 switch (cmd) { 750 case MOD_LOAD: 751 rc = t4_tom_mod_load(); 752 break; 753 754 case MOD_UNLOAD: 755 rc = t4_tom_mod_unload(); 756 break; 757 758 default: 759 rc = EINVAL; 760 } 761 #else 762 printf("t4_tom: compiled without TCP_OFFLOAD support.\n"); 763 rc = EOPNOTSUPP; 764 #endif 765 return (rc); 766 } 767 768 static moduledata_t t4_tom_moddata= { 769 "t4_tom", 770 t4_tom_modevent, 771 0 772 }; 773 774 MODULE_VERSION(t4_tom, 1); 775 MODULE_DEPEND(t4_tom, toecore, 1, 1, 1); 776 MODULE_DEPEND(t4_tom, t4nex, 1, 1, 1); 777 DECLARE_MODULE(t4_tom, t4_tom_moddata, SI_SUB_EXEC, SI_ORDER_ANY); 778