1 /*- 2 * Copyright (c) 2012 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __FBSDID("$FreeBSD$"); 30 31 #include "opt_inet.h" 32 #include "opt_inet6.h" 33 34 #include <sys/param.h> 35 #include <sys/types.h> 36 #include <sys/systm.h> 37 #include <sys/kernel.h> 38 #include <sys/ktr.h> 39 #include <sys/lock.h> 40 #include <sys/module.h> 41 #include <sys/protosw.h> 42 #include <sys/domain.h> 43 #include <sys/rmlock.h> 44 #include <sys/socket.h> 45 #include <sys/socketvar.h> 46 #include <sys/taskqueue.h> 47 #include <net/if.h> 48 #include <net/if_var.h> 49 #include <netinet/in.h> 50 #include <netinet/in_pcb.h> 51 #include <netinet/in_var.h> 52 #include <netinet/ip.h> 53 #include <netinet/ip6.h> 54 #include <netinet/tcp_var.h> 55 #include <netinet6/scope6_var.h> 56 #define TCPSTATES 57 #include <netinet/tcp_fsm.h> 58 #include <netinet/toecore.h> 59 60 #ifdef TCP_OFFLOAD 61 #include "common/common.h" 62 #include "common/t4_msg.h" 63 #include "common/t4_regs.h" 64 #include "common/t4_regs_values.h" 65 #include "common/t4_tcb.h" 66 #include "tom/t4_tom_l2t.h" 67 #include "tom/t4_tom.h" 68 69 static struct protosw ddp_protosw; 70 static struct pr_usrreqs ddp_usrreqs; 71 72 static struct protosw ddp6_protosw; 73 static struct pr_usrreqs ddp6_usrreqs; 74 75 /* Module ops */ 76 static int t4_tom_mod_load(void); 77 static int t4_tom_mod_unload(void); 78 static int t4_tom_modevent(module_t, int, void *); 79 80 /* ULD ops and helpers */ 81 static int t4_tom_activate(struct adapter *); 82 static int t4_tom_deactivate(struct adapter *); 83 84 static struct uld_info tom_uld_info = { 85 .uld_id = ULD_TOM, 86 .activate = t4_tom_activate, 87 .deactivate = t4_tom_deactivate, 88 }; 89 90 static void queue_tid_release(struct adapter *, int); 91 static void release_offload_resources(struct toepcb *); 92 static int alloc_tid_tabs(struct tid_info *); 93 static void free_tid_tabs(struct tid_info *); 94 static int add_lip(struct adapter *, struct in6_addr *); 95 static int delete_lip(struct adapter *, struct in6_addr *); 96 static struct clip_entry *search_lip(struct tom_data *, struct in6_addr *); 97 static void init_clip_table(struct adapter *, struct tom_data *); 98 static void update_clip(struct adapter *, void *); 99 static void t4_clip_task(void *, int); 100 static void update_clip_table(struct adapter *, struct tom_data *); 101 static void destroy_clip_table(struct adapter *, struct tom_data *); 102 static void free_tom_data(struct adapter *, struct tom_data *); 103 static void reclaim_wr_resources(void *, int); 104 105 static int in6_ifaddr_gen; 106 static eventhandler_tag ifaddr_evhandler; 107 static struct timeout_task clip_task; 108 109 struct toepcb * 110 alloc_toepcb(struct vi_info *vi, int txqid, int rxqid, int flags) 111 { 112 struct port_info *pi = vi->pi; 113 struct adapter *sc = pi->adapter; 114 struct toepcb *toep; 115 int tx_credits, txsd_total, len; 116 117 /* 118 * The firmware counts tx work request credits in units of 16 bytes 119 * each. Reserve room for an ABORT_REQ so the driver never has to worry 120 * about tx credits if it wants to abort a connection. 121 */ 122 tx_credits = sc->params.ofldq_wr_cred; 123 tx_credits -= howmany(sizeof(struct cpl_abort_req), 16); 124 125 /* 126 * Shortest possible tx work request is a fw_ofld_tx_data_wr + 1 byte 127 * immediate payload, and firmware counts tx work request credits in 128 * units of 16 byte. Calculate the maximum work requests possible. 129 */ 130 txsd_total = tx_credits / 131 howmany((sizeof(struct fw_ofld_tx_data_wr) + 1), 16); 132 133 if (txqid < 0) 134 txqid = (arc4random() % vi->nofldtxq) + vi->first_ofld_txq; 135 KASSERT(txqid >= vi->first_ofld_txq && 136 txqid < vi->first_ofld_txq + vi->nofldtxq, 137 ("%s: txqid %d for vi %p (first %d, n %d)", __func__, txqid, vi, 138 vi->first_ofld_txq, vi->nofldtxq)); 139 140 if (rxqid < 0) 141 rxqid = (arc4random() % vi->nofldrxq) + vi->first_ofld_rxq; 142 KASSERT(rxqid >= vi->first_ofld_rxq && 143 rxqid < vi->first_ofld_rxq + vi->nofldrxq, 144 ("%s: rxqid %d for vi %p (first %d, n %d)", __func__, rxqid, vi, 145 vi->first_ofld_rxq, vi->nofldrxq)); 146 147 len = offsetof(struct toepcb, txsd) + 148 txsd_total * sizeof(struct ofld_tx_sdesc); 149 150 toep = malloc(len, M_CXGBE, M_ZERO | flags); 151 if (toep == NULL) 152 return (NULL); 153 154 toep->td = sc->tom_softc; 155 toep->vi = vi; 156 toep->tx_total = tx_credits; 157 toep->tx_credits = tx_credits; 158 toep->ofld_txq = &sc->sge.ofld_txq[txqid]; 159 toep->ofld_rxq = &sc->sge.ofld_rxq[rxqid]; 160 toep->ctrlq = &sc->sge.ctrlq[pi->port_id]; 161 toep->txsd_total = txsd_total; 162 toep->txsd_avail = txsd_total; 163 toep->txsd_pidx = 0; 164 toep->txsd_cidx = 0; 165 166 return (toep); 167 } 168 169 void 170 free_toepcb(struct toepcb *toep) 171 { 172 173 KASSERT(!(toep->flags & TPF_ATTACHED), 174 ("%s: attached to an inpcb", __func__)); 175 KASSERT(!(toep->flags & TPF_CPL_PENDING), 176 ("%s: CPL pending", __func__)); 177 178 free(toep, M_CXGBE); 179 } 180 181 /* 182 * Set up the socket for TCP offload. 183 */ 184 void 185 offload_socket(struct socket *so, struct toepcb *toep) 186 { 187 struct tom_data *td = toep->td; 188 struct inpcb *inp = sotoinpcb(so); 189 struct tcpcb *tp = intotcpcb(inp); 190 struct sockbuf *sb; 191 192 INP_WLOCK_ASSERT(inp); 193 194 /* Update socket */ 195 sb = &so->so_snd; 196 SOCKBUF_LOCK(sb); 197 sb->sb_flags |= SB_NOCOALESCE; 198 SOCKBUF_UNLOCK(sb); 199 sb = &so->so_rcv; 200 SOCKBUF_LOCK(sb); 201 sb->sb_flags |= SB_NOCOALESCE; 202 if (toep->ulp_mode == ULP_MODE_TCPDDP) { 203 if (inp->inp_vflag & INP_IPV6) 204 so->so_proto = &ddp6_protosw; 205 else 206 so->so_proto = &ddp_protosw; 207 } 208 SOCKBUF_UNLOCK(sb); 209 210 /* Update TCP PCB */ 211 tp->tod = &td->tod; 212 tp->t_toe = toep; 213 tp->t_flags |= TF_TOE; 214 215 /* Install an extra hold on inp */ 216 toep->inp = inp; 217 toep->flags |= TPF_ATTACHED; 218 in_pcbref(inp); 219 220 /* Add the TOE PCB to the active list */ 221 mtx_lock(&td->toep_list_lock); 222 TAILQ_INSERT_HEAD(&td->toep_list, toep, link); 223 mtx_unlock(&td->toep_list_lock); 224 } 225 226 /* This is _not_ the normal way to "unoffload" a socket. */ 227 void 228 undo_offload_socket(struct socket *so) 229 { 230 struct inpcb *inp = sotoinpcb(so); 231 struct tcpcb *tp = intotcpcb(inp); 232 struct toepcb *toep = tp->t_toe; 233 struct tom_data *td = toep->td; 234 struct sockbuf *sb; 235 236 INP_WLOCK_ASSERT(inp); 237 238 sb = &so->so_snd; 239 SOCKBUF_LOCK(sb); 240 sb->sb_flags &= ~SB_NOCOALESCE; 241 SOCKBUF_UNLOCK(sb); 242 sb = &so->so_rcv; 243 SOCKBUF_LOCK(sb); 244 sb->sb_flags &= ~SB_NOCOALESCE; 245 SOCKBUF_UNLOCK(sb); 246 247 tp->tod = NULL; 248 tp->t_toe = NULL; 249 tp->t_flags &= ~TF_TOE; 250 251 toep->inp = NULL; 252 toep->flags &= ~TPF_ATTACHED; 253 if (in_pcbrele_wlocked(inp)) 254 panic("%s: inp freed.", __func__); 255 256 mtx_lock(&td->toep_list_lock); 257 TAILQ_REMOVE(&td->toep_list, toep, link); 258 mtx_unlock(&td->toep_list_lock); 259 } 260 261 static void 262 release_offload_resources(struct toepcb *toep) 263 { 264 struct tom_data *td = toep->td; 265 struct adapter *sc = td_adapter(td); 266 int tid = toep->tid; 267 268 KASSERT(!(toep->flags & TPF_CPL_PENDING), 269 ("%s: %p has CPL pending.", __func__, toep)); 270 KASSERT(!(toep->flags & TPF_ATTACHED), 271 ("%s: %p is still attached.", __func__, toep)); 272 273 CTR5(KTR_CXGBE, "%s: toep %p (tid %d, l2te %p, ce %p)", 274 __func__, toep, tid, toep->l2te, toep->ce); 275 276 if (toep->ulp_mode == ULP_MODE_TCPDDP) 277 release_ddp_resources(toep); 278 279 if (toep->l2te) 280 t4_l2t_release(toep->l2te); 281 282 if (tid >= 0) { 283 remove_tid(sc, tid); 284 release_tid(sc, tid, toep->ctrlq); 285 } 286 287 if (toep->ce) 288 release_lip(td, toep->ce); 289 290 mtx_lock(&td->toep_list_lock); 291 TAILQ_REMOVE(&td->toep_list, toep, link); 292 mtx_unlock(&td->toep_list_lock); 293 294 free_toepcb(toep); 295 } 296 297 /* 298 * The kernel is done with the TCP PCB and this is our opportunity to unhook the 299 * toepcb hanging off of it. If the TOE driver is also done with the toepcb (no 300 * pending CPL) then it is time to release all resources tied to the toepcb. 301 * 302 * Also gets called when an offloaded active open fails and the TOM wants the 303 * kernel to take the TCP PCB back. 304 */ 305 static void 306 t4_pcb_detach(struct toedev *tod __unused, struct tcpcb *tp) 307 { 308 #if defined(KTR) || defined(INVARIANTS) 309 struct inpcb *inp = tp->t_inpcb; 310 #endif 311 struct toepcb *toep = tp->t_toe; 312 313 INP_WLOCK_ASSERT(inp); 314 315 KASSERT(toep != NULL, ("%s: toep is NULL", __func__)); 316 KASSERT(toep->flags & TPF_ATTACHED, 317 ("%s: not attached", __func__)); 318 319 #ifdef KTR 320 if (tp->t_state == TCPS_SYN_SENT) { 321 CTR6(KTR_CXGBE, "%s: atid %d, toep %p (0x%x), inp %p (0x%x)", 322 __func__, toep->tid, toep, toep->flags, inp, 323 inp->inp_flags); 324 } else { 325 CTR6(KTR_CXGBE, 326 "t4_pcb_detach: tid %d (%s), toep %p (0x%x), inp %p (0x%x)", 327 toep->tid, tcpstates[tp->t_state], toep, toep->flags, inp, 328 inp->inp_flags); 329 } 330 #endif 331 332 tp->t_toe = NULL; 333 tp->t_flags &= ~TF_TOE; 334 toep->flags &= ~TPF_ATTACHED; 335 336 if (!(toep->flags & TPF_CPL_PENDING)) 337 release_offload_resources(toep); 338 } 339 340 /* 341 * setsockopt handler. 342 */ 343 static void 344 t4_ctloutput(struct toedev *tod, struct tcpcb *tp, int dir, int name) 345 { 346 struct adapter *sc = tod->tod_softc; 347 struct toepcb *toep = tp->t_toe; 348 349 if (dir == SOPT_GET) 350 return; 351 352 CTR4(KTR_CXGBE, "%s: tp %p, dir %u, name %u", __func__, tp, dir, name); 353 354 switch (name) { 355 case TCP_NODELAY: 356 t4_set_tcb_field(sc, toep, 1, W_TCB_T_FLAGS, V_TF_NAGLE(1), 357 V_TF_NAGLE(tp->t_flags & TF_NODELAY ? 0 : 1)); 358 break; 359 default: 360 break; 361 } 362 } 363 364 /* 365 * The TOE driver will not receive any more CPLs for the tid associated with the 366 * toepcb; release the hold on the inpcb. 367 */ 368 void 369 final_cpl_received(struct toepcb *toep) 370 { 371 struct inpcb *inp = toep->inp; 372 373 KASSERT(inp != NULL, ("%s: inp is NULL", __func__)); 374 INP_WLOCK_ASSERT(inp); 375 KASSERT(toep->flags & TPF_CPL_PENDING, 376 ("%s: CPL not pending already?", __func__)); 377 378 CTR6(KTR_CXGBE, "%s: tid %d, toep %p (0x%x), inp %p (0x%x)", 379 __func__, toep->tid, toep, toep->flags, inp, inp->inp_flags); 380 381 toep->inp = NULL; 382 toep->flags &= ~TPF_CPL_PENDING; 383 384 if (!(toep->flags & TPF_ATTACHED)) 385 release_offload_resources(toep); 386 387 if (!in_pcbrele_wlocked(inp)) 388 INP_WUNLOCK(inp); 389 } 390 391 void 392 insert_tid(struct adapter *sc, int tid, void *ctx) 393 { 394 struct tid_info *t = &sc->tids; 395 396 t->tid_tab[tid] = ctx; 397 atomic_add_int(&t->tids_in_use, 1); 398 } 399 400 void * 401 lookup_tid(struct adapter *sc, int tid) 402 { 403 struct tid_info *t = &sc->tids; 404 405 return (t->tid_tab[tid]); 406 } 407 408 void 409 update_tid(struct adapter *sc, int tid, void *ctx) 410 { 411 struct tid_info *t = &sc->tids; 412 413 t->tid_tab[tid] = ctx; 414 } 415 416 void 417 remove_tid(struct adapter *sc, int tid) 418 { 419 struct tid_info *t = &sc->tids; 420 421 t->tid_tab[tid] = NULL; 422 atomic_subtract_int(&t->tids_in_use, 1); 423 } 424 425 void 426 release_tid(struct adapter *sc, int tid, struct sge_wrq *ctrlq) 427 { 428 struct wrqe *wr; 429 struct cpl_tid_release *req; 430 431 wr = alloc_wrqe(sizeof(*req), ctrlq); 432 if (wr == NULL) { 433 queue_tid_release(sc, tid); /* defer */ 434 return; 435 } 436 req = wrtod(wr); 437 438 INIT_TP_WR_MIT_CPL(req, CPL_TID_RELEASE, tid); 439 440 t4_wrq_tx(sc, wr); 441 } 442 443 static void 444 queue_tid_release(struct adapter *sc, int tid) 445 { 446 447 CXGBE_UNIMPLEMENTED("deferred tid release"); 448 } 449 450 /* 451 * What mtu_idx to use, given a 4-tuple and/or an MSS cap 452 */ 453 int 454 find_best_mtu_idx(struct adapter *sc, struct in_conninfo *inc, int pmss) 455 { 456 unsigned short *mtus = &sc->params.mtus[0]; 457 int i, mss, n; 458 459 KASSERT(inc != NULL || pmss > 0, 460 ("%s: at least one of inc/pmss must be specified", __func__)); 461 462 mss = inc ? tcp_mssopt(inc) : pmss; 463 if (pmss > 0 && mss > pmss) 464 mss = pmss; 465 466 if (inc->inc_flags & INC_ISIPV6) 467 n = sizeof(struct ip6_hdr) + sizeof(struct tcphdr); 468 else 469 n = sizeof(struct ip) + sizeof(struct tcphdr); 470 471 for (i = 0; i < NMTUS - 1 && mtus[i + 1] <= mss + n; i++) 472 continue; 473 474 return (i); 475 } 476 477 /* 478 * Determine the receive window size for a socket. 479 */ 480 u_long 481 select_rcv_wnd(struct socket *so) 482 { 483 unsigned long wnd; 484 485 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 486 487 wnd = sbspace(&so->so_rcv); 488 if (wnd < MIN_RCV_WND) 489 wnd = MIN_RCV_WND; 490 491 return min(wnd, MAX_RCV_WND); 492 } 493 494 int 495 select_rcv_wscale(void) 496 { 497 int wscale = 0; 498 unsigned long space = sb_max; 499 500 if (space > MAX_RCV_WND) 501 space = MAX_RCV_WND; 502 503 while (wscale < TCP_MAX_WINSHIFT && (TCP_MAXWIN << wscale) < space) 504 wscale++; 505 506 return (wscale); 507 } 508 509 extern int always_keepalive; 510 #define VIID_SMACIDX(v) (((unsigned int)(v) & 0x7f) << 1) 511 512 /* 513 * socket so could be a listening socket too. 514 */ 515 uint64_t 516 calc_opt0(struct socket *so, struct vi_info *vi, struct l2t_entry *e, 517 int mtu_idx, int rscale, int rx_credits, int ulp_mode) 518 { 519 uint64_t opt0; 520 521 KASSERT(rx_credits <= M_RCV_BUFSIZ, 522 ("%s: rcv_bufsiz too high", __func__)); 523 524 opt0 = F_TCAM_BYPASS | V_WND_SCALE(rscale) | V_MSS_IDX(mtu_idx) | 525 V_ULP_MODE(ulp_mode) | V_RCV_BUFSIZ(rx_credits); 526 527 if (so != NULL) { 528 struct inpcb *inp = sotoinpcb(so); 529 struct tcpcb *tp = intotcpcb(inp); 530 int keepalive = always_keepalive || 531 so_options_get(so) & SO_KEEPALIVE; 532 533 opt0 |= V_NAGLE((tp->t_flags & TF_NODELAY) == 0); 534 opt0 |= V_KEEP_ALIVE(keepalive != 0); 535 } 536 537 if (e != NULL) 538 opt0 |= V_L2T_IDX(e->idx); 539 540 if (vi != NULL) { 541 opt0 |= V_SMAC_SEL(VIID_SMACIDX(vi->viid)); 542 opt0 |= V_TX_CHAN(vi->pi->tx_chan); 543 } 544 545 return htobe64(opt0); 546 } 547 548 uint64_t 549 select_ntuple(struct vi_info *vi, struct l2t_entry *e) 550 { 551 struct adapter *sc = vi->pi->adapter; 552 struct tp_params *tp = &sc->params.tp; 553 uint16_t viid = vi->viid; 554 uint64_t ntuple = 0; 555 556 /* 557 * Initialize each of the fields which we care about which are present 558 * in the Compressed Filter Tuple. 559 */ 560 if (tp->vlan_shift >= 0 && e->vlan != CPL_L2T_VLAN_NONE) 561 ntuple |= (uint64_t)(F_FT_VLAN_VLD | e->vlan) << tp->vlan_shift; 562 563 if (tp->port_shift >= 0) 564 ntuple |= (uint64_t)e->lport << tp->port_shift; 565 566 if (tp->protocol_shift >= 0) 567 ntuple |= (uint64_t)IPPROTO_TCP << tp->protocol_shift; 568 569 if (tp->vnic_shift >= 0) { 570 uint32_t vf = G_FW_VIID_VIN(viid); 571 uint32_t pf = G_FW_VIID_PFN(viid); 572 uint32_t vld = G_FW_VIID_VIVLD(viid); 573 574 ntuple |= (uint64_t)(V_FT_VNID_ID_VF(vf) | V_FT_VNID_ID_PF(pf) | 575 V_FT_VNID_ID_VLD(vld)) << tp->vnic_shift; 576 } 577 578 if (is_t4(sc)) 579 return (htobe32((uint32_t)ntuple)); 580 else 581 return (htobe64(V_FILTER_TUPLE(ntuple))); 582 } 583 584 void 585 set_tcpddp_ulp_mode(struct toepcb *toep) 586 { 587 588 toep->ulp_mode = ULP_MODE_TCPDDP; 589 toep->ddp_flags = DDP_OK; 590 toep->ddp_score = DDP_LOW_SCORE; 591 } 592 593 int 594 negative_advice(int status) 595 { 596 597 return (status == CPL_ERR_RTX_NEG_ADVICE || 598 status == CPL_ERR_PERSIST_NEG_ADVICE || 599 status == CPL_ERR_KEEPALV_NEG_ADVICE); 600 } 601 602 static int 603 alloc_tid_tabs(struct tid_info *t) 604 { 605 size_t size; 606 unsigned int i; 607 608 size = t->ntids * sizeof(*t->tid_tab) + 609 t->natids * sizeof(*t->atid_tab) + 610 t->nstids * sizeof(*t->stid_tab); 611 612 t->tid_tab = malloc(size, M_CXGBE, M_ZERO | M_NOWAIT); 613 if (t->tid_tab == NULL) 614 return (ENOMEM); 615 616 mtx_init(&t->atid_lock, "atid lock", NULL, MTX_DEF); 617 t->atid_tab = (union aopen_entry *)&t->tid_tab[t->ntids]; 618 t->afree = t->atid_tab; 619 t->atids_in_use = 0; 620 for (i = 1; i < t->natids; i++) 621 t->atid_tab[i - 1].next = &t->atid_tab[i]; 622 t->atid_tab[t->natids - 1].next = NULL; 623 624 mtx_init(&t->stid_lock, "stid lock", NULL, MTX_DEF); 625 t->stid_tab = (struct listen_ctx **)&t->atid_tab[t->natids]; 626 t->stids_in_use = 0; 627 TAILQ_INIT(&t->stids); 628 t->nstids_free_head = t->nstids; 629 630 atomic_store_rel_int(&t->tids_in_use, 0); 631 632 return (0); 633 } 634 635 static void 636 free_tid_tabs(struct tid_info *t) 637 { 638 KASSERT(t->tids_in_use == 0, 639 ("%s: %d tids still in use.", __func__, t->tids_in_use)); 640 KASSERT(t->atids_in_use == 0, 641 ("%s: %d atids still in use.", __func__, t->atids_in_use)); 642 KASSERT(t->stids_in_use == 0, 643 ("%s: %d tids still in use.", __func__, t->stids_in_use)); 644 645 free(t->tid_tab, M_CXGBE); 646 t->tid_tab = NULL; 647 648 if (mtx_initialized(&t->atid_lock)) 649 mtx_destroy(&t->atid_lock); 650 if (mtx_initialized(&t->stid_lock)) 651 mtx_destroy(&t->stid_lock); 652 } 653 654 static int 655 add_lip(struct adapter *sc, struct in6_addr *lip) 656 { 657 struct fw_clip_cmd c; 658 659 ASSERT_SYNCHRONIZED_OP(sc); 660 /* mtx_assert(&td->clip_table_lock, MA_OWNED); */ 661 662 memset(&c, 0, sizeof(c)); 663 c.op_to_write = htonl(V_FW_CMD_OP(FW_CLIP_CMD) | F_FW_CMD_REQUEST | 664 F_FW_CMD_WRITE); 665 c.alloc_to_len16 = htonl(F_FW_CLIP_CMD_ALLOC | FW_LEN16(c)); 666 c.ip_hi = *(uint64_t *)&lip->s6_addr[0]; 667 c.ip_lo = *(uint64_t *)&lip->s6_addr[8]; 668 669 return (-t4_wr_mbox_ns(sc, sc->mbox, &c, sizeof(c), &c)); 670 } 671 672 static int 673 delete_lip(struct adapter *sc, struct in6_addr *lip) 674 { 675 struct fw_clip_cmd c; 676 677 ASSERT_SYNCHRONIZED_OP(sc); 678 /* mtx_assert(&td->clip_table_lock, MA_OWNED); */ 679 680 memset(&c, 0, sizeof(c)); 681 c.op_to_write = htonl(V_FW_CMD_OP(FW_CLIP_CMD) | F_FW_CMD_REQUEST | 682 F_FW_CMD_READ); 683 c.alloc_to_len16 = htonl(F_FW_CLIP_CMD_FREE | FW_LEN16(c)); 684 c.ip_hi = *(uint64_t *)&lip->s6_addr[0]; 685 c.ip_lo = *(uint64_t *)&lip->s6_addr[8]; 686 687 return (-t4_wr_mbox_ns(sc, sc->mbox, &c, sizeof(c), &c)); 688 } 689 690 static struct clip_entry * 691 search_lip(struct tom_data *td, struct in6_addr *lip) 692 { 693 struct clip_entry *ce; 694 695 mtx_assert(&td->clip_table_lock, MA_OWNED); 696 697 TAILQ_FOREACH(ce, &td->clip_table, link) { 698 if (IN6_ARE_ADDR_EQUAL(&ce->lip, lip)) 699 return (ce); 700 } 701 702 return (NULL); 703 } 704 705 struct clip_entry * 706 hold_lip(struct tom_data *td, struct in6_addr *lip) 707 { 708 struct clip_entry *ce; 709 710 mtx_lock(&td->clip_table_lock); 711 ce = search_lip(td, lip); 712 if (ce != NULL) 713 ce->refcount++; 714 mtx_unlock(&td->clip_table_lock); 715 716 return (ce); 717 } 718 719 void 720 release_lip(struct tom_data *td, struct clip_entry *ce) 721 { 722 723 mtx_lock(&td->clip_table_lock); 724 KASSERT(search_lip(td, &ce->lip) == ce, 725 ("%s: CLIP entry %p p not in CLIP table.", __func__, ce)); 726 KASSERT(ce->refcount > 0, 727 ("%s: CLIP entry %p has refcount 0", __func__, ce)); 728 --ce->refcount; 729 mtx_unlock(&td->clip_table_lock); 730 } 731 732 static void 733 init_clip_table(struct adapter *sc, struct tom_data *td) 734 { 735 736 ASSERT_SYNCHRONIZED_OP(sc); 737 738 mtx_init(&td->clip_table_lock, "CLIP table lock", NULL, MTX_DEF); 739 TAILQ_INIT(&td->clip_table); 740 td->clip_gen = -1; 741 742 update_clip_table(sc, td); 743 } 744 745 static void 746 update_clip(struct adapter *sc, void *arg __unused) 747 { 748 749 if (begin_synchronized_op(sc, NULL, HOLD_LOCK, "t4tomuc")) 750 return; 751 752 if (uld_active(sc, ULD_TOM)) 753 update_clip_table(sc, sc->tom_softc); 754 755 end_synchronized_op(sc, LOCK_HELD); 756 } 757 758 static void 759 t4_clip_task(void *arg, int count) 760 { 761 762 t4_iterate(update_clip, NULL); 763 } 764 765 static void 766 update_clip_table(struct adapter *sc, struct tom_data *td) 767 { 768 struct rm_priotracker in6_ifa_tracker; 769 struct in6_ifaddr *ia; 770 struct in6_addr *lip, tlip; 771 struct clip_head stale; 772 struct clip_entry *ce, *ce_temp; 773 int rc, gen = atomic_load_acq_int(&in6_ifaddr_gen); 774 775 ASSERT_SYNCHRONIZED_OP(sc); 776 777 IN6_IFADDR_RLOCK(&in6_ifa_tracker); 778 mtx_lock(&td->clip_table_lock); 779 780 if (gen == td->clip_gen) 781 goto done; 782 783 TAILQ_INIT(&stale); 784 TAILQ_CONCAT(&stale, &td->clip_table, link); 785 786 TAILQ_FOREACH(ia, &V_in6_ifaddrhead, ia_link) { 787 lip = &ia->ia_addr.sin6_addr; 788 789 KASSERT(!IN6_IS_ADDR_MULTICAST(lip), 790 ("%s: mcast address in in6_ifaddr list", __func__)); 791 792 if (IN6_IS_ADDR_LOOPBACK(lip)) 793 continue; 794 if (IN6_IS_SCOPE_EMBED(lip)) { 795 /* Remove the embedded scope */ 796 tlip = *lip; 797 lip = &tlip; 798 in6_clearscope(lip); 799 } 800 /* 801 * XXX: how to weed out the link local address for the loopback 802 * interface? It's fe80::1 usually (always?). 803 */ 804 805 /* 806 * If it's in the main list then we already know it's not stale. 807 */ 808 TAILQ_FOREACH(ce, &td->clip_table, link) { 809 if (IN6_ARE_ADDR_EQUAL(&ce->lip, lip)) 810 goto next; 811 } 812 813 /* 814 * If it's in the stale list we should move it to the main list. 815 */ 816 TAILQ_FOREACH(ce, &stale, link) { 817 if (IN6_ARE_ADDR_EQUAL(&ce->lip, lip)) { 818 TAILQ_REMOVE(&stale, ce, link); 819 TAILQ_INSERT_TAIL(&td->clip_table, ce, link); 820 goto next; 821 } 822 } 823 824 /* A new IP6 address; add it to the CLIP table */ 825 ce = malloc(sizeof(*ce), M_CXGBE, M_NOWAIT); 826 memcpy(&ce->lip, lip, sizeof(ce->lip)); 827 ce->refcount = 0; 828 rc = add_lip(sc, lip); 829 if (rc == 0) 830 TAILQ_INSERT_TAIL(&td->clip_table, ce, link); 831 else { 832 char ip[INET6_ADDRSTRLEN]; 833 834 inet_ntop(AF_INET6, &ce->lip, &ip[0], sizeof(ip)); 835 log(LOG_ERR, "%s: could not add %s (%d)\n", 836 __func__, ip, rc); 837 free(ce, M_CXGBE); 838 } 839 next: 840 continue; 841 } 842 843 /* 844 * Remove stale addresses (those no longer in V_in6_ifaddrhead) that are 845 * no longer referenced by the driver. 846 */ 847 TAILQ_FOREACH_SAFE(ce, &stale, link, ce_temp) { 848 if (ce->refcount == 0) { 849 rc = delete_lip(sc, &ce->lip); 850 if (rc == 0) { 851 TAILQ_REMOVE(&stale, ce, link); 852 free(ce, M_CXGBE); 853 } else { 854 char ip[INET6_ADDRSTRLEN]; 855 856 inet_ntop(AF_INET6, &ce->lip, &ip[0], 857 sizeof(ip)); 858 log(LOG_ERR, "%s: could not delete %s (%d)\n", 859 __func__, ip, rc); 860 } 861 } 862 } 863 /* The ones that are still referenced need to stay in the CLIP table */ 864 TAILQ_CONCAT(&td->clip_table, &stale, link); 865 866 td->clip_gen = gen; 867 done: 868 mtx_unlock(&td->clip_table_lock); 869 IN6_IFADDR_RUNLOCK(&in6_ifa_tracker); 870 } 871 872 static void 873 destroy_clip_table(struct adapter *sc, struct tom_data *td) 874 { 875 struct clip_entry *ce, *ce_temp; 876 877 if (mtx_initialized(&td->clip_table_lock)) { 878 mtx_lock(&td->clip_table_lock); 879 TAILQ_FOREACH_SAFE(ce, &td->clip_table, link, ce_temp) { 880 KASSERT(ce->refcount == 0, 881 ("%s: CLIP entry %p still in use (%d)", __func__, 882 ce, ce->refcount)); 883 TAILQ_REMOVE(&td->clip_table, ce, link); 884 delete_lip(sc, &ce->lip); 885 free(ce, M_CXGBE); 886 } 887 mtx_unlock(&td->clip_table_lock); 888 mtx_destroy(&td->clip_table_lock); 889 } 890 } 891 892 static void 893 free_tom_data(struct adapter *sc, struct tom_data *td) 894 { 895 896 ASSERT_SYNCHRONIZED_OP(sc); 897 898 KASSERT(TAILQ_EMPTY(&td->toep_list), 899 ("%s: TOE PCB list is not empty.", __func__)); 900 KASSERT(td->lctx_count == 0, 901 ("%s: lctx hash table is not empty.", __func__)); 902 903 t4_uninit_l2t_cpl_handlers(sc); 904 t4_uninit_cpl_io_handlers(sc); 905 t4_uninit_ddp(sc, td); 906 destroy_clip_table(sc, td); 907 908 if (td->listen_mask != 0) 909 hashdestroy(td->listen_hash, M_CXGBE, td->listen_mask); 910 911 if (mtx_initialized(&td->unsent_wr_lock)) 912 mtx_destroy(&td->unsent_wr_lock); 913 if (mtx_initialized(&td->lctx_hash_lock)) 914 mtx_destroy(&td->lctx_hash_lock); 915 if (mtx_initialized(&td->toep_list_lock)) 916 mtx_destroy(&td->toep_list_lock); 917 918 free_tid_tabs(&sc->tids); 919 free(td, M_CXGBE); 920 } 921 922 static void 923 reclaim_wr_resources(void *arg, int count) 924 { 925 struct tom_data *td = arg; 926 STAILQ_HEAD(, wrqe) twr_list = STAILQ_HEAD_INITIALIZER(twr_list); 927 struct cpl_act_open_req *cpl; 928 u_int opcode, atid; 929 struct wrqe *wr; 930 struct adapter *sc; 931 932 mtx_lock(&td->unsent_wr_lock); 933 STAILQ_SWAP(&td->unsent_wr_list, &twr_list, wrqe); 934 mtx_unlock(&td->unsent_wr_lock); 935 936 while ((wr = STAILQ_FIRST(&twr_list)) != NULL) { 937 STAILQ_REMOVE_HEAD(&twr_list, link); 938 939 cpl = wrtod(wr); 940 opcode = GET_OPCODE(cpl); 941 942 switch (opcode) { 943 case CPL_ACT_OPEN_REQ: 944 case CPL_ACT_OPEN_REQ6: 945 atid = G_TID_TID(be32toh(OPCODE_TID(cpl))); 946 sc = td_adapter(td); 947 948 CTR2(KTR_CXGBE, "%s: atid %u ", __func__, atid); 949 act_open_failure_cleanup(sc, atid, EHOSTUNREACH); 950 free(wr, M_CXGBE); 951 break; 952 default: 953 log(LOG_ERR, "%s: leaked work request %p, wr_len %d, " 954 "opcode %x\n", __func__, wr, wr->wr_len, opcode); 955 /* WR not freed here; go look at it with a debugger. */ 956 } 957 } 958 } 959 960 /* 961 * Ground control to Major TOM 962 * Commencing countdown, engines on 963 */ 964 static int 965 t4_tom_activate(struct adapter *sc) 966 { 967 struct tom_data *td; 968 struct toedev *tod; 969 struct vi_info *vi; 970 int i, rc, v; 971 972 ASSERT_SYNCHRONIZED_OP(sc); 973 974 /* per-adapter softc for TOM */ 975 td = malloc(sizeof(*td), M_CXGBE, M_ZERO | M_NOWAIT); 976 if (td == NULL) 977 return (ENOMEM); 978 979 /* List of TOE PCBs and associated lock */ 980 mtx_init(&td->toep_list_lock, "PCB list lock", NULL, MTX_DEF); 981 TAILQ_INIT(&td->toep_list); 982 983 /* Listen context */ 984 mtx_init(&td->lctx_hash_lock, "lctx hash lock", NULL, MTX_DEF); 985 td->listen_hash = hashinit_flags(LISTEN_HASH_SIZE, M_CXGBE, 986 &td->listen_mask, HASH_NOWAIT); 987 988 /* List of WRs for which L2 resolution failed */ 989 mtx_init(&td->unsent_wr_lock, "Unsent WR list lock", NULL, MTX_DEF); 990 STAILQ_INIT(&td->unsent_wr_list); 991 TASK_INIT(&td->reclaim_wr_resources, 0, reclaim_wr_resources, td); 992 993 /* TID tables */ 994 rc = alloc_tid_tabs(&sc->tids); 995 if (rc != 0) 996 goto done; 997 998 /* DDP page pods and CPL handlers */ 999 t4_init_ddp(sc, td); 1000 1001 /* CLIP table for IPv6 offload */ 1002 init_clip_table(sc, td); 1003 1004 /* CPL handlers */ 1005 t4_init_connect_cpl_handlers(sc); 1006 t4_init_l2t_cpl_handlers(sc); 1007 t4_init_listen_cpl_handlers(sc); 1008 t4_init_cpl_io_handlers(sc); 1009 1010 /* toedev ops */ 1011 tod = &td->tod; 1012 init_toedev(tod); 1013 tod->tod_softc = sc; 1014 tod->tod_connect = t4_connect; 1015 tod->tod_listen_start = t4_listen_start; 1016 tod->tod_listen_stop = t4_listen_stop; 1017 tod->tod_rcvd = t4_rcvd; 1018 tod->tod_output = t4_tod_output; 1019 tod->tod_send_rst = t4_send_rst; 1020 tod->tod_send_fin = t4_send_fin; 1021 tod->tod_pcb_detach = t4_pcb_detach; 1022 tod->tod_l2_update = t4_l2_update; 1023 tod->tod_syncache_added = t4_syncache_added; 1024 tod->tod_syncache_removed = t4_syncache_removed; 1025 tod->tod_syncache_respond = t4_syncache_respond; 1026 tod->tod_offload_socket = t4_offload_socket; 1027 tod->tod_ctloutput = t4_ctloutput; 1028 1029 for_each_port(sc, i) { 1030 for_each_vi(sc->port[i], v, vi) { 1031 TOEDEV(vi->ifp) = &td->tod; 1032 } 1033 } 1034 1035 sc->tom_softc = td; 1036 register_toedev(sc->tom_softc); 1037 1038 done: 1039 if (rc != 0) 1040 free_tom_data(sc, td); 1041 return (rc); 1042 } 1043 1044 static int 1045 t4_tom_deactivate(struct adapter *sc) 1046 { 1047 int rc = 0; 1048 struct tom_data *td = sc->tom_softc; 1049 1050 ASSERT_SYNCHRONIZED_OP(sc); 1051 1052 if (td == NULL) 1053 return (0); /* XXX. KASSERT? */ 1054 1055 if (sc->offload_map != 0) 1056 return (EBUSY); /* at least one port has IFCAP_TOE enabled */ 1057 1058 if (uld_active(sc, ULD_IWARP) || uld_active(sc, ULD_ISCSI)) 1059 return (EBUSY); /* both iWARP and iSCSI rely on the TOE. */ 1060 1061 mtx_lock(&td->toep_list_lock); 1062 if (!TAILQ_EMPTY(&td->toep_list)) 1063 rc = EBUSY; 1064 mtx_unlock(&td->toep_list_lock); 1065 1066 mtx_lock(&td->lctx_hash_lock); 1067 if (td->lctx_count > 0) 1068 rc = EBUSY; 1069 mtx_unlock(&td->lctx_hash_lock); 1070 1071 taskqueue_drain(taskqueue_thread, &td->reclaim_wr_resources); 1072 mtx_lock(&td->unsent_wr_lock); 1073 if (!STAILQ_EMPTY(&td->unsent_wr_list)) 1074 rc = EBUSY; 1075 mtx_unlock(&td->unsent_wr_lock); 1076 1077 if (rc == 0) { 1078 unregister_toedev(sc->tom_softc); 1079 free_tom_data(sc, td); 1080 sc->tom_softc = NULL; 1081 } 1082 1083 return (rc); 1084 } 1085 1086 static void 1087 t4_tom_ifaddr_event(void *arg __unused, struct ifnet *ifp) 1088 { 1089 1090 atomic_add_rel_int(&in6_ifaddr_gen, 1); 1091 taskqueue_enqueue_timeout(taskqueue_thread, &clip_task, -hz / 4); 1092 } 1093 1094 static int 1095 t4_tom_mod_load(void) 1096 { 1097 int rc; 1098 struct protosw *tcp_protosw, *tcp6_protosw; 1099 1100 tcp_protosw = pffindproto(PF_INET, IPPROTO_TCP, SOCK_STREAM); 1101 if (tcp_protosw == NULL) 1102 return (ENOPROTOOPT); 1103 bcopy(tcp_protosw, &ddp_protosw, sizeof(ddp_protosw)); 1104 bcopy(tcp_protosw->pr_usrreqs, &ddp_usrreqs, sizeof(ddp_usrreqs)); 1105 ddp_usrreqs.pru_soreceive = t4_soreceive_ddp; 1106 ddp_protosw.pr_usrreqs = &ddp_usrreqs; 1107 1108 tcp6_protosw = pffindproto(PF_INET6, IPPROTO_TCP, SOCK_STREAM); 1109 if (tcp6_protosw == NULL) 1110 return (ENOPROTOOPT); 1111 bcopy(tcp6_protosw, &ddp6_protosw, sizeof(ddp6_protosw)); 1112 bcopy(tcp6_protosw->pr_usrreqs, &ddp6_usrreqs, sizeof(ddp6_usrreqs)); 1113 ddp6_usrreqs.pru_soreceive = t4_soreceive_ddp; 1114 ddp6_protosw.pr_usrreqs = &ddp6_usrreqs; 1115 1116 TIMEOUT_TASK_INIT(taskqueue_thread, &clip_task, 0, t4_clip_task, NULL); 1117 ifaddr_evhandler = EVENTHANDLER_REGISTER(ifaddr_event, 1118 t4_tom_ifaddr_event, NULL, EVENTHANDLER_PRI_ANY); 1119 1120 rc = t4_register_uld(&tom_uld_info); 1121 if (rc != 0) 1122 t4_tom_mod_unload(); 1123 1124 return (rc); 1125 } 1126 1127 static void 1128 tom_uninit(struct adapter *sc, void *arg __unused) 1129 { 1130 if (begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4tomun")) 1131 return; 1132 1133 /* Try to free resources (works only if no port has IFCAP_TOE) */ 1134 if (uld_active(sc, ULD_TOM)) 1135 t4_deactivate_uld(sc, ULD_TOM); 1136 1137 end_synchronized_op(sc, 0); 1138 } 1139 1140 static int 1141 t4_tom_mod_unload(void) 1142 { 1143 t4_iterate(tom_uninit, NULL); 1144 1145 if (t4_unregister_uld(&tom_uld_info) == EBUSY) 1146 return (EBUSY); 1147 1148 if (ifaddr_evhandler) { 1149 EVENTHANDLER_DEREGISTER(ifaddr_event, ifaddr_evhandler); 1150 taskqueue_cancel_timeout(taskqueue_thread, &clip_task, NULL); 1151 } 1152 1153 return (0); 1154 } 1155 #endif /* TCP_OFFLOAD */ 1156 1157 static int 1158 t4_tom_modevent(module_t mod, int cmd, void *arg) 1159 { 1160 int rc = 0; 1161 1162 #ifdef TCP_OFFLOAD 1163 switch (cmd) { 1164 case MOD_LOAD: 1165 rc = t4_tom_mod_load(); 1166 break; 1167 1168 case MOD_UNLOAD: 1169 rc = t4_tom_mod_unload(); 1170 break; 1171 1172 default: 1173 rc = EINVAL; 1174 } 1175 #else 1176 printf("t4_tom: compiled without TCP_OFFLOAD support.\n"); 1177 rc = EOPNOTSUPP; 1178 #endif 1179 return (rc); 1180 } 1181 1182 static moduledata_t t4_tom_moddata= { 1183 "t4_tom", 1184 t4_tom_modevent, 1185 0 1186 }; 1187 1188 MODULE_VERSION(t4_tom, 1); 1189 MODULE_DEPEND(t4_tom, toecore, 1, 1, 1); 1190 MODULE_DEPEND(t4_tom, t4nex, 1, 1, 1); 1191 DECLARE_MODULE(t4_tom, t4_tom_moddata, SI_SUB_EXEC, SI_ORDER_ANY); 1192