1 /*- 2 * Copyright (c) 2012, 2015 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __FBSDID("$FreeBSD$"); 30 31 #include "opt_inet.h" 32 33 #ifdef TCP_OFFLOAD 34 #include <sys/param.h> 35 #include <sys/types.h> 36 #include <sys/kernel.h> 37 #include <sys/ktr.h> 38 #include <sys/module.h> 39 #include <sys/protosw.h> 40 #include <sys/domain.h> 41 #include <sys/socket.h> 42 #include <sys/socketvar.h> 43 #include <sys/sglist.h> 44 #include <netinet/in.h> 45 #include <netinet/in_pcb.h> 46 #include <netinet/ip.h> 47 #include <netinet/ip6.h> 48 #define TCPSTATES 49 #include <netinet/tcp_fsm.h> 50 #include <netinet/tcp_seq.h> 51 #include <netinet/tcp_var.h> 52 #include <netinet/toecore.h> 53 54 #include "common/common.h" 55 #include "common/t4_msg.h" 56 #include "common/t4_regs.h" 57 #include "common/t4_tcb.h" 58 #include "tom/t4_tom_l2t.h" 59 #include "tom/t4_tom.h" 60 61 VNET_DECLARE(int, tcp_do_autosndbuf); 62 #define V_tcp_do_autosndbuf VNET(tcp_do_autosndbuf) 63 VNET_DECLARE(int, tcp_autosndbuf_inc); 64 #define V_tcp_autosndbuf_inc VNET(tcp_autosndbuf_inc) 65 VNET_DECLARE(int, tcp_autosndbuf_max); 66 #define V_tcp_autosndbuf_max VNET(tcp_autosndbuf_max) 67 VNET_DECLARE(int, tcp_do_autorcvbuf); 68 #define V_tcp_do_autorcvbuf VNET(tcp_do_autorcvbuf) 69 VNET_DECLARE(int, tcp_autorcvbuf_inc); 70 #define V_tcp_autorcvbuf_inc VNET(tcp_autorcvbuf_inc) 71 VNET_DECLARE(int, tcp_autorcvbuf_max); 72 #define V_tcp_autorcvbuf_max VNET(tcp_autorcvbuf_max) 73 74 void 75 send_flowc_wr(struct toepcb *toep, struct flowc_tx_params *ftxp) 76 { 77 struct wrqe *wr; 78 struct fw_flowc_wr *flowc; 79 unsigned int nparams = ftxp ? 8 : 6, flowclen; 80 struct vi_info *vi = toep->vi; 81 struct port_info *pi = vi->pi; 82 struct adapter *sc = pi->adapter; 83 unsigned int pfvf = G_FW_VIID_PFN(vi->viid) << S_FW_VIID_PFN; 84 struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx]; 85 86 KASSERT(!(toep->flags & TPF_FLOWC_WR_SENT), 87 ("%s: flowc for tid %u sent already", __func__, toep->tid)); 88 89 flowclen = sizeof(*flowc) + nparams * sizeof(struct fw_flowc_mnemval); 90 91 wr = alloc_wrqe(roundup2(flowclen, 16), toep->ofld_txq); 92 if (wr == NULL) { 93 /* XXX */ 94 panic("%s: allocation failure.", __func__); 95 } 96 flowc = wrtod(wr); 97 memset(flowc, 0, wr->wr_len); 98 99 flowc->op_to_nparams = htobe32(V_FW_WR_OP(FW_FLOWC_WR) | 100 V_FW_FLOWC_WR_NPARAMS(nparams)); 101 flowc->flowid_len16 = htonl(V_FW_WR_LEN16(howmany(flowclen, 16)) | 102 V_FW_WR_FLOWID(toep->tid)); 103 104 flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_PFNVFN; 105 flowc->mnemval[0].val = htobe32(pfvf); 106 flowc->mnemval[1].mnemonic = FW_FLOWC_MNEM_CH; 107 flowc->mnemval[1].val = htobe32(pi->tx_chan); 108 flowc->mnemval[2].mnemonic = FW_FLOWC_MNEM_PORT; 109 flowc->mnemval[2].val = htobe32(pi->tx_chan); 110 flowc->mnemval[3].mnemonic = FW_FLOWC_MNEM_IQID; 111 flowc->mnemval[3].val = htobe32(toep->ofld_rxq->iq.abs_id); 112 if (ftxp) { 113 uint32_t sndbuf = min(ftxp->snd_space, sc->tt.sndbuf); 114 115 flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_SNDNXT; 116 flowc->mnemval[4].val = htobe32(ftxp->snd_nxt); 117 flowc->mnemval[5].mnemonic = FW_FLOWC_MNEM_RCVNXT; 118 flowc->mnemval[5].val = htobe32(ftxp->rcv_nxt); 119 flowc->mnemval[6].mnemonic = FW_FLOWC_MNEM_SNDBUF; 120 flowc->mnemval[6].val = htobe32(sndbuf); 121 flowc->mnemval[7].mnemonic = FW_FLOWC_MNEM_MSS; 122 flowc->mnemval[7].val = htobe32(ftxp->mss); 123 124 CTR6(KTR_CXGBE, 125 "%s: tid %u, mss %u, sndbuf %u, snd_nxt 0x%x, rcv_nxt 0x%x", 126 __func__, toep->tid, ftxp->mss, sndbuf, ftxp->snd_nxt, 127 ftxp->rcv_nxt); 128 } else { 129 flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_SNDBUF; 130 flowc->mnemval[4].val = htobe32(512); 131 flowc->mnemval[5].mnemonic = FW_FLOWC_MNEM_MSS; 132 flowc->mnemval[5].val = htobe32(512); 133 134 CTR2(KTR_CXGBE, "%s: tid %u", __func__, toep->tid); 135 } 136 137 txsd->tx_credits = howmany(flowclen, 16); 138 txsd->plen = 0; 139 KASSERT(toep->tx_credits >= txsd->tx_credits && toep->txsd_avail > 0, 140 ("%s: not enough credits (%d)", __func__, toep->tx_credits)); 141 toep->tx_credits -= txsd->tx_credits; 142 if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) 143 toep->txsd_pidx = 0; 144 toep->txsd_avail--; 145 146 toep->flags |= TPF_FLOWC_WR_SENT; 147 t4_wrq_tx(sc, wr); 148 } 149 150 void 151 send_reset(struct adapter *sc, struct toepcb *toep, uint32_t snd_nxt) 152 { 153 struct wrqe *wr; 154 struct cpl_abort_req *req; 155 int tid = toep->tid; 156 struct inpcb *inp = toep->inp; 157 struct tcpcb *tp = intotcpcb(inp); /* don't use if INP_DROPPED */ 158 159 INP_WLOCK_ASSERT(inp); 160 161 CTR6(KTR_CXGBE, "%s: tid %d (%s), toep_flags 0x%x, inp_flags 0x%x%s", 162 __func__, toep->tid, 163 inp->inp_flags & INP_DROPPED ? "inp dropped" : 164 tcpstates[tp->t_state], 165 toep->flags, inp->inp_flags, 166 toep->flags & TPF_ABORT_SHUTDOWN ? 167 " (abort already in progress)" : ""); 168 169 if (toep->flags & TPF_ABORT_SHUTDOWN) 170 return; /* abort already in progress */ 171 172 toep->flags |= TPF_ABORT_SHUTDOWN; 173 174 KASSERT(toep->flags & TPF_FLOWC_WR_SENT, 175 ("%s: flowc_wr not sent for tid %d.", __func__, tid)); 176 177 wr = alloc_wrqe(sizeof(*req), toep->ofld_txq); 178 if (wr == NULL) { 179 /* XXX */ 180 panic("%s: allocation failure.", __func__); 181 } 182 req = wrtod(wr); 183 184 INIT_TP_WR_MIT_CPL(req, CPL_ABORT_REQ, tid); 185 if (inp->inp_flags & INP_DROPPED) 186 req->rsvd0 = htobe32(snd_nxt); 187 else 188 req->rsvd0 = htobe32(tp->snd_nxt); 189 req->rsvd1 = !(toep->flags & TPF_TX_DATA_SENT); 190 req->cmd = CPL_ABORT_SEND_RST; 191 192 /* 193 * XXX: What's the correct way to tell that the inp hasn't been detached 194 * from its socket? Should I even be flushing the snd buffer here? 195 */ 196 if ((inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) == 0) { 197 struct socket *so = inp->inp_socket; 198 199 if (so != NULL) /* because I'm not sure. See comment above */ 200 sbflush(&so->so_snd); 201 } 202 203 t4_l2t_send(sc, wr, toep->l2te); 204 } 205 206 /* 207 * Called when a connection is established to translate the TCP options 208 * reported by HW to FreeBSD's native format. 209 */ 210 static void 211 assign_rxopt(struct tcpcb *tp, unsigned int opt) 212 { 213 struct toepcb *toep = tp->t_toe; 214 struct inpcb *inp = tp->t_inpcb; 215 struct adapter *sc = td_adapter(toep->td); 216 int n; 217 218 INP_LOCK_ASSERT(inp); 219 220 if (inp->inp_inc.inc_flags & INC_ISIPV6) 221 n = sizeof(struct ip6_hdr) + sizeof(struct tcphdr); 222 else 223 n = sizeof(struct ip) + sizeof(struct tcphdr); 224 tp->t_maxseg = sc->params.mtus[G_TCPOPT_MSS(opt)] - n; 225 226 CTR4(KTR_CXGBE, "%s: tid %d, mtu_idx %u (%u)", __func__, toep->tid, 227 G_TCPOPT_MSS(opt), sc->params.mtus[G_TCPOPT_MSS(opt)]); 228 229 if (G_TCPOPT_TSTAMP(opt)) { 230 tp->t_flags |= TF_RCVD_TSTMP; /* timestamps ok */ 231 tp->ts_recent = 0; /* hmmm */ 232 tp->ts_recent_age = tcp_ts_getticks(); 233 } 234 235 if (G_TCPOPT_SACK(opt)) 236 tp->t_flags |= TF_SACK_PERMIT; /* should already be set */ 237 else 238 tp->t_flags &= ~TF_SACK_PERMIT; /* sack disallowed by peer */ 239 240 if (G_TCPOPT_WSCALE_OK(opt)) 241 tp->t_flags |= TF_RCVD_SCALE; 242 243 /* Doing window scaling? */ 244 if ((tp->t_flags & (TF_RCVD_SCALE | TF_REQ_SCALE)) == 245 (TF_RCVD_SCALE | TF_REQ_SCALE)) { 246 tp->rcv_scale = tp->request_r_scale; 247 tp->snd_scale = G_TCPOPT_SND_WSCALE(opt); 248 } 249 } 250 251 /* 252 * Completes some final bits of initialization for just established connections 253 * and changes their state to TCPS_ESTABLISHED. 254 * 255 * The ISNs are from after the exchange of SYNs. i.e., the true ISN + 1. 256 */ 257 void 258 make_established(struct toepcb *toep, uint32_t snd_isn, uint32_t rcv_isn, 259 uint16_t opt) 260 { 261 struct inpcb *inp = toep->inp; 262 struct socket *so = inp->inp_socket; 263 struct tcpcb *tp = intotcpcb(inp); 264 long bufsize; 265 uint32_t iss = be32toh(snd_isn) - 1; /* true ISS */ 266 uint32_t irs = be32toh(rcv_isn) - 1; /* true IRS */ 267 uint16_t tcpopt = be16toh(opt); 268 struct flowc_tx_params ftxp; 269 270 CURVNET_SET(so->so_vnet); 271 INP_WLOCK_ASSERT(inp); 272 KASSERT(tp->t_state == TCPS_SYN_SENT || 273 tp->t_state == TCPS_SYN_RECEIVED, 274 ("%s: TCP state %s", __func__, tcpstates[tp->t_state])); 275 276 CTR4(KTR_CXGBE, "%s: tid %d, toep %p, inp %p", 277 __func__, toep->tid, toep, inp); 278 279 tp->t_state = TCPS_ESTABLISHED; 280 tp->t_starttime = ticks; 281 TCPSTAT_INC(tcps_connects); 282 283 tp->irs = irs; 284 tcp_rcvseqinit(tp); 285 tp->rcv_wnd = toep->rx_credits << 10; 286 tp->rcv_adv += tp->rcv_wnd; 287 tp->last_ack_sent = tp->rcv_nxt; 288 289 /* 290 * If we were unable to send all rx credits via opt0, save the remainder 291 * in rx_credits so that they can be handed over with the next credit 292 * update. 293 */ 294 SOCKBUF_LOCK(&so->so_rcv); 295 bufsize = select_rcv_wnd(so); 296 SOCKBUF_UNLOCK(&so->so_rcv); 297 toep->rx_credits = bufsize - tp->rcv_wnd; 298 299 tp->iss = iss; 300 tcp_sendseqinit(tp); 301 tp->snd_una = iss + 1; 302 tp->snd_nxt = iss + 1; 303 tp->snd_max = iss + 1; 304 305 assign_rxopt(tp, tcpopt); 306 307 SOCKBUF_LOCK(&so->so_snd); 308 if (so->so_snd.sb_flags & SB_AUTOSIZE && V_tcp_do_autosndbuf) 309 bufsize = V_tcp_autosndbuf_max; 310 else 311 bufsize = sbspace(&so->so_snd); 312 SOCKBUF_UNLOCK(&so->so_snd); 313 314 ftxp.snd_nxt = tp->snd_nxt; 315 ftxp.rcv_nxt = tp->rcv_nxt; 316 ftxp.snd_space = bufsize; 317 ftxp.mss = tp->t_maxseg; 318 send_flowc_wr(toep, &ftxp); 319 320 soisconnected(so); 321 CURVNET_RESTORE(); 322 } 323 324 static int 325 send_rx_credits(struct adapter *sc, struct toepcb *toep, int credits) 326 { 327 struct wrqe *wr; 328 struct cpl_rx_data_ack *req; 329 uint32_t dack = F_RX_DACK_CHANGE | V_RX_DACK_MODE(1); 330 331 KASSERT(credits >= 0, ("%s: %d credits", __func__, credits)); 332 333 wr = alloc_wrqe(sizeof(*req), toep->ctrlq); 334 if (wr == NULL) 335 return (0); 336 req = wrtod(wr); 337 338 INIT_TP_WR_MIT_CPL(req, CPL_RX_DATA_ACK, toep->tid); 339 req->credit_dack = htobe32(dack | V_RX_CREDITS(credits)); 340 341 t4_wrq_tx(sc, wr); 342 return (credits); 343 } 344 345 void 346 t4_rcvd_locked(struct toedev *tod, struct tcpcb *tp) 347 { 348 struct adapter *sc = tod->tod_softc; 349 struct inpcb *inp = tp->t_inpcb; 350 struct socket *so = inp->inp_socket; 351 struct sockbuf *sb = &so->so_rcv; 352 struct toepcb *toep = tp->t_toe; 353 int credits; 354 355 INP_WLOCK_ASSERT(inp); 356 357 SOCKBUF_LOCK_ASSERT(sb); 358 KASSERT(toep->sb_cc >= sbused(sb), 359 ("%s: sb %p has more data (%d) than last time (%d).", 360 __func__, sb, sbused(sb), toep->sb_cc)); 361 362 toep->rx_credits += toep->sb_cc - sbused(sb); 363 toep->sb_cc = sbused(sb); 364 365 if (toep->rx_credits > 0 && 366 (tp->rcv_wnd <= 32 * 1024 || toep->rx_credits >= 64 * 1024 || 367 (toep->rx_credits >= 16 * 1024 && tp->rcv_wnd <= 128 * 1024) || 368 toep->sb_cc + tp->rcv_wnd < sb->sb_lowat)) { 369 370 credits = send_rx_credits(sc, toep, toep->rx_credits); 371 toep->rx_credits -= credits; 372 tp->rcv_wnd += credits; 373 tp->rcv_adv += credits; 374 } 375 } 376 377 void 378 t4_rcvd(struct toedev *tod, struct tcpcb *tp) 379 { 380 struct inpcb *inp = tp->t_inpcb; 381 struct socket *so = inp->inp_socket; 382 struct sockbuf *sb = &so->so_rcv; 383 384 SOCKBUF_LOCK(sb); 385 t4_rcvd_locked(tod, tp); 386 SOCKBUF_UNLOCK(sb); 387 } 388 389 /* 390 * Close a connection by sending a CPL_CLOSE_CON_REQ message. 391 */ 392 static int 393 close_conn(struct adapter *sc, struct toepcb *toep) 394 { 395 struct wrqe *wr; 396 struct cpl_close_con_req *req; 397 unsigned int tid = toep->tid; 398 399 CTR3(KTR_CXGBE, "%s: tid %u%s", __func__, toep->tid, 400 toep->flags & TPF_FIN_SENT ? ", IGNORED" : ""); 401 402 if (toep->flags & TPF_FIN_SENT) 403 return (0); 404 405 KASSERT(toep->flags & TPF_FLOWC_WR_SENT, 406 ("%s: flowc_wr not sent for tid %u.", __func__, tid)); 407 408 wr = alloc_wrqe(sizeof(*req), toep->ofld_txq); 409 if (wr == NULL) { 410 /* XXX */ 411 panic("%s: allocation failure.", __func__); 412 } 413 req = wrtod(wr); 414 415 req->wr.wr_hi = htonl(V_FW_WR_OP(FW_TP_WR) | 416 V_FW_WR_IMMDLEN(sizeof(*req) - sizeof(req->wr))); 417 req->wr.wr_mid = htonl(V_FW_WR_LEN16(howmany(sizeof(*req), 16)) | 418 V_FW_WR_FLOWID(tid)); 419 req->wr.wr_lo = cpu_to_be64(0); 420 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid)); 421 req->rsvd = 0; 422 423 toep->flags |= TPF_FIN_SENT; 424 toep->flags &= ~TPF_SEND_FIN; 425 t4_l2t_send(sc, wr, toep->l2te); 426 427 return (0); 428 } 429 430 #define MAX_OFLD_TX_CREDITS (SGE_MAX_WR_LEN / 16) 431 #define MIN_OFLD_TX_CREDITS (howmany(sizeof(struct fw_ofld_tx_data_wr) + 1, 16)) 432 433 /* Maximum amount of immediate data we could stuff in a WR */ 434 static inline int 435 max_imm_payload(int tx_credits) 436 { 437 const int n = 2; /* Use only up to 2 desc for imm. data WR */ 438 439 KASSERT(tx_credits >= 0 && 440 tx_credits <= MAX_OFLD_TX_CREDITS, 441 ("%s: %d credits", __func__, tx_credits)); 442 443 if (tx_credits < MIN_OFLD_TX_CREDITS) 444 return (0); 445 446 if (tx_credits >= (n * EQ_ESIZE) / 16) 447 return ((n * EQ_ESIZE) - sizeof(struct fw_ofld_tx_data_wr)); 448 else 449 return (tx_credits * 16 - sizeof(struct fw_ofld_tx_data_wr)); 450 } 451 452 /* Maximum number of SGL entries we could stuff in a WR */ 453 static inline int 454 max_dsgl_nsegs(int tx_credits) 455 { 456 int nseg = 1; /* ulptx_sgl has room for 1, rest ulp_tx_sge_pair */ 457 int sge_pair_credits = tx_credits - MIN_OFLD_TX_CREDITS; 458 459 KASSERT(tx_credits >= 0 && 460 tx_credits <= MAX_OFLD_TX_CREDITS, 461 ("%s: %d credits", __func__, tx_credits)); 462 463 if (tx_credits < MIN_OFLD_TX_CREDITS) 464 return (0); 465 466 nseg += 2 * (sge_pair_credits * 16 / 24); 467 if ((sge_pair_credits * 16) % 24 == 16) 468 nseg++; 469 470 return (nseg); 471 } 472 473 static inline void 474 write_tx_wr(void *dst, struct toepcb *toep, unsigned int immdlen, 475 unsigned int plen, uint8_t credits, int shove, int ulp_submode, int txalign) 476 { 477 struct fw_ofld_tx_data_wr *txwr = dst; 478 479 txwr->op_to_immdlen = htobe32(V_WR_OP(FW_OFLD_TX_DATA_WR) | 480 V_FW_WR_IMMDLEN(immdlen)); 481 txwr->flowid_len16 = htobe32(V_FW_WR_FLOWID(toep->tid) | 482 V_FW_WR_LEN16(credits)); 483 txwr->lsodisable_to_flags = htobe32(V_TX_ULP_MODE(toep->ulp_mode) | 484 V_TX_ULP_SUBMODE(ulp_submode) | V_TX_URG(0) | V_TX_SHOVE(shove)); 485 txwr->plen = htobe32(plen); 486 487 if (txalign > 0) { 488 struct tcpcb *tp = intotcpcb(toep->inp); 489 490 if (plen < 2 * tp->t_maxseg || is_10G_port(toep->vi->pi)) 491 txwr->lsodisable_to_flags |= 492 htobe32(F_FW_OFLD_TX_DATA_WR_LSODISABLE); 493 else 494 txwr->lsodisable_to_flags |= 495 htobe32(F_FW_OFLD_TX_DATA_WR_ALIGNPLD | 496 (tp->t_flags & TF_NODELAY ? 0 : 497 F_FW_OFLD_TX_DATA_WR_ALIGNPLDSHOVE)); 498 } 499 } 500 501 /* 502 * Generate a DSGL from a starting mbuf. The total number of segments and the 503 * maximum segments in any one mbuf are provided. 504 */ 505 static void 506 write_tx_sgl(void *dst, struct mbuf *start, struct mbuf *stop, int nsegs, int n) 507 { 508 struct mbuf *m; 509 struct ulptx_sgl *usgl = dst; 510 int i, j, rc; 511 struct sglist sg; 512 struct sglist_seg segs[n]; 513 514 KASSERT(nsegs > 0, ("%s: nsegs 0", __func__)); 515 516 sglist_init(&sg, n, segs); 517 usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) | 518 V_ULPTX_NSGE(nsegs)); 519 520 i = -1; 521 for (m = start; m != stop; m = m->m_next) { 522 rc = sglist_append(&sg, mtod(m, void *), m->m_len); 523 if (__predict_false(rc != 0)) 524 panic("%s: sglist_append %d", __func__, rc); 525 526 for (j = 0; j < sg.sg_nseg; i++, j++) { 527 if (i < 0) { 528 usgl->len0 = htobe32(segs[j].ss_len); 529 usgl->addr0 = htobe64(segs[j].ss_paddr); 530 } else { 531 usgl->sge[i / 2].len[i & 1] = 532 htobe32(segs[j].ss_len); 533 usgl->sge[i / 2].addr[i & 1] = 534 htobe64(segs[j].ss_paddr); 535 } 536 #ifdef INVARIANTS 537 nsegs--; 538 #endif 539 } 540 sglist_reset(&sg); 541 } 542 if (i & 1) 543 usgl->sge[i / 2].len[1] = htobe32(0); 544 KASSERT(nsegs == 0, ("%s: nsegs %d, start %p, stop %p", 545 __func__, nsegs, start, stop)); 546 } 547 548 /* 549 * Max number of SGL entries an offload tx work request can have. This is 41 550 * (1 + 40) for a full 512B work request. 551 * fw_ofld_tx_data_wr(16B) + ulptx_sgl(16B, 1) + ulptx_sge_pair(480B, 40) 552 */ 553 #define OFLD_SGL_LEN (41) 554 555 /* 556 * Send data and/or a FIN to the peer. 557 * 558 * The socket's so_snd buffer consists of a stream of data starting with sb_mb 559 * and linked together with m_next. sb_sndptr, if set, is the last mbuf that 560 * was transmitted. 561 * 562 * drop indicates the number of bytes that should be dropped from the head of 563 * the send buffer. It is an optimization that lets do_fw4_ack avoid creating 564 * contention on the send buffer lock (before this change it used to do 565 * sowwakeup and then t4_push_frames right after that when recovering from tx 566 * stalls). When drop is set this function MUST drop the bytes and wake up any 567 * writers. 568 */ 569 void 570 t4_push_frames(struct adapter *sc, struct toepcb *toep, int drop) 571 { 572 struct mbuf *sndptr, *m, *sb_sndptr; 573 struct fw_ofld_tx_data_wr *txwr; 574 struct wrqe *wr; 575 u_int plen, nsegs, credits, max_imm, max_nsegs, max_nsegs_1mbuf; 576 struct inpcb *inp = toep->inp; 577 struct tcpcb *tp = intotcpcb(inp); 578 struct socket *so = inp->inp_socket; 579 struct sockbuf *sb = &so->so_snd; 580 int tx_credits, shove, compl, sowwakeup; 581 struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx]; 582 583 INP_WLOCK_ASSERT(inp); 584 KASSERT(toep->flags & TPF_FLOWC_WR_SENT, 585 ("%s: flowc_wr not sent for tid %u.", __func__, toep->tid)); 586 587 KASSERT(toep->ulp_mode == ULP_MODE_NONE || 588 toep->ulp_mode == ULP_MODE_TCPDDP || 589 toep->ulp_mode == ULP_MODE_RDMA, 590 ("%s: ulp_mode %u for toep %p", __func__, toep->ulp_mode, toep)); 591 592 if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN)) 593 return; 594 595 /* 596 * This function doesn't resume by itself. Someone else must clear the 597 * flag and call this function. 598 */ 599 if (__predict_false(toep->flags & TPF_TX_SUSPENDED)) { 600 KASSERT(drop == 0, 601 ("%s: drop (%d) != 0 but tx is suspended", __func__, drop)); 602 return; 603 } 604 605 do { 606 tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS); 607 max_imm = max_imm_payload(tx_credits); 608 max_nsegs = max_dsgl_nsegs(tx_credits); 609 610 SOCKBUF_LOCK(sb); 611 sowwakeup = drop; 612 if (drop) { 613 sbdrop_locked(sb, drop); 614 drop = 0; 615 } 616 sb_sndptr = sb->sb_sndptr; 617 sndptr = sb_sndptr ? sb_sndptr->m_next : sb->sb_mb; 618 plen = 0; 619 nsegs = 0; 620 max_nsegs_1mbuf = 0; /* max # of SGL segments in any one mbuf */ 621 for (m = sndptr; m != NULL; m = m->m_next) { 622 int n = sglist_count(mtod(m, void *), m->m_len); 623 624 nsegs += n; 625 plen += m->m_len; 626 627 /* This mbuf sent us _over_ the nsegs limit, back out */ 628 if (plen > max_imm && nsegs > max_nsegs) { 629 nsegs -= n; 630 plen -= m->m_len; 631 if (plen == 0) { 632 /* Too few credits */ 633 toep->flags |= TPF_TX_SUSPENDED; 634 if (sowwakeup) 635 sowwakeup_locked(so); 636 else 637 SOCKBUF_UNLOCK(sb); 638 SOCKBUF_UNLOCK_ASSERT(sb); 639 return; 640 } 641 break; 642 } 643 644 if (max_nsegs_1mbuf < n) 645 max_nsegs_1mbuf = n; 646 sb_sndptr = m; /* new sb->sb_sndptr if all goes well */ 647 648 /* This mbuf put us right at the max_nsegs limit */ 649 if (plen > max_imm && nsegs == max_nsegs) { 650 m = m->m_next; 651 break; 652 } 653 } 654 655 if (sbused(sb) > sb->sb_hiwat * 5 / 8 && 656 toep->plen_nocompl + plen >= sb->sb_hiwat / 4) 657 compl = 1; 658 else 659 compl = 0; 660 661 if (sb->sb_flags & SB_AUTOSIZE && 662 V_tcp_do_autosndbuf && 663 sb->sb_hiwat < V_tcp_autosndbuf_max && 664 sbused(sb) >= sb->sb_hiwat * 7 / 8) { 665 int newsize = min(sb->sb_hiwat + V_tcp_autosndbuf_inc, 666 V_tcp_autosndbuf_max); 667 668 if (!sbreserve_locked(sb, newsize, so, NULL)) 669 sb->sb_flags &= ~SB_AUTOSIZE; 670 else 671 sowwakeup = 1; /* room available */ 672 } 673 if (sowwakeup) 674 sowwakeup_locked(so); 675 else 676 SOCKBUF_UNLOCK(sb); 677 SOCKBUF_UNLOCK_ASSERT(sb); 678 679 /* nothing to send */ 680 if (plen == 0) { 681 KASSERT(m == NULL, 682 ("%s: nothing to send, but m != NULL", __func__)); 683 break; 684 } 685 686 if (__predict_false(toep->flags & TPF_FIN_SENT)) 687 panic("%s: excess tx.", __func__); 688 689 shove = m == NULL && !(tp->t_flags & TF_MORETOCOME); 690 if (plen <= max_imm) { 691 692 /* Immediate data tx */ 693 694 wr = alloc_wrqe(roundup2(sizeof(*txwr) + plen, 16), 695 toep->ofld_txq); 696 if (wr == NULL) { 697 /* XXX: how will we recover from this? */ 698 toep->flags |= TPF_TX_SUSPENDED; 699 return; 700 } 701 txwr = wrtod(wr); 702 credits = howmany(wr->wr_len, 16); 703 write_tx_wr(txwr, toep, plen, plen, credits, shove, 0, 704 sc->tt.tx_align); 705 m_copydata(sndptr, 0, plen, (void *)(txwr + 1)); 706 nsegs = 0; 707 } else { 708 int wr_len; 709 710 /* DSGL tx */ 711 712 wr_len = sizeof(*txwr) + sizeof(struct ulptx_sgl) + 713 ((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8; 714 wr = alloc_wrqe(roundup2(wr_len, 16), toep->ofld_txq); 715 if (wr == NULL) { 716 /* XXX: how will we recover from this? */ 717 toep->flags |= TPF_TX_SUSPENDED; 718 return; 719 } 720 txwr = wrtod(wr); 721 credits = howmany(wr_len, 16); 722 write_tx_wr(txwr, toep, 0, plen, credits, shove, 0, 723 sc->tt.tx_align); 724 write_tx_sgl(txwr + 1, sndptr, m, nsegs, 725 max_nsegs_1mbuf); 726 if (wr_len & 0xf) { 727 uint64_t *pad = (uint64_t *) 728 ((uintptr_t)txwr + wr_len); 729 *pad = 0; 730 } 731 } 732 733 KASSERT(toep->tx_credits >= credits, 734 ("%s: not enough credits", __func__)); 735 736 toep->tx_credits -= credits; 737 toep->tx_nocompl += credits; 738 toep->plen_nocompl += plen; 739 if (toep->tx_credits <= toep->tx_total * 3 / 8 && 740 toep->tx_nocompl >= toep->tx_total / 4) 741 compl = 1; 742 743 if (compl || toep->ulp_mode == ULP_MODE_RDMA) { 744 txwr->op_to_immdlen |= htobe32(F_FW_WR_COMPL); 745 toep->tx_nocompl = 0; 746 toep->plen_nocompl = 0; 747 } 748 749 tp->snd_nxt += plen; 750 tp->snd_max += plen; 751 752 SOCKBUF_LOCK(sb); 753 KASSERT(sb_sndptr, ("%s: sb_sndptr is NULL", __func__)); 754 sb->sb_sndptr = sb_sndptr; 755 SOCKBUF_UNLOCK(sb); 756 757 toep->flags |= TPF_TX_DATA_SENT; 758 if (toep->tx_credits < MIN_OFLD_TX_CREDITS) 759 toep->flags |= TPF_TX_SUSPENDED; 760 761 KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__)); 762 txsd->plen = plen; 763 txsd->tx_credits = credits; 764 txsd++; 765 if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) { 766 toep->txsd_pidx = 0; 767 txsd = &toep->txsd[0]; 768 } 769 toep->txsd_avail--; 770 771 t4_l2t_send(sc, wr, toep->l2te); 772 } while (m != NULL); 773 774 /* Send a FIN if requested, but only if there's no more data to send */ 775 if (m == NULL && toep->flags & TPF_SEND_FIN) 776 close_conn(sc, toep); 777 } 778 779 static inline void 780 rqdrop_locked(struct mbufq *q, int plen) 781 { 782 struct mbuf *m; 783 784 while (plen > 0) { 785 m = mbufq_dequeue(q); 786 787 /* Too many credits. */ 788 MPASS(m != NULL); 789 M_ASSERTPKTHDR(m); 790 791 /* Partial credits. */ 792 MPASS(plen >= m->m_pkthdr.len); 793 794 plen -= m->m_pkthdr.len; 795 m_freem(m); 796 } 797 } 798 799 void 800 t4_push_pdus(struct adapter *sc, struct toepcb *toep, int drop) 801 { 802 struct mbuf *sndptr, *m; 803 struct fw_ofld_tx_data_wr *txwr; 804 struct wrqe *wr; 805 u_int plen, nsegs, credits, max_imm, max_nsegs, max_nsegs_1mbuf; 806 u_int adjusted_plen, ulp_submode; 807 struct inpcb *inp = toep->inp; 808 struct tcpcb *tp = intotcpcb(inp); 809 int tx_credits, shove; 810 struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx]; 811 struct mbufq *pduq = &toep->ulp_pduq; 812 static const u_int ulp_extra_len[] = {0, 4, 4, 8}; 813 814 INP_WLOCK_ASSERT(inp); 815 KASSERT(toep->flags & TPF_FLOWC_WR_SENT, 816 ("%s: flowc_wr not sent for tid %u.", __func__, toep->tid)); 817 KASSERT(toep->ulp_mode == ULP_MODE_ISCSI, 818 ("%s: ulp_mode %u for toep %p", __func__, toep->ulp_mode, toep)); 819 820 if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN)) 821 return; 822 823 /* 824 * This function doesn't resume by itself. Someone else must clear the 825 * flag and call this function. 826 */ 827 if (__predict_false(toep->flags & TPF_TX_SUSPENDED)) { 828 KASSERT(drop == 0, 829 ("%s: drop (%d) != 0 but tx is suspended", __func__, drop)); 830 return; 831 } 832 833 if (drop) 834 rqdrop_locked(&toep->ulp_pdu_reclaimq, drop); 835 836 while ((sndptr = mbufq_first(pduq)) != NULL) { 837 M_ASSERTPKTHDR(sndptr); 838 839 tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS); 840 max_imm = max_imm_payload(tx_credits); 841 max_nsegs = max_dsgl_nsegs(tx_credits); 842 843 plen = 0; 844 nsegs = 0; 845 max_nsegs_1mbuf = 0; /* max # of SGL segments in any one mbuf */ 846 for (m = sndptr; m != NULL; m = m->m_next) { 847 int n = sglist_count(mtod(m, void *), m->m_len); 848 849 nsegs += n; 850 plen += m->m_len; 851 852 /* 853 * This mbuf would send us _over_ the nsegs limit. 854 * Suspend tx because the PDU can't be sent out. 855 */ 856 if (plen > max_imm && nsegs > max_nsegs) { 857 toep->flags |= TPF_TX_SUSPENDED; 858 return; 859 } 860 861 if (max_nsegs_1mbuf < n) 862 max_nsegs_1mbuf = n; 863 } 864 865 if (__predict_false(toep->flags & TPF_FIN_SENT)) 866 panic("%s: excess tx.", __func__); 867 868 /* 869 * We have a PDU to send. All of it goes out in one WR so 'm' 870 * is NULL. A PDU's length is always a multiple of 4. 871 */ 872 MPASS(m == NULL); 873 MPASS((plen & 3) == 0); 874 MPASS(sndptr->m_pkthdr.len == plen); 875 876 shove = !(tp->t_flags & TF_MORETOCOME); 877 ulp_submode = mbuf_ulp_submode(sndptr); 878 MPASS(ulp_submode < nitems(ulp_extra_len)); 879 880 /* 881 * plen doesn't include header and data digests, which are 882 * generated and inserted in the right places by the TOE, but 883 * they do occupy TCP sequence space and need to be accounted 884 * for. 885 */ 886 adjusted_plen = plen + ulp_extra_len[ulp_submode]; 887 if (plen <= max_imm) { 888 889 /* Immediate data tx */ 890 891 wr = alloc_wrqe(roundup2(sizeof(*txwr) + plen, 16), 892 toep->ofld_txq); 893 if (wr == NULL) { 894 /* XXX: how will we recover from this? */ 895 toep->flags |= TPF_TX_SUSPENDED; 896 return; 897 } 898 txwr = wrtod(wr); 899 credits = howmany(wr->wr_len, 16); 900 write_tx_wr(txwr, toep, plen, adjusted_plen, credits, 901 shove, ulp_submode, sc->tt.tx_align); 902 m_copydata(sndptr, 0, plen, (void *)(txwr + 1)); 903 nsegs = 0; 904 } else { 905 int wr_len; 906 907 /* DSGL tx */ 908 wr_len = sizeof(*txwr) + sizeof(struct ulptx_sgl) + 909 ((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8; 910 wr = alloc_wrqe(roundup2(wr_len, 16), toep->ofld_txq); 911 if (wr == NULL) { 912 /* XXX: how will we recover from this? */ 913 toep->flags |= TPF_TX_SUSPENDED; 914 return; 915 } 916 txwr = wrtod(wr); 917 credits = howmany(wr_len, 16); 918 write_tx_wr(txwr, toep, 0, adjusted_plen, credits, 919 shove, ulp_submode, sc->tt.tx_align); 920 write_tx_sgl(txwr + 1, sndptr, m, nsegs, 921 max_nsegs_1mbuf); 922 if (wr_len & 0xf) { 923 uint64_t *pad = (uint64_t *) 924 ((uintptr_t)txwr + wr_len); 925 *pad = 0; 926 } 927 } 928 929 KASSERT(toep->tx_credits >= credits, 930 ("%s: not enough credits", __func__)); 931 932 m = mbufq_dequeue(pduq); 933 MPASS(m == sndptr); 934 mbufq_enqueue(&toep->ulp_pdu_reclaimq, m); 935 936 toep->tx_credits -= credits; 937 toep->tx_nocompl += credits; 938 toep->plen_nocompl += plen; 939 if (toep->tx_credits <= toep->tx_total * 3 / 8 && 940 toep->tx_nocompl >= toep->tx_total / 4) { 941 txwr->op_to_immdlen |= htobe32(F_FW_WR_COMPL); 942 toep->tx_nocompl = 0; 943 toep->plen_nocompl = 0; 944 } 945 946 tp->snd_nxt += adjusted_plen; 947 tp->snd_max += adjusted_plen; 948 949 toep->flags |= TPF_TX_DATA_SENT; 950 if (toep->tx_credits < MIN_OFLD_TX_CREDITS) 951 toep->flags |= TPF_TX_SUSPENDED; 952 953 KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__)); 954 txsd->plen = plen; 955 txsd->tx_credits = credits; 956 txsd++; 957 if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) { 958 toep->txsd_pidx = 0; 959 txsd = &toep->txsd[0]; 960 } 961 toep->txsd_avail--; 962 963 t4_l2t_send(sc, wr, toep->l2te); 964 } 965 966 /* Send a FIN if requested, but only if there are no more PDUs to send */ 967 if (mbufq_first(pduq) == NULL && toep->flags & TPF_SEND_FIN) 968 close_conn(sc, toep); 969 } 970 971 int 972 t4_tod_output(struct toedev *tod, struct tcpcb *tp) 973 { 974 struct adapter *sc = tod->tod_softc; 975 #ifdef INVARIANTS 976 struct inpcb *inp = tp->t_inpcb; 977 #endif 978 struct toepcb *toep = tp->t_toe; 979 980 INP_WLOCK_ASSERT(inp); 981 KASSERT((inp->inp_flags & INP_DROPPED) == 0, 982 ("%s: inp %p dropped.", __func__, inp)); 983 KASSERT(toep != NULL, ("%s: toep is NULL", __func__)); 984 985 if (toep->ulp_mode == ULP_MODE_ISCSI) 986 t4_push_pdus(sc, toep, 0); 987 else 988 t4_push_frames(sc, toep, 0); 989 990 return (0); 991 } 992 993 int 994 t4_send_fin(struct toedev *tod, struct tcpcb *tp) 995 { 996 struct adapter *sc = tod->tod_softc; 997 #ifdef INVARIANTS 998 struct inpcb *inp = tp->t_inpcb; 999 #endif 1000 struct toepcb *toep = tp->t_toe; 1001 1002 INP_WLOCK_ASSERT(inp); 1003 KASSERT((inp->inp_flags & INP_DROPPED) == 0, 1004 ("%s: inp %p dropped.", __func__, inp)); 1005 KASSERT(toep != NULL, ("%s: toep is NULL", __func__)); 1006 1007 toep->flags |= TPF_SEND_FIN; 1008 if (tp->t_state >= TCPS_ESTABLISHED) { 1009 if (toep->ulp_mode == ULP_MODE_ISCSI) 1010 t4_push_pdus(sc, toep, 0); 1011 else 1012 t4_push_frames(sc, toep, 0); 1013 } 1014 1015 return (0); 1016 } 1017 1018 int 1019 t4_send_rst(struct toedev *tod, struct tcpcb *tp) 1020 { 1021 struct adapter *sc = tod->tod_softc; 1022 #if defined(INVARIANTS) 1023 struct inpcb *inp = tp->t_inpcb; 1024 #endif 1025 struct toepcb *toep = tp->t_toe; 1026 1027 INP_WLOCK_ASSERT(inp); 1028 KASSERT((inp->inp_flags & INP_DROPPED) == 0, 1029 ("%s: inp %p dropped.", __func__, inp)); 1030 KASSERT(toep != NULL, ("%s: toep is NULL", __func__)); 1031 1032 /* hmmmm */ 1033 KASSERT(toep->flags & TPF_FLOWC_WR_SENT, 1034 ("%s: flowc for tid %u [%s] not sent already", 1035 __func__, toep->tid, tcpstates[tp->t_state])); 1036 1037 send_reset(sc, toep, 0); 1038 return (0); 1039 } 1040 1041 /* 1042 * Peer has sent us a FIN. 1043 */ 1044 static int 1045 do_peer_close(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 1046 { 1047 struct adapter *sc = iq->adapter; 1048 const struct cpl_peer_close *cpl = (const void *)(rss + 1); 1049 unsigned int tid = GET_TID(cpl); 1050 struct toepcb *toep = lookup_tid(sc, tid); 1051 struct inpcb *inp = toep->inp; 1052 struct tcpcb *tp = NULL; 1053 struct socket *so; 1054 #ifdef INVARIANTS 1055 unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl))); 1056 #endif 1057 1058 KASSERT(opcode == CPL_PEER_CLOSE, 1059 ("%s: unexpected opcode 0x%x", __func__, opcode)); 1060 KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__)); 1061 1062 if (__predict_false(toep->flags & TPF_SYNQE)) { 1063 #ifdef INVARIANTS 1064 struct synq_entry *synqe = (void *)toep; 1065 1066 INP_WLOCK(synqe->lctx->inp); 1067 if (synqe->flags & TPF_SYNQE_HAS_L2TE) { 1068 KASSERT(synqe->flags & TPF_ABORT_SHUTDOWN, 1069 ("%s: listen socket closed but tid %u not aborted.", 1070 __func__, tid)); 1071 } else { 1072 /* 1073 * do_pass_accept_req is still running and will 1074 * eventually take care of this tid. 1075 */ 1076 } 1077 INP_WUNLOCK(synqe->lctx->inp); 1078 #endif 1079 CTR4(KTR_CXGBE, "%s: tid %u, synqe %p (0x%x)", __func__, tid, 1080 toep, toep->flags); 1081 return (0); 1082 } 1083 1084 KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__)); 1085 1086 INP_INFO_RLOCK(&V_tcbinfo); 1087 INP_WLOCK(inp); 1088 tp = intotcpcb(inp); 1089 1090 CTR5(KTR_CXGBE, "%s: tid %u (%s), toep_flags 0x%x, inp %p", __func__, 1091 tid, tp ? tcpstates[tp->t_state] : "no tp", toep->flags, inp); 1092 1093 if (toep->flags & TPF_ABORT_SHUTDOWN) 1094 goto done; 1095 1096 tp->rcv_nxt++; /* FIN */ 1097 1098 so = inp->inp_socket; 1099 if (toep->ulp_mode == ULP_MODE_TCPDDP) { 1100 DDP_LOCK(toep); 1101 if (__predict_false(toep->ddp_flags & 1102 (DDP_BUF0_ACTIVE | DDP_BUF1_ACTIVE))) 1103 handle_ddp_close(toep, tp, cpl->rcv_nxt); 1104 DDP_UNLOCK(toep); 1105 } 1106 socantrcvmore(so); 1107 1108 if (toep->ulp_mode != ULP_MODE_RDMA) { 1109 KASSERT(tp->rcv_nxt == be32toh(cpl->rcv_nxt), 1110 ("%s: rcv_nxt mismatch: %u %u", __func__, tp->rcv_nxt, 1111 be32toh(cpl->rcv_nxt))); 1112 } 1113 1114 switch (tp->t_state) { 1115 case TCPS_SYN_RECEIVED: 1116 tp->t_starttime = ticks; 1117 /* FALLTHROUGH */ 1118 1119 case TCPS_ESTABLISHED: 1120 tp->t_state = TCPS_CLOSE_WAIT; 1121 break; 1122 1123 case TCPS_FIN_WAIT_1: 1124 tp->t_state = TCPS_CLOSING; 1125 break; 1126 1127 case TCPS_FIN_WAIT_2: 1128 tcp_twstart(tp); 1129 INP_UNLOCK_ASSERT(inp); /* safe, we have a ref on the inp */ 1130 INP_INFO_RUNLOCK(&V_tcbinfo); 1131 1132 INP_WLOCK(inp); 1133 final_cpl_received(toep); 1134 return (0); 1135 1136 default: 1137 log(LOG_ERR, "%s: TID %u received CPL_PEER_CLOSE in state %d\n", 1138 __func__, tid, tp->t_state); 1139 } 1140 done: 1141 INP_WUNLOCK(inp); 1142 INP_INFO_RUNLOCK(&V_tcbinfo); 1143 return (0); 1144 } 1145 1146 /* 1147 * Peer has ACK'd our FIN. 1148 */ 1149 static int 1150 do_close_con_rpl(struct sge_iq *iq, const struct rss_header *rss, 1151 struct mbuf *m) 1152 { 1153 struct adapter *sc = iq->adapter; 1154 const struct cpl_close_con_rpl *cpl = (const void *)(rss + 1); 1155 unsigned int tid = GET_TID(cpl); 1156 struct toepcb *toep = lookup_tid(sc, tid); 1157 struct inpcb *inp = toep->inp; 1158 struct tcpcb *tp = NULL; 1159 struct socket *so = NULL; 1160 #ifdef INVARIANTS 1161 unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl))); 1162 #endif 1163 1164 KASSERT(opcode == CPL_CLOSE_CON_RPL, 1165 ("%s: unexpected opcode 0x%x", __func__, opcode)); 1166 KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__)); 1167 KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__)); 1168 1169 INP_INFO_RLOCK(&V_tcbinfo); 1170 INP_WLOCK(inp); 1171 tp = intotcpcb(inp); 1172 1173 CTR4(KTR_CXGBE, "%s: tid %u (%s), toep_flags 0x%x", 1174 __func__, tid, tp ? tcpstates[tp->t_state] : "no tp", toep->flags); 1175 1176 if (toep->flags & TPF_ABORT_SHUTDOWN) 1177 goto done; 1178 1179 so = inp->inp_socket; 1180 tp->snd_una = be32toh(cpl->snd_nxt) - 1; /* exclude FIN */ 1181 1182 switch (tp->t_state) { 1183 case TCPS_CLOSING: /* see TCPS_FIN_WAIT_2 in do_peer_close too */ 1184 tcp_twstart(tp); 1185 release: 1186 INP_UNLOCK_ASSERT(inp); /* safe, we have a ref on the inp */ 1187 INP_INFO_RUNLOCK(&V_tcbinfo); 1188 1189 INP_WLOCK(inp); 1190 final_cpl_received(toep); /* no more CPLs expected */ 1191 1192 return (0); 1193 case TCPS_LAST_ACK: 1194 if (tcp_close(tp)) 1195 INP_WUNLOCK(inp); 1196 goto release; 1197 1198 case TCPS_FIN_WAIT_1: 1199 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) 1200 soisdisconnected(so); 1201 tp->t_state = TCPS_FIN_WAIT_2; 1202 break; 1203 1204 default: 1205 log(LOG_ERR, 1206 "%s: TID %u received CPL_CLOSE_CON_RPL in state %s\n", 1207 __func__, tid, tcpstates[tp->t_state]); 1208 } 1209 done: 1210 INP_WUNLOCK(inp); 1211 INP_INFO_RUNLOCK(&V_tcbinfo); 1212 return (0); 1213 } 1214 1215 void 1216 send_abort_rpl(struct adapter *sc, struct sge_wrq *ofld_txq, int tid, 1217 int rst_status) 1218 { 1219 struct wrqe *wr; 1220 struct cpl_abort_rpl *cpl; 1221 1222 wr = alloc_wrqe(sizeof(*cpl), ofld_txq); 1223 if (wr == NULL) { 1224 /* XXX */ 1225 panic("%s: allocation failure.", __func__); 1226 } 1227 cpl = wrtod(wr); 1228 1229 INIT_TP_WR_MIT_CPL(cpl, CPL_ABORT_RPL, tid); 1230 cpl->cmd = rst_status; 1231 1232 t4_wrq_tx(sc, wr); 1233 } 1234 1235 static int 1236 abort_status_to_errno(struct tcpcb *tp, unsigned int abort_reason) 1237 { 1238 switch (abort_reason) { 1239 case CPL_ERR_BAD_SYN: 1240 case CPL_ERR_CONN_RESET: 1241 return (tp->t_state == TCPS_CLOSE_WAIT ? EPIPE : ECONNRESET); 1242 case CPL_ERR_XMIT_TIMEDOUT: 1243 case CPL_ERR_PERSIST_TIMEDOUT: 1244 case CPL_ERR_FINWAIT2_TIMEDOUT: 1245 case CPL_ERR_KEEPALIVE_TIMEDOUT: 1246 return (ETIMEDOUT); 1247 default: 1248 return (EIO); 1249 } 1250 } 1251 1252 /* 1253 * TCP RST from the peer, timeout, or some other such critical error. 1254 */ 1255 static int 1256 do_abort_req(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 1257 { 1258 struct adapter *sc = iq->adapter; 1259 const struct cpl_abort_req_rss *cpl = (const void *)(rss + 1); 1260 unsigned int tid = GET_TID(cpl); 1261 struct toepcb *toep = lookup_tid(sc, tid); 1262 struct sge_wrq *ofld_txq = toep->ofld_txq; 1263 struct inpcb *inp; 1264 struct tcpcb *tp; 1265 #ifdef INVARIANTS 1266 unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl))); 1267 #endif 1268 1269 KASSERT(opcode == CPL_ABORT_REQ_RSS, 1270 ("%s: unexpected opcode 0x%x", __func__, opcode)); 1271 KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__)); 1272 1273 if (toep->flags & TPF_SYNQE) 1274 return (do_abort_req_synqe(iq, rss, m)); 1275 1276 KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__)); 1277 1278 if (negative_advice(cpl->status)) { 1279 CTR4(KTR_CXGBE, "%s: negative advice %d for tid %d (0x%x)", 1280 __func__, cpl->status, tid, toep->flags); 1281 return (0); /* Ignore negative advice */ 1282 } 1283 1284 inp = toep->inp; 1285 INP_INFO_RLOCK(&V_tcbinfo); /* for tcp_close */ 1286 INP_WLOCK(inp); 1287 1288 tp = intotcpcb(inp); 1289 1290 CTR6(KTR_CXGBE, 1291 "%s: tid %d (%s), toep_flags 0x%x, inp_flags 0x%x, status %d", 1292 __func__, tid, tp ? tcpstates[tp->t_state] : "no tp", toep->flags, 1293 inp->inp_flags, cpl->status); 1294 1295 /* 1296 * If we'd initiated an abort earlier the reply to it is responsible for 1297 * cleaning up resources. Otherwise we tear everything down right here 1298 * right now. We owe the T4 a CPL_ABORT_RPL no matter what. 1299 */ 1300 if (toep->flags & TPF_ABORT_SHUTDOWN) { 1301 INP_WUNLOCK(inp); 1302 goto done; 1303 } 1304 toep->flags |= TPF_ABORT_SHUTDOWN; 1305 1306 if ((inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) == 0) { 1307 struct socket *so = inp->inp_socket; 1308 1309 if (so != NULL) 1310 so_error_set(so, abort_status_to_errno(tp, 1311 cpl->status)); 1312 tp = tcp_close(tp); 1313 if (tp == NULL) 1314 INP_WLOCK(inp); /* re-acquire */ 1315 } 1316 1317 final_cpl_received(toep); 1318 done: 1319 INP_INFO_RUNLOCK(&V_tcbinfo); 1320 send_abort_rpl(sc, ofld_txq, tid, CPL_ABORT_NO_RST); 1321 return (0); 1322 } 1323 1324 /* 1325 * Reply to the CPL_ABORT_REQ (send_reset) 1326 */ 1327 static int 1328 do_abort_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 1329 { 1330 struct adapter *sc = iq->adapter; 1331 const struct cpl_abort_rpl_rss *cpl = (const void *)(rss + 1); 1332 unsigned int tid = GET_TID(cpl); 1333 struct toepcb *toep = lookup_tid(sc, tid); 1334 struct inpcb *inp = toep->inp; 1335 #ifdef INVARIANTS 1336 unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl))); 1337 #endif 1338 1339 KASSERT(opcode == CPL_ABORT_RPL_RSS, 1340 ("%s: unexpected opcode 0x%x", __func__, opcode)); 1341 KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__)); 1342 1343 if (toep->flags & TPF_SYNQE) 1344 return (do_abort_rpl_synqe(iq, rss, m)); 1345 1346 KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__)); 1347 1348 CTR5(KTR_CXGBE, "%s: tid %u, toep %p, inp %p, status %d", 1349 __func__, tid, toep, inp, cpl->status); 1350 1351 KASSERT(toep->flags & TPF_ABORT_SHUTDOWN, 1352 ("%s: wasn't expecting abort reply", __func__)); 1353 1354 INP_WLOCK(inp); 1355 final_cpl_received(toep); 1356 1357 return (0); 1358 } 1359 1360 static int 1361 do_rx_data(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 1362 { 1363 struct adapter *sc = iq->adapter; 1364 const struct cpl_rx_data *cpl = mtod(m, const void *); 1365 unsigned int tid = GET_TID(cpl); 1366 struct toepcb *toep = lookup_tid(sc, tid); 1367 struct inpcb *inp = toep->inp; 1368 struct tcpcb *tp; 1369 struct socket *so; 1370 struct sockbuf *sb; 1371 int len; 1372 uint32_t ddp_placed = 0; 1373 1374 if (__predict_false(toep->flags & TPF_SYNQE)) { 1375 #ifdef INVARIANTS 1376 struct synq_entry *synqe = (void *)toep; 1377 1378 INP_WLOCK(synqe->lctx->inp); 1379 if (synqe->flags & TPF_SYNQE_HAS_L2TE) { 1380 KASSERT(synqe->flags & TPF_ABORT_SHUTDOWN, 1381 ("%s: listen socket closed but tid %u not aborted.", 1382 __func__, tid)); 1383 } else { 1384 /* 1385 * do_pass_accept_req is still running and will 1386 * eventually take care of this tid. 1387 */ 1388 } 1389 INP_WUNLOCK(synqe->lctx->inp); 1390 #endif 1391 CTR4(KTR_CXGBE, "%s: tid %u, synqe %p (0x%x)", __func__, tid, 1392 toep, toep->flags); 1393 m_freem(m); 1394 return (0); 1395 } 1396 1397 KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__)); 1398 1399 /* strip off CPL header */ 1400 m_adj(m, sizeof(*cpl)); 1401 len = m->m_pkthdr.len; 1402 1403 INP_WLOCK(inp); 1404 if (inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) { 1405 CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x", 1406 __func__, tid, len, inp->inp_flags); 1407 INP_WUNLOCK(inp); 1408 m_freem(m); 1409 return (0); 1410 } 1411 1412 tp = intotcpcb(inp); 1413 1414 if (__predict_false(tp->rcv_nxt != be32toh(cpl->seq))) 1415 ddp_placed = be32toh(cpl->seq) - tp->rcv_nxt; 1416 1417 tp->rcv_nxt += len; 1418 if (tp->rcv_wnd < len) { 1419 KASSERT(toep->ulp_mode == ULP_MODE_RDMA, 1420 ("%s: negative window size", __func__)); 1421 } 1422 1423 tp->rcv_wnd -= len; 1424 tp->t_rcvtime = ticks; 1425 1426 if (toep->ulp_mode == ULP_MODE_TCPDDP) 1427 DDP_LOCK(toep); 1428 so = inp_inpcbtosocket(inp); 1429 sb = &so->so_rcv; 1430 SOCKBUF_LOCK(sb); 1431 1432 if (__predict_false(sb->sb_state & SBS_CANTRCVMORE)) { 1433 CTR3(KTR_CXGBE, "%s: tid %u, excess rx (%d bytes)", 1434 __func__, tid, len); 1435 m_freem(m); 1436 SOCKBUF_UNLOCK(sb); 1437 if (toep->ulp_mode == ULP_MODE_TCPDDP) 1438 DDP_UNLOCK(toep); 1439 INP_WUNLOCK(inp); 1440 1441 INP_INFO_RLOCK(&V_tcbinfo); 1442 INP_WLOCK(inp); 1443 tp = tcp_drop(tp, ECONNRESET); 1444 if (tp) 1445 INP_WUNLOCK(inp); 1446 INP_INFO_RUNLOCK(&V_tcbinfo); 1447 1448 return (0); 1449 } 1450 1451 /* receive buffer autosize */ 1452 CURVNET_SET(so->so_vnet); 1453 if (sb->sb_flags & SB_AUTOSIZE && 1454 V_tcp_do_autorcvbuf && 1455 sb->sb_hiwat < V_tcp_autorcvbuf_max && 1456 len > (sbspace(sb) / 8 * 7)) { 1457 unsigned int hiwat = sb->sb_hiwat; 1458 unsigned int newsize = min(hiwat + V_tcp_autorcvbuf_inc, 1459 V_tcp_autorcvbuf_max); 1460 1461 if (!sbreserve_locked(sb, newsize, so, NULL)) 1462 sb->sb_flags &= ~SB_AUTOSIZE; 1463 else 1464 toep->rx_credits += newsize - hiwat; 1465 } 1466 1467 if (toep->ddp_waiting_count != 0 || toep->ddp_active_count != 0) 1468 CTR3(KTR_CXGBE, "%s: tid %u, non-ddp rx (%d bytes)", __func__, 1469 tid, len); 1470 1471 if (toep->ulp_mode == ULP_MODE_TCPDDP) { 1472 int changed = !(toep->ddp_flags & DDP_ON) ^ cpl->ddp_off; 1473 1474 if (changed) { 1475 if (toep->ddp_flags & DDP_SC_REQ) 1476 toep->ddp_flags ^= DDP_ON | DDP_SC_REQ; 1477 else { 1478 KASSERT(cpl->ddp_off == 1, 1479 ("%s: DDP switched on by itself.", 1480 __func__)); 1481 1482 /* Fell out of DDP mode */ 1483 toep->ddp_flags &= ~DDP_ON; 1484 CTR1(KTR_CXGBE, "%s: fell out of DDP mode", 1485 __func__); 1486 1487 insert_ddp_data(toep, ddp_placed); 1488 } 1489 } 1490 1491 if (toep->ddp_flags & DDP_ON) { 1492 /* 1493 * CPL_RX_DATA with DDP on can only be an indicate. 1494 * Start posting queued AIO requests via DDP. The 1495 * payload that arrived in this indicate is appended 1496 * to the socket buffer as usual. 1497 */ 1498 handle_ddp_indicate(toep); 1499 } 1500 } 1501 1502 KASSERT(toep->sb_cc >= sbused(sb), 1503 ("%s: sb %p has more data (%d) than last time (%d).", 1504 __func__, sb, sbused(sb), toep->sb_cc)); 1505 toep->rx_credits += toep->sb_cc - sbused(sb); 1506 sbappendstream_locked(sb, m, 0); 1507 toep->sb_cc = sbused(sb); 1508 if (toep->rx_credits > 0 && toep->sb_cc + tp->rcv_wnd < sb->sb_lowat) { 1509 int credits; 1510 1511 credits = send_rx_credits(sc, toep, toep->rx_credits); 1512 toep->rx_credits -= credits; 1513 tp->rcv_wnd += credits; 1514 tp->rcv_adv += credits; 1515 } 1516 1517 if (toep->ddp_waiting_count > 0 && sbavail(sb) != 0) { 1518 CTR2(KTR_CXGBE, "%s: tid %u queueing AIO task", __func__, 1519 tid); 1520 ddp_queue_toep(toep); 1521 } 1522 sorwakeup_locked(so); 1523 SOCKBUF_UNLOCK_ASSERT(sb); 1524 if (toep->ulp_mode == ULP_MODE_TCPDDP) 1525 DDP_UNLOCK(toep); 1526 1527 INP_WUNLOCK(inp); 1528 CURVNET_RESTORE(); 1529 return (0); 1530 } 1531 1532 #define S_CPL_FW4_ACK_OPCODE 24 1533 #define M_CPL_FW4_ACK_OPCODE 0xff 1534 #define V_CPL_FW4_ACK_OPCODE(x) ((x) << S_CPL_FW4_ACK_OPCODE) 1535 #define G_CPL_FW4_ACK_OPCODE(x) \ 1536 (((x) >> S_CPL_FW4_ACK_OPCODE) & M_CPL_FW4_ACK_OPCODE) 1537 1538 #define S_CPL_FW4_ACK_FLOWID 0 1539 #define M_CPL_FW4_ACK_FLOWID 0xffffff 1540 #define V_CPL_FW4_ACK_FLOWID(x) ((x) << S_CPL_FW4_ACK_FLOWID) 1541 #define G_CPL_FW4_ACK_FLOWID(x) \ 1542 (((x) >> S_CPL_FW4_ACK_FLOWID) & M_CPL_FW4_ACK_FLOWID) 1543 1544 #define S_CPL_FW4_ACK_CR 24 1545 #define M_CPL_FW4_ACK_CR 0xff 1546 #define V_CPL_FW4_ACK_CR(x) ((x) << S_CPL_FW4_ACK_CR) 1547 #define G_CPL_FW4_ACK_CR(x) (((x) >> S_CPL_FW4_ACK_CR) & M_CPL_FW4_ACK_CR) 1548 1549 #define S_CPL_FW4_ACK_SEQVAL 0 1550 #define M_CPL_FW4_ACK_SEQVAL 0x1 1551 #define V_CPL_FW4_ACK_SEQVAL(x) ((x) << S_CPL_FW4_ACK_SEQVAL) 1552 #define G_CPL_FW4_ACK_SEQVAL(x) \ 1553 (((x) >> S_CPL_FW4_ACK_SEQVAL) & M_CPL_FW4_ACK_SEQVAL) 1554 #define F_CPL_FW4_ACK_SEQVAL V_CPL_FW4_ACK_SEQVAL(1U) 1555 1556 static int 1557 do_fw4_ack(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 1558 { 1559 struct adapter *sc = iq->adapter; 1560 const struct cpl_fw4_ack *cpl = (const void *)(rss + 1); 1561 unsigned int tid = G_CPL_FW4_ACK_FLOWID(be32toh(OPCODE_TID(cpl))); 1562 struct toepcb *toep = lookup_tid(sc, tid); 1563 struct inpcb *inp; 1564 struct tcpcb *tp; 1565 struct socket *so; 1566 uint8_t credits = cpl->credits; 1567 struct ofld_tx_sdesc *txsd; 1568 int plen; 1569 #ifdef INVARIANTS 1570 unsigned int opcode = G_CPL_FW4_ACK_OPCODE(be32toh(OPCODE_TID(cpl))); 1571 #endif 1572 1573 /* 1574 * Very unusual case: we'd sent a flowc + abort_req for a synq entry and 1575 * now this comes back carrying the credits for the flowc. 1576 */ 1577 if (__predict_false(toep->flags & TPF_SYNQE)) { 1578 KASSERT(toep->flags & TPF_ABORT_SHUTDOWN, 1579 ("%s: credits for a synq entry %p", __func__, toep)); 1580 return (0); 1581 } 1582 1583 inp = toep->inp; 1584 1585 KASSERT(opcode == CPL_FW4_ACK, 1586 ("%s: unexpected opcode 0x%x", __func__, opcode)); 1587 KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__)); 1588 KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__)); 1589 1590 INP_WLOCK(inp); 1591 1592 if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN)) { 1593 INP_WUNLOCK(inp); 1594 return (0); 1595 } 1596 1597 KASSERT((inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) == 0, 1598 ("%s: inp_flags 0x%x", __func__, inp->inp_flags)); 1599 1600 tp = intotcpcb(inp); 1601 1602 if (cpl->flags & CPL_FW4_ACK_FLAGS_SEQVAL) { 1603 tcp_seq snd_una = be32toh(cpl->snd_una); 1604 1605 #ifdef INVARIANTS 1606 if (__predict_false(SEQ_LT(snd_una, tp->snd_una))) { 1607 log(LOG_ERR, 1608 "%s: unexpected seq# %x for TID %u, snd_una %x\n", 1609 __func__, snd_una, toep->tid, tp->snd_una); 1610 } 1611 #endif 1612 1613 if (tp->snd_una != snd_una) { 1614 tp->snd_una = snd_una; 1615 tp->ts_recent_age = tcp_ts_getticks(); 1616 } 1617 } 1618 1619 so = inp->inp_socket; 1620 txsd = &toep->txsd[toep->txsd_cidx]; 1621 plen = 0; 1622 while (credits) { 1623 KASSERT(credits >= txsd->tx_credits, 1624 ("%s: too many (or partial) credits", __func__)); 1625 credits -= txsd->tx_credits; 1626 toep->tx_credits += txsd->tx_credits; 1627 plen += txsd->plen; 1628 txsd++; 1629 toep->txsd_avail++; 1630 KASSERT(toep->txsd_avail <= toep->txsd_total, 1631 ("%s: txsd avail > total", __func__)); 1632 if (__predict_false(++toep->txsd_cidx == toep->txsd_total)) { 1633 txsd = &toep->txsd[0]; 1634 toep->txsd_cidx = 0; 1635 } 1636 } 1637 1638 if (toep->tx_credits == toep->tx_total) { 1639 toep->tx_nocompl = 0; 1640 toep->plen_nocompl = 0; 1641 } 1642 1643 if (toep->flags & TPF_TX_SUSPENDED && 1644 toep->tx_credits >= toep->tx_total / 4) { 1645 toep->flags &= ~TPF_TX_SUSPENDED; 1646 if (toep->ulp_mode == ULP_MODE_ISCSI) 1647 t4_push_pdus(sc, toep, plen); 1648 else 1649 t4_push_frames(sc, toep, plen); 1650 } else if (plen > 0) { 1651 struct sockbuf *sb = &so->so_snd; 1652 int sbu; 1653 1654 SOCKBUF_LOCK(sb); 1655 sbu = sbused(sb); 1656 if (toep->ulp_mode == ULP_MODE_ISCSI) { 1657 1658 if (__predict_false(sbu > 0)) { 1659 /* 1660 * The data trasmitted before the tid's ULP mode 1661 * changed to ISCSI is still in so_snd. 1662 * Incoming credits should account for so_snd 1663 * first. 1664 */ 1665 sbdrop_locked(sb, min(sbu, plen)); 1666 plen -= min(sbu, plen); 1667 } 1668 sowwakeup_locked(so); /* unlocks so_snd */ 1669 rqdrop_locked(&toep->ulp_pdu_reclaimq, plen); 1670 } else { 1671 sbdrop_locked(sb, plen); 1672 sowwakeup_locked(so); /* unlocks so_snd */ 1673 } 1674 SOCKBUF_UNLOCK_ASSERT(sb); 1675 } 1676 1677 INP_WUNLOCK(inp); 1678 1679 return (0); 1680 } 1681 1682 static int 1683 do_set_tcb_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 1684 { 1685 struct adapter *sc = iq->adapter; 1686 const struct cpl_set_tcb_rpl *cpl = (const void *)(rss + 1); 1687 unsigned int tid = GET_TID(cpl); 1688 struct toepcb *toep; 1689 #ifdef INVARIANTS 1690 unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl))); 1691 #endif 1692 1693 KASSERT(opcode == CPL_SET_TCB_RPL, 1694 ("%s: unexpected opcode 0x%x", __func__, opcode)); 1695 KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__)); 1696 1697 if (is_ftid(sc, tid)) 1698 return (t4_filter_rpl(iq, rss, m)); /* TCB is a filter */ 1699 1700 toep = lookup_tid(sc, tid); 1701 if (toep->ulp_mode == ULP_MODE_TCPDDP) { 1702 handle_ddp_tcb_rpl(toep, cpl); 1703 return (0); 1704 } 1705 1706 /* 1707 * TOM and/or other ULPs don't request replies for CPL_SET_TCB or 1708 * CPL_SET_TCB_FIELD requests. This can easily change and when it does 1709 * the dispatch code will go here. 1710 */ 1711 #ifdef INVARIANTS 1712 panic("%s: Unexpected CPL_SET_TCB_RPL for tid %u on iq %p", __func__, 1713 tid, iq); 1714 #else 1715 log(LOG_ERR, "%s: Unexpected CPL_SET_TCB_RPL for tid %u on iq %p\n", 1716 __func__, tid, iq); 1717 #endif 1718 1719 return (0); 1720 } 1721 1722 void 1723 t4_set_tcb_field(struct adapter *sc, struct toepcb *toep, int ctrl, 1724 uint16_t word, uint64_t mask, uint64_t val) 1725 { 1726 struct wrqe *wr; 1727 struct cpl_set_tcb_field *req; 1728 1729 wr = alloc_wrqe(sizeof(*req), ctrl ? toep->ctrlq : toep->ofld_txq); 1730 if (wr == NULL) { 1731 /* XXX */ 1732 panic("%s: allocation failure.", __func__); 1733 } 1734 req = wrtod(wr); 1735 1736 INIT_TP_WR_MIT_CPL(req, CPL_SET_TCB_FIELD, toep->tid); 1737 req->reply_ctrl = htobe16(V_NO_REPLY(1) | 1738 V_QUEUENO(toep->ofld_rxq->iq.abs_id)); 1739 req->word_cookie = htobe16(V_WORD(word) | V_COOKIE(0)); 1740 req->mask = htobe64(mask); 1741 req->val = htobe64(val); 1742 1743 t4_wrq_tx(sc, wr); 1744 } 1745 1746 void 1747 t4_set_tcb_field_rpl(struct adapter *sc, struct toepcb *toep, int ctrl, 1748 uint16_t word, uint64_t mask, uint64_t val, uint8_t cookie) 1749 { 1750 struct wrqe *wr; 1751 struct cpl_set_tcb_field *req; 1752 1753 KASSERT((cookie & ~M_COOKIE) == 0, ("%s: invalid cookie %#x", __func__, 1754 cookie)); 1755 wr = alloc_wrqe(sizeof(*req), ctrl ? toep->ctrlq : toep->ofld_txq); 1756 if (wr == NULL) { 1757 /* XXX */ 1758 panic("%s: allocation failure.", __func__); 1759 } 1760 req = wrtod(wr); 1761 1762 INIT_TP_WR_MIT_CPL(req, CPL_SET_TCB_FIELD, toep->tid); 1763 req->reply_ctrl = htobe16(V_QUEUENO(toep->ofld_rxq->iq.abs_id)); 1764 req->word_cookie = htobe16(V_WORD(word) | V_COOKIE(cookie)); 1765 req->mask = htobe64(mask); 1766 req->val = htobe64(val); 1767 1768 t4_wrq_tx(sc, wr); 1769 } 1770 1771 void 1772 t4_init_cpl_io_handlers(struct adapter *sc) 1773 { 1774 1775 t4_register_cpl_handler(sc, CPL_PEER_CLOSE, do_peer_close); 1776 t4_register_cpl_handler(sc, CPL_CLOSE_CON_RPL, do_close_con_rpl); 1777 t4_register_cpl_handler(sc, CPL_ABORT_REQ_RSS, do_abort_req); 1778 t4_register_cpl_handler(sc, CPL_ABORT_RPL_RSS, do_abort_rpl); 1779 t4_register_cpl_handler(sc, CPL_RX_DATA, do_rx_data); 1780 t4_register_cpl_handler(sc, CPL_FW4_ACK, do_fw4_ack); 1781 t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, do_set_tcb_rpl); 1782 } 1783 1784 void 1785 t4_uninit_cpl_io_handlers(struct adapter *sc) 1786 { 1787 1788 t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl); 1789 } 1790 #endif 1791