xref: /freebsd/sys/dev/cxgbe/tom/t4_cpl_io.c (revision f5f7c05209ca2c3748fd8b27c5e80ffad49120eb)
1 /*-
2  * Copyright (c) 2012 Chelsio Communications, Inc.
3  * All rights reserved.
4  * Written by: Navdeep Parhar <np@FreeBSD.org>
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions
8  * are met:
9  * 1. Redistributions of source code must retain the above copyright
10  *    notice, this list of conditions and the following disclaimer.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  *
15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25  * SUCH DAMAGE.
26  */
27 
28 #include <sys/cdefs.h>
29 __FBSDID("$FreeBSD$");
30 
31 #include "opt_inet.h"
32 
33 #ifdef TCP_OFFLOAD
34 #include <sys/param.h>
35 #include <sys/types.h>
36 #include <sys/kernel.h>
37 #include <sys/ktr.h>
38 #include <sys/module.h>
39 #include <sys/protosw.h>
40 #include <sys/domain.h>
41 #include <sys/socket.h>
42 #include <sys/socketvar.h>
43 #include <sys/sglist.h>
44 #include <netinet/in.h>
45 #include <netinet/in_pcb.h>
46 #include <netinet/ip.h>
47 #include <netinet/tcp_var.h>
48 #define TCPSTATES
49 #include <netinet/tcp_fsm.h>
50 #include <netinet/tcp_seq.h>
51 #include <netinet/toecore.h>
52 
53 #include "common/common.h"
54 #include "common/t4_msg.h"
55 #include "common/t4_regs.h"
56 #include "common/t4_tcb.h"
57 #include "tom/t4_tom_l2t.h"
58 #include "tom/t4_tom.h"
59 
60 VNET_DECLARE(int, tcp_do_autosndbuf);
61 #define V_tcp_do_autosndbuf VNET(tcp_do_autosndbuf)
62 VNET_DECLARE(int, tcp_autosndbuf_inc);
63 #define V_tcp_autosndbuf_inc VNET(tcp_autosndbuf_inc)
64 VNET_DECLARE(int, tcp_autosndbuf_max);
65 #define V_tcp_autosndbuf_max VNET(tcp_autosndbuf_max)
66 VNET_DECLARE(int, tcp_do_autorcvbuf);
67 #define V_tcp_do_autorcvbuf VNET(tcp_do_autorcvbuf)
68 VNET_DECLARE(int, tcp_autorcvbuf_inc);
69 #define V_tcp_autorcvbuf_inc VNET(tcp_autorcvbuf_inc)
70 VNET_DECLARE(int, tcp_autorcvbuf_max);
71 #define V_tcp_autorcvbuf_max VNET(tcp_autorcvbuf_max)
72 
73 void
74 send_flowc_wr(struct toepcb *toep, struct flowc_tx_params *ftxp)
75 {
76 	struct wrqe *wr;
77 	struct fw_flowc_wr *flowc;
78 	unsigned int nparams = ftxp ? 8 : 6, flowclen;
79 	struct port_info *pi = toep->port;
80 	struct adapter *sc = pi->adapter;
81 	unsigned int pfvf = G_FW_VIID_PFN(pi->viid) << S_FW_VIID_PFN;
82 	struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx];
83 
84 	KASSERT(!(toep->flags & TPF_FLOWC_WR_SENT),
85 	    ("%s: flowc for tid %u sent already", __func__, toep->tid));
86 
87 	CTR2(KTR_CXGBE, "%s: tid %u", __func__, toep->tid);
88 
89 	flowclen = sizeof(*flowc) + nparams * sizeof(struct fw_flowc_mnemval);
90 
91 	wr = alloc_wrqe(roundup(flowclen, 16), toep->ofld_txq);
92 	if (wr == NULL) {
93 		/* XXX */
94 		panic("%s: allocation failure.", __func__);
95 	}
96 	flowc = wrtod(wr);
97 	memset(flowc, 0, wr->wr_len);
98 
99 	flowc->op_to_nparams = htobe32(V_FW_WR_OP(FW_FLOWC_WR) |
100 	    V_FW_FLOWC_WR_NPARAMS(nparams));
101 	flowc->flowid_len16 = htonl(V_FW_WR_LEN16(howmany(flowclen, 16)) |
102 	    V_FW_WR_FLOWID(toep->tid));
103 
104 	flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_PFNVFN;
105 	flowc->mnemval[0].val = htobe32(pfvf);
106 	flowc->mnemval[1].mnemonic = FW_FLOWC_MNEM_CH;
107 	flowc->mnemval[1].val = htobe32(pi->tx_chan);
108 	flowc->mnemval[2].mnemonic = FW_FLOWC_MNEM_PORT;
109 	flowc->mnemval[2].val = htobe32(pi->tx_chan);
110 	flowc->mnemval[3].mnemonic = FW_FLOWC_MNEM_IQID;
111 	flowc->mnemval[3].val = htobe32(toep->ofld_rxq->iq.abs_id);
112 	if (ftxp) {
113 		uint32_t sndbuf = min(ftxp->snd_space, sc->tt.sndbuf);
114 
115 		flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_SNDNXT;
116 		flowc->mnemval[4].val = htobe32(ftxp->snd_nxt);
117 		flowc->mnemval[5].mnemonic = FW_FLOWC_MNEM_RCVNXT;
118 		flowc->mnemval[5].val = htobe32(ftxp->rcv_nxt);
119 		flowc->mnemval[6].mnemonic = FW_FLOWC_MNEM_SNDBUF;
120 		flowc->mnemval[6].val = htobe32(sndbuf);
121 		flowc->mnemval[7].mnemonic = FW_FLOWC_MNEM_MSS;
122 		flowc->mnemval[7].val = htobe32(ftxp->mss);
123 	} else {
124 		flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_SNDBUF;
125 		flowc->mnemval[4].val = htobe32(512);
126 		flowc->mnemval[5].mnemonic = FW_FLOWC_MNEM_MSS;
127 		flowc->mnemval[5].val = htobe32(512);
128 	}
129 
130 	txsd->tx_credits = howmany(flowclen, 16);
131 	txsd->plen = 0;
132 	KASSERT(toep->tx_credits >= txsd->tx_credits && toep->txsd_avail > 0,
133 	    ("%s: not enough credits (%d)", __func__, toep->tx_credits));
134 	toep->tx_credits -= txsd->tx_credits;
135 	if (__predict_false(++toep->txsd_pidx == toep->txsd_total))
136 		toep->txsd_pidx = 0;
137 	toep->txsd_avail--;
138 
139 	toep->flags |= TPF_FLOWC_WR_SENT;
140         t4_wrq_tx(sc, wr);
141 }
142 
143 void
144 send_reset(struct adapter *sc, struct toepcb *toep, uint32_t snd_nxt)
145 {
146 	struct wrqe *wr;
147 	struct cpl_abort_req *req;
148 	int tid = toep->tid;
149 	struct inpcb *inp = toep->inp;
150 	struct tcpcb *tp = intotcpcb(inp);	/* don't use if INP_DROPPED */
151 
152 	INP_WLOCK_ASSERT(inp);
153 
154 	CTR6(KTR_CXGBE, "%s: tid %d (%s), toep_flags 0x%x, inp_flags 0x%x%s",
155 	    __func__, toep->tid,
156 	    inp->inp_flags & INP_DROPPED ? "inp dropped" :
157 	    tcpstates[tp->t_state],
158 	    toep->flags, inp->inp_flags,
159 	    toep->flags & TPF_ABORT_SHUTDOWN ?
160 	    " (abort already in progress)" : "");
161 
162 	if (toep->flags & TPF_ABORT_SHUTDOWN)
163 		return;	/* abort already in progress */
164 
165 	toep->flags |= TPF_ABORT_SHUTDOWN;
166 
167 	KASSERT(toep->flags & TPF_FLOWC_WR_SENT,
168 	    ("%s: flowc_wr not sent for tid %d.", __func__, tid));
169 
170 	wr = alloc_wrqe(sizeof(*req), toep->ofld_txq);
171 	if (wr == NULL) {
172 		/* XXX */
173 		panic("%s: allocation failure.", __func__);
174 	}
175 	req = wrtod(wr);
176 
177 	INIT_TP_WR_MIT_CPL(req, CPL_ABORT_REQ, tid);
178 	if (inp->inp_flags & INP_DROPPED)
179 		req->rsvd0 = htobe32(snd_nxt);
180 	else
181 		req->rsvd0 = htobe32(tp->snd_nxt);
182 	req->rsvd1 = !(toep->flags & TPF_TX_DATA_SENT);
183 	req->cmd = CPL_ABORT_SEND_RST;
184 
185 	/*
186 	 * XXX: What's the correct way to tell that the inp hasn't been detached
187 	 * from its socket?  Should I even be flushing the snd buffer here?
188 	 */
189 	if ((inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) == 0) {
190 		struct socket *so = inp->inp_socket;
191 
192 		if (so != NULL)	/* because I'm not sure.  See comment above */
193 			sbflush(&so->so_snd);
194 	}
195 
196 	t4_l2t_send(sc, wr, toep->l2te);
197 }
198 
199 /*
200  * Called when a connection is established to translate the TCP options
201  * reported by HW to FreeBSD's native format.
202  */
203 static void
204 assign_rxopt(struct tcpcb *tp, unsigned int opt)
205 {
206 	struct toepcb *toep = tp->t_toe;
207 	struct adapter *sc = td_adapter(toep->td);
208 
209 	INP_LOCK_ASSERT(tp->t_inpcb);
210 
211 	tp->t_maxseg = tp->t_maxopd = sc->params.mtus[G_TCPOPT_MSS(opt)] - 40;
212 
213 	if (G_TCPOPT_TSTAMP(opt)) {
214 		tp->t_flags |= TF_RCVD_TSTMP;	/* timestamps ok */
215 		tp->ts_recent = 0;		/* hmmm */
216 		tp->ts_recent_age = tcp_ts_getticks();
217 		tp->t_maxseg -= TCPOLEN_TSTAMP_APPA;
218 	}
219 
220 	if (G_TCPOPT_SACK(opt))
221 		tp->t_flags |= TF_SACK_PERMIT;	/* should already be set */
222 	else
223 		tp->t_flags &= ~TF_SACK_PERMIT;	/* sack disallowed by peer */
224 
225 	if (G_TCPOPT_WSCALE_OK(opt))
226 		tp->t_flags |= TF_RCVD_SCALE;
227 
228 	/* Doing window scaling? */
229 	if ((tp->t_flags & (TF_RCVD_SCALE | TF_REQ_SCALE)) ==
230 	    (TF_RCVD_SCALE | TF_REQ_SCALE)) {
231 		tp->rcv_scale = tp->request_r_scale;
232 		tp->snd_scale = G_TCPOPT_SND_WSCALE(opt);
233 	}
234 }
235 
236 /*
237  * Completes some final bits of initialization for just established connections
238  * and changes their state to TCPS_ESTABLISHED.
239  *
240  * The ISNs are from after the exchange of SYNs.  i.e., the true ISN + 1.
241  */
242 void
243 make_established(struct toepcb *toep, uint32_t snd_isn, uint32_t rcv_isn,
244     uint16_t opt)
245 {
246 	struct inpcb *inp = toep->inp;
247 	struct socket *so = inp->inp_socket;
248 	struct tcpcb *tp = intotcpcb(inp);
249 	long bufsize;
250 	uint32_t iss = be32toh(snd_isn) - 1;	/* true ISS */
251 	uint32_t irs = be32toh(rcv_isn) - 1;	/* true IRS */
252 	uint16_t tcpopt = be16toh(opt);
253 	struct flowc_tx_params ftxp;
254 
255 	INP_WLOCK_ASSERT(inp);
256 	KASSERT(tp->t_state == TCPS_SYN_SENT ||
257 	    tp->t_state == TCPS_SYN_RECEIVED,
258 	    ("%s: TCP state %s", __func__, tcpstates[tp->t_state]));
259 
260 	CTR4(KTR_CXGBE, "%s: tid %d, toep %p, inp %p",
261 	    __func__, toep->tid, toep, inp);
262 
263 	tp->t_state = TCPS_ESTABLISHED;
264 	tp->t_starttime = ticks;
265 	TCPSTAT_INC(tcps_connects);
266 
267 	tp->irs = irs;
268 	tcp_rcvseqinit(tp);
269 	tp->rcv_wnd = toep->rx_credits << 10;
270 	tp->rcv_adv += tp->rcv_wnd;
271 	tp->last_ack_sent = tp->rcv_nxt;
272 
273 	/*
274 	 * If we were unable to send all rx credits via opt0, save the remainder
275 	 * in rx_credits so that they can be handed over with the next credit
276 	 * update.
277 	 */
278 	SOCKBUF_LOCK(&so->so_rcv);
279 	bufsize = select_rcv_wnd(so);
280 	SOCKBUF_UNLOCK(&so->so_rcv);
281 	toep->rx_credits = bufsize - tp->rcv_wnd;
282 
283 	tp->iss = iss;
284 	tcp_sendseqinit(tp);
285 	tp->snd_una = iss + 1;
286 	tp->snd_nxt = iss + 1;
287 	tp->snd_max = iss + 1;
288 
289 	assign_rxopt(tp, tcpopt);
290 
291 	SOCKBUF_LOCK(&so->so_snd);
292 	if (so->so_snd.sb_flags & SB_AUTOSIZE && V_tcp_do_autosndbuf)
293 		bufsize = V_tcp_autosndbuf_max;
294 	else
295 		bufsize = sbspace(&so->so_snd);
296 	SOCKBUF_UNLOCK(&so->so_snd);
297 
298 	ftxp.snd_nxt = tp->snd_nxt;
299 	ftxp.rcv_nxt = tp->rcv_nxt;
300 	ftxp.snd_space = bufsize;
301 	ftxp.mss = tp->t_maxseg;
302 	send_flowc_wr(toep, &ftxp);
303 
304 	soisconnected(so);
305 }
306 
307 static int
308 send_rx_credits(struct adapter *sc, struct toepcb *toep, int credits)
309 {
310 	struct wrqe *wr;
311 	struct cpl_rx_data_ack *req;
312 	uint32_t dack = F_RX_DACK_CHANGE | V_RX_DACK_MODE(1);
313 
314 	KASSERT(credits >= 0, ("%s: %d credits", __func__, credits));
315 
316 	wr = alloc_wrqe(sizeof(*req), toep->ctrlq);
317 	if (wr == NULL)
318 		return (0);
319 	req = wrtod(wr);
320 
321 	INIT_TP_WR_MIT_CPL(req, CPL_RX_DATA_ACK, toep->tid);
322 	req->credit_dack = htobe32(dack | V_RX_CREDITS(credits));
323 
324 	t4_wrq_tx(sc, wr);
325 	return (credits);
326 }
327 
328 void
329 t4_rcvd(struct toedev *tod, struct tcpcb *tp)
330 {
331 	struct adapter *sc = tod->tod_softc;
332 	struct inpcb *inp = tp->t_inpcb;
333 	struct socket *so = inp->inp_socket;
334 	struct sockbuf *sb = &so->so_rcv;
335 	struct toepcb *toep = tp->t_toe;
336 	int credits;
337 
338 	INP_WLOCK_ASSERT(inp);
339 
340 	SOCKBUF_LOCK(sb);
341 	KASSERT(toep->sb_cc >= sb->sb_cc,
342 	    ("%s: sb %p has more data (%d) than last time (%d).",
343 	    __func__, sb, sb->sb_cc, toep->sb_cc));
344 	toep->rx_credits += toep->sb_cc - sb->sb_cc;
345 	toep->sb_cc = sb->sb_cc;
346 	credits = toep->rx_credits;
347 	SOCKBUF_UNLOCK(sb);
348 
349 	if (credits > 0 &&
350 	    (credits + 16384 >= tp->rcv_wnd || credits >= 15 * 1024)) {
351 
352 		credits = send_rx_credits(sc, toep, credits);
353 		SOCKBUF_LOCK(sb);
354 		toep->rx_credits -= credits;
355 		SOCKBUF_UNLOCK(sb);
356 		tp->rcv_wnd += credits;
357 		tp->rcv_adv += credits;
358 	}
359 }
360 
361 /*
362  * Close a connection by sending a CPL_CLOSE_CON_REQ message.
363  */
364 static int
365 close_conn(struct adapter *sc, struct toepcb *toep)
366 {
367 	struct wrqe *wr;
368 	struct cpl_close_con_req *req;
369 	unsigned int tid = toep->tid;
370 
371 	CTR3(KTR_CXGBE, "%s: tid %u%s", __func__, toep->tid,
372 	    toep->flags & TPF_FIN_SENT ? ", IGNORED" : "");
373 
374 	if (toep->flags & TPF_FIN_SENT)
375 		return (0);
376 
377 	KASSERT(toep->flags & TPF_FLOWC_WR_SENT,
378 	    ("%s: flowc_wr not sent for tid %u.", __func__, tid));
379 
380 	wr = alloc_wrqe(sizeof(*req), toep->ofld_txq);
381 	if (wr == NULL) {
382 		/* XXX */
383 		panic("%s: allocation failure.", __func__);
384 	}
385 	req = wrtod(wr);
386 
387         req->wr.wr_hi = htonl(V_FW_WR_OP(FW_TP_WR) |
388 	    V_FW_WR_IMMDLEN(sizeof(*req) - sizeof(req->wr)));
389 	req->wr.wr_mid = htonl(V_FW_WR_LEN16(howmany(sizeof(*req), 16)) |
390 	    V_FW_WR_FLOWID(tid));
391         req->wr.wr_lo = cpu_to_be64(0);
392         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid));
393 	req->rsvd = 0;
394 
395 	toep->flags |= TPF_FIN_SENT;
396 	toep->flags &= ~TPF_SEND_FIN;
397 	t4_l2t_send(sc, wr, toep->l2te);
398 
399 	return (0);
400 }
401 
402 #define MAX_OFLD_TX_CREDITS (SGE_MAX_WR_LEN / 16)
403 #define MIN_OFLD_TX_CREDITS (howmany(sizeof(struct fw_ofld_tx_data_wr) + 1, 16))
404 
405 /* Maximum amount of immediate data we could stuff in a WR */
406 static inline int
407 max_imm_payload(int tx_credits)
408 {
409 	const int n = 2;	/* Use only up to 2 desc for imm. data WR */
410 
411 	KASSERT(tx_credits >= 0 &&
412 		tx_credits <= MAX_OFLD_TX_CREDITS,
413 		("%s: %d credits", __func__, tx_credits));
414 
415 	if (tx_credits < MIN_OFLD_TX_CREDITS)
416 		return (0);
417 
418 	if (tx_credits >= (n * EQ_ESIZE) / 16)
419 		return ((n * EQ_ESIZE) - sizeof(struct fw_ofld_tx_data_wr));
420 	else
421 		return (tx_credits * 16 - sizeof(struct fw_ofld_tx_data_wr));
422 }
423 
424 /* Maximum number of SGL entries we could stuff in a WR */
425 static inline int
426 max_dsgl_nsegs(int tx_credits)
427 {
428 	int nseg = 1;	/* ulptx_sgl has room for 1, rest ulp_tx_sge_pair */
429 	int sge_pair_credits = tx_credits - MIN_OFLD_TX_CREDITS;
430 
431 	KASSERT(tx_credits >= 0 &&
432 		tx_credits <= MAX_OFLD_TX_CREDITS,
433 		("%s: %d credits", __func__, tx_credits));
434 
435 	if (tx_credits < MIN_OFLD_TX_CREDITS)
436 		return (0);
437 
438 	nseg += 2 * (sge_pair_credits * 16 / 24);
439 	if ((sge_pair_credits * 16) % 24 == 16)
440 		nseg++;
441 
442 	return (nseg);
443 }
444 
445 static inline void
446 write_tx_wr(void *dst, struct toepcb *toep, unsigned int immdlen,
447     unsigned int plen, uint8_t credits, int more_to_come)
448 {
449 	struct fw_ofld_tx_data_wr *txwr = dst;
450 	int shove = !more_to_come;
451 	int compl = 1;
452 
453 	/*
454 	 * We always request completion notifications from the firmware.  The
455 	 * only exception is when we know we'll get more data to send shortly
456 	 * and that we'll have some tx credits remaining to transmit that data.
457 	 */
458 	if (more_to_come && toep->tx_credits - credits >= MIN_OFLD_TX_CREDITS)
459 		compl = 0;
460 
461 	txwr->op_to_immdlen = htobe32(V_WR_OP(FW_OFLD_TX_DATA_WR) |
462 	    V_FW_WR_COMPL(compl) | V_FW_WR_IMMDLEN(immdlen));
463 	txwr->flowid_len16 = htobe32(V_FW_WR_FLOWID(toep->tid) |
464 	    V_FW_WR_LEN16(credits));
465 	txwr->tunnel_to_proxy =
466 	    htobe32(V_FW_OFLD_TX_DATA_WR_ULPMODE(toep->ulp_mode) |
467 		V_FW_OFLD_TX_DATA_WR_URGENT(0) |	/* XXX */
468 		V_FW_OFLD_TX_DATA_WR_SHOVE(shove));
469 	txwr->plen = htobe32(plen);
470 }
471 
472 /*
473  * Generate a DSGL from a starting mbuf.  The total number of segments and the
474  * maximum segments in any one mbuf are provided.
475  */
476 static void
477 write_tx_sgl(void *dst, struct mbuf *start, struct mbuf *stop, int nsegs, int n)
478 {
479 	struct mbuf *m;
480 	struct ulptx_sgl *usgl = dst;
481 	int i, j, rc;
482 	struct sglist sg;
483 	struct sglist_seg segs[n];
484 
485 	KASSERT(nsegs > 0, ("%s: nsegs 0", __func__));
486 
487 	sglist_init(&sg, n, segs);
488 	usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) |
489 	    V_ULPTX_NSGE(nsegs));
490 
491 	i = -1;
492 	for (m = start; m != stop; m = m->m_next) {
493 		rc = sglist_append(&sg, mtod(m, void *), m->m_len);
494 		if (__predict_false(rc != 0))
495 			panic("%s: sglist_append %d", __func__, rc);
496 
497 		for (j = 0; j < sg.sg_nseg; i++, j++) {
498 			if (i < 0) {
499 				usgl->len0 = htobe32(segs[j].ss_len);
500 				usgl->addr0 = htobe64(segs[j].ss_paddr);
501 			} else {
502 				usgl->sge[i / 2].len[i & 1] =
503 				    htobe32(segs[j].ss_len);
504 				usgl->sge[i / 2].addr[i & 1] =
505 				    htobe64(segs[j].ss_paddr);
506 			}
507 #ifdef INVARIANTS
508 			nsegs--;
509 #endif
510 		}
511 		sglist_reset(&sg);
512 	}
513 	if (i & 1)
514 		usgl->sge[i / 2].len[1] = htobe32(0);
515 	KASSERT(nsegs == 0, ("%s: nsegs %d, start %p, stop %p",
516 	    __func__, nsegs, start, stop));
517 }
518 
519 /*
520  * Max number of SGL entries an offload tx work request can have.  This is 41
521  * (1 + 40) for a full 512B work request.
522  * fw_ofld_tx_data_wr(16B) + ulptx_sgl(16B, 1) + ulptx_sge_pair(480B, 40)
523  */
524 #define OFLD_SGL_LEN (41)
525 
526 /*
527  * Send data and/or a FIN to the peer.
528  *
529  * The socket's so_snd buffer consists of a stream of data starting with sb_mb
530  * and linked together with m_next.  sb_sndptr, if set, is the last mbuf that
531  * was transmitted.
532  */
533 static void
534 t4_push_frames(struct adapter *sc, struct toepcb *toep)
535 {
536 	struct mbuf *sndptr, *m, *sb_sndptr;
537 	struct fw_ofld_tx_data_wr *txwr;
538 	struct wrqe *wr;
539 	unsigned int plen, nsegs, credits, max_imm, max_nsegs, max_nsegs_1mbuf;
540 	struct inpcb *inp = toep->inp;
541 	struct tcpcb *tp = intotcpcb(inp);
542 	struct socket *so = inp->inp_socket;
543 	struct sockbuf *sb = &so->so_snd;
544 	int tx_credits;
545 	struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx];
546 
547 	INP_WLOCK_ASSERT(inp);
548 	KASSERT(toep->flags & TPF_FLOWC_WR_SENT,
549 	    ("%s: flowc_wr not sent for tid %u.", __func__, toep->tid));
550 
551 	if (__predict_false(toep->ulp_mode != ULP_MODE_NONE &&
552 	    toep->ulp_mode != ULP_MODE_TCPDDP))
553 		CXGBE_UNIMPLEMENTED("ulp_mode");
554 
555 	/*
556 	 * This function doesn't resume by itself.  Someone else must clear the
557 	 * flag and call this function.
558 	 */
559 	if (__predict_false(toep->flags & TPF_TX_SUSPENDED))
560 		return;
561 
562 	do {
563 		tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS);
564 		max_imm = max_imm_payload(tx_credits);
565 		max_nsegs = max_dsgl_nsegs(tx_credits);
566 
567 		SOCKBUF_LOCK(sb);
568 		sb_sndptr = sb->sb_sndptr;
569 		sndptr = sb_sndptr ? sb_sndptr->m_next : sb->sb_mb;
570 		plen = 0;
571 		nsegs = 0;
572 		max_nsegs_1mbuf = 0; /* max # of SGL segments in any one mbuf */
573 		for (m = sndptr; m != NULL; m = m->m_next) {
574 			int n = sglist_count(mtod(m, void *), m->m_len);
575 
576 			nsegs += n;
577 			plen += m->m_len;
578 
579 			/* This mbuf sent us _over_ the nsegs limit, back out */
580 			if (plen > max_imm && nsegs > max_nsegs) {
581 				nsegs -= n;
582 				plen -= m->m_len;
583 				if (plen == 0) {
584 					/* Too few credits */
585 					toep->flags |= TPF_TX_SUSPENDED;
586 					SOCKBUF_UNLOCK(sb);
587 					return;
588 				}
589 				break;
590 			}
591 
592 			if (max_nsegs_1mbuf < n)
593 				max_nsegs_1mbuf = n;
594 			sb_sndptr = m;	/* new sb->sb_sndptr if all goes well */
595 
596 			/* This mbuf put us right at the max_nsegs limit */
597 			if (plen > max_imm && nsegs == max_nsegs) {
598 				m = m->m_next;
599 				break;
600 			}
601 		}
602 
603 		if (sb->sb_flags & SB_AUTOSIZE &&
604 		    V_tcp_do_autosndbuf &&
605 		    sb->sb_hiwat < V_tcp_autosndbuf_max &&
606 		    sbspace(sb) < sb->sb_hiwat / 8 * 7) {
607 			int newsize = min(sb->sb_hiwat + V_tcp_autosndbuf_inc,
608 			    V_tcp_autosndbuf_max);
609 
610 			if (!sbreserve_locked(sb, newsize, so, NULL))
611 				sb->sb_flags &= ~SB_AUTOSIZE;
612 			else {
613 				sowwakeup_locked(so);	/* room available */
614 				SOCKBUF_UNLOCK_ASSERT(sb);
615 				goto unlocked;
616 			}
617 		}
618 		SOCKBUF_UNLOCK(sb);
619 unlocked:
620 
621 		/* nothing to send */
622 		if (plen == 0) {
623 			KASSERT(m == NULL,
624 			    ("%s: nothing to send, but m != NULL", __func__));
625 			break;
626 		}
627 
628 		if (__predict_false(toep->flags & TPF_FIN_SENT))
629 			panic("%s: excess tx.", __func__);
630 
631 		if (plen <= max_imm) {
632 
633 			/* Immediate data tx */
634 
635 			wr = alloc_wrqe(roundup(sizeof(*txwr) + plen, 16),
636 					toep->ofld_txq);
637 			if (wr == NULL) {
638 				/* XXX: how will we recover from this? */
639 				toep->flags |= TPF_TX_SUSPENDED;
640 				return;
641 			}
642 			txwr = wrtod(wr);
643 			credits = howmany(wr->wr_len, 16);
644 			write_tx_wr(txwr, toep, plen, plen, credits,
645 			    tp->t_flags & TF_MORETOCOME);
646 			m_copydata(sndptr, 0, plen, (void *)(txwr + 1));
647 		} else {
648 			int wr_len;
649 
650 			/* DSGL tx */
651 
652 			wr_len = sizeof(*txwr) + sizeof(struct ulptx_sgl) +
653 			    ((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8;
654 			wr = alloc_wrqe(roundup(wr_len, 16), toep->ofld_txq);
655 			if (wr == NULL) {
656 				/* XXX: how will we recover from this? */
657 				toep->flags |= TPF_TX_SUSPENDED;
658 				return;
659 			}
660 			txwr = wrtod(wr);
661 			credits = howmany(wr_len, 16);
662 			write_tx_wr(txwr, toep, 0, plen, credits,
663 			    tp->t_flags & TF_MORETOCOME);
664 			write_tx_sgl(txwr + 1, sndptr, m, nsegs,
665 			    max_nsegs_1mbuf);
666 			if (wr_len & 0xf) {
667 				uint64_t *pad = (uint64_t *)
668 				    ((uintptr_t)txwr + wr_len);
669 				*pad = 0;
670 			}
671 		}
672 
673 		KASSERT(toep->tx_credits >= credits,
674 			("%s: not enough credits", __func__));
675 
676 		toep->tx_credits -= credits;
677 
678 		tp->snd_nxt += plen;
679 		tp->snd_max += plen;
680 
681 		SOCKBUF_LOCK(sb);
682 		KASSERT(sb_sndptr, ("%s: sb_sndptr is NULL", __func__));
683 		sb->sb_sndptr = sb_sndptr;
684 		SOCKBUF_UNLOCK(sb);
685 
686 		toep->flags |= TPF_TX_DATA_SENT;
687 
688 		KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__));
689 		txsd->plen = plen;
690 		txsd->tx_credits = credits;
691 		txsd++;
692 		if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) {
693 			toep->txsd_pidx = 0;
694 			txsd = &toep->txsd[0];
695 		}
696 		toep->txsd_avail--;
697 
698 		t4_l2t_send(sc, wr, toep->l2te);
699 	} while (m != NULL);
700 
701 	/* Send a FIN if requested, but only if there's no more data to send */
702 	if (m == NULL && toep->flags & TPF_SEND_FIN)
703 		close_conn(sc, toep);
704 }
705 
706 int
707 t4_tod_output(struct toedev *tod, struct tcpcb *tp)
708 {
709 	struct adapter *sc = tod->tod_softc;
710 #ifdef INVARIANTS
711 	struct inpcb *inp = tp->t_inpcb;
712 #endif
713 	struct toepcb *toep = tp->t_toe;
714 
715 	INP_WLOCK_ASSERT(inp);
716 	KASSERT((inp->inp_flags & INP_DROPPED) == 0,
717 	    ("%s: inp %p dropped.", __func__, inp));
718 	KASSERT(toep != NULL, ("%s: toep is NULL", __func__));
719 
720 	t4_push_frames(sc, toep);
721 
722 	return (0);
723 }
724 
725 int
726 t4_send_fin(struct toedev *tod, struct tcpcb *tp)
727 {
728 	struct adapter *sc = tod->tod_softc;
729 #ifdef INVARIANTS
730 	struct inpcb *inp = tp->t_inpcb;
731 #endif
732 	struct toepcb *toep = tp->t_toe;
733 
734 	INP_WLOCK_ASSERT(inp);
735 	KASSERT((inp->inp_flags & INP_DROPPED) == 0,
736 	    ("%s: inp %p dropped.", __func__, inp));
737 	KASSERT(toep != NULL, ("%s: toep is NULL", __func__));
738 
739 	toep->flags |= TPF_SEND_FIN;
740 	t4_push_frames(sc, toep);
741 
742 	return (0);
743 }
744 
745 int
746 t4_send_rst(struct toedev *tod, struct tcpcb *tp)
747 {
748 	struct adapter *sc = tod->tod_softc;
749 #if defined(INVARIANTS)
750 	struct inpcb *inp = tp->t_inpcb;
751 #endif
752 	struct toepcb *toep = tp->t_toe;
753 
754 	INP_WLOCK_ASSERT(inp);
755 	KASSERT((inp->inp_flags & INP_DROPPED) == 0,
756 	    ("%s: inp %p dropped.", __func__, inp));
757 	KASSERT(toep != NULL, ("%s: toep is NULL", __func__));
758 
759 	/* hmmmm */
760 	KASSERT(toep->flags & TPF_FLOWC_WR_SENT,
761 	    ("%s: flowc for tid %u [%s] not sent already",
762 	    __func__, toep->tid, tcpstates[tp->t_state]));
763 
764 	send_reset(sc, toep, 0);
765 	return (0);
766 }
767 
768 /*
769  * Peer has sent us a FIN.
770  */
771 static int
772 do_peer_close(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
773 {
774 	struct adapter *sc = iq->adapter;
775 	const struct cpl_peer_close *cpl = (const void *)(rss + 1);
776 	unsigned int tid = GET_TID(cpl);
777 	struct toepcb *toep = lookup_tid(sc, tid);
778 	struct inpcb *inp = toep->inp;
779 	struct tcpcb *tp = NULL;
780 	struct socket *so;
781 	struct sockbuf *sb;
782 #ifdef INVARIANTS
783 	unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl)));
784 #endif
785 
786 	KASSERT(opcode == CPL_PEER_CLOSE,
787 	    ("%s: unexpected opcode 0x%x", __func__, opcode));
788 	KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
789 
790 	if (__predict_false(toep->flags & TPF_SYNQE)) {
791 #ifdef INVARIANTS
792 		struct synq_entry *synqe = (void *)toep;
793 
794 		INP_WLOCK(synqe->lctx->inp);
795 		if (synqe->flags & TPF_SYNQE_HAS_L2TE) {
796 			KASSERT(synqe->flags & TPF_ABORT_SHUTDOWN,
797 			    ("%s: listen socket closed but tid %u not aborted.",
798 			    __func__, tid));
799 		} else {
800 			/*
801 			 * do_pass_accept_req is still running and will
802 			 * eventually take care of this tid.
803 			 */
804 		}
805 		INP_WUNLOCK(synqe->lctx->inp);
806 #endif
807 		CTR4(KTR_CXGBE, "%s: tid %u, synqe %p (0x%x)", __func__, tid,
808 		    toep, toep->flags);
809 		return (0);
810 	}
811 
812 	KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__));
813 
814 	INP_INFO_WLOCK(&V_tcbinfo);
815 	INP_WLOCK(inp);
816 	tp = intotcpcb(inp);
817 
818 	CTR5(KTR_CXGBE, "%s: tid %u (%s), toep_flags 0x%x, inp %p", __func__,
819 	    tid, tp ? tcpstates[tp->t_state] : "no tp", toep->flags, inp);
820 
821 	if (toep->flags & TPF_ABORT_SHUTDOWN)
822 		goto done;
823 
824 	tp->rcv_nxt++;	/* FIN */
825 
826 	so = inp->inp_socket;
827 	sb = &so->so_rcv;
828 	SOCKBUF_LOCK(sb);
829 	if (__predict_false(toep->ddp_flags & (DDP_BUF0_ACTIVE | DDP_BUF1_ACTIVE))) {
830 		m = m_get(M_NOWAIT, MT_DATA);
831 		if (m == NULL)
832 			CXGBE_UNIMPLEMENTED("mbuf alloc failure");
833 
834 		m->m_len = be32toh(cpl->rcv_nxt) - tp->rcv_nxt;
835 		m->m_flags |= M_DDP;	/* Data is already where it should be */
836 		m->m_data = "nothing to see here";
837 		tp->rcv_nxt = be32toh(cpl->rcv_nxt);
838 
839 		toep->ddp_flags &= ~(DDP_BUF0_ACTIVE | DDP_BUF1_ACTIVE);
840 
841 		KASSERT(toep->sb_cc >= sb->sb_cc,
842 		    ("%s: sb %p has more data (%d) than last time (%d).",
843 		    __func__, sb, sb->sb_cc, toep->sb_cc));
844 		toep->rx_credits += toep->sb_cc - sb->sb_cc;
845 #ifdef USE_DDP_RX_FLOW_CONTROL
846 		toep->rx_credits -= m->m_len;	/* adjust for F_RX_FC_DDP */
847 #endif
848 		sbappendstream_locked(sb, m);
849 		toep->sb_cc = sb->sb_cc;
850 	}
851 	socantrcvmore_locked(so);	/* unlocks the sockbuf */
852 
853 	KASSERT(tp->rcv_nxt == be32toh(cpl->rcv_nxt),
854 	    ("%s: rcv_nxt mismatch: %u %u", __func__, tp->rcv_nxt,
855 	    be32toh(cpl->rcv_nxt)));
856 
857 	switch (tp->t_state) {
858 	case TCPS_SYN_RECEIVED:
859 		tp->t_starttime = ticks;
860 		/* FALLTHROUGH */
861 
862 	case TCPS_ESTABLISHED:
863 		tp->t_state = TCPS_CLOSE_WAIT;
864 		break;
865 
866 	case TCPS_FIN_WAIT_1:
867 		tp->t_state = TCPS_CLOSING;
868 		break;
869 
870 	case TCPS_FIN_WAIT_2:
871 		tcp_twstart(tp);
872 		INP_UNLOCK_ASSERT(inp);	 /* safe, we have a ref on the inp */
873 		INP_INFO_WUNLOCK(&V_tcbinfo);
874 
875 		INP_WLOCK(inp);
876 		final_cpl_received(toep);
877 		return (0);
878 
879 	default:
880 		log(LOG_ERR, "%s: TID %u received CPL_PEER_CLOSE in state %d\n",
881 		    __func__, tid, tp->t_state);
882 	}
883 done:
884 	INP_WUNLOCK(inp);
885 	INP_INFO_WUNLOCK(&V_tcbinfo);
886 	return (0);
887 }
888 
889 /*
890  * Peer has ACK'd our FIN.
891  */
892 static int
893 do_close_con_rpl(struct sge_iq *iq, const struct rss_header *rss,
894     struct mbuf *m)
895 {
896 	struct adapter *sc = iq->adapter;
897 	const struct cpl_close_con_rpl *cpl = (const void *)(rss + 1);
898 	unsigned int tid = GET_TID(cpl);
899 	struct toepcb *toep = lookup_tid(sc, tid);
900 	struct inpcb *inp = toep->inp;
901 	struct tcpcb *tp = NULL;
902 	struct socket *so = NULL;
903 #ifdef INVARIANTS
904 	unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl)));
905 #endif
906 
907 	KASSERT(opcode == CPL_CLOSE_CON_RPL,
908 	    ("%s: unexpected opcode 0x%x", __func__, opcode));
909 	KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
910 	KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__));
911 
912 	INP_INFO_WLOCK(&V_tcbinfo);
913 	INP_WLOCK(inp);
914 	tp = intotcpcb(inp);
915 
916 	CTR4(KTR_CXGBE, "%s: tid %u (%s), toep_flags 0x%x",
917 	    __func__, tid, tp ? tcpstates[tp->t_state] : "no tp", toep->flags);
918 
919 	if (toep->flags & TPF_ABORT_SHUTDOWN)
920 		goto done;
921 
922 	so = inp->inp_socket;
923 	tp->snd_una = be32toh(cpl->snd_nxt) - 1;	/* exclude FIN */
924 
925 	switch (tp->t_state) {
926 	case TCPS_CLOSING:	/* see TCPS_FIN_WAIT_2 in do_peer_close too */
927 		tcp_twstart(tp);
928 release:
929 		INP_UNLOCK_ASSERT(inp);	/* safe, we have a ref on the  inp */
930 		INP_INFO_WUNLOCK(&V_tcbinfo);
931 
932 		INP_WLOCK(inp);
933 		final_cpl_received(toep);	/* no more CPLs expected */
934 
935 		return (0);
936 	case TCPS_LAST_ACK:
937 		if (tcp_close(tp))
938 			INP_WUNLOCK(inp);
939 		goto release;
940 
941 	case TCPS_FIN_WAIT_1:
942 		if (so->so_rcv.sb_state & SBS_CANTRCVMORE)
943 			soisdisconnected(so);
944 		tp->t_state = TCPS_FIN_WAIT_2;
945 		break;
946 
947 	default:
948 		log(LOG_ERR,
949 		    "%s: TID %u received CPL_CLOSE_CON_RPL in state %s\n",
950 		    __func__, tid, tcpstates[tp->t_state]);
951 	}
952 done:
953 	INP_WUNLOCK(inp);
954 	INP_INFO_WUNLOCK(&V_tcbinfo);
955 	return (0);
956 }
957 
958 void
959 send_abort_rpl(struct adapter *sc, struct sge_wrq *ofld_txq, int tid,
960     int rst_status)
961 {
962 	struct wrqe *wr;
963 	struct cpl_abort_rpl *cpl;
964 
965 	wr = alloc_wrqe(sizeof(*cpl), ofld_txq);
966 	if (wr == NULL) {
967 		/* XXX */
968 		panic("%s: allocation failure.", __func__);
969 	}
970 	cpl = wrtod(wr);
971 
972 	INIT_TP_WR_MIT_CPL(cpl, CPL_ABORT_RPL, tid);
973 	cpl->cmd = rst_status;
974 
975 	t4_wrq_tx(sc, wr);
976 }
977 
978 static int
979 abort_status_to_errno(struct tcpcb *tp, unsigned int abort_reason)
980 {
981 	switch (abort_reason) {
982 	case CPL_ERR_BAD_SYN:
983 	case CPL_ERR_CONN_RESET:
984 		return (tp->t_state == TCPS_CLOSE_WAIT ? EPIPE : ECONNRESET);
985 	case CPL_ERR_XMIT_TIMEDOUT:
986 	case CPL_ERR_PERSIST_TIMEDOUT:
987 	case CPL_ERR_FINWAIT2_TIMEDOUT:
988 	case CPL_ERR_KEEPALIVE_TIMEDOUT:
989 		return (ETIMEDOUT);
990 	default:
991 		return (EIO);
992 	}
993 }
994 
995 /*
996  * TCP RST from the peer, timeout, or some other such critical error.
997  */
998 static int
999 do_abort_req(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
1000 {
1001 	struct adapter *sc = iq->adapter;
1002 	const struct cpl_abort_req_rss *cpl = (const void *)(rss + 1);
1003 	unsigned int tid = GET_TID(cpl);
1004 	struct toepcb *toep = lookup_tid(sc, tid);
1005 	struct sge_wrq *ofld_txq = toep->ofld_txq;
1006 	struct inpcb *inp;
1007 	struct tcpcb *tp;
1008 #ifdef INVARIANTS
1009 	unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl)));
1010 #endif
1011 
1012 	KASSERT(opcode == CPL_ABORT_REQ_RSS,
1013 	    ("%s: unexpected opcode 0x%x", __func__, opcode));
1014 	KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
1015 
1016 	if (toep->flags & TPF_SYNQE)
1017 		return (do_abort_req_synqe(iq, rss, m));
1018 
1019 	KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__));
1020 
1021 	if (negative_advice(cpl->status)) {
1022 		CTR4(KTR_CXGBE, "%s: negative advice %d for tid %d (0x%x)",
1023 		    __func__, cpl->status, tid, toep->flags);
1024 		return (0);	/* Ignore negative advice */
1025 	}
1026 
1027 	inp = toep->inp;
1028 	INP_INFO_WLOCK(&V_tcbinfo);	/* for tcp_close */
1029 	INP_WLOCK(inp);
1030 
1031 	tp = intotcpcb(inp);
1032 
1033 	CTR6(KTR_CXGBE,
1034 	    "%s: tid %d (%s), toep_flags 0x%x, inp_flags 0x%x, status %d",
1035 	    __func__, tid, tp ? tcpstates[tp->t_state] : "no tp", toep->flags,
1036 	    inp->inp_flags, cpl->status);
1037 
1038 	/*
1039 	 * If we'd initiated an abort earlier the reply to it is responsible for
1040 	 * cleaning up resources.  Otherwise we tear everything down right here
1041 	 * right now.  We owe the T4 a CPL_ABORT_RPL no matter what.
1042 	 */
1043 	if (toep->flags & TPF_ABORT_SHUTDOWN) {
1044 		INP_WUNLOCK(inp);
1045 		goto done;
1046 	}
1047 	toep->flags |= TPF_ABORT_SHUTDOWN;
1048 
1049 	if ((inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) == 0) {
1050 		struct socket *so = inp->inp_socket;
1051 
1052 		if (so != NULL)
1053 			so_error_set(so, abort_status_to_errno(tp,
1054 			    cpl->status));
1055 		tp = tcp_close(tp);
1056 		if (tp == NULL)
1057 			INP_WLOCK(inp);	/* re-acquire */
1058 	}
1059 
1060 	final_cpl_received(toep);
1061 done:
1062 	INP_INFO_WUNLOCK(&V_tcbinfo);
1063 	send_abort_rpl(sc, ofld_txq, tid, CPL_ABORT_NO_RST);
1064 	return (0);
1065 }
1066 
1067 /*
1068  * Reply to the CPL_ABORT_REQ (send_reset)
1069  */
1070 static int
1071 do_abort_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
1072 {
1073 	struct adapter *sc = iq->adapter;
1074 	const struct cpl_abort_rpl_rss *cpl = (const void *)(rss + 1);
1075 	unsigned int tid = GET_TID(cpl);
1076 	struct toepcb *toep = lookup_tid(sc, tid);
1077 	struct inpcb *inp = toep->inp;
1078 #ifdef INVARIANTS
1079 	unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl)));
1080 #endif
1081 
1082 	KASSERT(opcode == CPL_ABORT_RPL_RSS,
1083 	    ("%s: unexpected opcode 0x%x", __func__, opcode));
1084 	KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
1085 
1086 	if (toep->flags & TPF_SYNQE)
1087 		return (do_abort_rpl_synqe(iq, rss, m));
1088 
1089 	KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__));
1090 
1091 	CTR5(KTR_CXGBE, "%s: tid %u, toep %p, inp %p, status %d",
1092 	    __func__, tid, toep, inp, cpl->status);
1093 
1094 	KASSERT(toep->flags & TPF_ABORT_SHUTDOWN,
1095 	    ("%s: wasn't expecting abort reply", __func__));
1096 
1097 	INP_WLOCK(inp);
1098 	final_cpl_received(toep);
1099 
1100 	return (0);
1101 }
1102 
1103 static int
1104 do_rx_data(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
1105 {
1106 	struct adapter *sc = iq->adapter;
1107 	const struct cpl_rx_data *cpl = mtod(m, const void *);
1108 	unsigned int tid = GET_TID(cpl);
1109 	struct toepcb *toep = lookup_tid(sc, tid);
1110 	struct inpcb *inp = toep->inp;
1111 	struct tcpcb *tp;
1112 	struct socket *so;
1113 	struct sockbuf *sb;
1114 	int len;
1115 	uint32_t ddp_placed = 0;
1116 
1117 	if (__predict_false(toep->flags & TPF_SYNQE)) {
1118 #ifdef INVARIANTS
1119 		struct synq_entry *synqe = (void *)toep;
1120 
1121 		INP_WLOCK(synqe->lctx->inp);
1122 		if (synqe->flags & TPF_SYNQE_HAS_L2TE) {
1123 			KASSERT(synqe->flags & TPF_ABORT_SHUTDOWN,
1124 			    ("%s: listen socket closed but tid %u not aborted.",
1125 			    __func__, tid));
1126 		} else {
1127 			/*
1128 			 * do_pass_accept_req is still running and will
1129 			 * eventually take care of this tid.
1130 			 */
1131 		}
1132 		INP_WUNLOCK(synqe->lctx->inp);
1133 #endif
1134 		CTR4(KTR_CXGBE, "%s: tid %u, synqe %p (0x%x)", __func__, tid,
1135 		    toep, toep->flags);
1136 		m_freem(m);
1137 		return (0);
1138 	}
1139 
1140 	KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__));
1141 
1142 	/* strip off CPL header */
1143 	m_adj(m, sizeof(*cpl));
1144 	len = m->m_pkthdr.len;
1145 
1146 	INP_WLOCK(inp);
1147 	if (inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) {
1148 		CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x",
1149 		    __func__, tid, len, inp->inp_flags);
1150 		INP_WUNLOCK(inp);
1151 		m_freem(m);
1152 		return (0);
1153 	}
1154 
1155 	tp = intotcpcb(inp);
1156 
1157 	if (__predict_false(tp->rcv_nxt != be32toh(cpl->seq)))
1158 		ddp_placed = be32toh(cpl->seq) - tp->rcv_nxt;
1159 
1160 	tp->rcv_nxt += len;
1161 	KASSERT(tp->rcv_wnd >= len, ("%s: negative window size", __func__));
1162 	tp->rcv_wnd -= len;
1163 	tp->t_rcvtime = ticks;
1164 
1165 	so = inp_inpcbtosocket(inp);
1166 	sb = &so->so_rcv;
1167 	SOCKBUF_LOCK(sb);
1168 
1169 	if (__predict_false(sb->sb_state & SBS_CANTRCVMORE)) {
1170 		CTR3(KTR_CXGBE, "%s: tid %u, excess rx (%d bytes)",
1171 		    __func__, tid, len);
1172 		m_freem(m);
1173 		SOCKBUF_UNLOCK(sb);
1174 		INP_WUNLOCK(inp);
1175 
1176 		INP_INFO_WLOCK(&V_tcbinfo);
1177 		INP_WLOCK(inp);
1178 		tp = tcp_drop(tp, ECONNRESET);
1179 		if (tp)
1180 			INP_WUNLOCK(inp);
1181 		INP_INFO_WUNLOCK(&V_tcbinfo);
1182 
1183 		return (0);
1184 	}
1185 
1186 	/* receive buffer autosize */
1187 	if (sb->sb_flags & SB_AUTOSIZE &&
1188 	    V_tcp_do_autorcvbuf &&
1189 	    sb->sb_hiwat < V_tcp_autorcvbuf_max &&
1190 	    len > (sbspace(sb) / 8 * 7)) {
1191 		unsigned int hiwat = sb->sb_hiwat;
1192 		unsigned int newsize = min(hiwat + V_tcp_autorcvbuf_inc,
1193 		    V_tcp_autorcvbuf_max);
1194 
1195 		if (!sbreserve_locked(sb, newsize, so, NULL))
1196 			sb->sb_flags &= ~SB_AUTOSIZE;
1197 		else
1198 			toep->rx_credits += newsize - hiwat;
1199 	}
1200 
1201 	if (toep->ulp_mode == ULP_MODE_TCPDDP) {
1202 		int changed = !(toep->ddp_flags & DDP_ON) ^ cpl->ddp_off;
1203 
1204 		if (changed) {
1205 			if (toep->ddp_flags & DDP_SC_REQ)
1206 				toep->ddp_flags ^= DDP_ON | DDP_SC_REQ;
1207 			else {
1208 				KASSERT(cpl->ddp_off == 1,
1209 				    ("%s: DDP switched on by itself.",
1210 				    __func__));
1211 
1212 				/* Fell out of DDP mode */
1213 				toep->ddp_flags &= ~(DDP_ON | DDP_BUF0_ACTIVE |
1214 				    DDP_BUF1_ACTIVE);
1215 
1216 				if (ddp_placed)
1217 					insert_ddp_data(toep, ddp_placed);
1218 			}
1219 		}
1220 
1221 		if ((toep->ddp_flags & DDP_OK) == 0 &&
1222 		    time_uptime >= toep->ddp_disabled + DDP_RETRY_WAIT) {
1223 			toep->ddp_score = DDP_LOW_SCORE;
1224 			toep->ddp_flags |= DDP_OK;
1225 			CTR3(KTR_CXGBE, "%s: tid %u DDP_OK @ %u",
1226 			    __func__, tid, time_uptime);
1227 		}
1228 
1229 		if (toep->ddp_flags & DDP_ON) {
1230 
1231 			/*
1232 			 * CPL_RX_DATA with DDP on can only be an indicate.  Ask
1233 			 * soreceive to post a buffer or disable DDP.  The
1234 			 * payload that arrived in this indicate is appended to
1235 			 * the socket buffer as usual.
1236 			 */
1237 
1238 #if 0
1239 			CTR5(KTR_CXGBE,
1240 			    "%s: tid %u (0x%x) DDP indicate (seq 0x%x, len %d)",
1241 			    __func__, tid, toep->flags, be32toh(cpl->seq), len);
1242 #endif
1243 			sb->sb_flags |= SB_DDP_INDICATE;
1244 		} else if ((toep->ddp_flags & (DDP_OK|DDP_SC_REQ)) == DDP_OK &&
1245 		    tp->rcv_wnd > DDP_RSVD_WIN && len >= sc->tt.ddp_thres) {
1246 
1247 			/*
1248 			 * DDP allowed but isn't on (and a request to switch it
1249 			 * on isn't pending either), and conditions are ripe for
1250 			 * it to work.  Switch it on.
1251 			 */
1252 
1253 			enable_ddp(sc, toep);
1254 		}
1255 	}
1256 
1257 	KASSERT(toep->sb_cc >= sb->sb_cc,
1258 	    ("%s: sb %p has more data (%d) than last time (%d).",
1259 	    __func__, sb, sb->sb_cc, toep->sb_cc));
1260 	toep->rx_credits += toep->sb_cc - sb->sb_cc;
1261 	sbappendstream_locked(sb, m);
1262 	toep->sb_cc = sb->sb_cc;
1263 	sorwakeup_locked(so);
1264 	SOCKBUF_UNLOCK_ASSERT(sb);
1265 
1266 	INP_WUNLOCK(inp);
1267 	return (0);
1268 }
1269 
1270 #define S_CPL_FW4_ACK_OPCODE    24
1271 #define M_CPL_FW4_ACK_OPCODE    0xff
1272 #define V_CPL_FW4_ACK_OPCODE(x) ((x) << S_CPL_FW4_ACK_OPCODE)
1273 #define G_CPL_FW4_ACK_OPCODE(x) \
1274     (((x) >> S_CPL_FW4_ACK_OPCODE) & M_CPL_FW4_ACK_OPCODE)
1275 
1276 #define S_CPL_FW4_ACK_FLOWID    0
1277 #define M_CPL_FW4_ACK_FLOWID    0xffffff
1278 #define V_CPL_FW4_ACK_FLOWID(x) ((x) << S_CPL_FW4_ACK_FLOWID)
1279 #define G_CPL_FW4_ACK_FLOWID(x) \
1280     (((x) >> S_CPL_FW4_ACK_FLOWID) & M_CPL_FW4_ACK_FLOWID)
1281 
1282 #define S_CPL_FW4_ACK_CR        24
1283 #define M_CPL_FW4_ACK_CR        0xff
1284 #define V_CPL_FW4_ACK_CR(x)     ((x) << S_CPL_FW4_ACK_CR)
1285 #define G_CPL_FW4_ACK_CR(x)     (((x) >> S_CPL_FW4_ACK_CR) & M_CPL_FW4_ACK_CR)
1286 
1287 #define S_CPL_FW4_ACK_SEQVAL    0
1288 #define M_CPL_FW4_ACK_SEQVAL    0x1
1289 #define V_CPL_FW4_ACK_SEQVAL(x) ((x) << S_CPL_FW4_ACK_SEQVAL)
1290 #define G_CPL_FW4_ACK_SEQVAL(x) \
1291     (((x) >> S_CPL_FW4_ACK_SEQVAL) & M_CPL_FW4_ACK_SEQVAL)
1292 #define F_CPL_FW4_ACK_SEQVAL    V_CPL_FW4_ACK_SEQVAL(1U)
1293 
1294 static int
1295 do_fw4_ack(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
1296 {
1297 	struct adapter *sc = iq->adapter;
1298 	const struct cpl_fw4_ack *cpl = (const void *)(rss + 1);
1299 	unsigned int tid = G_CPL_FW4_ACK_FLOWID(be32toh(OPCODE_TID(cpl)));
1300 	struct toepcb *toep = lookup_tid(sc, tid);
1301 	struct inpcb *inp;
1302 	struct tcpcb *tp;
1303 	struct socket *so;
1304 	uint8_t credits = cpl->credits;
1305 	struct ofld_tx_sdesc *txsd;
1306 	int plen;
1307 #ifdef INVARIANTS
1308 	unsigned int opcode = G_CPL_FW4_ACK_OPCODE(be32toh(OPCODE_TID(cpl)));
1309 #endif
1310 
1311 	/*
1312 	 * Very unusual case: we'd sent a flowc + abort_req for a synq entry and
1313 	 * now this comes back carrying the credits for the flowc.
1314 	 */
1315 	if (__predict_false(toep->flags & TPF_SYNQE)) {
1316 		KASSERT(toep->flags & TPF_ABORT_SHUTDOWN,
1317 		    ("%s: credits for a synq entry %p", __func__, toep));
1318 		return (0);
1319 	}
1320 
1321 	inp = toep->inp;
1322 
1323 	KASSERT(opcode == CPL_FW4_ACK,
1324 	    ("%s: unexpected opcode 0x%x", __func__, opcode));
1325 	KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
1326 	KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__));
1327 
1328 	INP_WLOCK(inp);
1329 
1330 	if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN)) {
1331 		INP_WUNLOCK(inp);
1332 		return (0);
1333 	}
1334 
1335 	KASSERT((inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) == 0,
1336 	    ("%s: inp_flags 0x%x", __func__, inp->inp_flags));
1337 
1338 	tp = intotcpcb(inp);
1339 
1340 	if (cpl->flags & CPL_FW4_ACK_FLAGS_SEQVAL) {
1341 		tcp_seq snd_una = be32toh(cpl->snd_una);
1342 
1343 #ifdef INVARIANTS
1344 		if (__predict_false(SEQ_LT(snd_una, tp->snd_una))) {
1345 			log(LOG_ERR,
1346 			    "%s: unexpected seq# %x for TID %u, snd_una %x\n",
1347 			    __func__, snd_una, toep->tid, tp->snd_una);
1348 		}
1349 #endif
1350 
1351 		if (tp->snd_una != snd_una) {
1352 			tp->snd_una = snd_una;
1353 			tp->ts_recent_age = tcp_ts_getticks();
1354 		}
1355 	}
1356 
1357 	so = inp->inp_socket;
1358 	txsd = &toep->txsd[toep->txsd_cidx];
1359 	plen = 0;
1360 	while (credits) {
1361 		KASSERT(credits >= txsd->tx_credits,
1362 		    ("%s: too many (or partial) credits", __func__));
1363 		credits -= txsd->tx_credits;
1364 		toep->tx_credits += txsd->tx_credits;
1365 		plen += txsd->plen;
1366 		txsd++;
1367 		toep->txsd_avail++;
1368 		KASSERT(toep->txsd_avail <= toep->txsd_total,
1369 		    ("%s: txsd avail > total", __func__));
1370 		if (__predict_false(++toep->txsd_cidx == toep->txsd_total)) {
1371 			txsd = &toep->txsd[0];
1372 			toep->txsd_cidx = 0;
1373 		}
1374 	}
1375 
1376 	if (plen > 0) {
1377 		struct sockbuf *sb = &so->so_snd;
1378 
1379 		SOCKBUF_LOCK(sb);
1380 		sbdrop_locked(sb, plen);
1381 		sowwakeup_locked(so);
1382 		SOCKBUF_UNLOCK_ASSERT(sb);
1383 	}
1384 
1385 	/* XXX */
1386 	if ((toep->flags & TPF_TX_SUSPENDED &&
1387 	    toep->tx_credits >= MIN_OFLD_TX_CREDITS) ||
1388 	    toep->tx_credits == toep->txsd_total *
1389 	    howmany((sizeof(struct fw_ofld_tx_data_wr) + 1), 16)) {
1390 		toep->flags &= ~TPF_TX_SUSPENDED;
1391 		t4_push_frames(sc, toep);
1392 	}
1393 	INP_WUNLOCK(inp);
1394 
1395 	return (0);
1396 }
1397 
1398 static int
1399 do_set_tcb_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
1400 {
1401 	struct adapter *sc = iq->adapter;
1402 	const struct cpl_set_tcb_rpl *cpl = (const void *)(rss + 1);
1403 	unsigned int tid = GET_TID(cpl);
1404 #ifdef INVARIANTS
1405 	unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl)));
1406 #endif
1407 
1408 	KASSERT(opcode == CPL_SET_TCB_RPL,
1409 	    ("%s: unexpected opcode 0x%x", __func__, opcode));
1410 	KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
1411 
1412 	if (tid >= sc->tids.ftid_base &&
1413 	    tid < sc->tids.ftid_base + sc->tids.nftids)
1414 		return (t4_filter_rpl(iq, rss, m)); /* TCB is a filter */
1415 
1416 	CXGBE_UNIMPLEMENTED(__func__);
1417 }
1418 
1419 void
1420 t4_set_tcb_field(struct adapter *sc, struct toepcb *toep, uint16_t word,
1421     uint64_t mask, uint64_t val)
1422 {
1423 	struct wrqe *wr;
1424 	struct cpl_set_tcb_field *req;
1425 
1426 	wr = alloc_wrqe(sizeof(*req), toep->ctrlq);
1427 	if (wr == NULL) {
1428 		/* XXX */
1429 		panic("%s: allocation failure.", __func__);
1430 	}
1431 	req = wrtod(wr);
1432 
1433 	INIT_TP_WR_MIT_CPL(req, CPL_SET_TCB_FIELD, toep->tid);
1434 	req->reply_ctrl = htobe16(V_NO_REPLY(1) |
1435 	    V_QUEUENO(toep->ofld_rxq->iq.abs_id));
1436 	req->word_cookie = htobe16(V_WORD(word) | V_COOKIE(0));
1437 	req->mask = htobe64(mask);
1438 	req->val = htobe64(val);
1439 
1440 	t4_wrq_tx(sc, wr);
1441 }
1442 
1443 void
1444 t4_init_cpl_io_handlers(struct adapter *sc)
1445 {
1446 
1447 	t4_register_cpl_handler(sc, CPL_PEER_CLOSE, do_peer_close);
1448 	t4_register_cpl_handler(sc, CPL_CLOSE_CON_RPL, do_close_con_rpl);
1449 	t4_register_cpl_handler(sc, CPL_ABORT_REQ_RSS, do_abort_req);
1450 	t4_register_cpl_handler(sc, CPL_ABORT_RPL_RSS, do_abort_rpl);
1451 	t4_register_cpl_handler(sc, CPL_RX_DATA, do_rx_data);
1452 	t4_register_cpl_handler(sc, CPL_FW4_ACK, do_fw4_ack);
1453 	t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, do_set_tcb_rpl);
1454 }
1455 
1456 void
1457 t4_uninit_cpl_io_handlers(struct adapter *sc)
1458 {
1459 
1460 	t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl);
1461 }
1462 #endif
1463