xref: /freebsd/sys/dev/cxgbe/tom/t4_cpl_io.c (revision 833a452e9f082a7982a31c21f0da437dbbe0a39d)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2012, 2015 Chelsio Communications, Inc.
5  * All rights reserved.
6  * Written by: Navdeep Parhar <np@FreeBSD.org>
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27  * SUCH DAMAGE.
28  */
29 
30 #include <sys/cdefs.h>
31 __FBSDID("$FreeBSD$");
32 
33 #include "opt_inet.h"
34 #include "opt_inet6.h"
35 #include "opt_kern_tls.h"
36 #include "opt_ratelimit.h"
37 
38 #ifdef TCP_OFFLOAD
39 #include <sys/param.h>
40 #include <sys/aio.h>
41 #include <sys/file.h>
42 #include <sys/kernel.h>
43 #include <sys/ktr.h>
44 #include <sys/module.h>
45 #include <sys/proc.h>
46 #include <sys/protosw.h>
47 #include <sys/domain.h>
48 #include <sys/socket.h>
49 #include <sys/socketvar.h>
50 #include <sys/sglist.h>
51 #include <sys/taskqueue.h>
52 #include <netinet/in.h>
53 #include <netinet/in_pcb.h>
54 #include <netinet/ip.h>
55 #include <netinet/ip6.h>
56 #define TCPSTATES
57 #include <netinet/tcp_fsm.h>
58 #include <netinet/tcp_seq.h>
59 #include <netinet/tcp_var.h>
60 #include <netinet/toecore.h>
61 
62 #include <security/mac/mac_framework.h>
63 
64 #include <vm/vm.h>
65 #include <vm/vm_extern.h>
66 #include <vm/pmap.h>
67 #include <vm/vm_map.h>
68 #include <vm/vm_page.h>
69 
70 #include <dev/iscsi/iscsi_proto.h>
71 
72 #include "common/common.h"
73 #include "common/t4_msg.h"
74 #include "common/t4_regs.h"
75 #include "common/t4_tcb.h"
76 #include "tom/t4_tom_l2t.h"
77 #include "tom/t4_tom.h"
78 
79 static void	t4_aiotx_cancel(struct kaiocb *job);
80 static void	t4_aiotx_queue_toep(struct socket *so, struct toepcb *toep);
81 
82 void
83 send_flowc_wr(struct toepcb *toep, struct tcpcb *tp)
84 {
85 	struct wrqe *wr;
86 	struct fw_flowc_wr *flowc;
87 	unsigned int nparams, flowclen, paramidx;
88 	struct vi_info *vi = toep->vi;
89 	struct port_info *pi = vi->pi;
90 	struct adapter *sc = pi->adapter;
91 	unsigned int pfvf = sc->pf << S_FW_VIID_PFN;
92 	struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx];
93 
94 	KASSERT(!(toep->flags & TPF_FLOWC_WR_SENT),
95 	    ("%s: flowc for tid %u sent already", __func__, toep->tid));
96 
97 	if (tp != NULL)
98 		nparams = 8;
99 	else
100 		nparams = 6;
101 	if (ulp_mode(toep) == ULP_MODE_TLS)
102 		nparams++;
103 	if (toep->tls.fcplenmax != 0)
104 		nparams++;
105 	if (toep->params.tc_idx != -1) {
106 		MPASS(toep->params.tc_idx >= 0 &&
107 		    toep->params.tc_idx < sc->params.nsched_cls);
108 		nparams++;
109 	}
110 
111 	flowclen = sizeof(*flowc) + nparams * sizeof(struct fw_flowc_mnemval);
112 
113 	wr = alloc_wrqe(roundup2(flowclen, 16), &toep->ofld_txq->wrq);
114 	if (wr == NULL) {
115 		/* XXX */
116 		panic("%s: allocation failure.", __func__);
117 	}
118 	flowc = wrtod(wr);
119 	memset(flowc, 0, wr->wr_len);
120 
121 	flowc->op_to_nparams = htobe32(V_FW_WR_OP(FW_FLOWC_WR) |
122 	    V_FW_FLOWC_WR_NPARAMS(nparams));
123 	flowc->flowid_len16 = htonl(V_FW_WR_LEN16(howmany(flowclen, 16)) |
124 	    V_FW_WR_FLOWID(toep->tid));
125 
126 #define FLOWC_PARAM(__m, __v) \
127 	do { \
128 		flowc->mnemval[paramidx].mnemonic = FW_FLOWC_MNEM_##__m; \
129 		flowc->mnemval[paramidx].val = htobe32(__v); \
130 		paramidx++; \
131 	} while (0)
132 
133 	paramidx = 0;
134 
135 	FLOWC_PARAM(PFNVFN, pfvf);
136 	FLOWC_PARAM(CH, pi->tx_chan);
137 	FLOWC_PARAM(PORT, pi->tx_chan);
138 	FLOWC_PARAM(IQID, toep->ofld_rxq->iq.abs_id);
139 	FLOWC_PARAM(SNDBUF, toep->params.sndbuf);
140 	if (tp) {
141 		FLOWC_PARAM(MSS, toep->params.emss);
142 		FLOWC_PARAM(SNDNXT, tp->snd_nxt);
143 		FLOWC_PARAM(RCVNXT, tp->rcv_nxt);
144 	} else
145 		FLOWC_PARAM(MSS, 512);
146 	CTR6(KTR_CXGBE,
147 	    "%s: tid %u, mss %u, sndbuf %u, snd_nxt 0x%x, rcv_nxt 0x%x",
148 	    __func__, toep->tid, toep->params.emss, toep->params.sndbuf,
149 	    tp ? tp->snd_nxt : 0, tp ? tp->rcv_nxt : 0);
150 
151 	if (ulp_mode(toep) == ULP_MODE_TLS)
152 		FLOWC_PARAM(ULP_MODE, ulp_mode(toep));
153 	if (toep->tls.fcplenmax != 0)
154 		FLOWC_PARAM(TXDATAPLEN_MAX, toep->tls.fcplenmax);
155 	if (toep->params.tc_idx != -1)
156 		FLOWC_PARAM(SCHEDCLASS, toep->params.tc_idx);
157 #undef FLOWC_PARAM
158 
159 	KASSERT(paramidx == nparams, ("nparams mismatch"));
160 
161 	txsd->tx_credits = howmany(flowclen, 16);
162 	txsd->plen = 0;
163 	KASSERT(toep->tx_credits >= txsd->tx_credits && toep->txsd_avail > 0,
164 	    ("%s: not enough credits (%d)", __func__, toep->tx_credits));
165 	toep->tx_credits -= txsd->tx_credits;
166 	if (__predict_false(++toep->txsd_pidx == toep->txsd_total))
167 		toep->txsd_pidx = 0;
168 	toep->txsd_avail--;
169 
170 	toep->flags |= TPF_FLOWC_WR_SENT;
171         t4_wrq_tx(sc, wr);
172 }
173 
174 #ifdef RATELIMIT
175 /*
176  * Input is Bytes/second (so_max_pacing_rate), chip counts in Kilobits/second.
177  */
178 static int
179 update_tx_rate_limit(struct adapter *sc, struct toepcb *toep, u_int Bps)
180 {
181 	int tc_idx, rc;
182 	const u_int kbps = (u_int) (uint64_t)Bps * 8ULL / 1000;
183 	const int port_id = toep->vi->pi->port_id;
184 
185 	CTR3(KTR_CXGBE, "%s: tid %u, rate %uKbps", __func__, toep->tid, kbps);
186 
187 	if (kbps == 0) {
188 		/* unbind */
189 		tc_idx = -1;
190 	} else {
191 		rc = t4_reserve_cl_rl_kbps(sc, port_id, kbps, &tc_idx);
192 		if (rc != 0)
193 			return (rc);
194 		MPASS(tc_idx >= 0 && tc_idx < sc->params.nsched_cls);
195 	}
196 
197 	if (toep->params.tc_idx != tc_idx) {
198 		struct wrqe *wr;
199 		struct fw_flowc_wr *flowc;
200 		int nparams = 1, flowclen, flowclen16;
201 		struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx];
202 
203 		flowclen = sizeof(*flowc) + nparams * sizeof(struct
204 		    fw_flowc_mnemval);
205 		flowclen16 = howmany(flowclen, 16);
206 		if (toep->tx_credits < flowclen16 || toep->txsd_avail == 0 ||
207 		    (wr = alloc_wrqe(roundup2(flowclen, 16),
208 		    &toep->ofld_txq->wrq)) == NULL) {
209 			if (tc_idx >= 0)
210 				t4_release_cl_rl(sc, port_id, tc_idx);
211 			return (ENOMEM);
212 		}
213 
214 		flowc = wrtod(wr);
215 		memset(flowc, 0, wr->wr_len);
216 
217 		flowc->op_to_nparams = htobe32(V_FW_WR_OP(FW_FLOWC_WR) |
218 		    V_FW_FLOWC_WR_NPARAMS(nparams));
219 		flowc->flowid_len16 = htonl(V_FW_WR_LEN16(flowclen16) |
220 		    V_FW_WR_FLOWID(toep->tid));
221 
222 		flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_SCHEDCLASS;
223 		if (tc_idx == -1)
224 			flowc->mnemval[0].val = htobe32(0xff);
225 		else
226 			flowc->mnemval[0].val = htobe32(tc_idx);
227 
228 		txsd->tx_credits = flowclen16;
229 		txsd->plen = 0;
230 		toep->tx_credits -= txsd->tx_credits;
231 		if (__predict_false(++toep->txsd_pidx == toep->txsd_total))
232 			toep->txsd_pidx = 0;
233 		toep->txsd_avail--;
234 		t4_wrq_tx(sc, wr);
235 	}
236 
237 	if (toep->params.tc_idx >= 0)
238 		t4_release_cl_rl(sc, port_id, toep->params.tc_idx);
239 	toep->params.tc_idx = tc_idx;
240 
241 	return (0);
242 }
243 #endif
244 
245 void
246 send_reset(struct adapter *sc, struct toepcb *toep, uint32_t snd_nxt)
247 {
248 	struct wrqe *wr;
249 	struct cpl_abort_req *req;
250 	int tid = toep->tid;
251 	struct inpcb *inp = toep->inp;
252 	struct tcpcb *tp = intotcpcb(inp);	/* don't use if INP_DROPPED */
253 
254 	INP_WLOCK_ASSERT(inp);
255 
256 	CTR6(KTR_CXGBE, "%s: tid %d (%s), toep_flags 0x%x, inp_flags 0x%x%s",
257 	    __func__, toep->tid,
258 	    inp->inp_flags & INP_DROPPED ? "inp dropped" :
259 	    tcpstates[tp->t_state],
260 	    toep->flags, inp->inp_flags,
261 	    toep->flags & TPF_ABORT_SHUTDOWN ?
262 	    " (abort already in progress)" : "");
263 
264 	if (toep->flags & TPF_ABORT_SHUTDOWN)
265 		return;	/* abort already in progress */
266 
267 	toep->flags |= TPF_ABORT_SHUTDOWN;
268 
269 	KASSERT(toep->flags & TPF_FLOWC_WR_SENT,
270 	    ("%s: flowc_wr not sent for tid %d.", __func__, tid));
271 
272 	wr = alloc_wrqe(sizeof(*req), &toep->ofld_txq->wrq);
273 	if (wr == NULL) {
274 		/* XXX */
275 		panic("%s: allocation failure.", __func__);
276 	}
277 	req = wrtod(wr);
278 
279 	INIT_TP_WR_MIT_CPL(req, CPL_ABORT_REQ, tid);
280 	if (inp->inp_flags & INP_DROPPED)
281 		req->rsvd0 = htobe32(snd_nxt);
282 	else
283 		req->rsvd0 = htobe32(tp->snd_nxt);
284 	req->rsvd1 = !(toep->flags & TPF_TX_DATA_SENT);
285 	req->cmd = CPL_ABORT_SEND_RST;
286 
287 	/*
288 	 * XXX: What's the correct way to tell that the inp hasn't been detached
289 	 * from its socket?  Should I even be flushing the snd buffer here?
290 	 */
291 	if ((inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) == 0) {
292 		struct socket *so = inp->inp_socket;
293 
294 		if (so != NULL)	/* because I'm not sure.  See comment above */
295 			sbflush(&so->so_snd);
296 	}
297 
298 	t4_l2t_send(sc, wr, toep->l2te);
299 }
300 
301 /*
302  * Called when a connection is established to translate the TCP options
303  * reported by HW to FreeBSD's native format.
304  */
305 static void
306 assign_rxopt(struct tcpcb *tp, uint16_t opt)
307 {
308 	struct toepcb *toep = tp->t_toe;
309 	struct inpcb *inp = tp->t_inpcb;
310 	struct adapter *sc = td_adapter(toep->td);
311 
312 	INP_LOCK_ASSERT(inp);
313 
314 	toep->params.mtu_idx = G_TCPOPT_MSS(opt);
315 	tp->t_maxseg = sc->params.mtus[toep->params.mtu_idx];
316 	if (inp->inp_inc.inc_flags & INC_ISIPV6)
317 		tp->t_maxseg -= sizeof(struct ip6_hdr) + sizeof(struct tcphdr);
318 	else
319 		tp->t_maxseg -= sizeof(struct ip) + sizeof(struct tcphdr);
320 
321 	toep->params.emss = tp->t_maxseg;
322 	if (G_TCPOPT_TSTAMP(opt)) {
323 		toep->params.tstamp = 1;
324 		toep->params.emss -= TCPOLEN_TSTAMP_APPA;
325 		tp->t_flags |= TF_RCVD_TSTMP;	/* timestamps ok */
326 		tp->ts_recent = 0;		/* hmmm */
327 		tp->ts_recent_age = tcp_ts_getticks();
328 	} else
329 		toep->params.tstamp = 0;
330 
331 	if (G_TCPOPT_SACK(opt)) {
332 		toep->params.sack = 1;
333 		tp->t_flags |= TF_SACK_PERMIT;	/* should already be set */
334 	} else {
335 		toep->params.sack = 0;
336 		tp->t_flags &= ~TF_SACK_PERMIT;	/* sack disallowed by peer */
337 	}
338 
339 	if (G_TCPOPT_WSCALE_OK(opt))
340 		tp->t_flags |= TF_RCVD_SCALE;
341 
342 	/* Doing window scaling? */
343 	if ((tp->t_flags & (TF_RCVD_SCALE | TF_REQ_SCALE)) ==
344 	    (TF_RCVD_SCALE | TF_REQ_SCALE)) {
345 		tp->rcv_scale = tp->request_r_scale;
346 		tp->snd_scale = G_TCPOPT_SND_WSCALE(opt);
347 	} else
348 		toep->params.wscale = 0;
349 
350 	CTR6(KTR_CXGBE,
351 	    "assign_rxopt: tid %d, mtu_idx %u, emss %u, ts %u, sack %u, wscale %u",
352 	    toep->tid, toep->params.mtu_idx, toep->params.emss,
353 	    toep->params.tstamp, toep->params.sack, toep->params.wscale);
354 }
355 
356 /*
357  * Completes some final bits of initialization for just established connections
358  * and changes their state to TCPS_ESTABLISHED.
359  *
360  * The ISNs are from the exchange of SYNs.
361  */
362 void
363 make_established(struct toepcb *toep, uint32_t iss, uint32_t irs, uint16_t opt)
364 {
365 	struct inpcb *inp = toep->inp;
366 	struct socket *so = inp->inp_socket;
367 	struct tcpcb *tp = intotcpcb(inp);
368 	uint16_t tcpopt = be16toh(opt);
369 
370 	INP_WLOCK_ASSERT(inp);
371 	KASSERT(tp->t_state == TCPS_SYN_SENT ||
372 	    tp->t_state == TCPS_SYN_RECEIVED,
373 	    ("%s: TCP state %s", __func__, tcpstates[tp->t_state]));
374 
375 	CTR6(KTR_CXGBE, "%s: tid %d, so %p, inp %p, tp %p, toep %p",
376 	    __func__, toep->tid, so, inp, tp, toep);
377 
378 	tcp_state_change(tp, TCPS_ESTABLISHED);
379 	tp->t_starttime = ticks;
380 	TCPSTAT_INC(tcps_connects);
381 
382 	tp->irs = irs;
383 	tcp_rcvseqinit(tp);
384 	tp->rcv_wnd = (u_int)toep->params.opt0_bufsize << 10;
385 	tp->rcv_adv += tp->rcv_wnd;
386 	tp->last_ack_sent = tp->rcv_nxt;
387 
388 	tp->iss = iss;
389 	tcp_sendseqinit(tp);
390 	tp->snd_una = iss + 1;
391 	tp->snd_nxt = iss + 1;
392 	tp->snd_max = iss + 1;
393 
394 	assign_rxopt(tp, tcpopt);
395 	send_flowc_wr(toep, tp);
396 
397 	soisconnected(so);
398 
399 	if (ulp_mode(toep) == ULP_MODE_TLS)
400 		tls_establish(toep);
401 }
402 
403 int
404 send_rx_credits(struct adapter *sc, struct toepcb *toep, int credits)
405 {
406 	struct wrqe *wr;
407 	struct cpl_rx_data_ack *req;
408 	uint32_t dack = F_RX_DACK_CHANGE | V_RX_DACK_MODE(1);
409 
410 	KASSERT(credits >= 0, ("%s: %d credits", __func__, credits));
411 
412 	wr = alloc_wrqe(sizeof(*req), toep->ctrlq);
413 	if (wr == NULL)
414 		return (0);
415 	req = wrtod(wr);
416 
417 	INIT_TP_WR_MIT_CPL(req, CPL_RX_DATA_ACK, toep->tid);
418 	req->credit_dack = htobe32(dack | V_RX_CREDITS(credits));
419 
420 	t4_wrq_tx(sc, wr);
421 	return (credits);
422 }
423 
424 void
425 send_rx_modulate(struct adapter *sc, struct toepcb *toep)
426 {
427 	struct wrqe *wr;
428 	struct cpl_rx_data_ack *req;
429 
430 	wr = alloc_wrqe(sizeof(*req), toep->ctrlq);
431 	if (wr == NULL)
432 		return;
433 	req = wrtod(wr);
434 
435 	INIT_TP_WR_MIT_CPL(req, CPL_RX_DATA_ACK, toep->tid);
436 	req->credit_dack = htobe32(F_RX_MODULATE_RX);
437 
438 	t4_wrq_tx(sc, wr);
439 }
440 
441 void
442 t4_rcvd_locked(struct toedev *tod, struct tcpcb *tp)
443 {
444 	struct adapter *sc = tod->tod_softc;
445 	struct inpcb *inp = tp->t_inpcb;
446 	struct socket *so = inp->inp_socket;
447 	struct sockbuf *sb = &so->so_rcv;
448 	struct toepcb *toep = tp->t_toe;
449 	int rx_credits;
450 
451 	INP_WLOCK_ASSERT(inp);
452 	SOCKBUF_LOCK_ASSERT(sb);
453 
454 	rx_credits = sbspace(sb) > tp->rcv_wnd ? sbspace(sb) - tp->rcv_wnd : 0;
455 	if (rx_credits > 0 &&
456 	    (tp->rcv_wnd <= 32 * 1024 || rx_credits >= 64 * 1024 ||
457 	    (rx_credits >= 16 * 1024 && tp->rcv_wnd <= 128 * 1024) ||
458 	    sbused(sb) + tp->rcv_wnd < sb->sb_lowat)) {
459 		rx_credits = send_rx_credits(sc, toep, rx_credits);
460 		tp->rcv_wnd += rx_credits;
461 		tp->rcv_adv += rx_credits;
462 	} else if (toep->flags & TPF_FORCE_CREDITS)
463 		send_rx_modulate(sc, toep);
464 }
465 
466 void
467 t4_rcvd(struct toedev *tod, struct tcpcb *tp)
468 {
469 	struct inpcb *inp = tp->t_inpcb;
470 	struct socket *so = inp->inp_socket;
471 	struct sockbuf *sb = &so->so_rcv;
472 
473 	SOCKBUF_LOCK(sb);
474 	t4_rcvd_locked(tod, tp);
475 	SOCKBUF_UNLOCK(sb);
476 }
477 
478 /*
479  * Close a connection by sending a CPL_CLOSE_CON_REQ message.
480  */
481 int
482 t4_close_conn(struct adapter *sc, struct toepcb *toep)
483 {
484 	struct wrqe *wr;
485 	struct cpl_close_con_req *req;
486 	unsigned int tid = toep->tid;
487 
488 	CTR3(KTR_CXGBE, "%s: tid %u%s", __func__, toep->tid,
489 	    toep->flags & TPF_FIN_SENT ? ", IGNORED" : "");
490 
491 	if (toep->flags & TPF_FIN_SENT)
492 		return (0);
493 
494 	KASSERT(toep->flags & TPF_FLOWC_WR_SENT,
495 	    ("%s: flowc_wr not sent for tid %u.", __func__, tid));
496 
497 	wr = alloc_wrqe(sizeof(*req), &toep->ofld_txq->wrq);
498 	if (wr == NULL) {
499 		/* XXX */
500 		panic("%s: allocation failure.", __func__);
501 	}
502 	req = wrtod(wr);
503 
504         req->wr.wr_hi = htonl(V_FW_WR_OP(FW_TP_WR) |
505 	    V_FW_WR_IMMDLEN(sizeof(*req) - sizeof(req->wr)));
506 	req->wr.wr_mid = htonl(V_FW_WR_LEN16(howmany(sizeof(*req), 16)) |
507 	    V_FW_WR_FLOWID(tid));
508         req->wr.wr_lo = cpu_to_be64(0);
509         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid));
510 	req->rsvd = 0;
511 
512 	toep->flags |= TPF_FIN_SENT;
513 	toep->flags &= ~TPF_SEND_FIN;
514 	t4_l2t_send(sc, wr, toep->l2te);
515 
516 	return (0);
517 }
518 
519 #define MAX_OFLD_TX_CREDITS (SGE_MAX_WR_LEN / 16)
520 #define MIN_OFLD_TX_CREDITS (howmany(sizeof(struct fw_ofld_tx_data_wr) + 1, 16))
521 #define MIN_ISO_TX_CREDITS  (howmany(sizeof(struct cpl_tx_data_iso), 16))
522 #define MIN_TX_CREDITS(iso)						\
523 	(MIN_OFLD_TX_CREDITS + ((iso) ? MIN_ISO_TX_CREDITS : 0))
524 
525 /* Maximum amount of immediate data we could stuff in a WR */
526 static inline int
527 max_imm_payload(int tx_credits, int iso)
528 {
529 	const int iso_cpl_size = iso ? sizeof(struct cpl_tx_data_iso) : 0;
530 	const int n = 1;	/* Use no more than one desc for imm. data WR */
531 
532 	KASSERT(tx_credits >= 0 &&
533 		tx_credits <= MAX_OFLD_TX_CREDITS,
534 		("%s: %d credits", __func__, tx_credits));
535 
536 	if (tx_credits < MIN_TX_CREDITS(iso))
537 		return (0);
538 
539 	if (tx_credits >= (n * EQ_ESIZE) / 16)
540 		return ((n * EQ_ESIZE) - sizeof(struct fw_ofld_tx_data_wr) -
541 		    iso_cpl_size);
542 	else
543 		return (tx_credits * 16 - sizeof(struct fw_ofld_tx_data_wr) -
544 		    iso_cpl_size);
545 }
546 
547 /* Maximum number of SGL entries we could stuff in a WR */
548 static inline int
549 max_dsgl_nsegs(int tx_credits, int iso)
550 {
551 	int nseg = 1;	/* ulptx_sgl has room for 1, rest ulp_tx_sge_pair */
552 	int sge_pair_credits = tx_credits - MIN_TX_CREDITS(iso);
553 
554 	KASSERT(tx_credits >= 0 &&
555 		tx_credits <= MAX_OFLD_TX_CREDITS,
556 		("%s: %d credits", __func__, tx_credits));
557 
558 	if (tx_credits < MIN_TX_CREDITS(iso))
559 		return (0);
560 
561 	nseg += 2 * (sge_pair_credits * 16 / 24);
562 	if ((sge_pair_credits * 16) % 24 == 16)
563 		nseg++;
564 
565 	return (nseg);
566 }
567 
568 static inline void
569 write_tx_wr(void *dst, struct toepcb *toep, int fw_wr_opcode,
570     unsigned int immdlen, unsigned int plen, uint8_t credits, int shove,
571     int ulp_submode)
572 {
573 	struct fw_ofld_tx_data_wr *txwr = dst;
574 
575 	txwr->op_to_immdlen = htobe32(V_WR_OP(fw_wr_opcode) |
576 	    V_FW_WR_IMMDLEN(immdlen));
577 	txwr->flowid_len16 = htobe32(V_FW_WR_FLOWID(toep->tid) |
578 	    V_FW_WR_LEN16(credits));
579 	txwr->lsodisable_to_flags = htobe32(V_TX_ULP_MODE(ulp_mode(toep)) |
580 	    V_TX_ULP_SUBMODE(ulp_submode) | V_TX_URG(0) | V_TX_SHOVE(shove));
581 	txwr->plen = htobe32(plen);
582 
583 	if (toep->params.tx_align > 0) {
584 		if (plen < 2 * toep->params.emss)
585 			txwr->lsodisable_to_flags |=
586 			    htobe32(F_FW_OFLD_TX_DATA_WR_LSODISABLE);
587 		else
588 			txwr->lsodisable_to_flags |=
589 			    htobe32(F_FW_OFLD_TX_DATA_WR_ALIGNPLD |
590 				(toep->params.nagle == 0 ? 0 :
591 				F_FW_OFLD_TX_DATA_WR_ALIGNPLDSHOVE));
592 	}
593 }
594 
595 /*
596  * Generate a DSGL from a starting mbuf.  The total number of segments and the
597  * maximum segments in any one mbuf are provided.
598  */
599 static void
600 write_tx_sgl(void *dst, struct mbuf *start, struct mbuf *stop, int nsegs, int n)
601 {
602 	struct mbuf *m;
603 	struct ulptx_sgl *usgl = dst;
604 	int i, j, rc;
605 	struct sglist sg;
606 	struct sglist_seg segs[n];
607 
608 	KASSERT(nsegs > 0, ("%s: nsegs 0", __func__));
609 
610 	sglist_init(&sg, n, segs);
611 	usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) |
612 	    V_ULPTX_NSGE(nsegs));
613 
614 	i = -1;
615 	for (m = start; m != stop; m = m->m_next) {
616 		if (m->m_flags & M_EXTPG)
617 			rc = sglist_append_mbuf_epg(&sg, m,
618 			    mtod(m, vm_offset_t), m->m_len);
619 		else
620 			rc = sglist_append(&sg, mtod(m, void *), m->m_len);
621 		if (__predict_false(rc != 0))
622 			panic("%s: sglist_append %d", __func__, rc);
623 
624 		for (j = 0; j < sg.sg_nseg; i++, j++) {
625 			if (i < 0) {
626 				usgl->len0 = htobe32(segs[j].ss_len);
627 				usgl->addr0 = htobe64(segs[j].ss_paddr);
628 			} else {
629 				usgl->sge[i / 2].len[i & 1] =
630 				    htobe32(segs[j].ss_len);
631 				usgl->sge[i / 2].addr[i & 1] =
632 				    htobe64(segs[j].ss_paddr);
633 			}
634 #ifdef INVARIANTS
635 			nsegs--;
636 #endif
637 		}
638 		sglist_reset(&sg);
639 	}
640 	if (i & 1)
641 		usgl->sge[i / 2].len[1] = htobe32(0);
642 	KASSERT(nsegs == 0, ("%s: nsegs %d, start %p, stop %p",
643 	    __func__, nsegs, start, stop));
644 }
645 
646 /*
647  * Max number of SGL entries an offload tx work request can have.  This is 41
648  * (1 + 40) for a full 512B work request.
649  * fw_ofld_tx_data_wr(16B) + ulptx_sgl(16B, 1) + ulptx_sge_pair(480B, 40)
650  */
651 #define OFLD_SGL_LEN (41)
652 
653 /*
654  * Send data and/or a FIN to the peer.
655  *
656  * The socket's so_snd buffer consists of a stream of data starting with sb_mb
657  * and linked together with m_next.  sb_sndptr, if set, is the last mbuf that
658  * was transmitted.
659  *
660  * drop indicates the number of bytes that should be dropped from the head of
661  * the send buffer.  It is an optimization that lets do_fw4_ack avoid creating
662  * contention on the send buffer lock (before this change it used to do
663  * sowwakeup and then t4_push_frames right after that when recovering from tx
664  * stalls).  When drop is set this function MUST drop the bytes and wake up any
665  * writers.
666  */
667 void
668 t4_push_frames(struct adapter *sc, struct toepcb *toep, int drop)
669 {
670 	struct mbuf *sndptr, *m, *sb_sndptr;
671 	struct fw_ofld_tx_data_wr *txwr;
672 	struct wrqe *wr;
673 	u_int plen, nsegs, credits, max_imm, max_nsegs, max_nsegs_1mbuf;
674 	struct inpcb *inp = toep->inp;
675 	struct tcpcb *tp = intotcpcb(inp);
676 	struct socket *so = inp->inp_socket;
677 	struct sockbuf *sb = &so->so_snd;
678 	int tx_credits, shove, compl, sowwakeup;
679 	struct ofld_tx_sdesc *txsd;
680 	bool nomap_mbuf_seen;
681 
682 	INP_WLOCK_ASSERT(inp);
683 	KASSERT(toep->flags & TPF_FLOWC_WR_SENT,
684 	    ("%s: flowc_wr not sent for tid %u.", __func__, toep->tid));
685 
686 	KASSERT(ulp_mode(toep) == ULP_MODE_NONE ||
687 	    ulp_mode(toep) == ULP_MODE_TCPDDP ||
688 	    ulp_mode(toep) == ULP_MODE_TLS ||
689 	    ulp_mode(toep) == ULP_MODE_RDMA,
690 	    ("%s: ulp_mode %u for toep %p", __func__, ulp_mode(toep), toep));
691 
692 #ifdef VERBOSE_TRACES
693 	CTR5(KTR_CXGBE, "%s: tid %d toep flags %#x tp flags %#x drop %d",
694 	    __func__, toep->tid, toep->flags, tp->t_flags, drop);
695 #endif
696 	if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN))
697 		return;
698 
699 #ifdef RATELIMIT
700 	if (__predict_false(inp->inp_flags2 & INP_RATE_LIMIT_CHANGED) &&
701 	    (update_tx_rate_limit(sc, toep, so->so_max_pacing_rate) == 0)) {
702 		inp->inp_flags2 &= ~INP_RATE_LIMIT_CHANGED;
703 	}
704 #endif
705 
706 	/*
707 	 * This function doesn't resume by itself.  Someone else must clear the
708 	 * flag and call this function.
709 	 */
710 	if (__predict_false(toep->flags & TPF_TX_SUSPENDED)) {
711 		KASSERT(drop == 0,
712 		    ("%s: drop (%d) != 0 but tx is suspended", __func__, drop));
713 		return;
714 	}
715 
716 	txsd = &toep->txsd[toep->txsd_pidx];
717 	do {
718 		tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS);
719 		max_imm = max_imm_payload(tx_credits, 0);
720 		max_nsegs = max_dsgl_nsegs(tx_credits, 0);
721 
722 		SOCKBUF_LOCK(sb);
723 		sowwakeup = drop;
724 		if (drop) {
725 			sbdrop_locked(sb, drop);
726 			drop = 0;
727 		}
728 		sb_sndptr = sb->sb_sndptr;
729 		sndptr = sb_sndptr ? sb_sndptr->m_next : sb->sb_mb;
730 		plen = 0;
731 		nsegs = 0;
732 		max_nsegs_1mbuf = 0; /* max # of SGL segments in any one mbuf */
733 		nomap_mbuf_seen = false;
734 		for (m = sndptr; m != NULL; m = m->m_next) {
735 			int n;
736 
737 			if ((m->m_flags & M_NOTAVAIL) != 0)
738 				break;
739 			if (m->m_flags & M_EXTPG) {
740 #ifdef KERN_TLS
741 				if (m->m_epg_tls != NULL) {
742 					toep->flags |= TPF_KTLS;
743 					if (plen == 0) {
744 						SOCKBUF_UNLOCK(sb);
745 						t4_push_ktls(sc, toep, 0);
746 						return;
747 					}
748 					break;
749 				}
750 #endif
751 				n = sglist_count_mbuf_epg(m,
752 				    mtod(m, vm_offset_t), m->m_len);
753 			} else
754 				n = sglist_count(mtod(m, void *), m->m_len);
755 
756 			nsegs += n;
757 			plen += m->m_len;
758 
759 			/* This mbuf sent us _over_ the nsegs limit, back out */
760 			if (plen > max_imm && nsegs > max_nsegs) {
761 				nsegs -= n;
762 				plen -= m->m_len;
763 				if (plen == 0) {
764 					/* Too few credits */
765 					toep->flags |= TPF_TX_SUSPENDED;
766 					if (sowwakeup) {
767 						if (!TAILQ_EMPTY(
768 						    &toep->aiotx_jobq))
769 							t4_aiotx_queue_toep(so,
770 							    toep);
771 						sowwakeup_locked(so);
772 					} else
773 						SOCKBUF_UNLOCK(sb);
774 					SOCKBUF_UNLOCK_ASSERT(sb);
775 					return;
776 				}
777 				break;
778 			}
779 
780 			if (m->m_flags & M_EXTPG)
781 				nomap_mbuf_seen = true;
782 			if (max_nsegs_1mbuf < n)
783 				max_nsegs_1mbuf = n;
784 			sb_sndptr = m;	/* new sb->sb_sndptr if all goes well */
785 
786 			/* This mbuf put us right at the max_nsegs limit */
787 			if (plen > max_imm && nsegs == max_nsegs) {
788 				m = m->m_next;
789 				break;
790 			}
791 		}
792 
793 		if (sbused(sb) > sb->sb_hiwat * 5 / 8 &&
794 		    toep->plen_nocompl + plen >= sb->sb_hiwat / 4)
795 			compl = 1;
796 		else
797 			compl = 0;
798 
799 		if (sb->sb_flags & SB_AUTOSIZE &&
800 		    V_tcp_do_autosndbuf &&
801 		    sb->sb_hiwat < V_tcp_autosndbuf_max &&
802 		    sbused(sb) >= sb->sb_hiwat * 7 / 8) {
803 			int newsize = min(sb->sb_hiwat + V_tcp_autosndbuf_inc,
804 			    V_tcp_autosndbuf_max);
805 
806 			if (!sbreserve_locked(sb, newsize, so, NULL))
807 				sb->sb_flags &= ~SB_AUTOSIZE;
808 			else
809 				sowwakeup = 1;	/* room available */
810 		}
811 		if (sowwakeup) {
812 			if (!TAILQ_EMPTY(&toep->aiotx_jobq))
813 				t4_aiotx_queue_toep(so, toep);
814 			sowwakeup_locked(so);
815 		} else
816 			SOCKBUF_UNLOCK(sb);
817 		SOCKBUF_UNLOCK_ASSERT(sb);
818 
819 		/* nothing to send */
820 		if (plen == 0) {
821 			KASSERT(m == NULL || (m->m_flags & M_NOTAVAIL) != 0,
822 			    ("%s: nothing to send, but m != NULL is ready",
823 			    __func__));
824 			break;
825 		}
826 
827 		if (__predict_false(toep->flags & TPF_FIN_SENT))
828 			panic("%s: excess tx.", __func__);
829 
830 		shove = m == NULL && !(tp->t_flags & TF_MORETOCOME);
831 		if (plen <= max_imm && !nomap_mbuf_seen) {
832 
833 			/* Immediate data tx */
834 
835 			wr = alloc_wrqe(roundup2(sizeof(*txwr) + plen, 16),
836 					&toep->ofld_txq->wrq);
837 			if (wr == NULL) {
838 				/* XXX: how will we recover from this? */
839 				toep->flags |= TPF_TX_SUSPENDED;
840 				return;
841 			}
842 			txwr = wrtod(wr);
843 			credits = howmany(wr->wr_len, 16);
844 			write_tx_wr(txwr, toep, FW_OFLD_TX_DATA_WR, plen, plen,
845 			    credits, shove, 0);
846 			m_copydata(sndptr, 0, plen, (void *)(txwr + 1));
847 			nsegs = 0;
848 		} else {
849 			int wr_len;
850 
851 			/* DSGL tx */
852 
853 			wr_len = sizeof(*txwr) + sizeof(struct ulptx_sgl) +
854 			    ((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8;
855 			wr = alloc_wrqe(roundup2(wr_len, 16),
856 			    &toep->ofld_txq->wrq);
857 			if (wr == NULL) {
858 				/* XXX: how will we recover from this? */
859 				toep->flags |= TPF_TX_SUSPENDED;
860 				return;
861 			}
862 			txwr = wrtod(wr);
863 			credits = howmany(wr_len, 16);
864 			write_tx_wr(txwr, toep, FW_OFLD_TX_DATA_WR, 0, plen,
865 			    credits, shove, 0);
866 			write_tx_sgl(txwr + 1, sndptr, m, nsegs,
867 			    max_nsegs_1mbuf);
868 			if (wr_len & 0xf) {
869 				uint64_t *pad = (uint64_t *)
870 				    ((uintptr_t)txwr + wr_len);
871 				*pad = 0;
872 			}
873 		}
874 
875 		KASSERT(toep->tx_credits >= credits,
876 			("%s: not enough credits", __func__));
877 
878 		toep->tx_credits -= credits;
879 		toep->tx_nocompl += credits;
880 		toep->plen_nocompl += plen;
881 		if (toep->tx_credits <= toep->tx_total * 3 / 8 &&
882 		    toep->tx_nocompl >= toep->tx_total / 4)
883 			compl = 1;
884 
885 		if (compl || ulp_mode(toep) == ULP_MODE_RDMA) {
886 			txwr->op_to_immdlen |= htobe32(F_FW_WR_COMPL);
887 			toep->tx_nocompl = 0;
888 			toep->plen_nocompl = 0;
889 		}
890 
891 		tp->snd_nxt += plen;
892 		tp->snd_max += plen;
893 
894 		SOCKBUF_LOCK(sb);
895 		KASSERT(sb_sndptr, ("%s: sb_sndptr is NULL", __func__));
896 		sb->sb_sndptr = sb_sndptr;
897 		SOCKBUF_UNLOCK(sb);
898 
899 		toep->flags |= TPF_TX_DATA_SENT;
900 		if (toep->tx_credits < MIN_OFLD_TX_CREDITS)
901 			toep->flags |= TPF_TX_SUSPENDED;
902 
903 		KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__));
904 		txsd->plen = plen;
905 		txsd->tx_credits = credits;
906 		txsd++;
907 		if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) {
908 			toep->txsd_pidx = 0;
909 			txsd = &toep->txsd[0];
910 		}
911 		toep->txsd_avail--;
912 
913 		t4_l2t_send(sc, wr, toep->l2te);
914 	} while (m != NULL && (m->m_flags & M_NOTAVAIL) == 0);
915 
916 	/* Send a FIN if requested, but only if there's no more data to send */
917 	if (m == NULL && toep->flags & TPF_SEND_FIN)
918 		t4_close_conn(sc, toep);
919 }
920 
921 static inline void
922 rqdrop_locked(struct mbufq *q, int plen)
923 {
924 	struct mbuf *m;
925 
926 	while (plen > 0) {
927 		m = mbufq_dequeue(q);
928 
929 		/* Too many credits. */
930 		MPASS(m != NULL);
931 		M_ASSERTPKTHDR(m);
932 
933 		/* Partial credits. */
934 		MPASS(plen >= m->m_pkthdr.len);
935 
936 		plen -= m->m_pkthdr.len;
937 		m_freem(m);
938 	}
939 }
940 
941 /*
942  * Not a bit in the TCB, but is a bit in the ulp_submode field of the
943  * CPL_TX_DATA flags field in FW_ISCSI_TX_DATA_WR.
944  */
945 #define	ULP_ISO		G_TX_ULP_SUBMODE(F_FW_ISCSI_TX_DATA_WR_ULPSUBMODE_ISO)
946 
947 static void
948 write_tx_data_iso(void *dst, u_int ulp_submode, uint8_t flags, uint16_t mss,
949     int len, int npdu)
950 {
951 	struct cpl_tx_data_iso *cpl;
952 	unsigned int burst_size;
953 	unsigned int last;
954 
955 	/*
956 	 * The firmware will set the 'F' bit on the last PDU when
957 	 * either condition is true:
958 	 *
959 	 * - this large PDU is marked as the "last" slice
960 	 *
961 	 * - the amount of data payload bytes equals the burst_size
962 	 *
963 	 * The strategy used here is to always set the burst_size
964 	 * artificially high (len includes the size of the template
965 	 * BHS) and only set the "last" flag if the original PDU had
966 	 * 'F' set.
967 	 */
968 	burst_size = len;
969 	last = !!(flags & CXGBE_ISO_F);
970 
971 	cpl = (struct cpl_tx_data_iso *)dst;
972 	cpl->op_to_scsi = htonl(V_CPL_TX_DATA_ISO_OP(CPL_TX_DATA_ISO) |
973 	    V_CPL_TX_DATA_ISO_FIRST(1) | V_CPL_TX_DATA_ISO_LAST(last) |
974 	    V_CPL_TX_DATA_ISO_CPLHDRLEN(0) |
975 	    V_CPL_TX_DATA_ISO_HDRCRC(!!(ulp_submode & ULP_CRC_HEADER)) |
976 	    V_CPL_TX_DATA_ISO_PLDCRC(!!(ulp_submode & ULP_CRC_DATA)) |
977 	    V_CPL_TX_DATA_ISO_IMMEDIATE(0) |
978 	    V_CPL_TX_DATA_ISO_SCSI(CXGBE_ISO_TYPE(flags)));
979 
980 	cpl->ahs_len = 0;
981 	cpl->mpdu = htons(DIV_ROUND_UP(mss, 4));
982 	cpl->burst_size = htonl(DIV_ROUND_UP(burst_size, 4));
983 	cpl->len = htonl(len);
984 	cpl->reserved2_seglen_offset = htonl(0);
985 	cpl->datasn_offset = htonl(0);
986 	cpl->buffer_offset = htonl(0);
987 	cpl->reserved3 = 0;
988 }
989 
990 static struct wrqe *
991 write_iscsi_mbuf_wr(struct toepcb *toep, struct mbuf *sndptr)
992 {
993 	struct mbuf *m;
994 	struct fw_ofld_tx_data_wr *txwr;
995 	struct cpl_tx_data_iso *cpl_iso;
996 	void *p;
997 	struct wrqe *wr;
998 	u_int plen, nsegs, credits, max_imm, max_nsegs, max_nsegs_1mbuf;
999 	u_int adjusted_plen, imm_data, ulp_submode;
1000 	struct inpcb *inp = toep->inp;
1001 	struct tcpcb *tp = intotcpcb(inp);
1002 	int tx_credits, shove, npdu, wr_len;
1003 	uint16_t iso_mss;
1004 	static const u_int ulp_extra_len[] = {0, 4, 4, 8};
1005 	bool iso;
1006 
1007 	M_ASSERTPKTHDR(sndptr);
1008 
1009 	tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS);
1010 	if (mbuf_raw_wr(sndptr)) {
1011 		plen = sndptr->m_pkthdr.len;
1012 		KASSERT(plen <= SGE_MAX_WR_LEN,
1013 		    ("raw WR len %u is greater than max WR len", plen));
1014 		if (plen > tx_credits * 16)
1015 			return (NULL);
1016 
1017 		wr = alloc_wrqe(roundup2(plen, 16), &toep->ofld_txq->wrq);
1018 		if (__predict_false(wr == NULL))
1019 			return (NULL);
1020 
1021 		m_copydata(sndptr, 0, plen, wrtod(wr));
1022 		return (wr);
1023 	}
1024 
1025 	iso = mbuf_iscsi_iso(sndptr);
1026 	max_imm = max_imm_payload(tx_credits, iso);
1027 	max_nsegs = max_dsgl_nsegs(tx_credits, iso);
1028 	iso_mss = mbuf_iscsi_iso_mss(sndptr);
1029 
1030 	plen = 0;
1031 	nsegs = 0;
1032 	max_nsegs_1mbuf = 0; /* max # of SGL segments in any one mbuf */
1033 	for (m = sndptr; m != NULL; m = m->m_next) {
1034 		int n = sglist_count(mtod(m, void *), m->m_len);
1035 
1036 		nsegs += n;
1037 		plen += m->m_len;
1038 
1039 		/*
1040 		 * This mbuf would send us _over_ the nsegs limit.
1041 		 * Suspend tx because the PDU can't be sent out.
1042 		 */
1043 		if (plen > max_imm && nsegs > max_nsegs)
1044 			return (NULL);
1045 
1046 		if (max_nsegs_1mbuf < n)
1047 			max_nsegs_1mbuf = n;
1048 	}
1049 
1050 	if (__predict_false(toep->flags & TPF_FIN_SENT))
1051 		panic("%s: excess tx.", __func__);
1052 
1053 	/*
1054 	 * We have a PDU to send.  All of it goes out in one WR so 'm'
1055 	 * is NULL.  A PDU's length is always a multiple of 4.
1056 	 */
1057 	MPASS(m == NULL);
1058 	MPASS((plen & 3) == 0);
1059 	MPASS(sndptr->m_pkthdr.len == plen);
1060 
1061 	shove = !(tp->t_flags & TF_MORETOCOME);
1062 
1063 	/*
1064 	 * plen doesn't include header and data digests, which are
1065 	 * generated and inserted in the right places by the TOE, but
1066 	 * they do occupy TCP sequence space and need to be accounted
1067 	 * for.
1068 	 */
1069 	ulp_submode = mbuf_ulp_submode(sndptr);
1070 	MPASS(ulp_submode < nitems(ulp_extra_len));
1071 	npdu = iso ? howmany(plen - ISCSI_BHS_SIZE, iso_mss) : 1;
1072 	adjusted_plen = plen + ulp_extra_len[ulp_submode] * npdu;
1073 	if (iso)
1074 		adjusted_plen += ISCSI_BHS_SIZE * (npdu - 1);
1075 	wr_len = sizeof(*txwr);
1076 	if (iso)
1077 		wr_len += sizeof(struct cpl_tx_data_iso);
1078 	if (plen <= max_imm) {
1079 		/* Immediate data tx */
1080 		imm_data = plen;
1081 		wr_len += plen;
1082 		nsegs = 0;
1083 	} else {
1084 		/* DSGL tx */
1085 		imm_data = 0;
1086 		wr_len += sizeof(struct ulptx_sgl) +
1087 		    ((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8;
1088 	}
1089 
1090 	wr = alloc_wrqe(roundup2(wr_len, 16), &toep->ofld_txq->wrq);
1091 	if (wr == NULL) {
1092 		/* XXX: how will we recover from this? */
1093 		return (NULL);
1094 	}
1095 	txwr = wrtod(wr);
1096 	credits = howmany(wr->wr_len, 16);
1097 
1098 	if (iso) {
1099 		write_tx_wr(txwr, toep, FW_ISCSI_TX_DATA_WR,
1100 		    imm_data + sizeof(struct cpl_tx_data_iso),
1101 		    adjusted_plen, credits, shove, ulp_submode | ULP_ISO);
1102 		cpl_iso = (struct cpl_tx_data_iso *)(txwr + 1);
1103 		MPASS(plen == sndptr->m_pkthdr.len);
1104 		write_tx_data_iso(cpl_iso, ulp_submode,
1105 		    mbuf_iscsi_iso_flags(sndptr), iso_mss, plen, npdu);
1106 		p = cpl_iso + 1;
1107 	} else {
1108 		write_tx_wr(txwr, toep, FW_OFLD_TX_DATA_WR, imm_data,
1109 		    adjusted_plen, credits, shove, ulp_submode);
1110 		p = txwr + 1;
1111 	}
1112 
1113 	if (imm_data != 0) {
1114 		m_copydata(sndptr, 0, plen, p);
1115 	} else {
1116 		write_tx_sgl(p, sndptr, m, nsegs, max_nsegs_1mbuf);
1117 		if (wr_len & 0xf) {
1118 			uint64_t *pad = (uint64_t *)((uintptr_t)txwr + wr_len);
1119 			*pad = 0;
1120 		}
1121 	}
1122 
1123 	KASSERT(toep->tx_credits >= credits,
1124 	    ("%s: not enough credits: credits %u "
1125 		"toep->tx_credits %u tx_credits %u nsegs %u "
1126 		"max_nsegs %u iso %d", __func__, credits,
1127 		toep->tx_credits, tx_credits, nsegs, max_nsegs, iso));
1128 
1129 	tp->snd_nxt += adjusted_plen;
1130 	tp->snd_max += adjusted_plen;
1131 
1132 	counter_u64_add(toep->ofld_txq->tx_iscsi_pdus, npdu);
1133 	counter_u64_add(toep->ofld_txq->tx_iscsi_octets, plen);
1134 	if (iso)
1135 		counter_u64_add(toep->ofld_txq->tx_iscsi_iso_wrs, 1);
1136 
1137 	return (wr);
1138 }
1139 
1140 void
1141 t4_push_pdus(struct adapter *sc, struct toepcb *toep, int drop)
1142 {
1143 	struct mbuf *sndptr, *m;
1144 	struct fw_wr_hdr *wrhdr;
1145 	struct wrqe *wr;
1146 	u_int plen, credits;
1147 	struct inpcb *inp = toep->inp;
1148 	struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx];
1149 	struct mbufq *pduq = &toep->ulp_pduq;
1150 
1151 	INP_WLOCK_ASSERT(inp);
1152 	KASSERT(toep->flags & TPF_FLOWC_WR_SENT,
1153 	    ("%s: flowc_wr not sent for tid %u.", __func__, toep->tid));
1154 	KASSERT(ulp_mode(toep) == ULP_MODE_ISCSI,
1155 	    ("%s: ulp_mode %u for toep %p", __func__, ulp_mode(toep), toep));
1156 
1157 	if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN))
1158 		return;
1159 
1160 	/*
1161 	 * This function doesn't resume by itself.  Someone else must clear the
1162 	 * flag and call this function.
1163 	 */
1164 	if (__predict_false(toep->flags & TPF_TX_SUSPENDED)) {
1165 		KASSERT(drop == 0,
1166 		    ("%s: drop (%d) != 0 but tx is suspended", __func__, drop));
1167 		return;
1168 	}
1169 
1170 	if (drop) {
1171 		struct socket *so = inp->inp_socket;
1172 		struct sockbuf *sb = &so->so_snd;
1173 		int sbu;
1174 
1175 		/*
1176 		 * An unlocked read is ok here as the data should only
1177 		 * transition from a non-zero value to either another
1178 		 * non-zero value or zero.  Once it is zero it should
1179 		 * stay zero.
1180 		 */
1181 		if (__predict_false(sbused(sb)) > 0) {
1182 			SOCKBUF_LOCK(sb);
1183 			sbu = sbused(sb);
1184 			if (sbu > 0) {
1185 				/*
1186 				 * The data transmitted before the
1187 				 * tid's ULP mode changed to ISCSI is
1188 				 * still in so_snd.  Incoming credits
1189 				 * should account for so_snd first.
1190 				 */
1191 				sbdrop_locked(sb, min(sbu, drop));
1192 				drop -= min(sbu, drop);
1193 			}
1194 			sowwakeup_locked(so);	/* unlocks so_snd */
1195 		}
1196 		rqdrop_locked(&toep->ulp_pdu_reclaimq, drop);
1197 	}
1198 
1199 	while ((sndptr = mbufq_first(pduq)) != NULL) {
1200 		wr = write_iscsi_mbuf_wr(toep, sndptr);
1201 		if (wr == NULL) {
1202 			toep->flags |= TPF_TX_SUSPENDED;
1203 			return;
1204 		}
1205 
1206 		plen = sndptr->m_pkthdr.len;
1207 		credits = howmany(wr->wr_len, 16);
1208 		KASSERT(toep->tx_credits >= credits,
1209 			("%s: not enough credits", __func__));
1210 
1211 		m = mbufq_dequeue(pduq);
1212 		MPASS(m == sndptr);
1213 		mbufq_enqueue(&toep->ulp_pdu_reclaimq, m);
1214 
1215 		toep->tx_credits -= credits;
1216 		toep->tx_nocompl += credits;
1217 		toep->plen_nocompl += plen;
1218 
1219 		/*
1220 		 * Ensure there are enough credits for a full-sized WR
1221 		 * as page pod WRs can be full-sized.
1222 		 */
1223 		if (toep->tx_credits <= SGE_MAX_WR_LEN * 5 / 4 &&
1224 		    toep->tx_nocompl >= toep->tx_total / 4) {
1225 			wrhdr = wrtod(wr);
1226 			wrhdr->hi |= htobe32(F_FW_WR_COMPL);
1227 			toep->tx_nocompl = 0;
1228 			toep->plen_nocompl = 0;
1229 		}
1230 
1231 		toep->flags |= TPF_TX_DATA_SENT;
1232 		if (toep->tx_credits < MIN_OFLD_TX_CREDITS)
1233 			toep->flags |= TPF_TX_SUSPENDED;
1234 
1235 		KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__));
1236 		txsd->plen = plen;
1237 		txsd->tx_credits = credits;
1238 		txsd++;
1239 		if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) {
1240 			toep->txsd_pidx = 0;
1241 			txsd = &toep->txsd[0];
1242 		}
1243 		toep->txsd_avail--;
1244 
1245 		t4_l2t_send(sc, wr, toep->l2te);
1246 	}
1247 
1248 	/* Send a FIN if requested, but only if there are no more PDUs to send */
1249 	if (mbufq_first(pduq) == NULL && toep->flags & TPF_SEND_FIN)
1250 		t4_close_conn(sc, toep);
1251 }
1252 
1253 static inline void
1254 t4_push_data(struct adapter *sc, struct toepcb *toep, int drop)
1255 {
1256 
1257 	if (ulp_mode(toep) == ULP_MODE_ISCSI)
1258 		t4_push_pdus(sc, toep, drop);
1259 	else if (toep->flags & TPF_KTLS)
1260 		t4_push_ktls(sc, toep, drop);
1261 	else
1262 		t4_push_frames(sc, toep, drop);
1263 }
1264 
1265 int
1266 t4_tod_output(struct toedev *tod, struct tcpcb *tp)
1267 {
1268 	struct adapter *sc = tod->tod_softc;
1269 #ifdef INVARIANTS
1270 	struct inpcb *inp = tp->t_inpcb;
1271 #endif
1272 	struct toepcb *toep = tp->t_toe;
1273 
1274 	INP_WLOCK_ASSERT(inp);
1275 	KASSERT((inp->inp_flags & INP_DROPPED) == 0,
1276 	    ("%s: inp %p dropped.", __func__, inp));
1277 	KASSERT(toep != NULL, ("%s: toep is NULL", __func__));
1278 
1279 	t4_push_data(sc, toep, 0);
1280 
1281 	return (0);
1282 }
1283 
1284 int
1285 t4_send_fin(struct toedev *tod, struct tcpcb *tp)
1286 {
1287 	struct adapter *sc = tod->tod_softc;
1288 #ifdef INVARIANTS
1289 	struct inpcb *inp = tp->t_inpcb;
1290 #endif
1291 	struct toepcb *toep = tp->t_toe;
1292 
1293 	INP_WLOCK_ASSERT(inp);
1294 	KASSERT((inp->inp_flags & INP_DROPPED) == 0,
1295 	    ("%s: inp %p dropped.", __func__, inp));
1296 	KASSERT(toep != NULL, ("%s: toep is NULL", __func__));
1297 
1298 	toep->flags |= TPF_SEND_FIN;
1299 	if (tp->t_state >= TCPS_ESTABLISHED)
1300 		t4_push_data(sc, toep, 0);
1301 
1302 	return (0);
1303 }
1304 
1305 int
1306 t4_send_rst(struct toedev *tod, struct tcpcb *tp)
1307 {
1308 	struct adapter *sc = tod->tod_softc;
1309 #if defined(INVARIANTS)
1310 	struct inpcb *inp = tp->t_inpcb;
1311 #endif
1312 	struct toepcb *toep = tp->t_toe;
1313 
1314 	INP_WLOCK_ASSERT(inp);
1315 	KASSERT((inp->inp_flags & INP_DROPPED) == 0,
1316 	    ("%s: inp %p dropped.", __func__, inp));
1317 	KASSERT(toep != NULL, ("%s: toep is NULL", __func__));
1318 
1319 	/* hmmmm */
1320 	KASSERT(toep->flags & TPF_FLOWC_WR_SENT,
1321 	    ("%s: flowc for tid %u [%s] not sent already",
1322 	    __func__, toep->tid, tcpstates[tp->t_state]));
1323 
1324 	send_reset(sc, toep, 0);
1325 	return (0);
1326 }
1327 
1328 /*
1329  * Peer has sent us a FIN.
1330  */
1331 static int
1332 do_peer_close(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
1333 {
1334 	struct adapter *sc = iq->adapter;
1335 	const struct cpl_peer_close *cpl = (const void *)(rss + 1);
1336 	unsigned int tid = GET_TID(cpl);
1337 	struct toepcb *toep = lookup_tid(sc, tid);
1338 	struct inpcb *inp = toep->inp;
1339 	struct tcpcb *tp = NULL;
1340 	struct socket *so;
1341 	struct epoch_tracker et;
1342 #ifdef INVARIANTS
1343 	unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl)));
1344 #endif
1345 
1346 	KASSERT(opcode == CPL_PEER_CLOSE,
1347 	    ("%s: unexpected opcode 0x%x", __func__, opcode));
1348 	KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
1349 
1350 	if (__predict_false(toep->flags & TPF_SYNQE)) {
1351 		/*
1352 		 * do_pass_establish must have run before do_peer_close and if
1353 		 * this is still a synqe instead of a toepcb then the connection
1354 		 * must be getting aborted.
1355 		 */
1356 		MPASS(toep->flags & TPF_ABORT_SHUTDOWN);
1357 		CTR4(KTR_CXGBE, "%s: tid %u, synqe %p (0x%x)", __func__, tid,
1358 		    toep, toep->flags);
1359 		return (0);
1360 	}
1361 
1362 	KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__));
1363 
1364 	CURVNET_SET(toep->vnet);
1365 	NET_EPOCH_ENTER(et);
1366 	INP_WLOCK(inp);
1367 	tp = intotcpcb(inp);
1368 
1369 	CTR6(KTR_CXGBE,
1370 	    "%s: tid %u (%s), toep_flags 0x%x, ddp_flags 0x%x, inp %p",
1371 	    __func__, tid, tp ? tcpstates[tp->t_state] : "no tp", toep->flags,
1372 	    toep->ddp.flags, inp);
1373 
1374 	if (toep->flags & TPF_ABORT_SHUTDOWN)
1375 		goto done;
1376 
1377 	so = inp->inp_socket;
1378 	socantrcvmore(so);
1379 	if (ulp_mode(toep) == ULP_MODE_TCPDDP) {
1380 		DDP_LOCK(toep);
1381 		if (__predict_false(toep->ddp.flags &
1382 		    (DDP_BUF0_ACTIVE | DDP_BUF1_ACTIVE)))
1383 			handle_ddp_close(toep, tp, cpl->rcv_nxt);
1384 		DDP_UNLOCK(toep);
1385 	}
1386 
1387 	if (ulp_mode(toep) == ULP_MODE_RDMA ||
1388 	    (ulp_mode(toep) == ULP_MODE_ISCSI && chip_id(sc) >= CHELSIO_T6)) {
1389 		/*
1390 		 * There might be data received via DDP before the FIN
1391 		 * not reported to the driver.  Just assume the
1392 		 * sequence number in the CPL is correct as the
1393 		 * sequence number of the FIN.
1394 		 */
1395 	} else {
1396 		KASSERT(tp->rcv_nxt + 1 == be32toh(cpl->rcv_nxt),
1397 		    ("%s: rcv_nxt mismatch: %u %u", __func__, tp->rcv_nxt,
1398 		    be32toh(cpl->rcv_nxt)));
1399 	}
1400 
1401 	tp->rcv_nxt = be32toh(cpl->rcv_nxt);
1402 
1403 	switch (tp->t_state) {
1404 	case TCPS_SYN_RECEIVED:
1405 		tp->t_starttime = ticks;
1406 		/* FALLTHROUGH */
1407 
1408 	case TCPS_ESTABLISHED:
1409 		tcp_state_change(tp, TCPS_CLOSE_WAIT);
1410 		break;
1411 
1412 	case TCPS_FIN_WAIT_1:
1413 		tcp_state_change(tp, TCPS_CLOSING);
1414 		break;
1415 
1416 	case TCPS_FIN_WAIT_2:
1417 		restore_so_proto(so, inp->inp_vflag & INP_IPV6);
1418 		tcp_twstart(tp);
1419 		INP_UNLOCK_ASSERT(inp);	 /* safe, we have a ref on the inp */
1420 		NET_EPOCH_EXIT(et);
1421 		CURVNET_RESTORE();
1422 
1423 		INP_WLOCK(inp);
1424 		final_cpl_received(toep);
1425 		return (0);
1426 
1427 	default:
1428 		log(LOG_ERR, "%s: TID %u received CPL_PEER_CLOSE in state %d\n",
1429 		    __func__, tid, tp->t_state);
1430 	}
1431 done:
1432 	INP_WUNLOCK(inp);
1433 	NET_EPOCH_EXIT(et);
1434 	CURVNET_RESTORE();
1435 	return (0);
1436 }
1437 
1438 /*
1439  * Peer has ACK'd our FIN.
1440  */
1441 static int
1442 do_close_con_rpl(struct sge_iq *iq, const struct rss_header *rss,
1443     struct mbuf *m)
1444 {
1445 	struct adapter *sc = iq->adapter;
1446 	const struct cpl_close_con_rpl *cpl = (const void *)(rss + 1);
1447 	unsigned int tid = GET_TID(cpl);
1448 	struct toepcb *toep = lookup_tid(sc, tid);
1449 	struct inpcb *inp = toep->inp;
1450 	struct tcpcb *tp = NULL;
1451 	struct socket *so = NULL;
1452 	struct epoch_tracker et;
1453 #ifdef INVARIANTS
1454 	unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl)));
1455 #endif
1456 
1457 	KASSERT(opcode == CPL_CLOSE_CON_RPL,
1458 	    ("%s: unexpected opcode 0x%x", __func__, opcode));
1459 	KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
1460 	KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__));
1461 
1462 	CURVNET_SET(toep->vnet);
1463 	NET_EPOCH_ENTER(et);
1464 	INP_WLOCK(inp);
1465 	tp = intotcpcb(inp);
1466 
1467 	CTR4(KTR_CXGBE, "%s: tid %u (%s), toep_flags 0x%x",
1468 	    __func__, tid, tp ? tcpstates[tp->t_state] : "no tp", toep->flags);
1469 
1470 	if (toep->flags & TPF_ABORT_SHUTDOWN)
1471 		goto done;
1472 
1473 	so = inp->inp_socket;
1474 	tp->snd_una = be32toh(cpl->snd_nxt) - 1;	/* exclude FIN */
1475 
1476 	switch (tp->t_state) {
1477 	case TCPS_CLOSING:	/* see TCPS_FIN_WAIT_2 in do_peer_close too */
1478 		restore_so_proto(so, inp->inp_vflag & INP_IPV6);
1479 		tcp_twstart(tp);
1480 release:
1481 		INP_UNLOCK_ASSERT(inp);	/* safe, we have a ref on the  inp */
1482 		NET_EPOCH_EXIT(et);
1483 		CURVNET_RESTORE();
1484 
1485 		INP_WLOCK(inp);
1486 		final_cpl_received(toep);	/* no more CPLs expected */
1487 
1488 		return (0);
1489 	case TCPS_LAST_ACK:
1490 		if (tcp_close(tp))
1491 			INP_WUNLOCK(inp);
1492 		goto release;
1493 
1494 	case TCPS_FIN_WAIT_1:
1495 		if (so->so_rcv.sb_state & SBS_CANTRCVMORE)
1496 			soisdisconnected(so);
1497 		tcp_state_change(tp, TCPS_FIN_WAIT_2);
1498 		break;
1499 
1500 	default:
1501 		log(LOG_ERR,
1502 		    "%s: TID %u received CPL_CLOSE_CON_RPL in state %s\n",
1503 		    __func__, tid, tcpstates[tp->t_state]);
1504 	}
1505 done:
1506 	INP_WUNLOCK(inp);
1507 	NET_EPOCH_EXIT(et);
1508 	CURVNET_RESTORE();
1509 	return (0);
1510 }
1511 
1512 void
1513 send_abort_rpl(struct adapter *sc, struct sge_ofld_txq *ofld_txq, int tid,
1514     int rst_status)
1515 {
1516 	struct wrqe *wr;
1517 	struct cpl_abort_rpl *cpl;
1518 
1519 	wr = alloc_wrqe(sizeof(*cpl), &ofld_txq->wrq);
1520 	if (wr == NULL) {
1521 		/* XXX */
1522 		panic("%s: allocation failure.", __func__);
1523 	}
1524 	cpl = wrtod(wr);
1525 
1526 	INIT_TP_WR_MIT_CPL(cpl, CPL_ABORT_RPL, tid);
1527 	cpl->cmd = rst_status;
1528 
1529 	t4_wrq_tx(sc, wr);
1530 }
1531 
1532 static int
1533 abort_status_to_errno(struct tcpcb *tp, unsigned int abort_reason)
1534 {
1535 	switch (abort_reason) {
1536 	case CPL_ERR_BAD_SYN:
1537 	case CPL_ERR_CONN_RESET:
1538 		return (tp->t_state == TCPS_CLOSE_WAIT ? EPIPE : ECONNRESET);
1539 	case CPL_ERR_XMIT_TIMEDOUT:
1540 	case CPL_ERR_PERSIST_TIMEDOUT:
1541 	case CPL_ERR_FINWAIT2_TIMEDOUT:
1542 	case CPL_ERR_KEEPALIVE_TIMEDOUT:
1543 		return (ETIMEDOUT);
1544 	default:
1545 		return (EIO);
1546 	}
1547 }
1548 
1549 /*
1550  * TCP RST from the peer, timeout, or some other such critical error.
1551  */
1552 static int
1553 do_abort_req(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
1554 {
1555 	struct adapter *sc = iq->adapter;
1556 	const struct cpl_abort_req_rss *cpl = (const void *)(rss + 1);
1557 	unsigned int tid = GET_TID(cpl);
1558 	struct toepcb *toep = lookup_tid(sc, tid);
1559 	struct sge_ofld_txq *ofld_txq = toep->ofld_txq;
1560 	struct inpcb *inp;
1561 	struct tcpcb *tp;
1562 	struct epoch_tracker et;
1563 #ifdef INVARIANTS
1564 	unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl)));
1565 #endif
1566 
1567 	KASSERT(opcode == CPL_ABORT_REQ_RSS,
1568 	    ("%s: unexpected opcode 0x%x", __func__, opcode));
1569 	KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
1570 
1571 	if (toep->flags & TPF_SYNQE)
1572 		return (do_abort_req_synqe(iq, rss, m));
1573 
1574 	KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__));
1575 
1576 	if (negative_advice(cpl->status)) {
1577 		CTR4(KTR_CXGBE, "%s: negative advice %d for tid %d (0x%x)",
1578 		    __func__, cpl->status, tid, toep->flags);
1579 		return (0);	/* Ignore negative advice */
1580 	}
1581 
1582 	inp = toep->inp;
1583 	CURVNET_SET(toep->vnet);
1584 	NET_EPOCH_ENTER(et);	/* for tcp_close */
1585 	INP_WLOCK(inp);
1586 
1587 	tp = intotcpcb(inp);
1588 
1589 	CTR6(KTR_CXGBE,
1590 	    "%s: tid %d (%s), toep_flags 0x%x, inp_flags 0x%x, status %d",
1591 	    __func__, tid, tp ? tcpstates[tp->t_state] : "no tp", toep->flags,
1592 	    inp->inp_flags, cpl->status);
1593 
1594 	/*
1595 	 * If we'd initiated an abort earlier the reply to it is responsible for
1596 	 * cleaning up resources.  Otherwise we tear everything down right here
1597 	 * right now.  We owe the T4 a CPL_ABORT_RPL no matter what.
1598 	 */
1599 	if (toep->flags & TPF_ABORT_SHUTDOWN) {
1600 		INP_WUNLOCK(inp);
1601 		goto done;
1602 	}
1603 	toep->flags |= TPF_ABORT_SHUTDOWN;
1604 
1605 	if ((inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) == 0) {
1606 		struct socket *so = inp->inp_socket;
1607 
1608 		if (so != NULL)
1609 			so_error_set(so, abort_status_to_errno(tp,
1610 			    cpl->status));
1611 		tp = tcp_close(tp);
1612 		if (tp == NULL)
1613 			INP_WLOCK(inp);	/* re-acquire */
1614 	}
1615 
1616 	final_cpl_received(toep);
1617 done:
1618 	NET_EPOCH_EXIT(et);
1619 	CURVNET_RESTORE();
1620 	send_abort_rpl(sc, ofld_txq, tid, CPL_ABORT_NO_RST);
1621 	return (0);
1622 }
1623 
1624 /*
1625  * Reply to the CPL_ABORT_REQ (send_reset)
1626  */
1627 static int
1628 do_abort_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
1629 {
1630 	struct adapter *sc = iq->adapter;
1631 	const struct cpl_abort_rpl_rss *cpl = (const void *)(rss + 1);
1632 	unsigned int tid = GET_TID(cpl);
1633 	struct toepcb *toep = lookup_tid(sc, tid);
1634 	struct inpcb *inp = toep->inp;
1635 #ifdef INVARIANTS
1636 	unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl)));
1637 #endif
1638 
1639 	KASSERT(opcode == CPL_ABORT_RPL_RSS,
1640 	    ("%s: unexpected opcode 0x%x", __func__, opcode));
1641 	KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
1642 
1643 	if (toep->flags & TPF_SYNQE)
1644 		return (do_abort_rpl_synqe(iq, rss, m));
1645 
1646 	KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__));
1647 
1648 	CTR5(KTR_CXGBE, "%s: tid %u, toep %p, inp %p, status %d",
1649 	    __func__, tid, toep, inp, cpl->status);
1650 
1651 	KASSERT(toep->flags & TPF_ABORT_SHUTDOWN,
1652 	    ("%s: wasn't expecting abort reply", __func__));
1653 
1654 	INP_WLOCK(inp);
1655 	final_cpl_received(toep);
1656 
1657 	return (0);
1658 }
1659 
1660 static int
1661 do_rx_data(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
1662 {
1663 	struct adapter *sc = iq->adapter;
1664 	const struct cpl_rx_data *cpl = mtod(m, const void *);
1665 	unsigned int tid = GET_TID(cpl);
1666 	struct toepcb *toep = lookup_tid(sc, tid);
1667 	struct inpcb *inp = toep->inp;
1668 	struct tcpcb *tp;
1669 	struct socket *so;
1670 	struct sockbuf *sb;
1671 	struct epoch_tracker et;
1672 	int len, rx_credits;
1673 	uint32_t ddp_placed = 0;
1674 
1675 	if (__predict_false(toep->flags & TPF_SYNQE)) {
1676 		/*
1677 		 * do_pass_establish must have run before do_rx_data and if this
1678 		 * is still a synqe instead of a toepcb then the connection must
1679 		 * be getting aborted.
1680 		 */
1681 		MPASS(toep->flags & TPF_ABORT_SHUTDOWN);
1682 		CTR4(KTR_CXGBE, "%s: tid %u, synqe %p (0x%x)", __func__, tid,
1683 		    toep, toep->flags);
1684 		m_freem(m);
1685 		return (0);
1686 	}
1687 
1688 	KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__));
1689 
1690 	/* strip off CPL header */
1691 	m_adj(m, sizeof(*cpl));
1692 	len = m->m_pkthdr.len;
1693 
1694 	INP_WLOCK(inp);
1695 	if (inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) {
1696 		CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x",
1697 		    __func__, tid, len, inp->inp_flags);
1698 		INP_WUNLOCK(inp);
1699 		m_freem(m);
1700 		return (0);
1701 	}
1702 
1703 	tp = intotcpcb(inp);
1704 
1705 	if (__predict_false(ulp_mode(toep) == ULP_MODE_TLS &&
1706 	   toep->flags & TPF_TLS_RECEIVE)) {
1707 		/* Received "raw" data on a TLS socket. */
1708 		CTR3(KTR_CXGBE, "%s: tid %u, raw TLS data (%d bytes)",
1709 		    __func__, tid, len);
1710 		do_rx_data_tls(cpl, toep, m);
1711 		return (0);
1712 	}
1713 
1714 	if (__predict_false(tp->rcv_nxt != be32toh(cpl->seq)))
1715 		ddp_placed = be32toh(cpl->seq) - tp->rcv_nxt;
1716 
1717 	tp->rcv_nxt += len;
1718 	if (tp->rcv_wnd < len) {
1719 		KASSERT(ulp_mode(toep) == ULP_MODE_RDMA,
1720 				("%s: negative window size", __func__));
1721 	}
1722 
1723 	tp->rcv_wnd -= len;
1724 	tp->t_rcvtime = ticks;
1725 
1726 	if (ulp_mode(toep) == ULP_MODE_TCPDDP)
1727 		DDP_LOCK(toep);
1728 	so = inp_inpcbtosocket(inp);
1729 	sb = &so->so_rcv;
1730 	SOCKBUF_LOCK(sb);
1731 
1732 	if (__predict_false(sb->sb_state & SBS_CANTRCVMORE)) {
1733 		CTR3(KTR_CXGBE, "%s: tid %u, excess rx (%d bytes)",
1734 		    __func__, tid, len);
1735 		m_freem(m);
1736 		SOCKBUF_UNLOCK(sb);
1737 		if (ulp_mode(toep) == ULP_MODE_TCPDDP)
1738 			DDP_UNLOCK(toep);
1739 		INP_WUNLOCK(inp);
1740 
1741 		CURVNET_SET(toep->vnet);
1742 		NET_EPOCH_ENTER(et);
1743 		INP_WLOCK(inp);
1744 		tp = tcp_drop(tp, ECONNRESET);
1745 		if (tp)
1746 			INP_WUNLOCK(inp);
1747 		NET_EPOCH_EXIT(et);
1748 		CURVNET_RESTORE();
1749 
1750 		return (0);
1751 	}
1752 
1753 	/* receive buffer autosize */
1754 	MPASS(toep->vnet == so->so_vnet);
1755 	CURVNET_SET(toep->vnet);
1756 	if (sb->sb_flags & SB_AUTOSIZE &&
1757 	    V_tcp_do_autorcvbuf &&
1758 	    sb->sb_hiwat < V_tcp_autorcvbuf_max &&
1759 	    len > (sbspace(sb) / 8 * 7)) {
1760 		unsigned int hiwat = sb->sb_hiwat;
1761 		unsigned int newsize = min(hiwat + sc->tt.autorcvbuf_inc,
1762 		    V_tcp_autorcvbuf_max);
1763 
1764 		if (!sbreserve_locked(sb, newsize, so, NULL))
1765 			sb->sb_flags &= ~SB_AUTOSIZE;
1766 	}
1767 
1768 	if (ulp_mode(toep) == ULP_MODE_TCPDDP) {
1769 		int changed = !(toep->ddp.flags & DDP_ON) ^ cpl->ddp_off;
1770 
1771 		if (toep->ddp.waiting_count != 0 || toep->ddp.active_count != 0)
1772 			CTR3(KTR_CXGBE, "%s: tid %u, non-ddp rx (%d bytes)",
1773 			    __func__, tid, len);
1774 
1775 		if (changed) {
1776 			if (toep->ddp.flags & DDP_SC_REQ)
1777 				toep->ddp.flags ^= DDP_ON | DDP_SC_REQ;
1778 			else {
1779 				KASSERT(cpl->ddp_off == 1,
1780 				    ("%s: DDP switched on by itself.",
1781 				    __func__));
1782 
1783 				/* Fell out of DDP mode */
1784 				toep->ddp.flags &= ~DDP_ON;
1785 				CTR1(KTR_CXGBE, "%s: fell out of DDP mode",
1786 				    __func__);
1787 
1788 				insert_ddp_data(toep, ddp_placed);
1789 			}
1790 		}
1791 
1792 		if (toep->ddp.flags & DDP_ON) {
1793 			/*
1794 			 * CPL_RX_DATA with DDP on can only be an indicate.
1795 			 * Start posting queued AIO requests via DDP.  The
1796 			 * payload that arrived in this indicate is appended
1797 			 * to the socket buffer as usual.
1798 			 */
1799 			handle_ddp_indicate(toep);
1800 		}
1801 	}
1802 
1803 	sbappendstream_locked(sb, m, 0);
1804 	rx_credits = sbspace(sb) > tp->rcv_wnd ? sbspace(sb) - tp->rcv_wnd : 0;
1805 	if (rx_credits > 0 && sbused(sb) + tp->rcv_wnd < sb->sb_lowat) {
1806 		rx_credits = send_rx_credits(sc, toep, rx_credits);
1807 		tp->rcv_wnd += rx_credits;
1808 		tp->rcv_adv += rx_credits;
1809 	}
1810 
1811 	if (ulp_mode(toep) == ULP_MODE_TCPDDP && toep->ddp.waiting_count > 0 &&
1812 	    sbavail(sb) != 0) {
1813 		CTR2(KTR_CXGBE, "%s: tid %u queueing AIO task", __func__,
1814 		    tid);
1815 		ddp_queue_toep(toep);
1816 	}
1817 	sorwakeup_locked(so);
1818 	SOCKBUF_UNLOCK_ASSERT(sb);
1819 	if (ulp_mode(toep) == ULP_MODE_TCPDDP)
1820 		DDP_UNLOCK(toep);
1821 
1822 	INP_WUNLOCK(inp);
1823 	CURVNET_RESTORE();
1824 	return (0);
1825 }
1826 
1827 static int
1828 do_fw4_ack(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
1829 {
1830 	struct adapter *sc = iq->adapter;
1831 	const struct cpl_fw4_ack *cpl = (const void *)(rss + 1);
1832 	unsigned int tid = G_CPL_FW4_ACK_FLOWID(be32toh(OPCODE_TID(cpl)));
1833 	struct toepcb *toep = lookup_tid(sc, tid);
1834 	struct inpcb *inp;
1835 	struct tcpcb *tp;
1836 	struct socket *so;
1837 	uint8_t credits = cpl->credits;
1838 	struct ofld_tx_sdesc *txsd;
1839 	int plen;
1840 #ifdef INVARIANTS
1841 	unsigned int opcode = G_CPL_FW4_ACK_OPCODE(be32toh(OPCODE_TID(cpl)));
1842 #endif
1843 
1844 	/*
1845 	 * Very unusual case: we'd sent a flowc + abort_req for a synq entry and
1846 	 * now this comes back carrying the credits for the flowc.
1847 	 */
1848 	if (__predict_false(toep->flags & TPF_SYNQE)) {
1849 		KASSERT(toep->flags & TPF_ABORT_SHUTDOWN,
1850 		    ("%s: credits for a synq entry %p", __func__, toep));
1851 		return (0);
1852 	}
1853 
1854 	inp = toep->inp;
1855 
1856 	KASSERT(opcode == CPL_FW4_ACK,
1857 	    ("%s: unexpected opcode 0x%x", __func__, opcode));
1858 	KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
1859 	KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__));
1860 
1861 	INP_WLOCK(inp);
1862 
1863 	if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN)) {
1864 		INP_WUNLOCK(inp);
1865 		return (0);
1866 	}
1867 
1868 	KASSERT((inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) == 0,
1869 	    ("%s: inp_flags 0x%x", __func__, inp->inp_flags));
1870 
1871 	tp = intotcpcb(inp);
1872 
1873 	if (cpl->flags & CPL_FW4_ACK_FLAGS_SEQVAL) {
1874 		tcp_seq snd_una = be32toh(cpl->snd_una);
1875 
1876 #ifdef INVARIANTS
1877 		if (__predict_false(SEQ_LT(snd_una, tp->snd_una))) {
1878 			log(LOG_ERR,
1879 			    "%s: unexpected seq# %x for TID %u, snd_una %x\n",
1880 			    __func__, snd_una, toep->tid, tp->snd_una);
1881 		}
1882 #endif
1883 
1884 		if (tp->snd_una != snd_una) {
1885 			tp->snd_una = snd_una;
1886 			tp->ts_recent_age = tcp_ts_getticks();
1887 		}
1888 	}
1889 
1890 #ifdef VERBOSE_TRACES
1891 	CTR3(KTR_CXGBE, "%s: tid %d credits %u", __func__, tid, credits);
1892 #endif
1893 	so = inp->inp_socket;
1894 	txsd = &toep->txsd[toep->txsd_cidx];
1895 	plen = 0;
1896 	while (credits) {
1897 		KASSERT(credits >= txsd->tx_credits,
1898 		    ("%s: too many (or partial) credits", __func__));
1899 		credits -= txsd->tx_credits;
1900 		toep->tx_credits += txsd->tx_credits;
1901 		plen += txsd->plen;
1902 		txsd++;
1903 		toep->txsd_avail++;
1904 		KASSERT(toep->txsd_avail <= toep->txsd_total,
1905 		    ("%s: txsd avail > total", __func__));
1906 		if (__predict_false(++toep->txsd_cidx == toep->txsd_total)) {
1907 			txsd = &toep->txsd[0];
1908 			toep->txsd_cidx = 0;
1909 		}
1910 	}
1911 
1912 	if (toep->tx_credits == toep->tx_total) {
1913 		toep->tx_nocompl = 0;
1914 		toep->plen_nocompl = 0;
1915 	}
1916 
1917 	if (toep->flags & TPF_TX_SUSPENDED &&
1918 	    toep->tx_credits >= toep->tx_total / 4) {
1919 #ifdef VERBOSE_TRACES
1920 		CTR2(KTR_CXGBE, "%s: tid %d calling t4_push_frames", __func__,
1921 		    tid);
1922 #endif
1923 		toep->flags &= ~TPF_TX_SUSPENDED;
1924 		CURVNET_SET(toep->vnet);
1925 		t4_push_data(sc, toep, plen);
1926 		CURVNET_RESTORE();
1927 	} else if (plen > 0) {
1928 		struct sockbuf *sb = &so->so_snd;
1929 		int sbu;
1930 
1931 		SOCKBUF_LOCK(sb);
1932 		sbu = sbused(sb);
1933 		if (ulp_mode(toep) == ULP_MODE_ISCSI) {
1934 			if (__predict_false(sbu > 0)) {
1935 				/*
1936 				 * The data transmitted before the
1937 				 * tid's ULP mode changed to ISCSI is
1938 				 * still in so_snd.  Incoming credits
1939 				 * should account for so_snd first.
1940 				 */
1941 				sbdrop_locked(sb, min(sbu, plen));
1942 				plen -= min(sbu, plen);
1943 			}
1944 			sowwakeup_locked(so);	/* unlocks so_snd */
1945 			rqdrop_locked(&toep->ulp_pdu_reclaimq, plen);
1946 		} else {
1947 #ifdef VERBOSE_TRACES
1948 			CTR3(KTR_CXGBE, "%s: tid %d dropped %d bytes", __func__,
1949 			    tid, plen);
1950 #endif
1951 			sbdrop_locked(sb, plen);
1952 			if (!TAILQ_EMPTY(&toep->aiotx_jobq))
1953 				t4_aiotx_queue_toep(so, toep);
1954 			sowwakeup_locked(so);	/* unlocks so_snd */
1955 		}
1956 		SOCKBUF_UNLOCK_ASSERT(sb);
1957 	}
1958 
1959 	INP_WUNLOCK(inp);
1960 
1961 	return (0);
1962 }
1963 
1964 void
1965 t4_set_tcb_field(struct adapter *sc, struct sge_wrq *wrq, struct toepcb *toep,
1966     uint16_t word, uint64_t mask, uint64_t val, int reply, int cookie)
1967 {
1968 	struct wrqe *wr;
1969 	struct cpl_set_tcb_field *req;
1970 	struct ofld_tx_sdesc *txsd;
1971 
1972 	MPASS((cookie & ~M_COOKIE) == 0);
1973 	if (reply) {
1974 		MPASS(cookie != CPL_COOKIE_RESERVED);
1975 	}
1976 
1977 	wr = alloc_wrqe(sizeof(*req), wrq);
1978 	if (wr == NULL) {
1979 		/* XXX */
1980 		panic("%s: allocation failure.", __func__);
1981 	}
1982 	req = wrtod(wr);
1983 
1984 	INIT_TP_WR_MIT_CPL(req, CPL_SET_TCB_FIELD, toep->tid);
1985 	req->reply_ctrl = htobe16(V_QUEUENO(toep->ofld_rxq->iq.abs_id));
1986 	if (reply == 0)
1987 		req->reply_ctrl |= htobe16(F_NO_REPLY);
1988 	req->word_cookie = htobe16(V_WORD(word) | V_COOKIE(cookie));
1989 	req->mask = htobe64(mask);
1990 	req->val = htobe64(val);
1991 	if (wrq->eq.type == EQ_OFLD) {
1992 		txsd = &toep->txsd[toep->txsd_pidx];
1993 		txsd->tx_credits = howmany(sizeof(*req), 16);
1994 		txsd->plen = 0;
1995 		KASSERT(toep->tx_credits >= txsd->tx_credits &&
1996 		    toep->txsd_avail > 0,
1997 		    ("%s: not enough credits (%d)", __func__,
1998 		    toep->tx_credits));
1999 		toep->tx_credits -= txsd->tx_credits;
2000 		if (__predict_false(++toep->txsd_pidx == toep->txsd_total))
2001 			toep->txsd_pidx = 0;
2002 		toep->txsd_avail--;
2003 	}
2004 
2005 	t4_wrq_tx(sc, wr);
2006 }
2007 
2008 void
2009 t4_init_cpl_io_handlers(void)
2010 {
2011 
2012 	t4_register_cpl_handler(CPL_PEER_CLOSE, do_peer_close);
2013 	t4_register_cpl_handler(CPL_CLOSE_CON_RPL, do_close_con_rpl);
2014 	t4_register_cpl_handler(CPL_ABORT_REQ_RSS, do_abort_req);
2015 	t4_register_shared_cpl_handler(CPL_ABORT_RPL_RSS, do_abort_rpl,
2016 	    CPL_COOKIE_TOM);
2017 	t4_register_cpl_handler(CPL_RX_DATA, do_rx_data);
2018 	t4_register_shared_cpl_handler(CPL_FW4_ACK, do_fw4_ack, CPL_COOKIE_TOM);
2019 }
2020 
2021 void
2022 t4_uninit_cpl_io_handlers(void)
2023 {
2024 
2025 	t4_register_cpl_handler(CPL_PEER_CLOSE, NULL);
2026 	t4_register_cpl_handler(CPL_CLOSE_CON_RPL, NULL);
2027 	t4_register_cpl_handler(CPL_ABORT_REQ_RSS, NULL);
2028 	t4_register_shared_cpl_handler(CPL_ABORT_RPL_RSS, NULL, CPL_COOKIE_TOM);
2029 	t4_register_cpl_handler(CPL_RX_DATA, NULL);
2030 	t4_register_shared_cpl_handler(CPL_FW4_ACK, NULL, CPL_COOKIE_TOM);
2031 }
2032 
2033 /*
2034  * Use the 'backend1' field in AIO jobs to hold an error that should
2035  * be reported when the job is completed, the 'backend3' field to
2036  * store the amount of data sent by the AIO job so far, and the
2037  * 'backend4' field to hold a reference count on the job.
2038  *
2039  * Each unmapped mbuf holds a reference on the job as does the queue
2040  * so long as the job is queued.
2041  */
2042 #define	aio_error	backend1
2043 #define	aio_sent	backend3
2044 #define	aio_refs	backend4
2045 
2046 #define	jobtotid(job)							\
2047 	(((struct toepcb *)(so_sototcpcb((job)->fd_file->f_data)->t_toe))->tid)
2048 
2049 static void
2050 aiotx_free_job(struct kaiocb *job)
2051 {
2052 	long status;
2053 	int error;
2054 
2055 	if (refcount_release(&job->aio_refs) == 0)
2056 		return;
2057 
2058 	error = (intptr_t)job->aio_error;
2059 	status = job->aio_sent;
2060 #ifdef VERBOSE_TRACES
2061 	CTR5(KTR_CXGBE, "%s: tid %d completed %p len %ld, error %d", __func__,
2062 	    jobtotid(job), job, status, error);
2063 #endif
2064 	if (error != 0 && status != 0)
2065 		error = 0;
2066 	if (error == ECANCELED)
2067 		aio_cancel(job);
2068 	else if (error)
2069 		aio_complete(job, -1, error);
2070 	else {
2071 		job->msgsnd = 1;
2072 		aio_complete(job, status, 0);
2073 	}
2074 }
2075 
2076 static void
2077 aiotx_free_pgs(struct mbuf *m)
2078 {
2079 	struct kaiocb *job;
2080 	vm_page_t pg;
2081 
2082 	M_ASSERTEXTPG(m);
2083 	job = m->m_ext.ext_arg1;
2084 #ifdef VERBOSE_TRACES
2085 	CTR3(KTR_CXGBE, "%s: completed %d bytes for tid %d", __func__,
2086 	    m->m_len, jobtotid(job));
2087 #endif
2088 
2089 	for (int i = 0; i < m->m_epg_npgs; i++) {
2090 		pg = PHYS_TO_VM_PAGE(m->m_epg_pa[i]);
2091 		vm_page_unwire(pg, PQ_ACTIVE);
2092 	}
2093 
2094 	aiotx_free_job(job);
2095 }
2096 
2097 /*
2098  * Allocate a chain of unmapped mbufs describing the next 'len' bytes
2099  * of an AIO job.
2100  */
2101 static struct mbuf *
2102 alloc_aiotx_mbuf(struct kaiocb *job, int len)
2103 {
2104 	struct vmspace *vm;
2105 	vm_page_t pgs[MBUF_PEXT_MAX_PGS];
2106 	struct mbuf *m, *top, *last;
2107 	vm_map_t map;
2108 	vm_offset_t start;
2109 	int i, mlen, npages, pgoff;
2110 
2111 	KASSERT(job->aio_sent + len <= job->uaiocb.aio_nbytes,
2112 	    ("%s(%p, %d): request to send beyond end of buffer", __func__,
2113 	    job, len));
2114 
2115 	/*
2116 	 * The AIO subsystem will cancel and drain all requests before
2117 	 * permitting a process to exit or exec, so p_vmspace should
2118 	 * be stable here.
2119 	 */
2120 	vm = job->userproc->p_vmspace;
2121 	map = &vm->vm_map;
2122 	start = (uintptr_t)job->uaiocb.aio_buf + job->aio_sent;
2123 	pgoff = start & PAGE_MASK;
2124 
2125 	top = NULL;
2126 	last = NULL;
2127 	while (len > 0) {
2128 		mlen = imin(len, MBUF_PEXT_MAX_PGS * PAGE_SIZE - pgoff);
2129 		KASSERT(mlen == len || ((start + mlen) & PAGE_MASK) == 0,
2130 		    ("%s: next start (%#jx + %#x) is not page aligned",
2131 		    __func__, (uintmax_t)start, mlen));
2132 
2133 		npages = vm_fault_quick_hold_pages(map, start, mlen,
2134 		    VM_PROT_WRITE, pgs, nitems(pgs));
2135 		if (npages < 0)
2136 			break;
2137 
2138 		m = mb_alloc_ext_pgs(M_WAITOK, aiotx_free_pgs);
2139 		if (m == NULL) {
2140 			vm_page_unhold_pages(pgs, npages);
2141 			break;
2142 		}
2143 
2144 		m->m_epg_1st_off = pgoff;
2145 		m->m_epg_npgs = npages;
2146 		if (npages == 1) {
2147 			KASSERT(mlen + pgoff <= PAGE_SIZE,
2148 			    ("%s: single page is too large (off %d len %d)",
2149 			    __func__, pgoff, mlen));
2150 			m->m_epg_last_len = mlen;
2151 		} else {
2152 			m->m_epg_last_len = mlen - (PAGE_SIZE - pgoff) -
2153 			    (npages - 2) * PAGE_SIZE;
2154 		}
2155 		for (i = 0; i < npages; i++)
2156 			m->m_epg_pa[i] = VM_PAGE_TO_PHYS(pgs[i]);
2157 
2158 		m->m_len = mlen;
2159 		m->m_ext.ext_size = npages * PAGE_SIZE;
2160 		m->m_ext.ext_arg1 = job;
2161 		refcount_acquire(&job->aio_refs);
2162 
2163 #ifdef VERBOSE_TRACES
2164 		CTR5(KTR_CXGBE, "%s: tid %d, new mbuf %p for job %p, npages %d",
2165 		    __func__, jobtotid(job), m, job, npages);
2166 #endif
2167 
2168 		if (top == NULL)
2169 			top = m;
2170 		else
2171 			last->m_next = m;
2172 		last = m;
2173 
2174 		len -= mlen;
2175 		start += mlen;
2176 		pgoff = 0;
2177 	}
2178 
2179 	return (top);
2180 }
2181 
2182 static void
2183 t4_aiotx_process_job(struct toepcb *toep, struct socket *so, struct kaiocb *job)
2184 {
2185 	struct sockbuf *sb;
2186 	struct file *fp;
2187 	struct inpcb *inp;
2188 	struct tcpcb *tp;
2189 	struct mbuf *m;
2190 	int error, len;
2191 	bool moretocome, sendmore;
2192 
2193 	sb = &so->so_snd;
2194 	SOCKBUF_UNLOCK(sb);
2195 	fp = job->fd_file;
2196 	m = NULL;
2197 
2198 #ifdef MAC
2199 	error = mac_socket_check_send(fp->f_cred, so);
2200 	if (error != 0)
2201 		goto out;
2202 #endif
2203 
2204 	/* Inline sosend_generic(). */
2205 
2206 	error = SOCK_IO_SEND_LOCK(so, SBL_WAIT);
2207 	MPASS(error == 0);
2208 
2209 sendanother:
2210 	SOCKBUF_LOCK(sb);
2211 	if (so->so_snd.sb_state & SBS_CANTSENDMORE) {
2212 		SOCKBUF_UNLOCK(sb);
2213 		SOCK_IO_SEND_UNLOCK(so);
2214 		if ((so->so_options & SO_NOSIGPIPE) == 0) {
2215 			PROC_LOCK(job->userproc);
2216 			kern_psignal(job->userproc, SIGPIPE);
2217 			PROC_UNLOCK(job->userproc);
2218 		}
2219 		error = EPIPE;
2220 		goto out;
2221 	}
2222 	if (so->so_error) {
2223 		error = so->so_error;
2224 		so->so_error = 0;
2225 		SOCKBUF_UNLOCK(sb);
2226 		SOCK_IO_SEND_UNLOCK(so);
2227 		goto out;
2228 	}
2229 	if ((so->so_state & SS_ISCONNECTED) == 0) {
2230 		SOCKBUF_UNLOCK(sb);
2231 		SOCK_IO_SEND_UNLOCK(so);
2232 		error = ENOTCONN;
2233 		goto out;
2234 	}
2235 	if (sbspace(sb) < sb->sb_lowat) {
2236 		MPASS(job->aio_sent == 0 || !(so->so_state & SS_NBIO));
2237 
2238 		/*
2239 		 * Don't block if there is too little room in the socket
2240 		 * buffer.  Instead, requeue the request.
2241 		 */
2242 		if (!aio_set_cancel_function(job, t4_aiotx_cancel)) {
2243 			SOCKBUF_UNLOCK(sb);
2244 			SOCK_IO_SEND_UNLOCK(so);
2245 			error = ECANCELED;
2246 			goto out;
2247 		}
2248 		TAILQ_INSERT_HEAD(&toep->aiotx_jobq, job, list);
2249 		SOCKBUF_UNLOCK(sb);
2250 		SOCK_IO_SEND_UNLOCK(so);
2251 		goto out;
2252 	}
2253 
2254 	/*
2255 	 * Write as much data as the socket permits, but no more than a
2256 	 * a single sndbuf at a time.
2257 	 */
2258 	len = sbspace(sb);
2259 	if (len > job->uaiocb.aio_nbytes - job->aio_sent) {
2260 		len = job->uaiocb.aio_nbytes - job->aio_sent;
2261 		moretocome = false;
2262 	} else
2263 		moretocome = true;
2264 	if (len > toep->params.sndbuf) {
2265 		len = toep->params.sndbuf;
2266 		sendmore = true;
2267 	} else
2268 		sendmore = false;
2269 
2270 	if (!TAILQ_EMPTY(&toep->aiotx_jobq))
2271 		moretocome = true;
2272 	SOCKBUF_UNLOCK(sb);
2273 	MPASS(len != 0);
2274 
2275 	m = alloc_aiotx_mbuf(job, len);
2276 	if (m == NULL) {
2277 		SOCK_IO_SEND_UNLOCK(so);
2278 		error = EFAULT;
2279 		goto out;
2280 	}
2281 
2282 	/* Inlined tcp_usr_send(). */
2283 
2284 	inp = toep->inp;
2285 	INP_WLOCK(inp);
2286 	if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) {
2287 		INP_WUNLOCK(inp);
2288 		SOCK_IO_SEND_UNLOCK(so);
2289 		error = ECONNRESET;
2290 		goto out;
2291 	}
2292 
2293 	job->aio_sent += m_length(m, NULL);
2294 
2295 	sbappendstream(sb, m, 0);
2296 	m = NULL;
2297 
2298 	if (!(inp->inp_flags & INP_DROPPED)) {
2299 		tp = intotcpcb(inp);
2300 		if (moretocome)
2301 			tp->t_flags |= TF_MORETOCOME;
2302 		error = tcp_output(tp);
2303 		if (error < 0) {
2304 			INP_UNLOCK_ASSERT(inp);
2305 			SOCK_IO_SEND_UNLOCK(so);
2306 			error = -error;
2307 			goto out;
2308 		}
2309 		if (moretocome)
2310 			tp->t_flags &= ~TF_MORETOCOME;
2311 	}
2312 
2313 	INP_WUNLOCK(inp);
2314 	if (sendmore)
2315 		goto sendanother;
2316 	SOCK_IO_SEND_UNLOCK(so);
2317 
2318 	if (error)
2319 		goto out;
2320 
2321 	/*
2322 	 * If this is a blocking socket and the request has not been
2323 	 * fully completed, requeue it until the socket is ready
2324 	 * again.
2325 	 */
2326 	if (job->aio_sent < job->uaiocb.aio_nbytes &&
2327 	    !(so->so_state & SS_NBIO)) {
2328 		SOCKBUF_LOCK(sb);
2329 		if (!aio_set_cancel_function(job, t4_aiotx_cancel)) {
2330 			SOCKBUF_UNLOCK(sb);
2331 			error = ECANCELED;
2332 			goto out;
2333 		}
2334 		TAILQ_INSERT_HEAD(&toep->aiotx_jobq, job, list);
2335 		return;
2336 	}
2337 
2338 	/*
2339 	 * If the request will not be requeued, drop the queue's
2340 	 * reference to the job.  Any mbufs in flight should still
2341 	 * hold a reference, but this drops the reference that the
2342 	 * queue owns while it is waiting to queue mbufs to the
2343 	 * socket.
2344 	 */
2345 	aiotx_free_job(job);
2346 
2347 out:
2348 	if (error) {
2349 		job->aio_error = (void *)(intptr_t)error;
2350 		aiotx_free_job(job);
2351 	}
2352 	m_freem(m);
2353 	SOCKBUF_LOCK(sb);
2354 }
2355 
2356 static void
2357 t4_aiotx_task(void *context, int pending)
2358 {
2359 	struct toepcb *toep = context;
2360 	struct socket *so;
2361 	struct kaiocb *job;
2362 	struct epoch_tracker et;
2363 
2364 	so = toep->aiotx_so;
2365 	CURVNET_SET(toep->vnet);
2366 	NET_EPOCH_ENTER(et);
2367 	SOCKBUF_LOCK(&so->so_snd);
2368 	while (!TAILQ_EMPTY(&toep->aiotx_jobq) && sowriteable(so)) {
2369 		job = TAILQ_FIRST(&toep->aiotx_jobq);
2370 		TAILQ_REMOVE(&toep->aiotx_jobq, job, list);
2371 		if (!aio_clear_cancel_function(job))
2372 			continue;
2373 
2374 		t4_aiotx_process_job(toep, so, job);
2375 	}
2376 	toep->aiotx_so = NULL;
2377 	SOCKBUF_UNLOCK(&so->so_snd);
2378 	NET_EPOCH_EXIT(et);
2379 
2380 	free_toepcb(toep);
2381 	sorele(so);
2382 	CURVNET_RESTORE();
2383 }
2384 
2385 static void
2386 t4_aiotx_queue_toep(struct socket *so, struct toepcb *toep)
2387 {
2388 
2389 	SOCKBUF_LOCK_ASSERT(&toep->inp->inp_socket->so_snd);
2390 #ifdef VERBOSE_TRACES
2391 	CTR3(KTR_CXGBE, "%s: queueing aiotx task for tid %d, active = %s",
2392 	    __func__, toep->tid, toep->aiotx_so != NULL ? "true" : "false");
2393 #endif
2394 	if (toep->aiotx_so != NULL)
2395 		return;
2396 	soref(so);
2397 	toep->aiotx_so = so;
2398 	hold_toepcb(toep);
2399 	soaio_enqueue(&toep->aiotx_task);
2400 }
2401 
2402 static void
2403 t4_aiotx_cancel(struct kaiocb *job)
2404 {
2405 	struct socket *so;
2406 	struct sockbuf *sb;
2407 	struct tcpcb *tp;
2408 	struct toepcb *toep;
2409 
2410 	so = job->fd_file->f_data;
2411 	tp = so_sototcpcb(so);
2412 	toep = tp->t_toe;
2413 	MPASS(job->uaiocb.aio_lio_opcode == LIO_WRITE);
2414 	sb = &so->so_snd;
2415 
2416 	SOCKBUF_LOCK(sb);
2417 	if (!aio_cancel_cleared(job))
2418 		TAILQ_REMOVE(&toep->aiotx_jobq, job, list);
2419 	SOCKBUF_UNLOCK(sb);
2420 
2421 	job->aio_error = (void *)(intptr_t)ECANCELED;
2422 	aiotx_free_job(job);
2423 }
2424 
2425 int
2426 t4_aio_queue_aiotx(struct socket *so, struct kaiocb *job)
2427 {
2428 	struct tcpcb *tp = so_sototcpcb(so);
2429 	struct toepcb *toep = tp->t_toe;
2430 	struct adapter *sc = td_adapter(toep->td);
2431 
2432 	/* This only handles writes. */
2433 	if (job->uaiocb.aio_lio_opcode != LIO_WRITE)
2434 		return (EOPNOTSUPP);
2435 
2436 	if (!sc->tt.tx_zcopy)
2437 		return (EOPNOTSUPP);
2438 
2439 	if (tls_tx_key(toep))
2440 		return (EOPNOTSUPP);
2441 
2442 	SOCKBUF_LOCK(&so->so_snd);
2443 #ifdef VERBOSE_TRACES
2444 	CTR3(KTR_CXGBE, "%s: queueing %p for tid %u", __func__, job, toep->tid);
2445 #endif
2446 	if (!aio_set_cancel_function(job, t4_aiotx_cancel))
2447 		panic("new job was cancelled");
2448 	refcount_init(&job->aio_refs, 1);
2449 	TAILQ_INSERT_TAIL(&toep->aiotx_jobq, job, list);
2450 	if (sowriteable(so))
2451 		t4_aiotx_queue_toep(so, toep);
2452 	SOCKBUF_UNLOCK(&so->so_snd);
2453 	return (0);
2454 }
2455 
2456 void
2457 aiotx_init_toep(struct toepcb *toep)
2458 {
2459 
2460 	TAILQ_INIT(&toep->aiotx_jobq);
2461 	TASK_INIT(&toep->aiotx_task, 0, t4_aiotx_task, toep);
2462 }
2463 #endif
2464