xref: /freebsd/sys/dev/cxgbe/cxgbei/cxgbei.c (revision ee28ad11b7635715ed7d2f5815ace34c433992d1)
1 /*-
2  * Copyright (c) 2012 Chelsio Communications, Inc.
3  * All rights reserved.
4  *
5  * Chelsio T5xx iSCSI driver
6  *
7  * Written by: Sreenivasa Honnur <shonnur@chelsio.com>
8  *
9  * Redistribution and use in source and binary forms, with or without
10  * modification, are permitted provided that the following conditions
11  * are met:
12  * 1. Redistributions of source code must retain the above copyright
13  *    notice, this list of conditions and the following disclaimer.
14  * 2. Redistributions in binary form must reproduce the above copyright
15  *    notice, this list of conditions and the following disclaimer in the
16  *    documentation and/or other materials provided with the distribution.
17  *
18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28  * SUCH DAMAGE.
29  */
30 
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD$");
33 
34 #include "opt_inet.h"
35 #include "opt_inet6.h"
36 
37 #include <sys/types.h>
38 #include <sys/param.h>
39 #include <sys/kernel.h>
40 #include <sys/ktr.h>
41 #include <sys/module.h>
42 #include <sys/systm.h>
43 
44 #ifdef TCP_OFFLOAD
45 #include <sys/errno.h>
46 #include <sys/gsb_crc32.h>
47 #include <sys/kthread.h>
48 #include <sys/smp.h>
49 #include <sys/socket.h>
50 #include <sys/socketvar.h>
51 #include <sys/mbuf.h>
52 #include <sys/lock.h>
53 #include <sys/mutex.h>
54 #include <sys/condvar.h>
55 #include <sys/uio.h>
56 
57 #include <netinet/in.h>
58 #include <netinet/in_pcb.h>
59 #include <netinet/toecore.h>
60 #include <netinet/tcp_var.h>
61 #include <netinet/tcp_fsm.h>
62 
63 #include <cam/scsi/scsi_all.h>
64 #include <cam/scsi/scsi_da.h>
65 #include <cam/ctl/ctl_io.h>
66 #include <cam/ctl/ctl.h>
67 #include <cam/ctl/ctl_backend.h>
68 #include <cam/ctl/ctl_error.h>
69 #include <cam/ctl/ctl_frontend.h>
70 #include <cam/ctl/ctl_debug.h>
71 #include <cam/ctl/ctl_ha.h>
72 #include <cam/ctl/ctl_ioctl.h>
73 
74 #include <dev/iscsi/icl.h>
75 #include <dev/iscsi/iscsi_proto.h>
76 #include <dev/iscsi/iscsi_ioctl.h>
77 #include <dev/iscsi/iscsi.h>
78 #include <cam/ctl/ctl_frontend_iscsi.h>
79 
80 #include <cam/cam.h>
81 #include <cam/cam_ccb.h>
82 #include <cam/cam_xpt.h>
83 #include <cam/cam_debug.h>
84 #include <cam/cam_sim.h>
85 #include <cam/cam_xpt_sim.h>
86 #include <cam/cam_xpt_periph.h>
87 #include <cam/cam_periph.h>
88 #include <cam/cam_compat.h>
89 #include <cam/scsi/scsi_message.h>
90 
91 #include "common/common.h"
92 #include "common/t4_msg.h"
93 #include "common/t4_regs.h"	/* for PCIE_MEM_ACCESS */
94 #include "tom/t4_tom.h"
95 #include "cxgbei.h"
96 
97 static void
98 read_pdu_limits(struct adapter *sc, uint32_t *max_tx_data_len,
99     uint32_t *max_rx_data_len, struct ppod_region *pr)
100 {
101 	uint32_t tx_len, rx_len, r, v;
102 
103 	rx_len = t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE);
104 	tx_len = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE);
105 
106 	r = t4_read_reg(sc, A_TP_PARA_REG2);
107 	rx_len = min(rx_len, G_MAXRXDATA(r));
108 	tx_len = min(tx_len, G_MAXRXDATA(r));
109 
110 	r = t4_read_reg(sc, A_TP_PARA_REG7);
111 	v = min(G_PMMAXXFERLEN0(r), G_PMMAXXFERLEN1(r));
112 	rx_len = min(rx_len, v);
113 	tx_len = min(tx_len, v);
114 
115 	/*
116 	 * AHS is not supported by the kernel so we'll not account for
117 	 * it either in our PDU len -> data segment len conversions.
118 	 */
119 	rx_len -= ISCSI_BHS_SIZE + ISCSI_HEADER_DIGEST_SIZE +
120 	    ISCSI_DATA_DIGEST_SIZE;
121 	tx_len -= ISCSI_BHS_SIZE + ISCSI_HEADER_DIGEST_SIZE +
122 	    ISCSI_DATA_DIGEST_SIZE;
123 
124 	/*
125 	 * DDP can place only 4 pages for a single PDU.  A single
126 	 * request might use larger pages than the smallest page size,
127 	 * but that cannot be guaranteed.  Assume the smallest DDP
128 	 * page size for this limit.
129 	 */
130 	rx_len = min(rx_len, 4 * (1U << pr->pr_page_shift[0]));
131 
132 	if (chip_id(sc) == CHELSIO_T5) {
133 		tx_len = min(tx_len, 15360);
134 
135 		rx_len = rounddown2(rx_len, 512);
136 		tx_len = rounddown2(tx_len, 512);
137 	}
138 
139 	*max_tx_data_len = tx_len;
140 	*max_rx_data_len = rx_len;
141 }
142 
143 /*
144  * Initialize the software state of the iSCSI ULP driver.
145  *
146  * ENXIO means firmware didn't set up something that it was supposed to.
147  */
148 static int
149 cxgbei_init(struct adapter *sc, struct cxgbei_data *ci)
150 {
151 	struct sysctl_oid *oid;
152 	struct sysctl_oid_list *children;
153 	struct ppod_region *pr;
154 	uint32_t r;
155 	int rc;
156 
157 	MPASS(sc->vres.iscsi.size > 0);
158 	MPASS(ci != NULL);
159 
160 	pr = &ci->pr;
161 	r = t4_read_reg(sc, A_ULP_RX_ISCSI_PSZ);
162 	rc = t4_init_ppod_region(pr, &sc->vres.iscsi, r, "iSCSI page pods");
163 	if (rc != 0) {
164 		device_printf(sc->dev,
165 		    "%s: failed to initialize the iSCSI page pod region: %u.\n",
166 		    __func__, rc);
167 		return (rc);
168 	}
169 
170 	r = t4_read_reg(sc, A_ULP_RX_ISCSI_TAGMASK);
171 	r &= V_ISCSITAGMASK(M_ISCSITAGMASK);
172 	if (r != pr->pr_tag_mask) {
173 		/*
174 		 * Recent firmwares are supposed to set up the iSCSI tagmask
175 		 * but we'll do it ourselves it the computed value doesn't match
176 		 * what's in the register.
177 		 */
178 		device_printf(sc->dev,
179 		    "tagmask 0x%08x does not match computed mask 0x%08x.\n", r,
180 		    pr->pr_tag_mask);
181 		t4_set_reg_field(sc, A_ULP_RX_ISCSI_TAGMASK,
182 		    V_ISCSITAGMASK(M_ISCSITAGMASK), pr->pr_tag_mask);
183 	}
184 
185 	read_pdu_limits(sc, &ci->max_tx_data_len, &ci->max_rx_data_len, pr);
186 
187 	sysctl_ctx_init(&ci->ctx);
188 	oid = device_get_sysctl_tree(sc->dev);	/* dev.t5nex.X */
189 	children = SYSCTL_CHILDREN(oid);
190 
191 	oid = SYSCTL_ADD_NODE(&ci->ctx, children, OID_AUTO, "iscsi",
192 	    CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "iSCSI ULP settings");
193 	children = SYSCTL_CHILDREN(oid);
194 
195 	ci->ddp_threshold = 2048;
196 	SYSCTL_ADD_UINT(&ci->ctx, children, OID_AUTO, "ddp_threshold",
197 	    CTLFLAG_RW, &ci->ddp_threshold, 0, "Rx zero copy threshold");
198 
199 	SYSCTL_ADD_UINT(&ci->ctx, children, OID_AUTO, "max_rx_data_len",
200 	    CTLFLAG_RD, &ci->max_rx_data_len, 0,
201 	    "Maximum receive data segment length");
202 	SYSCTL_ADD_UINT(&ci->ctx, children, OID_AUTO, "max_tx_data_len",
203 	    CTLFLAG_RD, &ci->max_tx_data_len, 0,
204 	    "Maximum transmit data segment length");
205 
206 	return (0);
207 }
208 
209 static int
210 do_rx_iscsi_hdr(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
211 {
212 	struct adapter *sc = iq->adapter;
213 	struct cpl_iscsi_hdr *cpl = mtod(m, struct cpl_iscsi_hdr *);
214 	u_int tid = GET_TID(cpl);
215 	struct toepcb *toep = lookup_tid(sc, tid);
216 	struct icl_pdu *ip;
217 	struct icl_cxgbei_pdu *icp;
218 	uint16_t len_ddp = be16toh(cpl->pdu_len_ddp);
219 	uint16_t len = be16toh(cpl->len);
220 
221 	M_ASSERTPKTHDR(m);
222 	MPASS(m->m_pkthdr.len == len + sizeof(*cpl));
223 
224 	ip = icl_cxgbei_new_pdu(M_NOWAIT);
225 	if (ip == NULL)
226 		CXGBE_UNIMPLEMENTED("PDU allocation failure");
227 	m_copydata(m, sizeof(*cpl), ISCSI_BHS_SIZE, (caddr_t)ip->ip_bhs);
228 	ip->ip_data_len = G_ISCSI_PDU_LEN(len_ddp) - len;
229 	icp = ip_to_icp(ip);
230 	icp->icp_seq = ntohl(cpl->seq);
231 	icp->icp_flags = ICPF_RX_HDR;
232 
233 	/* This is the start of a new PDU.  There should be no old state. */
234 	MPASS(toep->ulpcb2 == NULL);
235 	toep->ulpcb2 = icp;
236 
237 #if 0
238 	CTR5(KTR_CXGBE, "%s: tid %u, cpl->len %u, pdu_len_ddp 0x%04x, icp %p",
239 	    __func__, tid, len, len_ddp, icp);
240 #endif
241 
242 	m_freem(m);
243 	return (0);
244 }
245 
246 static int
247 do_rx_iscsi_data(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
248 {
249 	struct adapter *sc = iq->adapter;
250 	struct cpl_iscsi_data *cpl =  mtod(m, struct cpl_iscsi_data *);
251 	u_int tid = GET_TID(cpl);
252 	struct toepcb *toep = lookup_tid(sc, tid);
253 	struct icl_cxgbei_pdu *icp = toep->ulpcb2;
254 	struct icl_pdu *ip;
255 
256 	M_ASSERTPKTHDR(m);
257 	MPASS(m->m_pkthdr.len == be16toh(cpl->len) + sizeof(*cpl));
258 
259 	if (icp == NULL) {
260 		/*
261 		 * T6 completion enabled, start of a new pdu. Header
262 		 * will come in completion CPL.
263 		 */
264 	        ip = icl_cxgbei_new_pdu(M_NOWAIT);
265 	        if (ip == NULL)
266 			CXGBE_UNIMPLEMENTED("PDU allocation failure");
267 		icp = ip_to_icp(ip);
268 	} else {
269 		/* T5 mode, header is already received. */
270 		MPASS(icp->icp_flags == ICPF_RX_HDR);
271 		MPASS(icp->ip.ip_data_mbuf == NULL);
272 		MPASS(icp->ip.ip_data_len == m->m_pkthdr.len - sizeof(*cpl));
273 	}
274 
275 	/* Trim the cpl header from mbuf. */
276 	m_adj(m, sizeof(*cpl));
277 
278 	icp->icp_flags |= ICPF_RX_FLBUF;
279 	icp->ip.ip_data_mbuf = m;
280 	toep->ofld_rxq->rx_iscsi_fl_pdus++;
281 	toep->ofld_rxq->rx_iscsi_fl_octets += m->m_pkthdr.len;
282 
283 	/*
284 	 * For T6, save the icp for further processing in the
285 	 * completion handler.
286 	 */
287 	if (icp->icp_flags == ICPF_RX_FLBUF) {
288 		MPASS(toep->ulpcb2 == NULL);
289 		toep->ulpcb2 = icp;
290 	}
291 
292 #if 0
293 	CTR4(KTR_CXGBE, "%s: tid %u, cpl->len %u, icp %p", __func__, tid,
294 	    be16toh(cpl->len), icp);
295 #endif
296 
297 	return (0);
298 }
299 
300 static int
301 mbuf_crc32c_helper(void *arg, void *data, u_int len)
302 {
303 	uint32_t *digestp = arg;
304 
305 	*digestp = calculate_crc32c(*digestp, data, len);
306 	return (0);
307 }
308 
309 static struct icl_pdu *
310 parse_pdu(struct socket *so, struct toepcb *toep, struct icl_cxgbei_conn *icc,
311     struct sockbuf *sb, u_int total_len)
312 {
313 	struct uio uio;
314 	struct iovec iov[2];
315 	struct iscsi_bhs bhs;
316 	struct mbuf *m;
317 	struct icl_pdu *ip;
318 	u_int ahs_len, data_len, header_len, pdu_len;
319 	uint32_t calc_digest, wire_digest;
320 	int error;
321 
322 	uio.uio_segflg = UIO_SYSSPACE;
323 	uio.uio_rw = UIO_READ;
324 	uio.uio_td = curthread;
325 
326 	header_len = sizeof(struct iscsi_bhs);
327 	if (icc->ic.ic_header_crc32c)
328 		header_len += ISCSI_HEADER_DIGEST_SIZE;
329 
330 	if (total_len < header_len) {
331 		ICL_WARN("truncated pre-offload PDU with len %u", total_len);
332 		return (NULL);
333 	}
334 
335 	iov[0].iov_base = &bhs;
336 	iov[0].iov_len = sizeof(bhs);
337 	iov[1].iov_base = &wire_digest;
338 	iov[1].iov_len = sizeof(wire_digest);
339 	uio.uio_iov = iov;
340 	uio.uio_iovcnt = 1;
341 	uio.uio_offset = 0;
342 	uio.uio_resid = header_len;
343 	error = soreceive(so, NULL, &uio, NULL, NULL, NULL);
344 	if (error != 0) {
345 		ICL_WARN("failed to read BHS from pre-offload PDU: %d", error);
346 		return (NULL);
347 	}
348 
349 	ahs_len = bhs.bhs_total_ahs_len * 4;
350 	data_len = bhs.bhs_data_segment_len[0] << 16 |
351 	    bhs.bhs_data_segment_len[1] << 8 |
352 	    bhs.bhs_data_segment_len[2];
353 	pdu_len = header_len + ahs_len + roundup2(data_len, 4);
354 	if (icc->ic.ic_data_crc32c && data_len != 0)
355 		pdu_len += ISCSI_DATA_DIGEST_SIZE;
356 
357 	if (total_len < pdu_len) {
358 		ICL_WARN("truncated pre-offload PDU len %u vs %u", total_len,
359 		    pdu_len);
360 		return (NULL);
361 	}
362 
363 	if (ahs_len != 0) {
364 		ICL_WARN("received pre-offload PDU with AHS");
365 		return (NULL);
366 	}
367 
368 	if (icc->ic.ic_header_crc32c) {
369 		calc_digest = calculate_crc32c(0xffffffff, (caddr_t)&bhs,
370 		    sizeof(bhs));
371 		calc_digest ^= 0xffffffff;
372 		if (calc_digest != wire_digest) {
373 			ICL_WARN("received pre-offload PDU 0x%02x with "
374 			    "invalid header digest (0x%x vs 0x%x)",
375 			    bhs.bhs_opcode, wire_digest, calc_digest);
376 			toep->ofld_rxq->rx_iscsi_header_digest_errors++;
377 			return (NULL);
378 		}
379 	}
380 
381 	m = NULL;
382 	if (data_len != 0) {
383 		uio.uio_iov = NULL;
384 		uio.uio_resid = roundup2(data_len, 4);
385 		if (icc->ic.ic_data_crc32c)
386 			uio.uio_resid += ISCSI_DATA_DIGEST_SIZE;
387 
388 		error = soreceive(so, NULL, &uio, &m, NULL, NULL);
389 		if (error != 0) {
390 			ICL_WARN("failed to read data payload from "
391 			    "pre-offload PDU: %d", error);
392 			return (NULL);
393 		}
394 
395 		if (icc->ic.ic_data_crc32c) {
396 			m_copydata(m, roundup2(data_len, 4),
397 			    sizeof(wire_digest), (caddr_t)&wire_digest);
398 
399 			calc_digest = 0xffffffff;
400 			m_apply(m, 0, roundup2(data_len, 4), mbuf_crc32c_helper,
401 			    &calc_digest);
402 			calc_digest ^= 0xffffffff;
403 			if (calc_digest != wire_digest) {
404 				ICL_WARN("received pre-offload PDU 0x%02x "
405 				    "with invalid data digest (0x%x vs 0x%x)",
406 				    bhs.bhs_opcode, wire_digest, calc_digest);
407 				toep->ofld_rxq->rx_iscsi_data_digest_errors++;
408 				m_freem(m);
409 				return (NULL);
410 			}
411 		}
412 	}
413 
414 	ip = icl_cxgbei_new_pdu(M_WAITOK);
415 	icl_cxgbei_new_pdu_set_conn(ip, &icc->ic);
416 	*ip->ip_bhs = bhs;
417 	ip->ip_data_len = data_len;
418 	ip->ip_data_mbuf = m;
419 	return (ip);
420 }
421 
422 void
423 parse_pdus(struct icl_cxgbei_conn *icc, struct sockbuf *sb)
424 {
425 	struct icl_conn *ic = &icc->ic;
426 	struct socket *so = ic->ic_socket;
427 	struct toepcb *toep = icc->toep;
428 	struct icl_pdu *ip, *lastip;
429 	u_int total_len;
430 
431 	SOCKBUF_LOCK_ASSERT(sb);
432 
433 	CTR3(KTR_CXGBE, "%s: tid %u, %u bytes in so_rcv", __func__, toep->tid,
434 	    sbused(sb));
435 
436 	lastip = NULL;
437 	while (sbused(sb) != 0 && (sb->sb_state & SBS_CANTRCVMORE) == 0) {
438 		total_len = sbused(sb);
439 		SOCKBUF_UNLOCK(sb);
440 
441 		ip = parse_pdu(so, toep, icc, sb, total_len);
442 
443 		if (ip == NULL) {
444 			ic->ic_error(ic);
445 			SOCKBUF_LOCK(sb);
446 			return;
447 		}
448 
449 		if (lastip == NULL)
450 			STAILQ_INSERT_HEAD(&icc->rcvd_pdus, ip, ip_next);
451 		else
452 			STAILQ_INSERT_AFTER(&icc->rcvd_pdus, lastip, ip,
453 			    ip_next);
454 		lastip = ip;
455 
456 		SOCKBUF_LOCK(sb);
457 	}
458 }
459 
460 static int
461 do_rx_iscsi_ddp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
462 {
463 	struct adapter *sc = iq->adapter;
464 	const struct cpl_rx_data_ddp *cpl = (const void *)(rss + 1);
465 	u_int tid = GET_TID(cpl);
466 	struct toepcb *toep = lookup_tid(sc, tid);
467 	struct inpcb *inp = toep->inp;
468 	struct socket *so;
469 	struct sockbuf *sb;
470 	struct tcpcb *tp;
471 	struct icl_cxgbei_conn *icc;
472 	struct icl_conn *ic;
473 	struct icl_cxgbei_pdu *icp = toep->ulpcb2;
474 	struct icl_pdu *ip;
475 	u_int pdu_len, val;
476 	struct epoch_tracker et;
477 
478 	MPASS(m == NULL);
479 
480 	/* Must already be assembling a PDU. */
481 	MPASS(icp != NULL);
482 	MPASS(icp->icp_flags & ICPF_RX_HDR);	/* Data is optional. */
483 	MPASS((icp->icp_flags & ICPF_RX_STATUS) == 0);
484 
485 	pdu_len = be16toh(cpl->len);	/* includes everything. */
486 	val = be32toh(cpl->ddpvld);
487 
488 #if 0
489 	CTR5(KTR_CXGBE,
490 	    "%s: tid %u, cpl->len %u, ddpvld 0x%08x, icp_flags 0x%08x",
491 	    __func__, tid, pdu_len, val, icp->icp_flags);
492 #endif
493 
494 	icp->icp_flags |= ICPF_RX_STATUS;
495 	ip = &icp->ip;
496 	if (val & F_DDP_PADDING_ERR) {
497 		ICL_WARN("received PDU 0x%02x with invalid padding",
498 		    ip->ip_bhs->bhs_opcode);
499 		toep->ofld_rxq->rx_iscsi_padding_errors++;
500 	}
501 	if (val & F_DDP_HDRCRC_ERR) {
502 		ICL_WARN("received PDU 0x%02x with invalid header digest",
503 		    ip->ip_bhs->bhs_opcode);
504 		toep->ofld_rxq->rx_iscsi_header_digest_errors++;
505 	}
506 	if (val & F_DDP_DATACRC_ERR) {
507 		ICL_WARN("received PDU 0x%02x with invalid data digest",
508 		    ip->ip_bhs->bhs_opcode);
509 		toep->ofld_rxq->rx_iscsi_data_digest_errors++;
510 	}
511 	if (val & F_DDP_PDU && ip->ip_data_mbuf == NULL) {
512 		MPASS((icp->icp_flags & ICPF_RX_FLBUF) == 0);
513 		MPASS(ip->ip_data_len > 0);
514 		icp->icp_flags |= ICPF_RX_DDP;
515 		toep->ofld_rxq->rx_iscsi_ddp_pdus++;
516 		toep->ofld_rxq->rx_iscsi_ddp_octets += ip->ip_data_len;
517 	}
518 
519 	INP_WLOCK(inp);
520 	if (__predict_false(inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT))) {
521 		CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x",
522 		    __func__, tid, pdu_len, inp->inp_flags);
523 		INP_WUNLOCK(inp);
524 		icl_cxgbei_conn_pdu_free(NULL, ip);
525 		toep->ulpcb2 = NULL;
526 		return (0);
527 	}
528 
529 	/*
530 	 * T6+ does not report data PDUs received via DDP without F
531 	 * set.  This can result in gaps in the TCP sequence space.
532 	 */
533 	tp = intotcpcb(inp);
534 	MPASS(chip_id(sc) >= CHELSIO_T6 || icp->icp_seq == tp->rcv_nxt);
535 	tp->rcv_nxt = icp->icp_seq + pdu_len;
536 	tp->t_rcvtime = ticks;
537 
538 	/*
539 	 * Don't update the window size or return credits since RX
540 	 * flow control is disabled.
541 	 */
542 
543 	so = inp->inp_socket;
544 	sb = &so->so_rcv;
545 	SOCKBUF_LOCK(sb);
546 
547 	icc = toep->ulpcb;
548 	if (__predict_false(icc == NULL || sb->sb_state & SBS_CANTRCVMORE)) {
549 		CTR5(KTR_CXGBE,
550 		    "%s: tid %u, excess rx (%d bytes), icc %p, sb_state 0x%x",
551 		    __func__, tid, pdu_len, icc, sb->sb_state);
552 		SOCKBUF_UNLOCK(sb);
553 		INP_WUNLOCK(inp);
554 
555 		CURVNET_SET(so->so_vnet);
556 		NET_EPOCH_ENTER(et);
557 		INP_WLOCK(inp);
558 		tp = tcp_drop(tp, ECONNRESET);
559 		if (tp)
560 			INP_WUNLOCK(inp);
561 		NET_EPOCH_EXIT(et);
562 		CURVNET_RESTORE();
563 
564 		icl_cxgbei_conn_pdu_free(NULL, ip);
565 		toep->ulpcb2 = NULL;
566 		return (0);
567 	}
568 	MPASS(icc->icc_signature == CXGBEI_CONN_SIGNATURE);
569 	ic = &icc->ic;
570 	if ((val & (F_DDP_PADDING_ERR | F_DDP_HDRCRC_ERR |
571 	    F_DDP_DATACRC_ERR)) != 0) {
572 		SOCKBUF_UNLOCK(sb);
573 		INP_WUNLOCK(inp);
574 
575 		icl_cxgbei_conn_pdu_free(NULL, ip);
576 		toep->ulpcb2 = NULL;
577 		ic->ic_error(ic);
578 		return (0);
579 	}
580 
581 	icl_cxgbei_new_pdu_set_conn(ip, ic);
582 
583 	STAILQ_INSERT_TAIL(&icc->rcvd_pdus, ip, ip_next);
584 	if (!icc->rx_active) {
585 		icc->rx_active = true;
586 		wakeup(&icc->rx_active);
587 	}
588 	SOCKBUF_UNLOCK(sb);
589 	INP_WUNLOCK(inp);
590 
591 	toep->ulpcb2 = NULL;
592 
593 	return (0);
594 }
595 
596 static int
597 do_rx_iscsi_cmp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
598 {
599 	struct epoch_tracker et;
600 	struct adapter *sc = iq->adapter;
601 	struct cpl_rx_iscsi_cmp *cpl = mtod(m, struct cpl_rx_iscsi_cmp *);
602 	u_int tid = GET_TID(cpl);
603 	struct toepcb *toep = lookup_tid(sc, tid);
604 	struct icl_cxgbei_pdu *icp = toep->ulpcb2;
605 	struct icl_pdu *ip;
606 	struct cxgbei_cmp *cmp;
607 	struct inpcb *inp = toep->inp;
608 #ifdef INVARIANTS
609 	uint16_t len = be16toh(cpl->len);
610 	u_int data_digest_len;
611 #endif
612 	struct socket *so;
613 	struct sockbuf *sb;
614 	struct tcpcb *tp;
615 	struct icl_cxgbei_conn *icc;
616 	struct icl_conn *ic;
617 	struct iscsi_bhs_data_out *bhsdo;
618 	u_int val = be32toh(cpl->ddpvld);
619 	u_int npdus, pdu_len;
620 	uint32_t prev_seg_len;
621 
622 	M_ASSERTPKTHDR(m);
623 	MPASS(m->m_pkthdr.len == len + sizeof(*cpl));
624 
625 	if ((val & F_DDP_PDU) == 0) {
626 		MPASS(icp != NULL);
627 		MPASS((icp->icp_flags & ICPF_RX_STATUS) == 0);
628 		ip = &icp->ip;
629 	}
630 
631 	if (icp == NULL) {
632 		/* T6 completion enabled, start of a new PDU. */
633 		ip = icl_cxgbei_new_pdu(M_NOWAIT);
634 		if (ip == NULL)
635 			CXGBE_UNIMPLEMENTED("PDU allocation failure");
636 		icp = ip_to_icp(ip);
637 	}
638 	pdu_len = G_ISCSI_PDU_LEN(be16toh(cpl->pdu_len_ddp));
639 
640 #if 0
641 	CTR5(KTR_CXGBE,
642 	    "%s: tid %u, cpl->len %u, ddpvld 0x%08x, icp %p",
643 	    __func__, tid, pdu_len, val, icp);
644 #endif
645 
646 	/* Copy header */
647 	m_copydata(m, sizeof(*cpl), ISCSI_BHS_SIZE, (caddr_t)ip->ip_bhs);
648 	bhsdo = (struct iscsi_bhs_data_out *)ip->ip_bhs;
649 	ip->ip_data_len = bhsdo->bhsdo_data_segment_len[0] << 16 |
650 	    bhsdo->bhsdo_data_segment_len[1] << 8 |
651 	    bhsdo->bhsdo_data_segment_len[2];
652 	icp->icp_seq = ntohl(cpl->seq);
653 	icp->icp_flags |= ICPF_RX_HDR;
654 	icp->icp_flags |= ICPF_RX_STATUS;
655 
656 	if (val & F_DDP_PADDING_ERR) {
657 		ICL_WARN("received PDU 0x%02x with invalid padding",
658 		    ip->ip_bhs->bhs_opcode);
659 		toep->ofld_rxq->rx_iscsi_padding_errors++;
660 	}
661 	if (val & F_DDP_HDRCRC_ERR) {
662 		ICL_WARN("received PDU 0x%02x with invalid header digest",
663 		    ip->ip_bhs->bhs_opcode);
664 		toep->ofld_rxq->rx_iscsi_header_digest_errors++;
665 	}
666 	if (val & F_DDP_DATACRC_ERR) {
667 		ICL_WARN("received PDU 0x%02x with invalid data digest",
668 		    ip->ip_bhs->bhs_opcode);
669 		toep->ofld_rxq->rx_iscsi_data_digest_errors++;
670 	}
671 
672 	INP_WLOCK(inp);
673 	if (__predict_false(inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT))) {
674 		CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x",
675 		    __func__, tid, pdu_len, inp->inp_flags);
676 		INP_WUNLOCK(inp);
677 		icl_cxgbei_conn_pdu_free(NULL, ip);
678 		toep->ulpcb2 = NULL;
679 		m_freem(m);
680 		return (0);
681 	}
682 
683 	tp = intotcpcb(inp);
684 
685 	/*
686 	 * If icc is NULL, the connection is being closed in
687 	 * icl_cxgbei_conn_close(), just drop this data.
688 	 */
689 	icc = toep->ulpcb;
690 	if (__predict_false(icc == NULL)) {
691 		CTR4(KTR_CXGBE, "%s: tid %u, excess rx (%d bytes), icc %p",
692 		    __func__, tid, pdu_len, icc);
693 
694 		/*
695 		 * Update rcv_nxt so the sequence number of the FIN
696 		 * doesn't appear wrong.
697 		 */
698 		tp->rcv_nxt = icp->icp_seq + pdu_len;
699 		tp->t_rcvtime = ticks;
700 		INP_WUNLOCK(inp);
701 
702 		icl_cxgbei_conn_pdu_free(NULL, ip);
703 		toep->ulpcb2 = NULL;
704 		m_freem(m);
705 		return (0);
706 	}
707 
708 	MPASS(icc->icc_signature == CXGBEI_CONN_SIGNATURE);
709 	ic = &icc->ic;
710 	if ((val & (F_DDP_PADDING_ERR | F_DDP_HDRCRC_ERR |
711 	    F_DDP_DATACRC_ERR)) != 0) {
712 		INP_WUNLOCK(inp);
713 
714 		icl_cxgbei_conn_pdu_free(NULL, ip);
715 		toep->ulpcb2 = NULL;
716 		m_freem(m);
717 		ic->ic_error(ic);
718 		return (0);
719 	}
720 
721 #ifdef INVARIANTS
722 	data_digest_len = (icc->ulp_submode & ULP_CRC_DATA) ?
723 	    ISCSI_DATA_DIGEST_SIZE : 0;
724 	MPASS(roundup2(ip->ip_data_len, 4) == pdu_len - len - data_digest_len);
725 #endif
726 
727 	if (val & F_DDP_PDU && ip->ip_data_mbuf == NULL) {
728 		MPASS((icp->icp_flags & ICPF_RX_FLBUF) == 0);
729 		MPASS(ip->ip_data_len > 0);
730 		icp->icp_flags |= ICPF_RX_DDP;
731 		bhsdo = (struct iscsi_bhs_data_out *)ip->ip_bhs;
732 
733 		switch (ip->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) {
734 		case ISCSI_BHS_OPCODE_SCSI_DATA_IN:
735 			cmp = cxgbei_find_cmp(icc,
736 			    be32toh(bhsdo->bhsdo_initiator_task_tag));
737 			break;
738 		case ISCSI_BHS_OPCODE_SCSI_DATA_OUT:
739 			cmp = cxgbei_find_cmp(icc,
740 			    be32toh(bhsdo->bhsdo_target_transfer_tag));
741 			break;
742 		default:
743 			__assert_unreachable();
744 		}
745 		MPASS(cmp != NULL);
746 
747 		/*
748 		 * The difference between the end of the last burst
749 		 * and the offset of the last PDU in this burst is
750 		 * the additional data received via DDP.
751 		 */
752 		prev_seg_len = be32toh(bhsdo->bhsdo_buffer_offset) -
753 		    cmp->next_buffer_offset;
754 
755 		if (prev_seg_len != 0) {
756 			uint32_t orig_datasn;
757 
758 			/*
759 			 * Return a "large" PDU representing the burst
760 			 * of PDUs.  Adjust the offset and length of
761 			 * this PDU to represent the entire burst.
762 			 */
763 			ip->ip_data_len += prev_seg_len;
764 			bhsdo->bhsdo_data_segment_len[2] = ip->ip_data_len;
765 			bhsdo->bhsdo_data_segment_len[1] = ip->ip_data_len >> 8;
766 			bhsdo->bhsdo_data_segment_len[0] = ip->ip_data_len >> 16;
767 			bhsdo->bhsdo_buffer_offset =
768 			    htobe32(cmp->next_buffer_offset);
769 
770 			orig_datasn = htobe32(bhsdo->bhsdo_datasn);
771 			npdus = orig_datasn - cmp->last_datasn;
772 			bhsdo->bhsdo_datasn = htobe32(cmp->last_datasn + 1);
773 			cmp->last_datasn = orig_datasn;
774 			ip->ip_additional_pdus = npdus - 1;
775 		} else {
776 			MPASS(htobe32(bhsdo->bhsdo_datasn) ==
777 			    cmp->last_datasn + 1);
778 			npdus = 1;
779 			cmp->last_datasn = htobe32(bhsdo->bhsdo_datasn);
780 		}
781 
782 		cmp->next_buffer_offset += ip->ip_data_len;
783 		toep->ofld_rxq->rx_iscsi_ddp_pdus += npdus;
784 		toep->ofld_rxq->rx_iscsi_ddp_octets += ip->ip_data_len;
785 	} else {
786 		MPASS(icp->icp_flags & (ICPF_RX_FLBUF));
787 		MPASS(ip->ip_data_len == ip->ip_data_mbuf->m_pkthdr.len);
788 	}
789 
790 	tp->rcv_nxt = icp->icp_seq + pdu_len;
791 	tp->t_rcvtime = ticks;
792 
793 	/*
794 	 * Don't update the window size or return credits since RX
795 	 * flow control is disabled.
796 	 */
797 
798 	so = inp->inp_socket;
799 	sb = &so->so_rcv;
800 	SOCKBUF_LOCK(sb);
801 	if (__predict_false(sb->sb_state & SBS_CANTRCVMORE)) {
802 		CTR5(KTR_CXGBE,
803 		    "%s: tid %u, excess rx (%d bytes), icc %p, sb_state 0x%x",
804 		    __func__, tid, pdu_len, icc, sb->sb_state);
805 		SOCKBUF_UNLOCK(sb);
806 		INP_WUNLOCK(inp);
807 
808 		CURVNET_SET(so->so_vnet);
809 		NET_EPOCH_ENTER(et);
810 		INP_WLOCK(inp);
811 		tp = tcp_drop(tp, ECONNRESET);
812 		if (tp != NULL)
813 			INP_WUNLOCK(inp);
814 		NET_EPOCH_EXIT(et);
815 		CURVNET_RESTORE();
816 
817 		icl_cxgbei_conn_pdu_free(NULL, ip);
818 		toep->ulpcb2 = NULL;
819 		m_freem(m);
820 		return (0);
821 	}
822 
823 	icl_cxgbei_new_pdu_set_conn(ip, ic);
824 
825 	/* Enqueue the PDU to the received pdus queue. */
826 	STAILQ_INSERT_TAIL(&icc->rcvd_pdus, ip, ip_next);
827 	if (!icc->rx_active) {
828 		icc->rx_active = true;
829 		wakeup(&icc->rx_active);
830 	}
831 	SOCKBUF_UNLOCK(sb);
832 	INP_WUNLOCK(inp);
833 
834 	toep->ulpcb2 = NULL;
835 	m_freem(m);
836 
837 	return (0);
838 }
839 
840 static int
841 cxgbei_activate(struct adapter *sc)
842 {
843 	struct cxgbei_data *ci;
844 	int rc;
845 
846 	ASSERT_SYNCHRONIZED_OP(sc);
847 
848 	if (uld_active(sc, ULD_ISCSI)) {
849 		KASSERT(0, ("%s: iSCSI offload already enabled on adapter %p",
850 		    __func__, sc));
851 		return (0);
852 	}
853 
854 	if (sc->iscsicaps == 0 || sc->vres.iscsi.size == 0) {
855 		device_printf(sc->dev,
856 		    "not iSCSI offload capable, or capability disabled.\n");
857 		return (ENOSYS);
858 	}
859 
860 	/* per-adapter softc for iSCSI */
861 	ci = malloc(sizeof(*ci), M_CXGBE, M_ZERO | M_WAITOK);
862 	if (ci == NULL)
863 		return (ENOMEM);
864 
865 	rc = cxgbei_init(sc, ci);
866 	if (rc != 0) {
867 		free(ci, M_CXGBE);
868 		return (rc);
869 	}
870 
871 	sc->iscsi_ulp_softc = ci;
872 
873 	return (0);
874 }
875 
876 static int
877 cxgbei_deactivate(struct adapter *sc)
878 {
879 	struct cxgbei_data *ci = sc->iscsi_ulp_softc;
880 
881 	ASSERT_SYNCHRONIZED_OP(sc);
882 
883 	if (ci != NULL) {
884 		sysctl_ctx_free(&ci->ctx);
885 		t4_free_ppod_region(&ci->pr);
886 		free(ci, M_CXGBE);
887 		sc->iscsi_ulp_softc = NULL;
888 	}
889 
890 	return (0);
891 }
892 
893 static void
894 cxgbei_activate_all(struct adapter *sc, void *arg __unused)
895 {
896 
897 	if (begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4isact") != 0)
898 		return;
899 
900 	/* Activate iSCSI if any port on this adapter has IFCAP_TOE enabled. */
901 	if (sc->offload_map && !uld_active(sc, ULD_ISCSI))
902 		(void) t4_activate_uld(sc, ULD_ISCSI);
903 
904 	end_synchronized_op(sc, 0);
905 }
906 
907 static void
908 cxgbei_deactivate_all(struct adapter *sc, void *arg __unused)
909 {
910 
911 	if (begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4isdea") != 0)
912 		return;
913 
914 	if (uld_active(sc, ULD_ISCSI))
915 	    (void) t4_deactivate_uld(sc, ULD_ISCSI);
916 
917 	end_synchronized_op(sc, 0);
918 }
919 
920 static struct uld_info cxgbei_uld_info = {
921 	.uld_id = ULD_ISCSI,
922 	.activate = cxgbei_activate,
923 	.deactivate = cxgbei_deactivate,
924 };
925 
926 static int
927 cxgbei_mod_load(void)
928 {
929 	int rc;
930 
931 	t4_register_cpl_handler(CPL_ISCSI_HDR, do_rx_iscsi_hdr);
932 	t4_register_cpl_handler(CPL_ISCSI_DATA, do_rx_iscsi_data);
933 	t4_register_cpl_handler(CPL_RX_ISCSI_DDP, do_rx_iscsi_ddp);
934 	t4_register_cpl_handler(CPL_RX_ISCSI_CMP, do_rx_iscsi_cmp);
935 
936 	rc = t4_register_uld(&cxgbei_uld_info);
937 	if (rc != 0)
938 		return (rc);
939 
940 	t4_iterate(cxgbei_activate_all, NULL);
941 
942 	return (rc);
943 }
944 
945 static int
946 cxgbei_mod_unload(void)
947 {
948 
949 	t4_iterate(cxgbei_deactivate_all, NULL);
950 
951 	if (t4_unregister_uld(&cxgbei_uld_info) == EBUSY)
952 		return (EBUSY);
953 
954 	t4_register_cpl_handler(CPL_ISCSI_HDR, NULL);
955 	t4_register_cpl_handler(CPL_ISCSI_DATA, NULL);
956 	t4_register_cpl_handler(CPL_RX_ISCSI_DDP, NULL);
957 	t4_register_cpl_handler(CPL_RX_ISCSI_CMP, NULL);
958 
959 	return (0);
960 }
961 #endif
962 
963 static int
964 cxgbei_modevent(module_t mod, int cmd, void *arg)
965 {
966 	int rc = 0;
967 
968 #ifdef TCP_OFFLOAD
969 	switch (cmd) {
970 	case MOD_LOAD:
971 		rc = cxgbei_mod_load();
972 		if (rc == 0)
973 			rc = icl_cxgbei_mod_load();
974 		break;
975 
976 	case MOD_UNLOAD:
977 		rc = icl_cxgbei_mod_unload();
978 		if (rc == 0)
979 			rc = cxgbei_mod_unload();
980 		break;
981 
982 	default:
983 		rc = EINVAL;
984 	}
985 #else
986 	printf("cxgbei: compiled without TCP_OFFLOAD support.\n");
987 	rc = EOPNOTSUPP;
988 #endif
989 
990 	return (rc);
991 }
992 
993 static moduledata_t cxgbei_mod = {
994 	"cxgbei",
995 	cxgbei_modevent,
996 	NULL,
997 };
998 
999 MODULE_VERSION(cxgbei, 1);
1000 DECLARE_MODULE(cxgbei, cxgbei_mod, SI_SUB_EXEC, SI_ORDER_ANY);
1001 MODULE_DEPEND(cxgbei, t4_tom, 1, 1, 1);
1002 MODULE_DEPEND(cxgbei, cxgbe, 1, 1, 1);
1003 MODULE_DEPEND(cxgbei, icl, 1, 1, 1);
1004