xref: /freebsd/sys/dev/cxgbe/cxgbei/icl_cxgbei.c (revision 2eb0e53a6b5ec1a72be70e966d4e562e1a8d4e88)
1 /*-
2  * Copyright (c) 2012 The FreeBSD Foundation
3  * Copyright (c) 2015 Chelsio Communications, Inc.
4  * All rights reserved.
5  *
6  * This software was developed by Edward Tomasz Napierala under sponsorship
7  * from the FreeBSD Foundation.
8  *
9  * Redistribution and use in source and binary forms, with or without
10  * modification, are permitted provided that the following conditions
11  * are met:
12  * 1. Redistributions of source code must retain the above copyright
13  *    notice, this list of conditions and the following disclaimer.
14  * 2. Redistributions in binary form must reproduce the above copyright
15  *    notice, this list of conditions and the following disclaimer in the
16  *    documentation and/or other materials provided with the distribution.
17  *
18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28  * SUCH DAMAGE.
29  *
30  */
31 
32 /*
33  * cxgbei implementation of iSCSI Common Layer kobj(9) interface.
34  */
35 
36 #include <sys/cdefs.h>
37 __FBSDID("$FreeBSD$");
38 
39 #include "opt_inet.h"
40 #include "opt_inet6.h"
41 
42 #ifdef TCP_OFFLOAD
43 #include <sys/param.h>
44 #include <sys/capsicum.h>
45 #include <sys/condvar.h>
46 #include <sys/conf.h>
47 #include <sys/file.h>
48 #include <sys/kernel.h>
49 #include <sys/kthread.h>
50 #include <sys/ktr.h>
51 #include <sys/lock.h>
52 #include <sys/mbuf.h>
53 #include <sys/mutex.h>
54 #include <sys/module.h>
55 #include <sys/protosw.h>
56 #include <sys/socket.h>
57 #include <sys/socketvar.h>
58 #include <sys/sysctl.h>
59 #include <sys/systm.h>
60 #include <sys/sx.h>
61 #include <sys/uio.h>
62 #include <machine/bus.h>
63 #include <vm/vm.h>
64 #include <vm/pmap.h>
65 #include <netinet/in.h>
66 #include <netinet/in_pcb.h>
67 #include <netinet/tcp.h>
68 #include <netinet/tcp_var.h>
69 #include <netinet/toecore.h>
70 
71 #include <dev/iscsi/icl.h>
72 #include <dev/iscsi/iscsi_proto.h>
73 #include <icl_conn_if.h>
74 
75 #include <cam/scsi/scsi_all.h>
76 #include <cam/scsi/scsi_da.h>
77 #include <cam/ctl/ctl_io.h>
78 #include <cam/ctl/ctl.h>
79 #include <cam/ctl/ctl_backend.h>
80 #include <cam/ctl/ctl_error.h>
81 #include <cam/ctl/ctl_frontend.h>
82 #include <cam/ctl/ctl_debug.h>
83 #include <cam/ctl/ctl_ha.h>
84 #include <cam/ctl/ctl_ioctl.h>
85 
86 #include <cam/cam.h>
87 #include <cam/cam_ccb.h>
88 #include <cam/cam_xpt.h>
89 #include <cam/cam_debug.h>
90 #include <cam/cam_sim.h>
91 #include <cam/cam_xpt_sim.h>
92 #include <cam/cam_xpt_periph.h>
93 #include <cam/cam_periph.h>
94 #include <cam/cam_compat.h>
95 #include <cam/scsi/scsi_message.h>
96 
97 #include "common/common.h"
98 #include "common/t4_tcb.h"
99 #include "tom/t4_tom.h"
100 #include "cxgbei.h"
101 
102 /*
103  * Use the page pod tag for the TT hash.
104  */
105 #define	TT_HASH(icc, tt)	(G_PPOD_TAG(tt) & (icc)->cmp_hash_mask)
106 
107 struct cxgbei_ddp_state {
108 	struct ppod_reservation prsv;
109 	struct cxgbei_cmp cmp;
110 };
111 
112 static MALLOC_DEFINE(M_CXGBEI, "cxgbei", "cxgbei(4)");
113 
114 SYSCTL_NODE(_kern_icl, OID_AUTO, cxgbei, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
115     "Chelsio iSCSI offload");
116 static int first_burst_length = 8192;
117 SYSCTL_INT(_kern_icl_cxgbei, OID_AUTO, first_burst_length, CTLFLAG_RWTUN,
118     &first_burst_length, 0, "First burst length");
119 static int max_burst_length = 2 * 1024 * 1024;
120 SYSCTL_INT(_kern_icl_cxgbei, OID_AUTO, max_burst_length, CTLFLAG_RWTUN,
121     &max_burst_length, 0, "Maximum burst length");
122 static int sendspace = 1048576;
123 SYSCTL_INT(_kern_icl_cxgbei, OID_AUTO, sendspace, CTLFLAG_RWTUN,
124     &sendspace, 0, "Default send socket buffer size");
125 static int recvspace = 1048576;
126 SYSCTL_INT(_kern_icl_cxgbei, OID_AUTO, recvspace, CTLFLAG_RWTUN,
127     &recvspace, 0, "Default receive socket buffer size");
128 
129 static volatile u_int icl_cxgbei_ncons;
130 
131 #define ICL_CONN_LOCK(X)		mtx_lock(X->ic_lock)
132 #define ICL_CONN_UNLOCK(X)		mtx_unlock(X->ic_lock)
133 #define ICL_CONN_LOCK_ASSERT(X)		mtx_assert(X->ic_lock, MA_OWNED)
134 #define ICL_CONN_LOCK_ASSERT_NOT(X)	mtx_assert(X->ic_lock, MA_NOTOWNED)
135 
136 static icl_conn_new_pdu_t	icl_cxgbei_conn_new_pdu;
137 static icl_conn_pdu_data_segment_length_t
138 				    icl_cxgbei_conn_pdu_data_segment_length;
139 static icl_conn_pdu_append_data_t	icl_cxgbei_conn_pdu_append_data;
140 static icl_conn_pdu_get_data_t	icl_cxgbei_conn_pdu_get_data;
141 static icl_conn_pdu_queue_t	icl_cxgbei_conn_pdu_queue;
142 static icl_conn_pdu_queue_cb_t	icl_cxgbei_conn_pdu_queue_cb;
143 static icl_conn_handoff_t	icl_cxgbei_conn_handoff;
144 static icl_conn_free_t		icl_cxgbei_conn_free;
145 static icl_conn_close_t		icl_cxgbei_conn_close;
146 static icl_conn_task_setup_t	icl_cxgbei_conn_task_setup;
147 static icl_conn_task_done_t	icl_cxgbei_conn_task_done;
148 static icl_conn_transfer_setup_t	icl_cxgbei_conn_transfer_setup;
149 static icl_conn_transfer_done_t	icl_cxgbei_conn_transfer_done;
150 
151 static kobj_method_t icl_cxgbei_methods[] = {
152 	KOBJMETHOD(icl_conn_new_pdu, icl_cxgbei_conn_new_pdu),
153 	KOBJMETHOD(icl_conn_pdu_free, icl_cxgbei_conn_pdu_free),
154 	KOBJMETHOD(icl_conn_pdu_data_segment_length,
155 	    icl_cxgbei_conn_pdu_data_segment_length),
156 	KOBJMETHOD(icl_conn_pdu_append_data, icl_cxgbei_conn_pdu_append_data),
157 	KOBJMETHOD(icl_conn_pdu_get_data, icl_cxgbei_conn_pdu_get_data),
158 	KOBJMETHOD(icl_conn_pdu_queue, icl_cxgbei_conn_pdu_queue),
159 	KOBJMETHOD(icl_conn_pdu_queue_cb, icl_cxgbei_conn_pdu_queue_cb),
160 	KOBJMETHOD(icl_conn_handoff, icl_cxgbei_conn_handoff),
161 	KOBJMETHOD(icl_conn_free, icl_cxgbei_conn_free),
162 	KOBJMETHOD(icl_conn_close, icl_cxgbei_conn_close),
163 	KOBJMETHOD(icl_conn_task_setup, icl_cxgbei_conn_task_setup),
164 	KOBJMETHOD(icl_conn_task_done, icl_cxgbei_conn_task_done),
165 	KOBJMETHOD(icl_conn_transfer_setup, icl_cxgbei_conn_transfer_setup),
166 	KOBJMETHOD(icl_conn_transfer_done, icl_cxgbei_conn_transfer_done),
167 	{ 0, 0 }
168 };
169 
170 DEFINE_CLASS(icl_cxgbei, icl_cxgbei_methods, sizeof(struct icl_cxgbei_conn));
171 
172 void
173 icl_cxgbei_conn_pdu_free(struct icl_conn *ic, struct icl_pdu *ip)
174 {
175 	struct icl_cxgbei_pdu *icp = ip_to_icp(ip);
176 
177 	KASSERT(icp->ref_cnt != 0, ("freeing deleted PDU"));
178 	MPASS(icp->icp_signature == CXGBEI_PDU_SIGNATURE);
179 	MPASS(ic == ip->ip_conn);
180 
181 	m_freem(ip->ip_ahs_mbuf);
182 	m_freem(ip->ip_data_mbuf);
183 	m_freem(ip->ip_bhs_mbuf);
184 
185 	KASSERT(ic != NULL || icp->ref_cnt == 1,
186 	    ("orphaned PDU has oustanding references"));
187 
188 	if (atomic_fetchadd_int(&icp->ref_cnt, -1) != 1)
189 		return;
190 
191 	free(icp, M_CXGBEI);
192 #ifdef DIAGNOSTIC
193 	if (__predict_true(ic != NULL))
194 		refcount_release(&ic->ic_outstanding_pdus);
195 #endif
196 }
197 
198 static void
199 icl_cxgbei_pdu_call_cb(struct icl_pdu *ip)
200 {
201 	struct icl_cxgbei_pdu *icp = ip_to_icp(ip);
202 
203 	MPASS(icp->icp_signature == CXGBEI_PDU_SIGNATURE);
204 
205 	if (icp->cb != NULL)
206 		icp->cb(ip, icp->error);
207 #ifdef DIAGNOSTIC
208 	if (__predict_true(ip->ip_conn != NULL))
209 		refcount_release(&ip->ip_conn->ic_outstanding_pdus);
210 #endif
211 	free(icp, M_CXGBEI);
212 }
213 
214 static void
215 icl_cxgbei_pdu_done(struct icl_pdu *ip, int error)
216 {
217 	struct icl_cxgbei_pdu *icp = ip_to_icp(ip);
218 
219 	if (error != 0)
220 		icp->error = error;
221 
222 	m_freem(ip->ip_ahs_mbuf);
223 	ip->ip_ahs_mbuf = NULL;
224 	m_freem(ip->ip_data_mbuf);
225 	ip->ip_data_mbuf = NULL;
226 	m_freem(ip->ip_bhs_mbuf);
227 	ip->ip_bhs_mbuf = NULL;
228 
229 	/*
230 	 * All other references to this PDU should have been dropped
231 	 * by the m_freem() of ip_data_mbuf.
232 	 */
233 	if (atomic_fetchadd_int(&icp->ref_cnt, -1) == 1)
234 		icl_cxgbei_pdu_call_cb(ip);
235 	else
236 		__assert_unreachable();
237 }
238 
239 static void
240 icl_cxgbei_mbuf_done(struct mbuf *mb)
241 {
242 
243 	struct icl_cxgbei_pdu *icp = (struct icl_cxgbei_pdu *)mb->m_ext.ext_arg1;
244 
245 	/*
246 	 * NB: mb_free_mext() might leave ref_cnt as 1 without
247 	 * decrementing it if it hits the fast path in the ref_cnt
248 	 * check.
249 	 */
250 	icl_cxgbei_pdu_call_cb(&icp->ip);
251 }
252 
253 struct icl_pdu *
254 icl_cxgbei_new_pdu(int flags)
255 {
256 	struct icl_cxgbei_pdu *icp;
257 	struct icl_pdu *ip;
258 	struct mbuf *m;
259 
260 	icp = malloc(sizeof(*icp), M_CXGBEI, flags | M_ZERO);
261 	if (__predict_false(icp == NULL))
262 		return (NULL);
263 
264 	icp->icp_signature = CXGBEI_PDU_SIGNATURE;
265 	icp->ref_cnt = 1;
266 	ip = &icp->ip;
267 
268 	m = m_gethdr(flags, MT_DATA);
269 	if (__predict_false(m == NULL)) {
270 		free(icp, M_CXGBEI);
271 		return (NULL);
272 	}
273 
274 	ip->ip_bhs_mbuf = m;
275 	ip->ip_bhs = mtod(m, struct iscsi_bhs *);
276 	memset(ip->ip_bhs, 0, sizeof(*ip->ip_bhs));
277 	m->m_len = sizeof(struct iscsi_bhs);
278 	m->m_pkthdr.len = m->m_len;
279 
280 	return (ip);
281 }
282 
283 void
284 icl_cxgbei_new_pdu_set_conn(struct icl_pdu *ip, struct icl_conn *ic)
285 {
286 
287 	ip->ip_conn = ic;
288 #ifdef DIAGNOSTIC
289 	refcount_acquire(&ic->ic_outstanding_pdus);
290 #endif
291 }
292 
293 /*
294  * Allocate icl_pdu with empty BHS to fill up by the caller.
295  */
296 static struct icl_pdu *
297 icl_cxgbei_conn_new_pdu(struct icl_conn *ic, int flags)
298 {
299 	struct icl_pdu *ip;
300 
301 	ip = icl_cxgbei_new_pdu(flags);
302 	if (__predict_false(ip == NULL))
303 		return (NULL);
304 	icl_cxgbei_new_pdu_set_conn(ip, ic);
305 
306 	return (ip);
307 }
308 
309 static size_t
310 icl_pdu_data_segment_length(const struct icl_pdu *request)
311 {
312 	uint32_t len = 0;
313 
314 	len += request->ip_bhs->bhs_data_segment_len[0];
315 	len <<= 8;
316 	len += request->ip_bhs->bhs_data_segment_len[1];
317 	len <<= 8;
318 	len += request->ip_bhs->bhs_data_segment_len[2];
319 
320 	return (len);
321 }
322 
323 size_t
324 icl_cxgbei_conn_pdu_data_segment_length(struct icl_conn *ic,
325     const struct icl_pdu *request)
326 {
327 
328 	return (icl_pdu_data_segment_length(request));
329 }
330 
331 static struct mbuf *
332 finalize_pdu(struct icl_cxgbei_conn *icc, struct icl_cxgbei_pdu *icp)
333 {
334 	struct icl_pdu *ip = &icp->ip;
335 	uint8_t ulp_submode, padding;
336 	struct mbuf *m, *last;
337 	struct iscsi_bhs *bhs;
338 	int data_len;
339 
340 	/*
341 	 * Fix up the data segment mbuf first.
342 	 */
343 	m = ip->ip_data_mbuf;
344 	ulp_submode = icc->ulp_submode;
345 	if (m != NULL) {
346 		last = m_last(m);
347 
348 		/*
349 		 * Round up the data segment to a 4B boundary.	Pad with 0 if
350 		 * necessary.  There will definitely be room in the mbuf.
351 		 */
352 		padding = roundup2(ip->ip_data_len, 4) - ip->ip_data_len;
353 		if (padding != 0) {
354 			MPASS(padding <= M_TRAILINGSPACE(last));
355 			bzero(mtod(last, uint8_t *) + last->m_len, padding);
356 			last->m_len += padding;
357 		}
358 	} else {
359 		MPASS(ip->ip_data_len == 0);
360 		ulp_submode &= ~ULP_CRC_DATA;
361 		padding = 0;
362 	}
363 
364 	/*
365 	 * Now the header mbuf that has the BHS.
366 	 */
367 	m = ip->ip_bhs_mbuf;
368 	MPASS(m->m_pkthdr.len == sizeof(struct iscsi_bhs));
369 	MPASS(m->m_len == sizeof(struct iscsi_bhs));
370 
371 	bhs = ip->ip_bhs;
372 	data_len = ip->ip_data_len;
373 	if (data_len > icc->ic.ic_max_send_data_segment_length) {
374 		struct iscsi_bhs_data_in *bhsdi;
375 		int flags;
376 
377 		KASSERT(padding == 0, ("%s: ISO with padding %d for icp %p",
378 		    __func__, padding, icp));
379 		switch (bhs->bhs_opcode) {
380 		case ISCSI_BHS_OPCODE_SCSI_DATA_OUT:
381 			flags = 1;
382 			break;
383 		case ISCSI_BHS_OPCODE_SCSI_DATA_IN:
384 			flags = 2;
385 			break;
386 		default:
387 			panic("invalid opcode %#x for ISO", bhs->bhs_opcode);
388 		}
389 		data_len = icc->ic.ic_max_send_data_segment_length;
390 		bhsdi = (struct iscsi_bhs_data_in *)bhs;
391 		if (bhsdi->bhsdi_flags & BHSDI_FLAGS_F) {
392 			/*
393 			 * Firmware will set F on the final PDU in the
394 			 * burst.
395 			 */
396 			flags |= CXGBE_ISO_F;
397 			bhsdi->bhsdi_flags &= ~BHSDI_FLAGS_F;
398 		}
399 		set_mbuf_iscsi_iso(m, true);
400 		set_mbuf_iscsi_iso_flags(m, flags);
401 		set_mbuf_iscsi_iso_mss(m, data_len);
402 	}
403 
404 	bhs->bhs_data_segment_len[2] = data_len;
405 	bhs->bhs_data_segment_len[1] = data_len >> 8;
406 	bhs->bhs_data_segment_len[0] = data_len >> 16;
407 
408 	/*
409 	 * Extract mbuf chain from PDU.
410 	 */
411 	m->m_pkthdr.len += ip->ip_data_len + padding;
412 	m->m_next = ip->ip_data_mbuf;
413 	set_mbuf_ulp_submode(m, ulp_submode);
414 	ip->ip_bhs_mbuf = NULL;
415 	ip->ip_data_mbuf = NULL;
416 	ip->ip_bhs = NULL;
417 
418 	/*
419 	 * Drop PDU reference on icp.  Additional references might
420 	 * still be held by zero-copy PDU buffers (ICL_NOCOPY).
421 	 */
422 	if (atomic_fetchadd_int(&icp->ref_cnt, -1) == 1)
423 		icl_cxgbei_pdu_call_cb(ip);
424 
425 	return (m);
426 }
427 
428 int
429 icl_cxgbei_conn_pdu_append_data(struct icl_conn *ic, struct icl_pdu *ip,
430     const void *addr, size_t len, int flags)
431 {
432 	struct icl_cxgbei_pdu *icp = ip_to_icp(ip);
433 	struct mbuf *m, *m_tail;
434 	const char *src;
435 
436 	MPASS(icp->icp_signature == CXGBEI_PDU_SIGNATURE);
437 	MPASS(ic == ip->ip_conn);
438 	KASSERT(len > 0, ("%s: len is %jd", __func__, (intmax_t)len));
439 
440 	m_tail = ip->ip_data_mbuf;
441 	if (m_tail != NULL)
442 		for (; m_tail->m_next != NULL; m_tail = m_tail->m_next)
443 			;
444 
445 	if (flags & ICL_NOCOPY) {
446 		m = m_get(flags & ~ICL_NOCOPY, MT_DATA);
447 		if (m == NULL) {
448 			ICL_WARN("failed to allocate mbuf");
449 			return (ENOMEM);
450 		}
451 
452 		m->m_flags |= M_RDONLY;
453 		m_extaddref(m, __DECONST(char *, addr), len, &icp->ref_cnt,
454 		    icl_cxgbei_mbuf_done, icp, NULL);
455 		m->m_len = len;
456 		if (ip->ip_data_mbuf == NULL) {
457 			ip->ip_data_mbuf = m;
458 			ip->ip_data_len = len;
459 		} else {
460 			m_tail->m_next = m;
461 			m_tail = m_tail->m_next;
462 			ip->ip_data_len += len;
463 		}
464 
465 		return (0);
466 	}
467 
468 	src = (const char *)addr;
469 
470 	/* Allocate as jumbo mbufs of size MJUM16BYTES. */
471 	while (len >= MJUM16BYTES) {
472 		m = m_getjcl(M_NOWAIT, MT_DATA, 0, MJUM16BYTES);
473 		if (__predict_false(m == NULL)) {
474 			if ((flags & M_WAITOK) != 0) {
475 				/* Fall back to non-jumbo mbufs. */
476 				break;
477 			}
478 			return (ENOMEM);
479 		}
480 		memcpy(mtod(m, void *), src, MJUM16BYTES);
481 		m->m_len = MJUM16BYTES;
482 		if (ip->ip_data_mbuf == NULL) {
483 			ip->ip_data_mbuf = m_tail = m;
484 			ip->ip_data_len = MJUM16BYTES;
485 		} else {
486 			m_tail->m_next = m;
487 			m_tail = m_tail->m_next;
488 			ip->ip_data_len += MJUM16BYTES;
489 		}
490 		src += MJUM16BYTES;
491 		len -= MJUM16BYTES;
492 	}
493 
494 	/* Allocate mbuf chain for the remaining data. */
495 	if (len != 0) {
496 		m = m_getm2(NULL, len, flags, MT_DATA, 0);
497 		if (__predict_false(m == NULL))
498 			return (ENOMEM);
499 		if (ip->ip_data_mbuf == NULL) {
500 			ip->ip_data_mbuf = m;
501 			ip->ip_data_len = len;
502 		} else {
503 			m_tail->m_next = m;
504 			ip->ip_data_len += len;
505 		}
506 		for (; m != NULL; m = m->m_next) {
507 			m->m_len = min(len, M_SIZE(m));
508 			memcpy(mtod(m, void *), src, m->m_len);
509 			src += m->m_len;
510 			len -= m->m_len;
511 		}
512 		MPASS(len == 0);
513 	}
514 	MPASS(ip->ip_data_len <= max(ic->ic_max_send_data_segment_length,
515 	    ic->ic_hw_isomax));
516 
517 	return (0);
518 }
519 
520 void
521 icl_cxgbei_conn_pdu_get_data(struct icl_conn *ic, struct icl_pdu *ip,
522     size_t off, void *addr, size_t len)
523 {
524 	struct icl_cxgbei_pdu *icp = ip_to_icp(ip);
525 
526 	if (icp->icp_flags & ICPF_RX_DDP)
527 		return; /* data is DDP'ed, no need to copy */
528 	m_copydata(ip->ip_data_mbuf, off, len, addr);
529 }
530 
531 void
532 icl_cxgbei_conn_pdu_queue(struct icl_conn *ic, struct icl_pdu *ip)
533 {
534 	icl_cxgbei_conn_pdu_queue_cb(ic, ip, NULL);
535 }
536 
537 void
538 icl_cxgbei_conn_pdu_queue_cb(struct icl_conn *ic, struct icl_pdu *ip,
539 			     icl_pdu_cb cb)
540 {
541 	struct epoch_tracker et;
542 	struct icl_cxgbei_conn *icc = ic_to_icc(ic);
543 	struct icl_cxgbei_pdu *icp = ip_to_icp(ip);
544 	struct socket *so = ic->ic_socket;
545 	struct toepcb *toep = icc->toep;
546 	struct inpcb *inp;
547 	struct mbuf *m;
548 
549 	MPASS(ic == ip->ip_conn);
550 	MPASS(ip->ip_bhs_mbuf != NULL);
551 	/* The kernel doesn't generate PDUs with AHS. */
552 	MPASS(ip->ip_ahs_mbuf == NULL && ip->ip_ahs_len == 0);
553 
554 	ICL_CONN_LOCK_ASSERT(ic);
555 
556 	icp->cb = cb;
557 
558 	/* NOTE: sowriteable without so_snd lock is a mostly harmless race. */
559 	if (ic->ic_disconnecting || so == NULL || !sowriteable(so)) {
560 		icl_cxgbei_pdu_done(ip, ENOTCONN);
561 		return;
562 	}
563 
564 	m = finalize_pdu(icc, icp);
565 	M_ASSERTPKTHDR(m);
566 	MPASS((m->m_pkthdr.len & 3) == 0);
567 
568 	/*
569 	 * Do not get inp from toep->inp as the toepcb might have detached
570 	 * already.
571 	 */
572 	inp = sotoinpcb(so);
573 	CURVNET_SET(toep->vnet);
574 	NET_EPOCH_ENTER(et);
575 	INP_WLOCK(inp);
576 	if (__predict_false(inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) ||
577 	    __predict_false((toep->flags & TPF_ATTACHED) == 0))
578 		m_freem(m);
579 	else {
580 		mbufq_enqueue(&toep->ulp_pduq, m);
581 		t4_push_pdus(icc->sc, toep, 0);
582 	}
583 	INP_WUNLOCK(inp);
584 	NET_EPOCH_EXIT(et);
585 	CURVNET_RESTORE();
586 }
587 
588 static struct icl_conn *
589 icl_cxgbei_new_conn(const char *name, struct mtx *lock)
590 {
591 	struct icl_cxgbei_conn *icc;
592 	struct icl_conn *ic;
593 
594 	refcount_acquire(&icl_cxgbei_ncons);
595 
596 	icc = (struct icl_cxgbei_conn *)kobj_create(&icl_cxgbei_class, M_CXGBE,
597 	    M_WAITOK | M_ZERO);
598 	icc->icc_signature = CXGBEI_CONN_SIGNATURE;
599 	STAILQ_INIT(&icc->rcvd_pdus);
600 
601 	icc->cmp_table = hashinit(64, M_CXGBEI, &icc->cmp_hash_mask);
602 	mtx_init(&icc->cmp_lock, "cxgbei_cmp", NULL, MTX_DEF);
603 
604 	ic = &icc->ic;
605 	ic->ic_lock = lock;
606 
607 #ifdef DIAGNOSTIC
608 	refcount_init(&ic->ic_outstanding_pdus, 0);
609 #endif
610 	ic->ic_name = name;
611 	ic->ic_offload = "cxgbei";
612 	ic->ic_unmapped = false;
613 
614 	CTR2(KTR_CXGBE, "%s: icc %p", __func__, icc);
615 
616 	return (ic);
617 }
618 
619 void
620 icl_cxgbei_conn_free(struct icl_conn *ic)
621 {
622 	struct icl_cxgbei_conn *icc = ic_to_icc(ic);
623 
624 	MPASS(icc->icc_signature == CXGBEI_CONN_SIGNATURE);
625 
626 	CTR2(KTR_CXGBE, "%s: icc %p", __func__, icc);
627 
628 	mtx_destroy(&icc->cmp_lock);
629 	hashdestroy(icc->cmp_table, M_CXGBEI, icc->cmp_hash_mask);
630 	kobj_delete((struct kobj *)icc, M_CXGBE);
631 	refcount_release(&icl_cxgbei_ncons);
632 }
633 
634 static int
635 icl_cxgbei_setsockopt(struct icl_conn *ic, struct socket *so, int sspace,
636     int rspace)
637 {
638 	struct sockopt opt;
639 	int error, one = 1, ss, rs;
640 
641 	ss = max(sendspace, sspace);
642 	rs = max(recvspace, rspace);
643 
644 	error = soreserve(so, ss, rs);
645 	if (error != 0) {
646 		icl_cxgbei_conn_close(ic);
647 		return (error);
648 	}
649 	SOCKBUF_LOCK(&so->so_snd);
650 	so->so_snd.sb_flags |= SB_AUTOSIZE;
651 	SOCKBUF_UNLOCK(&so->so_snd);
652 	SOCKBUF_LOCK(&so->so_rcv);
653 	so->so_rcv.sb_flags |= SB_AUTOSIZE;
654 	SOCKBUF_UNLOCK(&so->so_rcv);
655 
656 	/*
657 	 * Disable Nagle.
658 	 */
659 	bzero(&opt, sizeof(opt));
660 	opt.sopt_dir = SOPT_SET;
661 	opt.sopt_level = IPPROTO_TCP;
662 	opt.sopt_name = TCP_NODELAY;
663 	opt.sopt_val = &one;
664 	opt.sopt_valsize = sizeof(one);
665 	error = sosetopt(so, &opt);
666 	if (error != 0) {
667 		icl_cxgbei_conn_close(ic);
668 		return (error);
669 	}
670 
671 	return (0);
672 }
673 
674 /*
675  * Request/response structure used to find out the adapter offloading a socket.
676  */
677 struct find_ofld_adapter_rr {
678 	struct socket *so;
679 	struct adapter *sc;	/* result */
680 };
681 
682 static void
683 find_offload_adapter(struct adapter *sc, void *arg)
684 {
685 	struct find_ofld_adapter_rr *fa = arg;
686 	struct socket *so = fa->so;
687 	struct tom_data *td = sc->tom_softc;
688 	struct tcpcb *tp;
689 	struct inpcb *inp;
690 
691 	/* Non-TCP were filtered out earlier. */
692 	MPASS(so->so_proto->pr_protocol == IPPROTO_TCP);
693 
694 	if (fa->sc != NULL)
695 		return;	/* Found already. */
696 
697 	if (td == NULL)
698 		return;	/* TOE not enabled on this adapter. */
699 
700 	inp = sotoinpcb(so);
701 	INP_WLOCK(inp);
702 	if ((inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) == 0) {
703 		tp = intotcpcb(inp);
704 		if (tp->t_flags & TF_TOE && tp->tod == &td->tod)
705 			fa->sc = sc;	/* Found. */
706 	}
707 	INP_WUNLOCK(inp);
708 }
709 
710 /* XXXNP: move this to t4_tom. */
711 static void
712 send_iscsi_flowc_wr(struct adapter *sc, struct toepcb *toep, int maxlen)
713 {
714 	struct wrqe *wr;
715 	struct fw_flowc_wr *flowc;
716 	const u_int nparams = 1;
717 	u_int flowclen;
718 	struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx];
719 
720 	flowclen = sizeof(*flowc) + nparams * sizeof(struct fw_flowc_mnemval);
721 
722 	wr = alloc_wrqe(roundup2(flowclen, 16), &toep->ofld_txq->wrq);
723 	if (wr == NULL) {
724 		/* XXX */
725 		panic("%s: allocation failure.", __func__);
726 	}
727 	flowc = wrtod(wr);
728 	memset(flowc, 0, wr->wr_len);
729 
730 	flowc->op_to_nparams = htobe32(V_FW_WR_OP(FW_FLOWC_WR) |
731 	    V_FW_FLOWC_WR_NPARAMS(nparams));
732 	flowc->flowid_len16 = htonl(V_FW_WR_LEN16(howmany(flowclen, 16)) |
733 	    V_FW_WR_FLOWID(toep->tid));
734 
735 	flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_TXDATAPLEN_MAX;
736 	flowc->mnemval[0].val = htobe32(maxlen);
737 
738 	txsd->tx_credits = howmany(flowclen, 16);
739 	txsd->plen = 0;
740 	KASSERT(toep->tx_credits >= txsd->tx_credits && toep->txsd_avail > 0,
741 	    ("%s: not enough credits (%d)", __func__, toep->tx_credits));
742 	toep->tx_credits -= txsd->tx_credits;
743 	if (__predict_false(++toep->txsd_pidx == toep->txsd_total))
744 		toep->txsd_pidx = 0;
745 	toep->txsd_avail--;
746 
747 	t4_wrq_tx(sc, wr);
748 }
749 
750 static void
751 set_ulp_mode_iscsi(struct adapter *sc, struct toepcb *toep, u_int ulp_submode)
752 {
753 	uint64_t val;
754 
755 	CTR3(KTR_CXGBE, "%s: tid %u, ULP_MODE_ISCSI, submode=%#x",
756 	    __func__, toep->tid, ulp_submode);
757 
758 	val = V_TCB_ULP_TYPE(ULP_MODE_ISCSI) | V_TCB_ULP_RAW(ulp_submode);
759 	t4_set_tcb_field(sc, toep->ctrlq, toep, W_TCB_ULP_TYPE,
760 	    V_TCB_ULP_TYPE(M_TCB_ULP_TYPE) | V_TCB_ULP_RAW(M_TCB_ULP_RAW), val,
761 	    0, 0);
762 
763 	val = V_TF_RX_FLOW_CONTROL_DISABLE(1ULL);
764 	t4_set_tcb_field(sc, toep->ctrlq, toep, W_TCB_T_FLAGS, val, val, 0, 0);
765 }
766 
767 /*
768  * XXXNP: Who is responsible for cleaning up the socket if this returns with an
769  * error?  Review all error paths.
770  *
771  * XXXNP: What happens to the socket's fd reference if the operation is
772  * successful, and how does that affect the socket's life cycle?
773  */
774 int
775 icl_cxgbei_conn_handoff(struct icl_conn *ic, int fd)
776 {
777 	struct icl_cxgbei_conn *icc = ic_to_icc(ic);
778 	struct cxgbei_data *ci;
779 	struct find_ofld_adapter_rr fa;
780 	struct file *fp;
781 	struct socket *so;
782 	struct inpcb *inp;
783 	struct tcpcb *tp;
784 	struct toepcb *toep;
785 	cap_rights_t rights;
786 	int error, max_iso_pdus;
787 
788 	MPASS(icc->icc_signature == CXGBEI_CONN_SIGNATURE);
789 	ICL_CONN_LOCK_ASSERT_NOT(ic);
790 
791 	/*
792 	 * Steal the socket from userland.
793 	 */
794 	error = fget(curthread, fd,
795 	    cap_rights_init_one(&rights, CAP_SOCK_CLIENT), &fp);
796 	if (error != 0)
797 		return (error);
798 	if (fp->f_type != DTYPE_SOCKET) {
799 		fdrop(fp, curthread);
800 		return (EINVAL);
801 	}
802 	so = fp->f_data;
803 	if (so->so_type != SOCK_STREAM ||
804 	    so->so_proto->pr_protocol != IPPROTO_TCP) {
805 		fdrop(fp, curthread);
806 		return (EINVAL);
807 	}
808 
809 	ICL_CONN_LOCK(ic);
810 	if (ic->ic_socket != NULL) {
811 		ICL_CONN_UNLOCK(ic);
812 		fdrop(fp, curthread);
813 		return (EBUSY);
814 	}
815 	ic->ic_disconnecting = false;
816 	ic->ic_socket = so;
817 	fp->f_ops = &badfileops;
818 	fp->f_data = NULL;
819 	fdrop(fp, curthread);
820 	ICL_CONN_UNLOCK(ic);
821 
822 	/* Find the adapter offloading this socket. */
823 	fa.sc = NULL;
824 	fa.so = so;
825 	t4_iterate(find_offload_adapter, &fa);
826 	if (fa.sc == NULL)
827 		return (EINVAL);
828 	icc->sc = fa.sc;
829 	ci = icc->sc->iscsi_ulp_softc;
830 
831 	inp = sotoinpcb(so);
832 	INP_WLOCK(inp);
833 	tp = intotcpcb(inp);
834 	if (inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT))
835 		error = EBUSY;
836 	else {
837 		/*
838 		 * socket could not have been "unoffloaded" if here.
839 		 */
840 		MPASS(tp->t_flags & TF_TOE);
841 		MPASS(tp->tod != NULL);
842 		MPASS(tp->t_toe != NULL);
843 		toep = tp->t_toe;
844 		MPASS(toep->vi->adapter == icc->sc);
845 		icc->toep = toep;
846 		icc->cwt = cxgbei_select_worker_thread(icc);
847 
848 		icc->ulp_submode = 0;
849 		if (ic->ic_header_crc32c)
850 			icc->ulp_submode |= ULP_CRC_HEADER;
851 		if (ic->ic_data_crc32c)
852 			icc->ulp_submode |= ULP_CRC_DATA;
853 
854 		if (icc->sc->tt.iso && chip_id(icc->sc) >= CHELSIO_T5) {
855 			max_iso_pdus = CXGBEI_MAX_ISO_PAYLOAD /
856 			    ci->max_tx_pdu_len;
857 			ic->ic_hw_isomax = max_iso_pdus *
858 			    ic->ic_max_send_data_segment_length;
859 		} else
860 			max_iso_pdus = 1;
861 
862 		so->so_options |= SO_NO_DDP;
863 		toep->params.ulp_mode = ULP_MODE_ISCSI;
864 		toep->ulpcb = icc;
865 
866 		send_iscsi_flowc_wr(icc->sc, toep,
867 		    roundup(max_iso_pdus * ci->max_tx_pdu_len, tp->t_maxseg));
868 		set_ulp_mode_iscsi(icc->sc, toep, icc->ulp_submode);
869 		error = 0;
870 	}
871 	INP_WUNLOCK(inp);
872 
873 	if (error == 0) {
874 		error = icl_cxgbei_setsockopt(ic, so, ci->max_tx_pdu_len,
875 		    ci->max_rx_pdu_len);
876 	}
877 
878 	return (error);
879 }
880 
881 void
882 icl_cxgbei_conn_close(struct icl_conn *ic)
883 {
884 	struct icl_cxgbei_conn *icc = ic_to_icc(ic);
885 	struct icl_pdu *ip;
886 	struct socket *so;
887 	struct sockbuf *sb;
888 	struct inpcb *inp;
889 	struct toepcb *toep = icc->toep;
890 
891 	MPASS(icc->icc_signature == CXGBEI_CONN_SIGNATURE);
892 	ICL_CONN_LOCK_ASSERT_NOT(ic);
893 
894 	ICL_CONN_LOCK(ic);
895 	so = ic->ic_socket;
896 	if (ic->ic_disconnecting || so == NULL) {
897 		CTR4(KTR_CXGBE, "%s: icc %p (disconnecting = %d), so %p",
898 		    __func__, icc, ic->ic_disconnecting, so);
899 		ICL_CONN_UNLOCK(ic);
900 		return;
901 	}
902 	ic->ic_disconnecting = true;
903 
904 #ifdef DIAGNOSTIC
905 	KASSERT(ic->ic_outstanding_pdus == 0,
906 	    ("destroying session with %d outstanding PDUs",
907 	     ic->ic_outstanding_pdus));
908 #endif
909 	ICL_CONN_UNLOCK(ic);
910 
911 	CTR3(KTR_CXGBE, "%s: tid %d, icc %p", __func__, toep ? toep->tid : -1,
912 	    icc);
913 	inp = sotoinpcb(so);
914 	sb = &so->so_rcv;
915 	INP_WLOCK(inp);
916 	if (toep != NULL) {	/* NULL if connection was never offloaded. */
917 		toep->ulpcb = NULL;
918 
919 		/* Discard PDUs queued for TX. */
920 		mbufq_drain(&toep->ulp_pduq);
921 
922 		/*
923 		 * Wait for the cwt threads to stop processing this
924 		 * connection.
925 		 */
926 		SOCKBUF_LOCK(sb);
927 		if (icc->rx_flags & RXF_ACTIVE) {
928 			volatile u_int *p = &icc->rx_flags;
929 
930 			SOCKBUF_UNLOCK(sb);
931 			INP_WUNLOCK(inp);
932 
933 			while (*p & RXF_ACTIVE)
934 				pause("conclo", 1);
935 
936 			INP_WLOCK(inp);
937 			SOCKBUF_LOCK(sb);
938 		}
939 
940 		/*
941 		 * Discard received PDUs not passed to the iSCSI
942 		 * layer.
943 		 */
944 		while (!STAILQ_EMPTY(&icc->rcvd_pdus)) {
945 			ip = STAILQ_FIRST(&icc->rcvd_pdus);
946 			STAILQ_REMOVE_HEAD(&icc->rcvd_pdus, ip_next);
947 			icl_cxgbei_pdu_done(ip, ENOTCONN);
948 		}
949 		SOCKBUF_UNLOCK(sb);
950 
951 		/*
952 		 * Grab a reference to use when waiting for the final
953 		 * CPL to be received.  If toep->inp is NULL, then
954 		 * final_cpl_received() has already been called (e.g.
955 		 * due to the peer sending a RST).
956 		 */
957 		if (toep->inp != NULL) {
958 			toep = hold_toepcb(toep);
959 			toep->flags |= TPF_WAITING_FOR_FINAL;
960 		} else
961 			toep = NULL;
962 	}
963 	INP_WUNLOCK(inp);
964 
965 	ICL_CONN_LOCK(ic);
966 	ic->ic_socket = NULL;
967 	ICL_CONN_UNLOCK(ic);
968 
969 	/*
970 	 * XXXNP: we should send RST instead of FIN when PDUs held in various
971 	 * queues were purged instead of delivered reliably but soabort isn't
972 	 * really general purpose and wouldn't do the right thing here.
973 	 */
974 	soclose(so);
975 
976 	/*
977 	 * Wait for the socket to fully close.  This ensures any
978 	 * pending received data has been received (and in particular,
979 	 * any data that would be received by DDP has been handled).
980 	 * Callers assume that it is safe to free buffers for tasks
981 	 * and transfers after this function returns.
982 	 */
983 	if (toep != NULL) {
984 		struct mtx *lock = mtx_pool_find(mtxpool_sleep, toep);
985 
986 		mtx_lock(lock);
987 		while ((toep->flags & TPF_WAITING_FOR_FINAL) != 0)
988 			mtx_sleep(toep, lock, PSOCK, "conclo2", 0);
989 		mtx_unlock(lock);
990 		free_toepcb(toep);
991 	}
992 }
993 
994 static void
995 cxgbei_insert_cmp(struct icl_cxgbei_conn *icc, struct cxgbei_cmp *cmp,
996     uint32_t tt)
997 {
998 #ifdef INVARIANTS
999 	struct cxgbei_cmp *cmp2;
1000 #endif
1001 
1002 	cmp->tt = tt;
1003 
1004 	mtx_lock(&icc->cmp_lock);
1005 #ifdef INVARIANTS
1006 	LIST_FOREACH(cmp2, &icc->cmp_table[TT_HASH(icc, tt)], link) {
1007 		KASSERT(cmp2->tt != tt, ("%s: duplicate cmp", __func__));
1008 	}
1009 #endif
1010 	LIST_INSERT_HEAD(&icc->cmp_table[TT_HASH(icc, tt)], cmp, link);
1011 	mtx_unlock(&icc->cmp_lock);
1012 }
1013 
1014 struct cxgbei_cmp *
1015 cxgbei_find_cmp(struct icl_cxgbei_conn *icc, uint32_t tt)
1016 {
1017 	struct cxgbei_cmp *cmp;
1018 
1019 	mtx_lock(&icc->cmp_lock);
1020 	LIST_FOREACH(cmp, &icc->cmp_table[TT_HASH(icc, tt)], link) {
1021 		if (cmp->tt == tt)
1022 			break;
1023 	}
1024 	mtx_unlock(&icc->cmp_lock);
1025 	return (cmp);
1026 }
1027 
1028 static void
1029 cxgbei_rm_cmp(struct icl_cxgbei_conn *icc, struct cxgbei_cmp *cmp)
1030 {
1031 #ifdef INVARIANTS
1032 	struct cxgbei_cmp *cmp2;
1033 #endif
1034 
1035 	mtx_lock(&icc->cmp_lock);
1036 
1037 #ifdef INVARIANTS
1038 	LIST_FOREACH(cmp2, &icc->cmp_table[TT_HASH(icc, cmp->tt)], link) {
1039 		if (cmp2 == cmp)
1040 			goto found;
1041 	}
1042 	panic("%s: could not find cmp", __func__);
1043 found:
1044 #endif
1045 	LIST_REMOVE(cmp, link);
1046 	mtx_unlock(&icc->cmp_lock);
1047 }
1048 
1049 int
1050 icl_cxgbei_conn_task_setup(struct icl_conn *ic, struct icl_pdu *ip,
1051     struct ccb_scsiio *csio, uint32_t *ittp, void **arg)
1052 {
1053 	struct icl_cxgbei_conn *icc = ic_to_icc(ic);
1054 	struct toepcb *toep = icc->toep;
1055 	struct adapter *sc = icc->sc;
1056 	struct cxgbei_data *ci = sc->iscsi_ulp_softc;
1057 	struct ppod_region *pr = &ci->pr;
1058 	struct cxgbei_ddp_state *ddp;
1059 	struct ppod_reservation *prsv;
1060 	struct inpcb *inp;
1061 	struct mbufq mq;
1062 	uint32_t itt;
1063 	int rc = 0;
1064 
1065 	ICL_CONN_LOCK_ASSERT(ic);
1066 
1067 	/* This is for the offload driver's state.  Must not be set already. */
1068 	MPASS(arg != NULL);
1069 	MPASS(*arg == NULL);
1070 
1071 	if (ic->ic_disconnecting || ic->ic_socket == NULL)
1072 		return (ECONNRESET);
1073 
1074 	if ((csio->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_IN ||
1075 	    csio->dxfer_len < ci->ddp_threshold) {
1076 no_ddp:
1077 		/*
1078 		 * No DDP for this I/O.	 Allocate an ITT (based on the one
1079 		 * passed in) that cannot be a valid hardware DDP tag in the
1080 		 * iSCSI region.
1081 		 */
1082 		itt = *ittp & M_PPOD_TAG;
1083 		itt = V_PPOD_TAG(itt) | pr->pr_invalid_bit;
1084 		*ittp = htobe32(itt);
1085 		MPASS(*arg == NULL);	/* State is maintained for DDP only. */
1086 		if (rc != 0)
1087 			counter_u64_add(
1088 			    toep->ofld_rxq->rx_iscsi_ddp_setup_error, 1);
1089 		return (0);
1090 	}
1091 
1092 	/*
1093 	 * Reserve resources for DDP, update the itt that should be used in the
1094 	 * PDU, and save DDP specific state for this I/O in *arg.
1095 	 */
1096 	ddp = malloc(sizeof(*ddp), M_CXGBEI, M_NOWAIT | M_ZERO);
1097 	if (ddp == NULL) {
1098 		rc = ENOMEM;
1099 		goto no_ddp;
1100 	}
1101 	prsv = &ddp->prsv;
1102 
1103 	/* XXX add support for all CAM_DATA_ types */
1104 	MPASS((csio->ccb_h.flags & CAM_DATA_MASK) == CAM_DATA_VADDR);
1105 	rc = t4_alloc_page_pods_for_buf(pr, (vm_offset_t)csio->data_ptr,
1106 	    csio->dxfer_len, prsv);
1107 	if (rc != 0) {
1108 		free(ddp, M_CXGBEI);
1109 		goto no_ddp;
1110 	}
1111 
1112 	mbufq_init(&mq, INT_MAX);
1113 	rc = t4_write_page_pods_for_buf(sc, toep, prsv,
1114 	    (vm_offset_t)csio->data_ptr, csio->dxfer_len, &mq);
1115 	if (__predict_false(rc != 0)) {
1116 		mbufq_drain(&mq);
1117 		t4_free_page_pods(prsv);
1118 		free(ddp, M_CXGBEI);
1119 		goto no_ddp;
1120 	}
1121 
1122 	/*
1123 	 * Do not get inp from toep->inp as the toepcb might have
1124 	 * detached already.
1125 	 */
1126 	inp = sotoinpcb(ic->ic_socket);
1127 	INP_WLOCK(inp);
1128 	if ((inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) != 0) {
1129 		INP_WUNLOCK(inp);
1130 		mbufq_drain(&mq);
1131 		t4_free_page_pods(prsv);
1132 		free(ddp, M_CXGBEI);
1133 		return (ECONNRESET);
1134 	}
1135 	mbufq_concat(&toep->ulp_pduq, &mq);
1136 	INP_WUNLOCK(inp);
1137 
1138 	ddp->cmp.last_datasn = -1;
1139 	cxgbei_insert_cmp(icc, &ddp->cmp, prsv->prsv_tag);
1140 	*ittp = htobe32(prsv->prsv_tag);
1141 	*arg = prsv;
1142 	counter_u64_add(toep->ofld_rxq->rx_iscsi_ddp_setup_ok, 1);
1143 	return (0);
1144 }
1145 
1146 void
1147 icl_cxgbei_conn_task_done(struct icl_conn *ic, void *arg)
1148 {
1149 
1150 	if (arg != NULL) {
1151 		struct cxgbei_ddp_state *ddp = arg;
1152 
1153 		cxgbei_rm_cmp(ic_to_icc(ic), &ddp->cmp);
1154 		t4_free_page_pods(&ddp->prsv);
1155 		free(ddp, M_CXGBEI);
1156 	}
1157 }
1158 
1159 static inline bool
1160 ddp_sgl_check(struct ctl_sg_entry *sg, int entries, int xferlen)
1161 {
1162 	int total_len = 0;
1163 
1164 	MPASS(entries > 0);
1165 	if (((vm_offset_t)sg[--entries].addr & 3U) != 0)
1166 		return (false);
1167 
1168 	total_len += sg[entries].len;
1169 
1170 	while (--entries >= 0) {
1171 		if (((vm_offset_t)sg[entries].addr & PAGE_MASK) != 0 ||
1172 		    (sg[entries].len % PAGE_SIZE) != 0)
1173 			return (false);
1174 		total_len += sg[entries].len;
1175 	}
1176 
1177 	MPASS(total_len == xferlen);
1178 	return (true);
1179 }
1180 
1181 /* XXXNP: PDU should be passed in as parameter, like on the initiator. */
1182 #define io_to_request_pdu(io) ((io)->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr)
1183 #define io_to_ddp_state(io) ((io)->io_hdr.ctl_private[CTL_PRIV_FRONTEND2].ptr)
1184 
1185 int
1186 icl_cxgbei_conn_transfer_setup(struct icl_conn *ic, union ctl_io *io,
1187     uint32_t *tttp, void **arg)
1188 {
1189 	struct icl_cxgbei_conn *icc = ic_to_icc(ic);
1190 	struct toepcb *toep = icc->toep;
1191 	struct ctl_scsiio *ctsio = &io->scsiio;
1192 	struct adapter *sc = icc->sc;
1193 	struct cxgbei_data *ci = sc->iscsi_ulp_softc;
1194 	struct ppod_region *pr = &ci->pr;
1195 	struct cxgbei_ddp_state *ddp;
1196 	struct ppod_reservation *prsv;
1197 	struct ctl_sg_entry *sgl, sg_entry;
1198 	struct inpcb *inp;
1199 	struct mbufq mq;
1200 	int sg_entries = ctsio->kern_sg_entries;
1201 	uint32_t ttt;
1202 	int xferlen, rc = 0, alias;
1203 
1204 	/* This is for the offload driver's state.  Must not be set already. */
1205 	MPASS(arg != NULL);
1206 	MPASS(*arg == NULL);
1207 
1208 	if (ctsio->ext_data_filled == 0) {
1209 		int first_burst;
1210 		struct icl_pdu *ip = io_to_request_pdu(io);
1211 #ifdef INVARIANTS
1212 		struct icl_cxgbei_pdu *icp = ip_to_icp(ip);
1213 
1214 		MPASS(icp->icp_signature == CXGBEI_PDU_SIGNATURE);
1215 		MPASS(ic == ip->ip_conn);
1216 		MPASS(ip->ip_bhs_mbuf != NULL);
1217 #endif
1218 		first_burst = icl_pdu_data_segment_length(ip);
1219 
1220 		/*
1221 		 * Note that ICL calls conn_transfer_setup even if the first
1222 		 * burst had everything and there's nothing left to transfer.
1223 		 *
1224 		 * NB: The CTL frontend might have provided a buffer
1225 		 * whose length (kern_data_len) is smaller than the
1226 		 * FirstBurstLength of unsolicited data.  Treat those
1227 		 * as an empty transfer.
1228 		 */
1229 		xferlen = ctsio->kern_data_len;
1230 		if (xferlen < first_burst ||
1231 		    xferlen - first_burst < ci->ddp_threshold) {
1232 no_ddp:
1233 			/*
1234 			 * No DDP for this transfer.  Allocate a TTT (based on
1235 			 * the one passed in) that cannot be a valid hardware
1236 			 * DDP tag in the iSCSI region.
1237 			 */
1238 			ttt = *tttp & M_PPOD_TAG;
1239 			ttt = V_PPOD_TAG(ttt) | pr->pr_invalid_bit;
1240 			*tttp = htobe32(ttt);
1241 			MPASS(io_to_ddp_state(io) == NULL);
1242 			if (rc != 0)
1243 				counter_u64_add(
1244 				    toep->ofld_rxq->rx_iscsi_ddp_setup_error, 1);
1245 			return (0);
1246 		}
1247 
1248 		if (sg_entries == 0) {
1249 			sgl = &sg_entry;
1250 			sgl->len = xferlen;
1251 			sgl->addr = (void *)ctsio->kern_data_ptr;
1252 			sg_entries = 1;
1253 		} else
1254 			sgl = (void *)ctsio->kern_data_ptr;
1255 
1256 		if (!ddp_sgl_check(sgl, sg_entries, xferlen))
1257 			goto no_ddp;
1258 
1259 		/*
1260 		 * Reserve resources for DDP, update the ttt that should be used
1261 		 * in the PDU, and save DDP specific state for this I/O.
1262 		 */
1263 		MPASS(io_to_ddp_state(io) == NULL);
1264 		ddp = malloc(sizeof(*ddp), M_CXGBEI, M_NOWAIT | M_ZERO);
1265 		if (ddp == NULL) {
1266 			rc = ENOMEM;
1267 			goto no_ddp;
1268 		}
1269 		prsv = &ddp->prsv;
1270 
1271 		rc = t4_alloc_page_pods_for_sgl(pr, sgl, sg_entries, prsv);
1272 		if (rc != 0) {
1273 			free(ddp, M_CXGBEI);
1274 			goto no_ddp;
1275 		}
1276 
1277 		mbufq_init(&mq, INT_MAX);
1278 		rc = t4_write_page_pods_for_sgl(sc, toep, prsv, sgl, sg_entries,
1279 		    xferlen, &mq);
1280 		if (__predict_false(rc != 0)) {
1281 			mbufq_drain(&mq);
1282 			t4_free_page_pods(prsv);
1283 			free(ddp, M_CXGBEI);
1284 			goto no_ddp;
1285 		}
1286 
1287 		/*
1288 		 * Do not get inp from toep->inp as the toepcb might
1289 		 * have detached already.
1290 		 */
1291 		ICL_CONN_LOCK(ic);
1292 		if (ic->ic_disconnecting || ic->ic_socket == NULL) {
1293 			ICL_CONN_UNLOCK(ic);
1294 			mbufq_drain(&mq);
1295 			t4_free_page_pods(prsv);
1296 			free(ddp, M_CXGBEI);
1297 			return (ECONNRESET);
1298 		}
1299 		inp = sotoinpcb(ic->ic_socket);
1300 		INP_WLOCK(inp);
1301 		ICL_CONN_UNLOCK(ic);
1302 		if ((inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) != 0) {
1303 			INP_WUNLOCK(inp);
1304 			mbufq_drain(&mq);
1305 			t4_free_page_pods(prsv);
1306 			free(ddp, M_CXGBEI);
1307 			return (ECONNRESET);
1308 		}
1309 		mbufq_concat(&toep->ulp_pduq, &mq);
1310 		INP_WUNLOCK(inp);
1311 
1312 		ddp->cmp.next_buffer_offset = ctsio->kern_rel_offset +
1313 		    first_burst;
1314 		ddp->cmp.last_datasn = -1;
1315 		cxgbei_insert_cmp(icc, &ddp->cmp, prsv->prsv_tag);
1316 		*tttp = htobe32(prsv->prsv_tag);
1317 		io_to_ddp_state(io) = ddp;
1318 		*arg = ctsio;
1319 		counter_u64_add(toep->ofld_rxq->rx_iscsi_ddp_setup_ok, 1);
1320 		return (0);
1321 	}
1322 
1323 	/*
1324 	 * In the middle of an I/O.  A non-NULL page pod reservation indicates
1325 	 * that a DDP buffer is being used for the I/O.
1326 	 */
1327 	ddp = io_to_ddp_state(ctsio);
1328 	if (ddp == NULL)
1329 		goto no_ddp;
1330 	prsv = &ddp->prsv;
1331 
1332 	alias = (prsv->prsv_tag & pr->pr_alias_mask) >> pr->pr_alias_shift;
1333 	alias++;
1334 	prsv->prsv_tag &= ~pr->pr_alias_mask;
1335 	prsv->prsv_tag |= alias << pr->pr_alias_shift & pr->pr_alias_mask;
1336 
1337 	ddp->cmp.next_datasn = 0;
1338 	ddp->cmp.last_datasn = -1;
1339 	cxgbei_insert_cmp(icc, &ddp->cmp, prsv->prsv_tag);
1340 	*tttp = htobe32(prsv->prsv_tag);
1341 	*arg = ctsio;
1342 
1343 	return (0);
1344 }
1345 
1346 void
1347 icl_cxgbei_conn_transfer_done(struct icl_conn *ic, void *arg)
1348 {
1349 	struct ctl_scsiio *ctsio = arg;
1350 
1351 	if (ctsio != NULL) {
1352 		struct cxgbei_ddp_state *ddp;
1353 
1354 		ddp = io_to_ddp_state(ctsio);
1355 		MPASS(ddp != NULL);
1356 
1357 		cxgbei_rm_cmp(ic_to_icc(ic), &ddp->cmp);
1358 		if (ctsio->kern_data_len == ctsio->ext_data_filled ||
1359 		    ic->ic_disconnecting) {
1360 			t4_free_page_pods(&ddp->prsv);
1361 			free(ddp, M_CXGBEI);
1362 			io_to_ddp_state(ctsio) = NULL;
1363 		}
1364 	}
1365 }
1366 
1367 static void
1368 cxgbei_limits(struct adapter *sc, void *arg)
1369 {
1370 	struct icl_drv_limits *idl = arg;
1371 	struct cxgbei_data *ci;
1372 	int max_dsl;
1373 
1374 	if (begin_synchronized_op(sc, NULL, HOLD_LOCK, "t4lims") != 0)
1375 		return;
1376 
1377 	if (uld_active(sc, ULD_ISCSI)) {
1378 		ci = sc->iscsi_ulp_softc;
1379 		MPASS(ci != NULL);
1380 
1381 		/*
1382 		 * AHS is not supported by the kernel so we'll not account for
1383 		 * it either in our PDU len -> data segment len conversions.
1384 		 */
1385 
1386 		max_dsl = ci->max_rx_pdu_len - ISCSI_BHS_SIZE -
1387 		    ISCSI_HEADER_DIGEST_SIZE - ISCSI_DATA_DIGEST_SIZE;
1388 		if (idl->idl_max_recv_data_segment_length > max_dsl)
1389 			idl->idl_max_recv_data_segment_length = max_dsl;
1390 
1391 		max_dsl = ci->max_tx_pdu_len - ISCSI_BHS_SIZE -
1392 		    ISCSI_HEADER_DIGEST_SIZE - ISCSI_DATA_DIGEST_SIZE;
1393 		if (idl->idl_max_send_data_segment_length > max_dsl)
1394 			idl->idl_max_send_data_segment_length = max_dsl;
1395 	}
1396 
1397 	end_synchronized_op(sc, LOCK_HELD);
1398 }
1399 
1400 static int
1401 icl_cxgbei_limits(struct icl_drv_limits *idl)
1402 {
1403 
1404 	/* Maximum allowed by the RFC.	cxgbei_limits will clip them. */
1405 	idl->idl_max_recv_data_segment_length = (1 << 24) - 1;
1406 	idl->idl_max_send_data_segment_length = (1 << 24) - 1;
1407 
1408 	/* These are somewhat arbitrary. */
1409 	idl->idl_max_burst_length = max_burst_length;
1410 	idl->idl_first_burst_length = first_burst_length;
1411 
1412 	t4_iterate(cxgbei_limits, idl);
1413 
1414 	return (0);
1415 }
1416 
1417 int
1418 icl_cxgbei_mod_load(void)
1419 {
1420 	int rc;
1421 
1422 	refcount_init(&icl_cxgbei_ncons, 0);
1423 
1424 	rc = icl_register("cxgbei", false, -100, icl_cxgbei_limits,
1425 	    icl_cxgbei_new_conn);
1426 
1427 	return (rc);
1428 }
1429 
1430 int
1431 icl_cxgbei_mod_unload(void)
1432 {
1433 
1434 	if (icl_cxgbei_ncons != 0)
1435 		return (EBUSY);
1436 
1437 	icl_unregister("cxgbei", false);
1438 
1439 	return (0);
1440 }
1441 #endif
1442