xref: /freebsd/sys/dev/cxgbe/tom/t4_tom.c (revision ddd5b8e9b4d8957fce018c520657cdfa4ecffad3)
1 /*-
2  * Copyright (c) 2012 Chelsio Communications, Inc.
3  * All rights reserved.
4  * Written by: Navdeep Parhar <np@FreeBSD.org>
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions
8  * are met:
9  * 1. Redistributions of source code must retain the above copyright
10  *    notice, this list of conditions and the following disclaimer.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  *
15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25  * SUCH DAMAGE.
26  */
27 
28 #include <sys/cdefs.h>
29 __FBSDID("$FreeBSD$");
30 
31 #include "opt_inet.h"
32 #include "opt_inet6.h"
33 
34 #include <sys/param.h>
35 #include <sys/types.h>
36 #include <sys/systm.h>
37 #include <sys/kernel.h>
38 #include <sys/ktr.h>
39 #include <sys/module.h>
40 #include <sys/protosw.h>
41 #include <sys/domain.h>
42 #include <sys/socket.h>
43 #include <sys/socketvar.h>
44 #include <net/if.h>
45 #include <netinet/in.h>
46 #include <netinet/in_pcb.h>
47 #include <netinet/in_var.h>
48 #include <netinet/ip.h>
49 #include <netinet/ip6.h>
50 #include <netinet/tcp_var.h>
51 #include <netinet6/scope6_var.h>
52 #define TCPSTATES
53 #include <netinet/tcp_fsm.h>
54 #include <netinet/toecore.h>
55 
56 #ifdef TCP_OFFLOAD
57 #include "common/common.h"
58 #include "common/t4_msg.h"
59 #include "common/t4_regs.h"
60 #include "tom/t4_tom_l2t.h"
61 #include "tom/t4_tom.h"
62 
63 static struct protosw ddp_protosw;
64 static struct pr_usrreqs ddp_usrreqs;
65 
66 static struct protosw ddp6_protosw;
67 static struct pr_usrreqs ddp6_usrreqs;
68 
69 /* Module ops */
70 static int t4_tom_mod_load(void);
71 static int t4_tom_mod_unload(void);
72 static int t4_tom_modevent(module_t, int, void *);
73 
74 /* ULD ops and helpers */
75 static int t4_tom_activate(struct adapter *);
76 static int t4_tom_deactivate(struct adapter *);
77 
78 static struct uld_info tom_uld_info = {
79 	.uld_id = ULD_TOM,
80 	.activate = t4_tom_activate,
81 	.deactivate = t4_tom_deactivate,
82 };
83 
84 static void queue_tid_release(struct adapter *, int);
85 static void release_offload_resources(struct toepcb *);
86 static int alloc_tid_tabs(struct tid_info *);
87 static void free_tid_tabs(struct tid_info *);
88 static int add_lip(struct adapter *, struct in6_addr *);
89 static int delete_lip(struct adapter *, struct in6_addr *);
90 static struct clip_entry *search_lip(struct tom_data *, struct in6_addr *);
91 static void init_clip_table(struct adapter *, struct tom_data *);
92 static void destroy_clip_table(struct adapter *, struct tom_data *);
93 static void free_tom_data(struct adapter *, struct tom_data *);
94 
95 struct toepcb *
96 alloc_toepcb(struct port_info *pi, int txqid, int rxqid, int flags)
97 {
98 	struct adapter *sc = pi->adapter;
99 	struct toepcb *toep;
100 	int tx_credits, txsd_total, len;
101 
102 	/*
103 	 * The firmware counts tx work request credits in units of 16 bytes
104 	 * each.  Reserve room for an ABORT_REQ so the driver never has to worry
105 	 * about tx credits if it wants to abort a connection.
106 	 */
107 	tx_credits = sc->params.ofldq_wr_cred;
108 	tx_credits -= howmany(sizeof(struct cpl_abort_req), 16);
109 
110 	/*
111 	 * Shortest possible tx work request is a fw_ofld_tx_data_wr + 1 byte
112 	 * immediate payload, and firmware counts tx work request credits in
113 	 * units of 16 byte.  Calculate the maximum work requests possible.
114 	 */
115 	txsd_total = tx_credits /
116 	    howmany((sizeof(struct fw_ofld_tx_data_wr) + 1), 16);
117 
118 	if (txqid < 0)
119 		txqid = (arc4random() % pi->nofldtxq) + pi->first_ofld_txq;
120 	KASSERT(txqid >= pi->first_ofld_txq &&
121 	    txqid < pi->first_ofld_txq + pi->nofldtxq,
122 	    ("%s: txqid %d for port %p (first %d, n %d)", __func__, txqid, pi,
123 		pi->first_ofld_txq, pi->nofldtxq));
124 
125 	if (rxqid < 0)
126 		rxqid = (arc4random() % pi->nofldrxq) + pi->first_ofld_rxq;
127 	KASSERT(rxqid >= pi->first_ofld_rxq &&
128 	    rxqid < pi->first_ofld_rxq + pi->nofldrxq,
129 	    ("%s: rxqid %d for port %p (first %d, n %d)", __func__, rxqid, pi,
130 		pi->first_ofld_rxq, pi->nofldrxq));
131 
132 	len = offsetof(struct toepcb, txsd) +
133 	    txsd_total * sizeof(struct ofld_tx_sdesc);
134 
135 	toep = malloc(len, M_CXGBE, M_ZERO | flags);
136 	if (toep == NULL)
137 		return (NULL);
138 
139 	toep->td = sc->tom_softc;
140 	toep->port = pi;
141 	toep->tx_credits = tx_credits;
142 	toep->ofld_txq = &sc->sge.ofld_txq[txqid];
143 	toep->ofld_rxq = &sc->sge.ofld_rxq[rxqid];
144 	toep->ctrlq = &sc->sge.ctrlq[pi->port_id];
145 	toep->txsd_total = txsd_total;
146 	toep->txsd_avail = txsd_total;
147 	toep->txsd_pidx = 0;
148 	toep->txsd_cidx = 0;
149 
150 	return (toep);
151 }
152 
153 void
154 free_toepcb(struct toepcb *toep)
155 {
156 
157 	KASSERT(!(toep->flags & TPF_ATTACHED),
158 	    ("%s: attached to an inpcb", __func__));
159 	KASSERT(!(toep->flags & TPF_CPL_PENDING),
160 	    ("%s: CPL pending", __func__));
161 
162 	free(toep, M_CXGBE);
163 }
164 
165 /*
166  * Set up the socket for TCP offload.
167  */
168 void
169 offload_socket(struct socket *so, struct toepcb *toep)
170 {
171 	struct tom_data *td = toep->td;
172 	struct inpcb *inp = sotoinpcb(so);
173 	struct tcpcb *tp = intotcpcb(inp);
174 	struct sockbuf *sb;
175 
176 	INP_WLOCK_ASSERT(inp);
177 
178 	/* Update socket */
179 	sb = &so->so_snd;
180 	SOCKBUF_LOCK(sb);
181 	sb->sb_flags |= SB_NOCOALESCE;
182 	SOCKBUF_UNLOCK(sb);
183 	sb = &so->so_rcv;
184 	SOCKBUF_LOCK(sb);
185 	sb->sb_flags |= SB_NOCOALESCE;
186 	if (toep->ulp_mode == ULP_MODE_TCPDDP) {
187 		if (inp->inp_vflag & INP_IPV6)
188 			so->so_proto = &ddp6_protosw;
189 		else
190 			so->so_proto = &ddp_protosw;
191 	}
192 	SOCKBUF_UNLOCK(sb);
193 
194 	/* Update TCP PCB */
195 	tp->tod = &td->tod;
196 	tp->t_toe = toep;
197 	tp->t_flags |= TF_TOE;
198 
199 	/* Install an extra hold on inp */
200 	toep->inp = inp;
201 	toep->flags |= TPF_ATTACHED;
202 	in_pcbref(inp);
203 
204 	/* Add the TOE PCB to the active list */
205 	mtx_lock(&td->toep_list_lock);
206 	TAILQ_INSERT_HEAD(&td->toep_list, toep, link);
207 	mtx_unlock(&td->toep_list_lock);
208 }
209 
210 /* This is _not_ the normal way to "unoffload" a socket. */
211 void
212 undo_offload_socket(struct socket *so)
213 {
214 	struct inpcb *inp = sotoinpcb(so);
215 	struct tcpcb *tp = intotcpcb(inp);
216 	struct toepcb *toep = tp->t_toe;
217 	struct tom_data *td = toep->td;
218 	struct sockbuf *sb;
219 
220 	INP_WLOCK_ASSERT(inp);
221 
222 	sb = &so->so_snd;
223 	SOCKBUF_LOCK(sb);
224 	sb->sb_flags &= ~SB_NOCOALESCE;
225 	SOCKBUF_UNLOCK(sb);
226 	sb = &so->so_rcv;
227 	SOCKBUF_LOCK(sb);
228 	sb->sb_flags &= ~SB_NOCOALESCE;
229 	SOCKBUF_UNLOCK(sb);
230 
231 	tp->tod = NULL;
232 	tp->t_toe = NULL;
233 	tp->t_flags &= ~TF_TOE;
234 
235 	toep->inp = NULL;
236 	toep->flags &= ~TPF_ATTACHED;
237 	if (in_pcbrele_wlocked(inp))
238 		panic("%s: inp freed.", __func__);
239 
240 	mtx_lock(&td->toep_list_lock);
241 	TAILQ_REMOVE(&td->toep_list, toep, link);
242 	mtx_unlock(&td->toep_list_lock);
243 }
244 
245 static void
246 release_offload_resources(struct toepcb *toep)
247 {
248 	struct tom_data *td = toep->td;
249 	struct adapter *sc = td_adapter(td);
250 	int tid = toep->tid;
251 
252 	KASSERT(!(toep->flags & TPF_CPL_PENDING),
253 	    ("%s: %p has CPL pending.", __func__, toep));
254 	KASSERT(!(toep->flags & TPF_ATTACHED),
255 	    ("%s: %p is still attached.", __func__, toep));
256 
257 	CTR5(KTR_CXGBE, "%s: toep %p (tid %d, l2te %p, ce %p)",
258 	    __func__, toep, tid, toep->l2te, toep->ce);
259 
260 	if (toep->ulp_mode == ULP_MODE_TCPDDP)
261 		release_ddp_resources(toep);
262 
263 	if (toep->l2te)
264 		t4_l2t_release(toep->l2te);
265 
266 	if (tid >= 0) {
267 		remove_tid(sc, tid);
268 		release_tid(sc, tid, toep->ctrlq);
269 	}
270 
271 	if (toep->ce)
272 		release_lip(td, toep->ce);
273 
274 	mtx_lock(&td->toep_list_lock);
275 	TAILQ_REMOVE(&td->toep_list, toep, link);
276 	mtx_unlock(&td->toep_list_lock);
277 
278 	free_toepcb(toep);
279 }
280 
281 /*
282  * The kernel is done with the TCP PCB and this is our opportunity to unhook the
283  * toepcb hanging off of it.  If the TOE driver is also done with the toepcb (no
284  * pending CPL) then it is time to release all resources tied to the toepcb.
285  *
286  * Also gets called when an offloaded active open fails and the TOM wants the
287  * kernel to take the TCP PCB back.
288  */
289 static void
290 t4_pcb_detach(struct toedev *tod __unused, struct tcpcb *tp)
291 {
292 #if defined(KTR) || defined(INVARIANTS)
293 	struct inpcb *inp = tp->t_inpcb;
294 #endif
295 	struct toepcb *toep = tp->t_toe;
296 
297 	INP_WLOCK_ASSERT(inp);
298 
299 	KASSERT(toep != NULL, ("%s: toep is NULL", __func__));
300 	KASSERT(toep->flags & TPF_ATTACHED,
301 	    ("%s: not attached", __func__));
302 
303 #ifdef KTR
304 	if (tp->t_state == TCPS_SYN_SENT) {
305 		CTR6(KTR_CXGBE, "%s: atid %d, toep %p (0x%x), inp %p (0x%x)",
306 		    __func__, toep->tid, toep, toep->flags, inp,
307 		    inp->inp_flags);
308 	} else {
309 		CTR6(KTR_CXGBE,
310 		    "t4_pcb_detach: tid %d (%s), toep %p (0x%x), inp %p (0x%x)",
311 		    toep->tid, tcpstates[tp->t_state], toep, toep->flags, inp,
312 		    inp->inp_flags);
313 	}
314 #endif
315 
316 	tp->t_toe = NULL;
317 	tp->t_flags &= ~TF_TOE;
318 	toep->flags &= ~TPF_ATTACHED;
319 
320 	if (!(toep->flags & TPF_CPL_PENDING))
321 		release_offload_resources(toep);
322 }
323 
324 /*
325  * The TOE driver will not receive any more CPLs for the tid associated with the
326  * toepcb; release the hold on the inpcb.
327  */
328 void
329 final_cpl_received(struct toepcb *toep)
330 {
331 	struct inpcb *inp = toep->inp;
332 
333 	KASSERT(inp != NULL, ("%s: inp is NULL", __func__));
334 	INP_WLOCK_ASSERT(inp);
335 	KASSERT(toep->flags & TPF_CPL_PENDING,
336 	    ("%s: CPL not pending already?", __func__));
337 
338 	CTR6(KTR_CXGBE, "%s: tid %d, toep %p (0x%x), inp %p (0x%x)",
339 	    __func__, toep->tid, toep, toep->flags, inp, inp->inp_flags);
340 
341 	toep->inp = NULL;
342 	toep->flags &= ~TPF_CPL_PENDING;
343 
344 	if (!(toep->flags & TPF_ATTACHED))
345 		release_offload_resources(toep);
346 
347 	if (!in_pcbrele_wlocked(inp))
348 		INP_WUNLOCK(inp);
349 }
350 
351 void
352 insert_tid(struct adapter *sc, int tid, void *ctx)
353 {
354 	struct tid_info *t = &sc->tids;
355 
356 	t->tid_tab[tid] = ctx;
357 	atomic_add_int(&t->tids_in_use, 1);
358 }
359 
360 void *
361 lookup_tid(struct adapter *sc, int tid)
362 {
363 	struct tid_info *t = &sc->tids;
364 
365 	return (t->tid_tab[tid]);
366 }
367 
368 void
369 update_tid(struct adapter *sc, int tid, void *ctx)
370 {
371 	struct tid_info *t = &sc->tids;
372 
373 	t->tid_tab[tid] = ctx;
374 }
375 
376 void
377 remove_tid(struct adapter *sc, int tid)
378 {
379 	struct tid_info *t = &sc->tids;
380 
381 	t->tid_tab[tid] = NULL;
382 	atomic_subtract_int(&t->tids_in_use, 1);
383 }
384 
385 void
386 release_tid(struct adapter *sc, int tid, struct sge_wrq *ctrlq)
387 {
388 	struct wrqe *wr;
389 	struct cpl_tid_release *req;
390 
391 	wr = alloc_wrqe(sizeof(*req), ctrlq);
392 	if (wr == NULL) {
393 		queue_tid_release(sc, tid);	/* defer */
394 		return;
395 	}
396 	req = wrtod(wr);
397 
398 	INIT_TP_WR_MIT_CPL(req, CPL_TID_RELEASE, tid);
399 
400 	t4_wrq_tx(sc, wr);
401 }
402 
403 static void
404 queue_tid_release(struct adapter *sc, int tid)
405 {
406 
407 	CXGBE_UNIMPLEMENTED("deferred tid release");
408 }
409 
410 /*
411  * What mtu_idx to use, given a 4-tuple and/or an MSS cap
412  */
413 int
414 find_best_mtu_idx(struct adapter *sc, struct in_conninfo *inc, int pmss)
415 {
416 	unsigned short *mtus = &sc->params.mtus[0];
417 	int i, mss, n;
418 
419 	KASSERT(inc != NULL || pmss > 0,
420 	    ("%s: at least one of inc/pmss must be specified", __func__));
421 
422 	mss = inc ? tcp_mssopt(inc) : pmss;
423 	if (pmss > 0 && mss > pmss)
424 		mss = pmss;
425 
426 	if (inc->inc_flags & INC_ISIPV6)
427 		n = sizeof(struct ip6_hdr) + sizeof(struct tcphdr);
428 	else
429 		n = sizeof(struct ip) + sizeof(struct tcphdr);
430 
431 	for (i = 0; i < NMTUS - 1 && mtus[i + 1] <= mss + n; i++)
432 		continue;
433 
434 	return (i);
435 }
436 
437 /*
438  * Determine the receive window size for a socket.
439  */
440 u_long
441 select_rcv_wnd(struct socket *so)
442 {
443 	unsigned long wnd;
444 
445 	SOCKBUF_LOCK_ASSERT(&so->so_rcv);
446 
447 	wnd = sbspace(&so->so_rcv);
448 	if (wnd < MIN_RCV_WND)
449 		wnd = MIN_RCV_WND;
450 
451 	return min(wnd, MAX_RCV_WND);
452 }
453 
454 int
455 select_rcv_wscale(void)
456 {
457 	int wscale = 0;
458 	unsigned long space = sb_max;
459 
460 	if (space > MAX_RCV_WND)
461 		space = MAX_RCV_WND;
462 
463 	while (wscale < TCP_MAX_WINSHIFT && (TCP_MAXWIN << wscale) < space)
464 		wscale++;
465 
466 	return (wscale);
467 }
468 
469 extern int always_keepalive;
470 #define VIID_SMACIDX(v)	(((unsigned int)(v) & 0x7f) << 1)
471 
472 /*
473  * socket so could be a listening socket too.
474  */
475 uint64_t
476 calc_opt0(struct socket *so, struct port_info *pi, struct l2t_entry *e,
477     int mtu_idx, int rscale, int rx_credits, int ulp_mode)
478 {
479 	uint64_t opt0;
480 
481 	KASSERT(rx_credits <= M_RCV_BUFSIZ,
482 	    ("%s: rcv_bufsiz too high", __func__));
483 
484 	opt0 = F_TCAM_BYPASS | V_WND_SCALE(rscale) | V_MSS_IDX(mtu_idx) |
485 	    V_ULP_MODE(ulp_mode) | V_RCV_BUFSIZ(rx_credits);
486 
487 	if (so != NULL) {
488 		struct inpcb *inp = sotoinpcb(so);
489 		struct tcpcb *tp = intotcpcb(inp);
490 		int keepalive = always_keepalive ||
491 		    so_options_get(so) & SO_KEEPALIVE;
492 
493 		opt0 |= V_NAGLE((tp->t_flags & TF_NODELAY) == 0);
494 		opt0 |= V_KEEP_ALIVE(keepalive != 0);
495 	}
496 
497 	if (e != NULL)
498 		opt0 |= V_L2T_IDX(e->idx);
499 
500 	if (pi != NULL) {
501 		opt0 |= V_SMAC_SEL(VIID_SMACIDX(pi->viid));
502 		opt0 |= V_TX_CHAN(pi->tx_chan);
503 	}
504 
505 	return htobe64(opt0);
506 }
507 
508 #define FILTER_SEL_WIDTH_P_FC (3 + 1)
509 #define FILTER_SEL_WIDTH_VIN_P_FC (6 + 7 + FILTER_SEL_WIDTH_P_FC)
510 #define FILTER_SEL_WIDTH_TAG_P_FC (3 + FILTER_SEL_WIDTH_VIN_P_FC)
511 #define FILTER_SEL_WIDTH_VLD_TAG_P_FC (1 + FILTER_SEL_WIDTH_TAG_P_FC)
512 #define VLAN_NONE 0xfff
513 #define FILTER_SEL_VLAN_NONE 0xffff
514 
515 uint64_t
516 select_ntuple(struct port_info *pi, struct l2t_entry *e, uint32_t filter_mode)
517 {
518 	uint16_t viid = pi->viid;
519 	uint32_t ntuple = 0;
520 
521 	if (filter_mode == HW_TPL_FR_MT_PR_IV_P_FC) {
522                 if (e->vlan == VLAN_NONE)
523 			ntuple |= FILTER_SEL_VLAN_NONE << FILTER_SEL_WIDTH_P_FC;
524                 else {
525                         ntuple |= e->vlan << FILTER_SEL_WIDTH_P_FC;
526                         ntuple |= 1 << FILTER_SEL_WIDTH_VLD_TAG_P_FC;
527                 }
528                 ntuple |= e->lport << S_PORT;
529 		ntuple |= IPPROTO_TCP << FILTER_SEL_WIDTH_VLD_TAG_P_FC;
530 	} else if (filter_mode == HW_TPL_FR_MT_PR_OV_P_FC) {
531                 ntuple |= G_FW_VIID_VIN(viid) << FILTER_SEL_WIDTH_P_FC;
532                 ntuple |= G_FW_VIID_PFN(viid) << FILTER_SEL_WIDTH_VIN_P_FC;
533                 ntuple |= G_FW_VIID_VIVLD(viid) << FILTER_SEL_WIDTH_TAG_P_FC;
534                 ntuple |= e->lport << S_PORT;
535 		ntuple |= IPPROTO_TCP << FILTER_SEL_WIDTH_VLD_TAG_P_FC;
536         }
537 
538 	if (is_t4(pi->adapter))
539 		return (htobe32(ntuple));
540 	else
541 		return (htobe64(V_FILTER_TUPLE(ntuple)));
542 }
543 
544 void
545 set_tcpddp_ulp_mode(struct toepcb *toep)
546 {
547 
548 	toep->ulp_mode = ULP_MODE_TCPDDP;
549 	toep->ddp_flags = DDP_OK;
550 	toep->ddp_score = DDP_LOW_SCORE;
551 }
552 
553 int
554 negative_advice(int status)
555 {
556 
557 	return (status == CPL_ERR_RTX_NEG_ADVICE ||
558 	    status == CPL_ERR_PERSIST_NEG_ADVICE ||
559 	    status == CPL_ERR_KEEPALV_NEG_ADVICE);
560 }
561 
562 static int
563 alloc_tid_tabs(struct tid_info *t)
564 {
565 	size_t size;
566 	unsigned int i;
567 
568 	size = t->ntids * sizeof(*t->tid_tab) +
569 	    t->natids * sizeof(*t->atid_tab) +
570 	    t->nstids * sizeof(*t->stid_tab);
571 
572 	t->tid_tab = malloc(size, M_CXGBE, M_ZERO | M_NOWAIT);
573 	if (t->tid_tab == NULL)
574 		return (ENOMEM);
575 
576 	mtx_init(&t->atid_lock, "atid lock", NULL, MTX_DEF);
577 	t->atid_tab = (union aopen_entry *)&t->tid_tab[t->ntids];
578 	t->afree = t->atid_tab;
579 	t->atids_in_use = 0;
580 	for (i = 1; i < t->natids; i++)
581 		t->atid_tab[i - 1].next = &t->atid_tab[i];
582 	t->atid_tab[t->natids - 1].next = NULL;
583 
584 	mtx_init(&t->stid_lock, "stid lock", NULL, MTX_DEF);
585 	t->stid_tab = (struct listen_ctx **)&t->atid_tab[t->natids];
586 	t->stids_in_use = 0;
587 	TAILQ_INIT(&t->stids);
588 	t->nstids_free_head = t->nstids;
589 
590 	atomic_store_rel_int(&t->tids_in_use, 0);
591 
592 	return (0);
593 }
594 
595 static void
596 free_tid_tabs(struct tid_info *t)
597 {
598 	KASSERT(t->tids_in_use == 0,
599 	    ("%s: %d tids still in use.", __func__, t->tids_in_use));
600 	KASSERT(t->atids_in_use == 0,
601 	    ("%s: %d atids still in use.", __func__, t->atids_in_use));
602 	KASSERT(t->stids_in_use == 0,
603 	    ("%s: %d tids still in use.", __func__, t->stids_in_use));
604 
605 	free(t->tid_tab, M_CXGBE);
606 	t->tid_tab = NULL;
607 
608 	if (mtx_initialized(&t->atid_lock))
609 		mtx_destroy(&t->atid_lock);
610 	if (mtx_initialized(&t->stid_lock))
611 		mtx_destroy(&t->stid_lock);
612 }
613 
614 static int
615 add_lip(struct adapter *sc, struct in6_addr *lip)
616 {
617         struct fw_clip_cmd c;
618 
619 	ASSERT_SYNCHRONIZED_OP(sc);
620 	/* mtx_assert(&td->clip_table_lock, MA_OWNED); */
621 
622         memset(&c, 0, sizeof(c));
623 	c.op_to_write = htonl(V_FW_CMD_OP(FW_CLIP_CMD) | F_FW_CMD_REQUEST |
624 	    F_FW_CMD_WRITE);
625         c.alloc_to_len16 = htonl(F_FW_CLIP_CMD_ALLOC | FW_LEN16(c));
626         c.ip_hi = *(uint64_t *)&lip->s6_addr[0];
627         c.ip_lo = *(uint64_t *)&lip->s6_addr[8];
628 
629 	return (t4_wr_mbox_ns(sc, sc->mbox, &c, sizeof(c), &c));
630 }
631 
632 static int
633 delete_lip(struct adapter *sc, struct in6_addr *lip)
634 {
635 	struct fw_clip_cmd c;
636 
637 	ASSERT_SYNCHRONIZED_OP(sc);
638 	/* mtx_assert(&td->clip_table_lock, MA_OWNED); */
639 
640 	memset(&c, 0, sizeof(c));
641 	c.op_to_write = htonl(V_FW_CMD_OP(FW_CLIP_CMD) | F_FW_CMD_REQUEST |
642 	    F_FW_CMD_READ);
643         c.alloc_to_len16 = htonl(F_FW_CLIP_CMD_FREE | FW_LEN16(c));
644         c.ip_hi = *(uint64_t *)&lip->s6_addr[0];
645         c.ip_lo = *(uint64_t *)&lip->s6_addr[8];
646 
647 	return (t4_wr_mbox_ns(sc, sc->mbox, &c, sizeof(c), &c));
648 }
649 
650 static struct clip_entry *
651 search_lip(struct tom_data *td, struct in6_addr *lip)
652 {
653 	struct clip_entry *ce;
654 
655 	mtx_assert(&td->clip_table_lock, MA_OWNED);
656 
657 	TAILQ_FOREACH(ce, &td->clip_table, link) {
658 		if (IN6_ARE_ADDR_EQUAL(&ce->lip, lip))
659 			return (ce);
660 	}
661 
662 	return (NULL);
663 }
664 
665 struct clip_entry *
666 hold_lip(struct tom_data *td, struct in6_addr *lip)
667 {
668 	struct clip_entry *ce;
669 
670 	mtx_lock(&td->clip_table_lock);
671 	ce = search_lip(td, lip);
672 	if (ce != NULL)
673 		ce->refcount++;
674 	mtx_unlock(&td->clip_table_lock);
675 
676 	return (ce);
677 }
678 
679 void
680 release_lip(struct tom_data *td, struct clip_entry *ce)
681 {
682 
683 	mtx_lock(&td->clip_table_lock);
684 	KASSERT(search_lip(td, &ce->lip) == ce,
685 	    ("%s: CLIP entry %p p not in CLIP table.", __func__, ce));
686 	KASSERT(ce->refcount > 0,
687 	    ("%s: CLIP entry %p has refcount 0", __func__, ce));
688 	--ce->refcount;
689 	mtx_unlock(&td->clip_table_lock);
690 }
691 
692 static void
693 init_clip_table(struct adapter *sc, struct tom_data *td)
694 {
695 	struct in6_ifaddr *ia;
696 	struct in6_addr *lip, tlip;
697 	struct clip_entry *ce;
698 
699 	ASSERT_SYNCHRONIZED_OP(sc);
700 
701 	mtx_init(&td->clip_table_lock, "CLIP table lock", NULL, MTX_DEF);
702 	TAILQ_INIT(&td->clip_table);
703 
704 	IN6_IFADDR_RLOCK();
705 	TAILQ_FOREACH(ia, &V_in6_ifaddrhead, ia_link) {
706 		lip = &ia->ia_addr.sin6_addr;
707 
708 		KASSERT(!IN6_IS_ADDR_MULTICAST(lip),
709 		    ("%s: mcast address in in6_ifaddr list", __func__));
710 
711 		if (IN6_IS_ADDR_LOOPBACK(lip))
712 			continue;
713 		if (IN6_IS_SCOPE_EMBED(lip)) {
714 			/* Remove the embedded scope */
715 			tlip = *lip;
716 			lip = &tlip;
717 			in6_clearscope(lip);
718 		}
719 		/*
720 		 * XXX: how to weed out the link local address for the loopback
721 		 * interface?  It's fe80::1 usually (always?).
722 		 */
723 
724 		mtx_lock(&td->clip_table_lock);
725 		if (search_lip(td, lip) == NULL) {
726 			ce = malloc(sizeof(*ce), M_CXGBE, M_NOWAIT);
727 			memcpy(&ce->lip, lip, sizeof(ce->lip));
728 			ce->refcount = 0;
729 			if (add_lip(sc, lip) == 0)
730 				TAILQ_INSERT_TAIL(&td->clip_table, ce, link);
731 			else
732 				free(ce, M_CXGBE);
733 		}
734 		mtx_unlock(&td->clip_table_lock);
735 	}
736 	IN6_IFADDR_RUNLOCK();
737 }
738 
739 static void
740 destroy_clip_table(struct adapter *sc, struct tom_data *td)
741 {
742 	struct clip_entry *ce, *ce_temp;
743 
744 	if (mtx_initialized(&td->clip_table_lock)) {
745 		mtx_lock(&td->clip_table_lock);
746 		TAILQ_FOREACH_SAFE(ce, &td->clip_table, link, ce_temp) {
747 			KASSERT(ce->refcount == 0,
748 			    ("%s: CLIP entry %p still in use (%d)", __func__,
749 			    ce, ce->refcount));
750 			TAILQ_REMOVE(&td->clip_table, ce, link);
751 			delete_lip(sc, &ce->lip);
752 			free(ce, M_CXGBE);
753 		}
754 		mtx_unlock(&td->clip_table_lock);
755 		mtx_destroy(&td->clip_table_lock);
756 	}
757 }
758 
759 static void
760 free_tom_data(struct adapter *sc, struct tom_data *td)
761 {
762 
763 	ASSERT_SYNCHRONIZED_OP(sc);
764 
765 	KASSERT(TAILQ_EMPTY(&td->toep_list),
766 	    ("%s: TOE PCB list is not empty.", __func__));
767 	KASSERT(td->lctx_count == 0,
768 	    ("%s: lctx hash table is not empty.", __func__));
769 
770 	t4_uninit_l2t_cpl_handlers(sc);
771 	t4_uninit_cpl_io_handlers(sc);
772 	t4_uninit_ddp(sc, td);
773 	destroy_clip_table(sc, td);
774 
775 	if (td->listen_mask != 0)
776 		hashdestroy(td->listen_hash, M_CXGBE, td->listen_mask);
777 
778 	if (mtx_initialized(&td->lctx_hash_lock))
779 		mtx_destroy(&td->lctx_hash_lock);
780 	if (mtx_initialized(&td->toep_list_lock))
781 		mtx_destroy(&td->toep_list_lock);
782 
783 	free_tid_tabs(&sc->tids);
784 	free(td, M_CXGBE);
785 }
786 
787 /*
788  * Ground control to Major TOM
789  * Commencing countdown, engines on
790  */
791 static int
792 t4_tom_activate(struct adapter *sc)
793 {
794 	struct tom_data *td;
795 	struct toedev *tod;
796 	int i, rc;
797 
798 	ASSERT_SYNCHRONIZED_OP(sc);
799 
800 	/* per-adapter softc for TOM */
801 	td = malloc(sizeof(*td), M_CXGBE, M_ZERO | M_NOWAIT);
802 	if (td == NULL)
803 		return (ENOMEM);
804 
805 	/* List of TOE PCBs and associated lock */
806 	mtx_init(&td->toep_list_lock, "PCB list lock", NULL, MTX_DEF);
807 	TAILQ_INIT(&td->toep_list);
808 
809 	/* Listen context */
810 	mtx_init(&td->lctx_hash_lock, "lctx hash lock", NULL, MTX_DEF);
811 	td->listen_hash = hashinit_flags(LISTEN_HASH_SIZE, M_CXGBE,
812 	    &td->listen_mask, HASH_NOWAIT);
813 
814 	/* TID tables */
815 	rc = alloc_tid_tabs(&sc->tids);
816 	if (rc != 0)
817 		goto done;
818 
819 	/* DDP page pods and CPL handlers */
820 	t4_init_ddp(sc, td);
821 
822 	/* CLIP table for IPv6 offload */
823 	init_clip_table(sc, td);
824 
825 	/* CPL handlers */
826 	t4_init_connect_cpl_handlers(sc);
827 	t4_init_l2t_cpl_handlers(sc);
828 	t4_init_listen_cpl_handlers(sc);
829 	t4_init_cpl_io_handlers(sc);
830 
831 	/* toedev ops */
832 	tod = &td->tod;
833 	init_toedev(tod);
834 	tod->tod_softc = sc;
835 	tod->tod_connect = t4_connect;
836 	tod->tod_listen_start = t4_listen_start;
837 	tod->tod_listen_stop = t4_listen_stop;
838 	tod->tod_rcvd = t4_rcvd;
839 	tod->tod_output = t4_tod_output;
840 	tod->tod_send_rst = t4_send_rst;
841 	tod->tod_send_fin = t4_send_fin;
842 	tod->tod_pcb_detach = t4_pcb_detach;
843 	tod->tod_l2_update = t4_l2_update;
844 	tod->tod_syncache_added = t4_syncache_added;
845 	tod->tod_syncache_removed = t4_syncache_removed;
846 	tod->tod_syncache_respond = t4_syncache_respond;
847 	tod->tod_offload_socket = t4_offload_socket;
848 
849 	for_each_port(sc, i)
850 		TOEDEV(sc->port[i]->ifp) = &td->tod;
851 
852 	sc->tom_softc = td;
853 	sc->flags |= TOM_INIT_DONE;
854 	register_toedev(sc->tom_softc);
855 
856 done:
857 	if (rc != 0)
858 		free_tom_data(sc, td);
859 	return (rc);
860 }
861 
862 static int
863 t4_tom_deactivate(struct adapter *sc)
864 {
865 	int rc = 0;
866 	struct tom_data *td = sc->tom_softc;
867 
868 	ASSERT_SYNCHRONIZED_OP(sc);
869 
870 	if (td == NULL)
871 		return (0);	/* XXX. KASSERT? */
872 
873 	if (sc->offload_map != 0)
874 		return (EBUSY);	/* at least one port has IFCAP_TOE enabled */
875 
876 	mtx_lock(&td->toep_list_lock);
877 	if (!TAILQ_EMPTY(&td->toep_list))
878 		rc = EBUSY;
879 	mtx_unlock(&td->toep_list_lock);
880 
881 	mtx_lock(&td->lctx_hash_lock);
882 	if (td->lctx_count > 0)
883 		rc = EBUSY;
884 	mtx_unlock(&td->lctx_hash_lock);
885 
886 	if (rc == 0) {
887 		unregister_toedev(sc->tom_softc);
888 		free_tom_data(sc, td);
889 		sc->tom_softc = NULL;
890 		sc->flags &= ~TOM_INIT_DONE;
891 	}
892 
893 	return (rc);
894 }
895 
896 static int
897 t4_tom_mod_load(void)
898 {
899 	int rc;
900 	struct protosw *tcp_protosw, *tcp6_protosw;
901 
902 	tcp_protosw = pffindproto(PF_INET, IPPROTO_TCP, SOCK_STREAM);
903 	if (tcp_protosw == NULL)
904 		return (ENOPROTOOPT);
905 	bcopy(tcp_protosw, &ddp_protosw, sizeof(ddp_protosw));
906 	bcopy(tcp_protosw->pr_usrreqs, &ddp_usrreqs, sizeof(ddp_usrreqs));
907 	ddp_usrreqs.pru_soreceive = t4_soreceive_ddp;
908 	ddp_protosw.pr_usrreqs = &ddp_usrreqs;
909 
910 	tcp6_protosw = pffindproto(PF_INET6, IPPROTO_TCP, SOCK_STREAM);
911 	if (tcp6_protosw == NULL)
912 		return (ENOPROTOOPT);
913 	bcopy(tcp6_protosw, &ddp6_protosw, sizeof(ddp6_protosw));
914 	bcopy(tcp6_protosw->pr_usrreqs, &ddp6_usrreqs, sizeof(ddp6_usrreqs));
915 	ddp6_usrreqs.pru_soreceive = t4_soreceive_ddp;
916 	ddp6_protosw.pr_usrreqs = &ddp6_usrreqs;
917 
918 	rc = t4_register_uld(&tom_uld_info);
919 	if (rc != 0)
920 		t4_tom_mod_unload();
921 
922 	return (rc);
923 }
924 
925 static void
926 tom_uninit(struct adapter *sc, void *arg __unused)
927 {
928 	if (begin_synchronized_op(sc, NULL, HOLD_LOCK, "t4tomun"))
929 		return;
930 
931 	/* Try to free resources (works only if no port has IFCAP_TOE) */
932 	if (sc->flags & TOM_INIT_DONE)
933 		t4_deactivate_uld(sc, ULD_TOM);
934 
935 	end_synchronized_op(sc, LOCK_HELD);
936 }
937 
938 static int
939 t4_tom_mod_unload(void)
940 {
941 	t4_iterate(tom_uninit, NULL);
942 
943 	if (t4_unregister_uld(&tom_uld_info) == EBUSY)
944 		return (EBUSY);
945 
946 	return (0);
947 }
948 #endif	/* TCP_OFFLOAD */
949 
950 static int
951 t4_tom_modevent(module_t mod, int cmd, void *arg)
952 {
953 	int rc = 0;
954 
955 #ifdef TCP_OFFLOAD
956 	switch (cmd) {
957 	case MOD_LOAD:
958 		rc = t4_tom_mod_load();
959 		break;
960 
961 	case MOD_UNLOAD:
962 		rc = t4_tom_mod_unload();
963 		break;
964 
965 	default:
966 		rc = EINVAL;
967 	}
968 #else
969 	printf("t4_tom: compiled without TCP_OFFLOAD support.\n");
970 	rc = EOPNOTSUPP;
971 #endif
972 	return (rc);
973 }
974 
975 static moduledata_t t4_tom_moddata= {
976 	"t4_tom",
977 	t4_tom_modevent,
978 	0
979 };
980 
981 MODULE_VERSION(t4_tom, 1);
982 MODULE_DEPEND(t4_tom, toecore, 1, 1, 1);
983 MODULE_DEPEND(t4_tom, t4nex, 1, 1, 1);
984 DECLARE_MODULE(t4_tom, t4_tom_moddata, SI_SUB_EXEC, SI_ORDER_ANY);
985