1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2012 Chelsio Communications, Inc.
5 * All rights reserved.
6 * Written by: Navdeep Parhar <np@FreeBSD.org>
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30 #include <sys/cdefs.h>
31 #include "opt_inet.h"
32 #include "opt_inet6.h"
33
34 #ifdef TCP_OFFLOAD
35 #include <sys/param.h>
36 #include <sys/systm.h>
37 #include <sys/kernel.h>
38 #include <sys/ktr.h>
39 #include <sys/module.h>
40 #include <sys/protosw.h>
41 #include <sys/domain.h>
42 #include <sys/socket.h>
43 #include <sys/socketvar.h>
44 #include <sys/sysctl.h>
45 #include <net/ethernet.h>
46 #include <net/if.h>
47 #include <net/if_types.h>
48 #include <net/if_vlan_var.h>
49 #include <net/route.h>
50 #include <net/route/nhop.h>
51 #include <netinet/in.h>
52 #include <netinet/in_pcb.h>
53 #include <netinet/ip.h>
54 #define TCPSTATES
55 #include <netinet/tcp_fsm.h>
56 #include <netinet/tcp_var.h>
57 #include <netinet/toecore.h>
58 #include <netinet/cc/cc.h>
59
60 #include "common/common.h"
61 #include "common/t4_msg.h"
62 #include "common/t4_regs.h"
63 #include "common/t4_regs_values.h"
64 #include "t4_clip.h"
65 #include "tom/t4_tom_l2t.h"
66 #include "tom/t4_tom.h"
67
68 /*
69 * Active open succeeded.
70 */
71 static int
do_act_establish(struct sge_iq * iq,const struct rss_header * rss,struct mbuf * m)72 do_act_establish(struct sge_iq *iq, const struct rss_header *rss,
73 struct mbuf *m)
74 {
75 struct adapter *sc = iq->adapter;
76 const struct cpl_act_establish *cpl = (const void *)(rss + 1);
77 u_int tid = GET_TID(cpl);
78 u_int atid = G_TID_TID(ntohl(cpl->tos_atid));
79 struct toepcb *toep = lookup_atid(sc, atid);
80 struct inpcb *inp = toep->inp;
81 struct tcpcb *tp = intotcpcb(inp);
82
83 KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
84 KASSERT(toep->tid == atid, ("%s: toep tid/atid mismatch", __func__));
85
86 CTR3(KTR_CXGBE, "%s: atid %u, tid %u", __func__, atid, tid);
87 free_atid(sc, atid);
88
89 CURVNET_SET(toep->vnet);
90 INP_WLOCK(inp);
91 toep->tid = tid;
92 insert_tid(sc, tid, toep, inp->inp_vflag & INP_IPV6 ? 2 : 1);
93 if (sc->params.tid_qid_sel_mask != 0) {
94 update_tid_qid_sel(toep->vi, &toep->params, tid);
95 toep->ofld_txq = &sc->sge.ofld_txq[toep->params.txq_idx];
96 toep->ctrlq = &sc->sge.ctrlq[toep->params.ctrlq_idx];
97 }
98
99 if (tp->t_flags & TF_DISCONNECTED) {
100
101 /* socket closed by the kernel before hw told us it connected */
102
103 send_flowc_wr(toep, NULL);
104 send_reset(sc, toep, be32toh(cpl->snd_isn));
105 goto done;
106 }
107
108 make_established(toep, be32toh(cpl->snd_isn) - 1,
109 be32toh(cpl->rcv_isn) - 1, cpl->tcp_opt);
110 inp->inp_flowtype = M_HASHTYPE_OPAQUE;
111 inp->inp_flowid = tid;
112
113 done:
114 INP_WUNLOCK(inp);
115 CURVNET_RESTORE();
116 return (0);
117 }
118
119 void
act_open_failure_cleanup(struct adapter * sc,struct toepcb * toep,u_int status)120 act_open_failure_cleanup(struct adapter *sc, struct toepcb *toep, u_int status)
121 {
122 struct inpcb *inp = toep->inp;
123 struct toedev *tod = &toep->td->tod;
124 struct epoch_tracker et;
125 struct tom_data *td = sc->tom_softc;
126
127 if (toep->tid >= 0) {
128 free_atid(sc, toep->tid);
129 toep->tid = -1;
130 mtx_lock(&td->toep_list_lock);
131 if (toep->flags & TPF_IN_TOEP_LIST) {
132 toep->flags &= ~TPF_IN_TOEP_LIST;
133 TAILQ_REMOVE(&td->toep_list, toep, link);
134 }
135 mtx_unlock(&td->toep_list_lock);
136 }
137
138 CURVNET_SET(toep->vnet);
139 if (status != EAGAIN)
140 NET_EPOCH_ENTER(et);
141 INP_WLOCK(inp);
142 toe_connect_failed(tod, inp, status);
143 final_cpl_received(toep); /* unlocks inp */
144 if (status != EAGAIN)
145 NET_EPOCH_EXIT(et);
146 CURVNET_RESTORE();
147 }
148
149 /*
150 * Active open failed.
151 */
152 static int
do_act_open_rpl(struct sge_iq * iq,const struct rss_header * rss,struct mbuf * m)153 do_act_open_rpl(struct sge_iq *iq, const struct rss_header *rss,
154 struct mbuf *m)
155 {
156 struct adapter *sc = iq->adapter;
157 const struct cpl_act_open_rpl *cpl = (const void *)(rss + 1);
158 u_int atid = G_TID_TID(G_AOPEN_ATID(be32toh(cpl->atid_status)));
159 u_int status = G_AOPEN_STATUS(be32toh(cpl->atid_status));
160 struct toepcb *toep = lookup_atid(sc, atid);
161 int rc;
162
163 KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
164 KASSERT(toep->tid == atid, ("%s: toep tid/atid mismatch", __func__));
165
166 CTR3(KTR_CXGBE, "%s: atid %u, status %u ", __func__, atid, status);
167
168 /* Ignore negative advice */
169 if (negative_advice(status))
170 return (0);
171
172 if (status && act_open_has_tid(status))
173 release_tid(sc, GET_TID(cpl), toep->ctrlq);
174
175 rc = act_open_rpl_status_to_errno(status);
176 act_open_failure_cleanup(sc, toep, rc);
177
178 return (0);
179 }
180
181 void
t4_init_connect_cpl_handlers(void)182 t4_init_connect_cpl_handlers(void)
183 {
184
185 t4_register_cpl_handler(CPL_ACT_ESTABLISH, do_act_establish);
186 t4_register_shared_cpl_handler(CPL_ACT_OPEN_RPL, do_act_open_rpl,
187 CPL_COOKIE_TOM);
188 }
189
190 void
t4_uninit_connect_cpl_handlers(void)191 t4_uninit_connect_cpl_handlers(void)
192 {
193
194 t4_register_cpl_handler(CPL_ACT_ESTABLISH, NULL);
195 t4_register_shared_cpl_handler(CPL_ACT_OPEN_RPL, NULL, CPL_COOKIE_TOM);
196 }
197
198 #ifdef KTR
199 #define DONT_OFFLOAD_ACTIVE_OPEN(x) do { \
200 reason = __LINE__; \
201 rc = (x); \
202 goto failed; \
203 } while (0)
204 #else
205 #define DONT_OFFLOAD_ACTIVE_OPEN(x) do { \
206 rc = (x); \
207 goto failed; \
208 } while (0)
209 #endif
210
211 static inline int
act_open_cpl_size(struct adapter * sc,int isipv6)212 act_open_cpl_size(struct adapter *sc, int isipv6)
213 {
214 int idx;
215 static const int sz_table[4][2] = {
216 {
217 sizeof (struct cpl_act_open_req),
218 sizeof (struct cpl_act_open_req6)
219 },
220 {
221 sizeof (struct cpl_t5_act_open_req),
222 sizeof (struct cpl_t5_act_open_req6)
223 },
224 {
225 sizeof (struct cpl_t6_act_open_req),
226 sizeof (struct cpl_t6_act_open_req6)
227 },
228 {
229 sizeof (struct cpl_t7_act_open_req),
230 sizeof (struct cpl_t7_act_open_req6)
231 },
232 };
233
234 MPASS(chip_id(sc) >= CHELSIO_T4);
235 idx = min(chip_id(sc) - CHELSIO_T4, 3);
236
237 return (sz_table[idx][!!isipv6]);
238 }
239
240 /*
241 * active open (soconnect).
242 *
243 * State of affairs on entry:
244 * soisconnecting (so_state |= SS_ISCONNECTING)
245 * tcbinfo not locked (This has changed - used to be WLOCKed)
246 * inp WLOCKed
247 * tp->t_state = TCPS_SYN_SENT
248 * rtalloc1, RT_UNLOCK on rt.
249 */
250 int
t4_connect(struct toedev * tod,struct socket * so,struct nhop_object * nh,struct sockaddr * nam)251 t4_connect(struct toedev *tod, struct socket *so, struct nhop_object *nh,
252 struct sockaddr *nam)
253 {
254 struct adapter *sc = tod->tod_softc;
255 struct tom_data *td;
256 struct toepcb *toep = NULL;
257 struct wrqe *wr = NULL;
258 if_t rt_ifp = nh->nh_ifp;
259 struct vi_info *vi;
260 int qid_atid, rc, isipv6;
261 struct inpcb *inp = sotoinpcb(so);
262 struct tcpcb *tp = intotcpcb(inp);
263 #ifdef KTR
264 int reason;
265 #endif
266 struct offload_settings settings;
267 struct epoch_tracker et;
268 uint16_t vid = 0xfff, pcp = 0;
269 uint64_t ntuple;
270
271 INP_WLOCK_ASSERT(inp);
272 KASSERT(nam->sa_family == AF_INET || nam->sa_family == AF_INET6,
273 ("%s: dest addr %p has family %u", __func__, nam, nam->sa_family));
274
275 if (if_gettype(rt_ifp) == IFT_ETHER)
276 vi = if_getsoftc(rt_ifp);
277 else if (if_gettype(rt_ifp) == IFT_L2VLAN) {
278 if_t ifp = VLAN_TRUNKDEV(rt_ifp);
279
280 vi = if_getsoftc(ifp);
281 VLAN_TAG(rt_ifp, &vid);
282 VLAN_PCP(rt_ifp, &pcp);
283 } else if (if_gettype(rt_ifp) == IFT_IEEE8023ADLAG)
284 DONT_OFFLOAD_ACTIVE_OPEN(ENOSYS); /* XXX: implement lagg+TOE */
285 else
286 DONT_OFFLOAD_ACTIVE_OPEN(ENOTSUP);
287 if (sc->flags & KERN_TLS_ON)
288 DONT_OFFLOAD_ACTIVE_OPEN(ENOTSUP);
289
290 rw_rlock(&sc->policy_lock);
291 settings = *lookup_offload_policy(sc, OPEN_TYPE_ACTIVE, NULL,
292 EVL_MAKETAG(vid, pcp, 0), inp);
293 rw_runlock(&sc->policy_lock);
294 if (!settings.offload)
295 DONT_OFFLOAD_ACTIVE_OPEN(EPERM);
296
297 toep = alloc_toepcb(vi, M_NOWAIT);
298 if (toep == NULL)
299 DONT_OFFLOAD_ACTIVE_OPEN(ENOMEM);
300
301 toep->tid = alloc_atid(sc, toep);
302 if (toep->tid < 0)
303 DONT_OFFLOAD_ACTIVE_OPEN(ENOMEM);
304
305 toep->l2te = t4_l2t_get(vi->pi, rt_ifp,
306 nh->nh_flags & NHF_GATEWAY ? &nh->gw_sa : nam);
307 if (toep->l2te == NULL)
308 DONT_OFFLOAD_ACTIVE_OPEN(ENOMEM);
309
310 toep->vnet = so->so_vnet;
311 init_conn_params(vi, &settings, &inp->inp_inc, so, NULL,
312 toep->l2te->idx, &toep->params);
313 init_toepcb(vi, toep);
314
315 isipv6 = nam->sa_family == AF_INET6;
316 wr = alloc_wrqe(act_open_cpl_size(sc, isipv6), toep->ctrlq);
317 if (wr == NULL)
318 DONT_OFFLOAD_ACTIVE_OPEN(ENOMEM);
319
320 qid_atid = V_TID_QID(toep->ofld_rxq->iq.abs_id) | V_TID_TID(toep->tid) |
321 V_TID_COOKIE(CPL_COOKIE_TOM);
322
323 ntuple = select_ntuple(vi, toep->l2te);
324 if (isipv6) {
325 struct cpl_act_open_req6 *cpl = wrtod(wr);
326 struct cpl_t5_act_open_req6 *cpl5 = (void *)cpl;
327 struct cpl_t6_act_open_req6 *cpl6 = (void *)cpl;
328 struct cpl_t7_act_open_req6 *cpl7 = (void *)cpl;
329
330 if ((inp->inp_vflag & INP_IPV6) == 0)
331 DONT_OFFLOAD_ACTIVE_OPEN(ENOTSUP);
332
333 toep->ce = t4_get_clip_entry(sc, &inp->in6p_laddr, true);
334 if (toep->ce == NULL)
335 DONT_OFFLOAD_ACTIVE_OPEN(ENOENT);
336
337 switch (chip_id(sc)) {
338 case CHELSIO_T4:
339 INIT_TP_WR(cpl, 0);
340 cpl->params = htobe32((uint32_t)ntuple);
341 break;
342 case CHELSIO_T5:
343 INIT_TP_WR(cpl5, 0);
344 cpl5->iss = htobe32(tp->iss);
345 cpl5->params = htobe64(V_FILTER_TUPLE(ntuple));
346 break;
347 case CHELSIO_T6:
348 INIT_TP_WR(cpl6, 0);
349 cpl6->iss = htobe32(tp->iss);
350 cpl6->params = htobe64(V_FILTER_TUPLE(ntuple));
351 break;
352 case CHELSIO_T7:
353 default:
354 INIT_TP_WR(cpl7, 0);
355 cpl7->iss = htobe32(tp->iss);
356 cpl7->params = htobe64(V_T7_FILTER_TUPLE(ntuple));
357 break;
358 }
359 OPCODE_TID(cpl) = htobe32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6,
360 qid_atid));
361 cpl->local_port = inp->inp_lport;
362 cpl->local_ip_hi = *(uint64_t *)&inp->in6p_laddr.s6_addr[0];
363 cpl->local_ip_lo = *(uint64_t *)&inp->in6p_laddr.s6_addr[8];
364 cpl->peer_port = inp->inp_fport;
365 cpl->peer_ip_hi = *(uint64_t *)&inp->in6p_faddr.s6_addr[0];
366 cpl->peer_ip_lo = *(uint64_t *)&inp->in6p_faddr.s6_addr[8];
367 cpl->opt0 = calc_options0(vi, &toep->params);
368 cpl->opt2 = calc_options2(vi, &toep->params);
369
370 CTR6(KTR_CXGBE,
371 "%s: atid %u, toep %p, inp %p, opt0 %#016lx, opt2 %#08x",
372 __func__, toep->tid, toep, inp, be64toh(cpl->opt0),
373 be32toh(cpl->opt2));
374 } else {
375 struct cpl_act_open_req *cpl = wrtod(wr);
376 struct cpl_t5_act_open_req *cpl5 = (void *)cpl;
377 struct cpl_t6_act_open_req *cpl6 = (void *)cpl;
378 struct cpl_t7_act_open_req *cpl7 = (void *)cpl;
379
380 switch (chip_id(sc)) {
381 case CHELSIO_T4:
382 INIT_TP_WR(cpl, 0);
383 cpl->params = htobe32((uint32_t)ntuple);
384 break;
385 case CHELSIO_T5:
386 INIT_TP_WR(cpl5, 0);
387 cpl5->iss = htobe32(tp->iss);
388 cpl5->params = htobe64(V_FILTER_TUPLE(ntuple));
389 break;
390 case CHELSIO_T6:
391 INIT_TP_WR(cpl6, 0);
392 cpl6->iss = htobe32(tp->iss);
393 cpl6->params = htobe64(V_FILTER_TUPLE(ntuple));
394 break;
395 case CHELSIO_T7:
396 default:
397 INIT_TP_WR(cpl7, 0);
398 cpl7->iss = htobe32(tp->iss);
399 cpl7->params = htobe64(V_T7_FILTER_TUPLE(ntuple));
400 }
401 OPCODE_TID(cpl) = htobe32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ,
402 qid_atid));
403 inp_4tuple_get(inp, &cpl->local_ip, &cpl->local_port,
404 &cpl->peer_ip, &cpl->peer_port);
405 cpl->opt0 = calc_options0(vi, &toep->params);
406 cpl->opt2 = calc_options2(vi, &toep->params);
407
408 CTR6(KTR_CXGBE,
409 "%s: atid %u, toep %p, inp %p, opt0 %#016lx, opt2 %#08x",
410 __func__, toep->tid, toep, inp, be64toh(cpl->opt0),
411 be32toh(cpl->opt2));
412 }
413
414 offload_socket(so, toep);
415 /* Add the TOE PCB to the active list */
416 td = toep->td;
417 mtx_lock(&td->toep_list_lock);
418 TAILQ_INSERT_TAIL(&td->toep_list, toep, link);
419 toep->flags |= TPF_IN_TOEP_LIST;
420 mtx_unlock(&td->toep_list_lock);
421 NET_EPOCH_ENTER(et);
422 rc = t4_l2t_send(sc, wr, toep->l2te);
423 NET_EPOCH_EXIT(et);
424 if (rc == 0) {
425 toep->flags |= TPF_CPL_PENDING;
426 return (0);
427 }
428
429 undo_offload_socket(so);
430 #if defined(KTR)
431 reason = __LINE__;
432 #endif
433 failed:
434 CTR3(KTR_CXGBE, "%s: not offloading (%d), rc %d", __func__, reason, rc);
435
436 if (wr)
437 free_wrqe(wr);
438
439 if (toep) {
440 if (toep->tid >= 0)
441 free_atid(sc, toep->tid);
442 if (toep->l2te)
443 t4_l2t_release(toep->l2te);
444 if (toep->ce)
445 t4_release_clip_entry(sc, toep->ce);
446 free_toepcb(toep);
447 }
448
449 return (rc);
450 }
451 #endif
452