1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2012 Chelsio Communications, Inc. 5 * All rights reserved. 6 * Written by: Navdeep Parhar <np@FreeBSD.org> 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include <sys/cdefs.h> 31 #include "opt_inet.h" 32 #include "opt_inet6.h" 33 34 #ifdef TCP_OFFLOAD 35 #include <sys/param.h> 36 #include <sys/systm.h> 37 #include <sys/kernel.h> 38 #include <sys/ktr.h> 39 #include <sys/module.h> 40 #include <sys/protosw.h> 41 #include <sys/domain.h> 42 #include <sys/socket.h> 43 #include <sys/socketvar.h> 44 #include <sys/sysctl.h> 45 #include <net/ethernet.h> 46 #include <net/if.h> 47 #include <net/if_types.h> 48 #include <net/if_vlan_var.h> 49 #include <net/route.h> 50 #include <net/route/nhop.h> 51 #include <netinet/in.h> 52 #include <netinet/in_pcb.h> 53 #include <netinet/ip.h> 54 #define TCPSTATES 55 #include <netinet/tcp_fsm.h> 56 #include <netinet/tcp_var.h> 57 #include <netinet/toecore.h> 58 #include <netinet/cc/cc.h> 59 60 #include "common/common.h" 61 #include "common/t4_msg.h" 62 #include "common/t4_regs.h" 63 #include "common/t4_regs_values.h" 64 #include "t4_clip.h" 65 #include "tom/t4_tom_l2t.h" 66 #include "tom/t4_tom.h" 67 68 /* 69 * Active open succeeded. 70 */ 71 static int 72 do_act_establish(struct sge_iq *iq, const struct rss_header *rss, 73 struct mbuf *m) 74 { 75 struct adapter *sc = iq->adapter; 76 const struct cpl_act_establish *cpl = (const void *)(rss + 1); 77 u_int tid = GET_TID(cpl); 78 u_int atid = G_TID_TID(ntohl(cpl->tos_atid)); 79 struct toepcb *toep = lookup_atid(sc, atid); 80 struct inpcb *inp = toep->inp; 81 struct tcpcb *tp = intotcpcb(inp); 82 83 KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__)); 84 KASSERT(toep->tid == atid, ("%s: toep tid/atid mismatch", __func__)); 85 86 CTR3(KTR_CXGBE, "%s: atid %u, tid %u", __func__, atid, tid); 87 free_atid(sc, atid); 88 89 CURVNET_SET(toep->vnet); 90 INP_WLOCK(inp); 91 toep->tid = tid; 92 insert_tid(sc, tid, toep, inp->inp_vflag & INP_IPV6 ? 2 : 1); 93 if (sc->params.tid_qid_sel_mask != 0) { 94 update_tid_qid_sel(toep->vi, &toep->params, tid); 95 toep->ofld_txq = &sc->sge.ofld_txq[toep->params.txq_idx]; 96 toep->ctrlq = &sc->sge.ctrlq[toep->params.ctrlq_idx]; 97 } 98 99 if (tp->t_flags & TF_DISCONNECTED) { 100 101 /* socket closed by the kernel before hw told us it connected */ 102 103 send_flowc_wr(toep, NULL); 104 send_reset(sc, toep, be32toh(cpl->snd_isn)); 105 goto done; 106 } 107 108 make_established(toep, be32toh(cpl->snd_isn) - 1, 109 be32toh(cpl->rcv_isn) - 1, cpl->tcp_opt); 110 inp->inp_flowtype = M_HASHTYPE_OPAQUE; 111 inp->inp_flowid = tid; 112 113 done: 114 INP_WUNLOCK(inp); 115 CURVNET_RESTORE(); 116 return (0); 117 } 118 119 void 120 act_open_failure_cleanup(struct adapter *sc, struct toepcb *toep, u_int status) 121 { 122 struct inpcb *inp = toep->inp; 123 struct toedev *tod = &toep->td->tod; 124 struct epoch_tracker et; 125 struct tom_data *td = sc->tom_softc; 126 127 if (toep->tid >= 0) { 128 free_atid(sc, toep->tid); 129 toep->tid = -1; 130 mtx_lock(&td->toep_list_lock); 131 if (toep->flags & TPF_IN_TOEP_LIST) { 132 toep->flags &= ~TPF_IN_TOEP_LIST; 133 TAILQ_REMOVE(&td->toep_list, toep, link); 134 } 135 mtx_unlock(&td->toep_list_lock); 136 } 137 138 CURVNET_SET(toep->vnet); 139 if (status != EAGAIN) 140 NET_EPOCH_ENTER(et); 141 INP_WLOCK(inp); 142 toe_connect_failed(tod, inp, status); 143 final_cpl_received(toep); /* unlocks inp */ 144 if (status != EAGAIN) 145 NET_EPOCH_EXIT(et); 146 CURVNET_RESTORE(); 147 } 148 149 /* 150 * Active open failed. 151 */ 152 static int 153 do_act_open_rpl(struct sge_iq *iq, const struct rss_header *rss, 154 struct mbuf *m) 155 { 156 struct adapter *sc = iq->adapter; 157 const struct cpl_act_open_rpl *cpl = (const void *)(rss + 1); 158 u_int atid = G_TID_TID(G_AOPEN_ATID(be32toh(cpl->atid_status))); 159 u_int status = G_AOPEN_STATUS(be32toh(cpl->atid_status)); 160 struct toepcb *toep = lookup_atid(sc, atid); 161 int rc; 162 163 KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__)); 164 KASSERT(toep->tid == atid, ("%s: toep tid/atid mismatch", __func__)); 165 166 CTR3(KTR_CXGBE, "%s: atid %u, status %u ", __func__, atid, status); 167 168 /* Ignore negative advice */ 169 if (negative_advice(status)) 170 return (0); 171 172 if (status && act_open_has_tid(status)) 173 release_tid(sc, GET_TID(cpl), toep->ctrlq); 174 175 rc = act_open_rpl_status_to_errno(status); 176 act_open_failure_cleanup(sc, toep, rc); 177 178 return (0); 179 } 180 181 void 182 t4_init_connect_cpl_handlers(void) 183 { 184 185 t4_register_cpl_handler(CPL_ACT_ESTABLISH, do_act_establish); 186 t4_register_shared_cpl_handler(CPL_ACT_OPEN_RPL, do_act_open_rpl, 187 CPL_COOKIE_TOM); 188 } 189 190 void 191 t4_uninit_connect_cpl_handlers(void) 192 { 193 194 t4_register_cpl_handler(CPL_ACT_ESTABLISH, NULL); 195 t4_register_shared_cpl_handler(CPL_ACT_OPEN_RPL, NULL, CPL_COOKIE_TOM); 196 } 197 198 #ifdef KTR 199 #define DONT_OFFLOAD_ACTIVE_OPEN(x) do { \ 200 reason = __LINE__; \ 201 rc = (x); \ 202 goto failed; \ 203 } while (0) 204 #else 205 #define DONT_OFFLOAD_ACTIVE_OPEN(x) do { \ 206 rc = (x); \ 207 goto failed; \ 208 } while (0) 209 #endif 210 211 static inline int 212 act_open_cpl_size(struct adapter *sc, int isipv6) 213 { 214 int idx; 215 static const int sz_table[4][2] = { 216 { 217 sizeof (struct cpl_act_open_req), 218 sizeof (struct cpl_act_open_req6) 219 }, 220 { 221 sizeof (struct cpl_t5_act_open_req), 222 sizeof (struct cpl_t5_act_open_req6) 223 }, 224 { 225 sizeof (struct cpl_t6_act_open_req), 226 sizeof (struct cpl_t6_act_open_req6) 227 }, 228 { 229 sizeof (struct cpl_t7_act_open_req), 230 sizeof (struct cpl_t7_act_open_req6) 231 }, 232 }; 233 234 MPASS(chip_id(sc) >= CHELSIO_T4); 235 idx = min(chip_id(sc) - CHELSIO_T4, 3); 236 237 return (sz_table[idx][!!isipv6]); 238 } 239 240 /* 241 * active open (soconnect). 242 * 243 * State of affairs on entry: 244 * soisconnecting (so_state |= SS_ISCONNECTING) 245 * tcbinfo not locked (This has changed - used to be WLOCKed) 246 * inp WLOCKed 247 * tp->t_state = TCPS_SYN_SENT 248 * rtalloc1, RT_UNLOCK on rt. 249 */ 250 int 251 t4_connect(struct toedev *tod, struct socket *so, struct nhop_object *nh, 252 struct sockaddr *nam) 253 { 254 struct adapter *sc = tod->tod_softc; 255 struct tom_data *td; 256 struct toepcb *toep = NULL; 257 struct wrqe *wr = NULL; 258 if_t rt_ifp = nh->nh_ifp; 259 struct vi_info *vi; 260 int qid_atid, rc, isipv6; 261 struct inpcb *inp = sotoinpcb(so); 262 struct tcpcb *tp = intotcpcb(inp); 263 #ifdef KTR 264 int reason; 265 #endif 266 struct offload_settings settings; 267 struct epoch_tracker et; 268 uint16_t vid = 0xfff, pcp = 0; 269 uint64_t ntuple; 270 271 INP_WLOCK_ASSERT(inp); 272 KASSERT(nam->sa_family == AF_INET || nam->sa_family == AF_INET6, 273 ("%s: dest addr %p has family %u", __func__, nam, nam->sa_family)); 274 275 if (if_gettype(rt_ifp) == IFT_ETHER) 276 vi = if_getsoftc(rt_ifp); 277 else if (if_gettype(rt_ifp) == IFT_L2VLAN) { 278 if_t ifp = VLAN_TRUNKDEV(rt_ifp); 279 280 vi = if_getsoftc(ifp); 281 VLAN_TAG(rt_ifp, &vid); 282 VLAN_PCP(rt_ifp, &pcp); 283 } else if (if_gettype(rt_ifp) == IFT_IEEE8023ADLAG) 284 DONT_OFFLOAD_ACTIVE_OPEN(ENOSYS); /* XXX: implement lagg+TOE */ 285 else 286 DONT_OFFLOAD_ACTIVE_OPEN(ENOTSUP); 287 if (sc->flags & KERN_TLS_ON) 288 DONT_OFFLOAD_ACTIVE_OPEN(ENOTSUP); 289 290 rw_rlock(&sc->policy_lock); 291 settings = *lookup_offload_policy(sc, OPEN_TYPE_ACTIVE, NULL, 292 EVL_MAKETAG(vid, pcp, 0), inp); 293 rw_runlock(&sc->policy_lock); 294 if (!settings.offload) 295 DONT_OFFLOAD_ACTIVE_OPEN(EPERM); 296 297 toep = alloc_toepcb(vi, M_NOWAIT); 298 if (toep == NULL) 299 DONT_OFFLOAD_ACTIVE_OPEN(ENOMEM); 300 301 toep->tid = alloc_atid(sc, toep); 302 if (toep->tid < 0) 303 DONT_OFFLOAD_ACTIVE_OPEN(ENOMEM); 304 305 toep->l2te = t4_l2t_get(vi->pi, rt_ifp, 306 nh->nh_flags & NHF_GATEWAY ? &nh->gw_sa : nam); 307 if (toep->l2te == NULL) 308 DONT_OFFLOAD_ACTIVE_OPEN(ENOMEM); 309 310 toep->vnet = so->so_vnet; 311 init_conn_params(vi, &settings, &inp->inp_inc, so, NULL, 312 toep->l2te->idx, &toep->params); 313 init_toepcb(vi, toep); 314 315 isipv6 = nam->sa_family == AF_INET6; 316 wr = alloc_wrqe(act_open_cpl_size(sc, isipv6), toep->ctrlq); 317 if (wr == NULL) 318 DONT_OFFLOAD_ACTIVE_OPEN(ENOMEM); 319 320 qid_atid = V_TID_QID(toep->ofld_rxq->iq.abs_id) | V_TID_TID(toep->tid) | 321 V_TID_COOKIE(CPL_COOKIE_TOM); 322 323 ntuple = select_ntuple(vi, toep->l2te); 324 if (isipv6) { 325 struct cpl_act_open_req6 *cpl = wrtod(wr); 326 struct cpl_t5_act_open_req6 *cpl5 = (void *)cpl; 327 struct cpl_t6_act_open_req6 *cpl6 = (void *)cpl; 328 struct cpl_t7_act_open_req6 *cpl7 = (void *)cpl; 329 330 if ((inp->inp_vflag & INP_IPV6) == 0) 331 DONT_OFFLOAD_ACTIVE_OPEN(ENOTSUP); 332 333 toep->ce = t4_get_clip_entry(sc, &inp->in6p_laddr, true); 334 if (toep->ce == NULL) 335 DONT_OFFLOAD_ACTIVE_OPEN(ENOENT); 336 337 switch (chip_id(sc)) { 338 case CHELSIO_T4: 339 INIT_TP_WR(cpl, 0); 340 cpl->params = htobe32((uint32_t)ntuple); 341 break; 342 case CHELSIO_T5: 343 INIT_TP_WR(cpl5, 0); 344 cpl5->iss = htobe32(tp->iss); 345 cpl5->params = htobe64(V_FILTER_TUPLE(ntuple)); 346 break; 347 case CHELSIO_T6: 348 INIT_TP_WR(cpl6, 0); 349 cpl6->iss = htobe32(tp->iss); 350 cpl6->params = htobe64(V_FILTER_TUPLE(ntuple)); 351 break; 352 case CHELSIO_T7: 353 default: 354 INIT_TP_WR(cpl7, 0); 355 cpl7->iss = htobe32(tp->iss); 356 cpl7->params = htobe64(V_T7_FILTER_TUPLE(ntuple)); 357 break; 358 } 359 OPCODE_TID(cpl) = htobe32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6, 360 qid_atid)); 361 cpl->local_port = inp->inp_lport; 362 cpl->local_ip_hi = *(uint64_t *)&inp->in6p_laddr.s6_addr[0]; 363 cpl->local_ip_lo = *(uint64_t *)&inp->in6p_laddr.s6_addr[8]; 364 cpl->peer_port = inp->inp_fport; 365 cpl->peer_ip_hi = *(uint64_t *)&inp->in6p_faddr.s6_addr[0]; 366 cpl->peer_ip_lo = *(uint64_t *)&inp->in6p_faddr.s6_addr[8]; 367 cpl->opt0 = calc_options0(vi, &toep->params); 368 cpl->opt2 = calc_options2(vi, &toep->params); 369 370 CTR6(KTR_CXGBE, 371 "%s: atid %u, toep %p, inp %p, opt0 %#016lx, opt2 %#08x", 372 __func__, toep->tid, toep, inp, be64toh(cpl->opt0), 373 be32toh(cpl->opt2)); 374 } else { 375 struct cpl_act_open_req *cpl = wrtod(wr); 376 struct cpl_t5_act_open_req *cpl5 = (void *)cpl; 377 struct cpl_t6_act_open_req *cpl6 = (void *)cpl; 378 struct cpl_t7_act_open_req *cpl7 = (void *)cpl; 379 380 switch (chip_id(sc)) { 381 case CHELSIO_T4: 382 INIT_TP_WR(cpl, 0); 383 cpl->params = htobe32((uint32_t)ntuple); 384 break; 385 case CHELSIO_T5: 386 INIT_TP_WR(cpl5, 0); 387 cpl5->iss = htobe32(tp->iss); 388 cpl5->params = htobe64(V_FILTER_TUPLE(ntuple)); 389 break; 390 case CHELSIO_T6: 391 INIT_TP_WR(cpl6, 0); 392 cpl6->iss = htobe32(tp->iss); 393 cpl6->params = htobe64(V_FILTER_TUPLE(ntuple)); 394 break; 395 case CHELSIO_T7: 396 default: 397 INIT_TP_WR(cpl7, 0); 398 cpl7->iss = htobe32(tp->iss); 399 cpl7->params = htobe64(V_T7_FILTER_TUPLE(ntuple)); 400 } 401 OPCODE_TID(cpl) = htobe32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, 402 qid_atid)); 403 inp_4tuple_get(inp, &cpl->local_ip, &cpl->local_port, 404 &cpl->peer_ip, &cpl->peer_port); 405 cpl->opt0 = calc_options0(vi, &toep->params); 406 cpl->opt2 = calc_options2(vi, &toep->params); 407 408 CTR6(KTR_CXGBE, 409 "%s: atid %u, toep %p, inp %p, opt0 %#016lx, opt2 %#08x", 410 __func__, toep->tid, toep, inp, be64toh(cpl->opt0), 411 be32toh(cpl->opt2)); 412 } 413 414 offload_socket(so, toep); 415 /* Add the TOE PCB to the active list */ 416 td = toep->td; 417 mtx_lock(&td->toep_list_lock); 418 TAILQ_INSERT_TAIL(&td->toep_list, toep, link); 419 toep->flags |= TPF_IN_TOEP_LIST; 420 mtx_unlock(&td->toep_list_lock); 421 NET_EPOCH_ENTER(et); 422 rc = t4_l2t_send(sc, wr, toep->l2te); 423 NET_EPOCH_EXIT(et); 424 if (rc == 0) { 425 toep->flags |= TPF_CPL_PENDING; 426 return (0); 427 } 428 429 undo_offload_socket(so); 430 #if defined(KTR) 431 reason = __LINE__; 432 #endif 433 failed: 434 CTR3(KTR_CXGBE, "%s: not offloading (%d), rc %d", __func__, reason, rc); 435 436 if (wr) 437 free_wrqe(wr); 438 439 if (toep) { 440 if (toep->tid >= 0) 441 free_atid(sc, toep->tid); 442 if (toep->l2te) 443 t4_l2t_release(toep->l2te); 444 if (toep->ce) 445 t4_release_clip_entry(sc, toep->ce); 446 free_toepcb(toep); 447 } 448 449 return (rc); 450 } 451 #endif 452