1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2017-2018 Chelsio Communications, Inc. 5 * All rights reserved. 6 * Written by: John Baldwin <jhb@FreeBSD.org> 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include "opt_inet.h" 31 #include "opt_kern_tls.h" 32 33 #include <sys/cdefs.h> 34 __FBSDID("$FreeBSD$"); 35 36 #ifdef KERN_TLS 37 #include <sys/param.h> 38 #include <sys/ktr.h> 39 #include <sys/ktls.h> 40 #include <sys/sglist.h> 41 #include <sys/socket.h> 42 #include <sys/socketvar.h> 43 #include <sys/systm.h> 44 #include <netinet/in.h> 45 #include <netinet/in_pcb.h> 46 #include <netinet/tcp_var.h> 47 #include <netinet/toecore.h> 48 #include <opencrypto/cryptodev.h> 49 #include <opencrypto/xform.h> 50 51 #ifdef TCP_OFFLOAD 52 #include "common/common.h" 53 #include "common/t4_tcb.h" 54 #include "crypto/t4_crypto.h" 55 #include "tom/t4_tom_l2t.h" 56 #include "tom/t4_tom.h" 57 58 /* 59 * The TCP sequence number of a CPL_TLS_DATA mbuf is saved here while 60 * the mbuf is in the ulp_pdu_reclaimq. 61 */ 62 #define tls_tcp_seq PH_loc.thirtytwo[0] 63 64 static void 65 t4_set_tls_tcb_field(struct toepcb *toep, uint16_t word, uint64_t mask, 66 uint64_t val) 67 { 68 struct adapter *sc = td_adapter(toep->td); 69 70 t4_set_tcb_field(sc, &toep->ofld_txq->wrq, toep, word, mask, val, 0, 0); 71 } 72 73 /* TLS and DTLS common routines */ 74 bool 75 can_tls_offload(struct adapter *sc) 76 { 77 78 return (sc->tt.tls && sc->cryptocaps & FW_CAPS_CONFIG_TLSKEYS); 79 } 80 81 int 82 tls_tx_key(struct toepcb *toep) 83 { 84 struct tls_ofld_info *tls_ofld = &toep->tls; 85 86 return (tls_ofld->tx_key_addr >= 0); 87 } 88 89 /* Set TLS Key-Id in TCB */ 90 static void 91 t4_set_tls_keyid(struct toepcb *toep, unsigned int key_id) 92 { 93 94 t4_set_tls_tcb_field(toep, W_TCB_RX_TLS_KEY_TAG, 95 V_TCB_RX_TLS_KEY_TAG(M_TCB_RX_TLS_BUF_TAG), 96 V_TCB_RX_TLS_KEY_TAG(key_id)); 97 } 98 99 /* Clear TF_RX_QUIESCE to re-enable receive. */ 100 static void 101 t4_clear_rx_quiesce(struct toepcb *toep) 102 { 103 104 t4_set_tls_tcb_field(toep, W_TCB_T_FLAGS, V_TF_RX_QUIESCE(1), 0); 105 } 106 107 static void 108 tls_clr_ofld_mode(struct toepcb *toep) 109 { 110 111 tls_stop_handshake_timer(toep); 112 113 KASSERT(toep->tls.rx_key_addr == -1, 114 ("%s: tid %d has RX key", __func__, toep->tid)); 115 116 /* Switch to plain TOE mode. */ 117 t4_set_tls_tcb_field(toep, W_TCB_ULP_RAW, 118 V_TCB_ULP_RAW(V_TF_TLS_ENABLE(1)), 119 V_TCB_ULP_RAW(V_TF_TLS_ENABLE(0))); 120 t4_set_tls_tcb_field(toep, W_TCB_ULP_TYPE, 121 V_TCB_ULP_TYPE(M_TCB_ULP_TYPE), V_TCB_ULP_TYPE(ULP_MODE_NONE)); 122 t4_clear_rx_quiesce(toep); 123 124 toep->flags &= ~(TPF_FORCE_CREDITS | TPF_TLS_ESTABLISHED); 125 toep->params.ulp_mode = ULP_MODE_NONE; 126 } 127 128 /* TLS/DTLS content type for CPL SFO */ 129 static inline unsigned char 130 tls_content_type(unsigned char content_type) 131 { 132 switch (content_type) { 133 case CONTENT_TYPE_CCS: 134 return CPL_TX_TLS_SFO_TYPE_CCS; 135 case CONTENT_TYPE_ALERT: 136 return CPL_TX_TLS_SFO_TYPE_ALERT; 137 case CONTENT_TYPE_HANDSHAKE: 138 return CPL_TX_TLS_SFO_TYPE_HANDSHAKE; 139 case CONTENT_TYPE_APP_DATA: 140 return CPL_TX_TLS_SFO_TYPE_DATA; 141 default: 142 return CPL_TX_TLS_SFO_TYPE_CUSTOM; 143 } 144 } 145 146 /* TLS Key memory management */ 147 static void 148 clear_tls_keyid(struct toepcb *toep) 149 { 150 struct tls_ofld_info *tls_ofld = &toep->tls; 151 struct adapter *sc = td_adapter(toep->td); 152 153 if (tls_ofld->rx_key_addr >= 0) { 154 t4_free_tls_keyid(sc, tls_ofld->rx_key_addr); 155 tls_ofld->rx_key_addr = -1; 156 } 157 if (tls_ofld->tx_key_addr >= 0) { 158 t4_free_tls_keyid(sc, tls_ofld->tx_key_addr); 159 tls_ofld->tx_key_addr = -1; 160 } 161 } 162 163 static int 164 get_tp_plen_max(struct ktls_session *tls) 165 { 166 int plen = ((min(3*4096, TP_TX_PG_SZ))/1448) * 1448; 167 168 return (tls->params.max_frame_len <= 8192 ? plen : FC_TP_PLEN_MAX); 169 } 170 171 /* Send request to get the key-id */ 172 static int 173 tls_program_key_id(struct toepcb *toep, struct ktls_session *tls, 174 int direction) 175 { 176 struct tls_ofld_info *tls_ofld = &toep->tls; 177 struct adapter *sc = td_adapter(toep->td); 178 struct ofld_tx_sdesc *txsd; 179 int keyid; 180 struct wrqe *wr; 181 struct tls_key_req *kwr; 182 struct tls_keyctx *kctx; 183 184 #ifdef INVARIANTS 185 int kwrlen, kctxlen, len; 186 187 kwrlen = sizeof(*kwr); 188 kctxlen = roundup2(sizeof(*kctx), 32); 189 len = roundup2(kwrlen + kctxlen, 16); 190 MPASS(TLS_KEY_WR_SZ == len); 191 #endif 192 if (toep->txsd_avail == 0) 193 return (EAGAIN); 194 195 if ((keyid = t4_alloc_tls_keyid(sc)) < 0) { 196 return (ENOSPC); 197 } 198 199 wr = alloc_wrqe(TLS_KEY_WR_SZ, &toep->ofld_txq->wrq); 200 if (wr == NULL) { 201 t4_free_tls_keyid(sc, keyid); 202 return (ENOMEM); 203 } 204 kwr = wrtod(wr); 205 memset(kwr, 0, TLS_KEY_WR_SZ); 206 207 t4_write_tlskey_wr(tls, direction, toep->tid, F_FW_WR_COMPL, keyid, 208 kwr); 209 kctx = (struct tls_keyctx *)(kwr + 1); 210 if (direction == KTLS_TX) 211 tls_ofld->tx_key_addr = keyid; 212 else 213 tls_ofld->rx_key_addr = keyid; 214 t4_tls_key_ctx(tls, direction, kctx); 215 216 txsd = &toep->txsd[toep->txsd_pidx]; 217 txsd->tx_credits = DIV_ROUND_UP(TLS_KEY_WR_SZ, 16); 218 txsd->plen = 0; 219 toep->tx_credits -= txsd->tx_credits; 220 if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) 221 toep->txsd_pidx = 0; 222 toep->txsd_avail--; 223 224 t4_wrq_tx(sc, wr); 225 226 return (0); 227 } 228 229 /* 230 * In some cases a client connection can hang without sending the 231 * ServerHelloDone message from the NIC to the host. Send a dummy 232 * RX_DATA_ACK with RX_MODULATE to unstick the connection. 233 */ 234 static void 235 tls_send_handshake_ack(void *arg) 236 { 237 struct toepcb *toep = arg; 238 struct tls_ofld_info *tls_ofld = &toep->tls; 239 struct adapter *sc = td_adapter(toep->td); 240 241 /* Bail without rescheduling if the connection has closed. */ 242 if ((toep->flags & (TPF_FIN_SENT | TPF_ABORT_SHUTDOWN)) != 0) 243 return; 244 245 /* 246 * If this connection has timed out without receiving more 247 * data, downgrade to plain TOE mode and don't re-arm the 248 * timer. 249 */ 250 if (sc->tt.tls_rx_timeout != 0) { 251 struct inpcb *inp; 252 struct tcpcb *tp; 253 254 inp = toep->inp; 255 tp = intotcpcb(inp); 256 if ((ticks - tp->t_rcvtime) >= sc->tt.tls_rx_timeout) { 257 CTR2(KTR_CXGBE, "%s: tid %d clr_ofld_mode", __func__, 258 toep->tid); 259 tls_clr_ofld_mode(toep); 260 return; 261 } 262 } 263 264 /* 265 * XXX: Does not have the t4_get_tcb() checks to refine the 266 * workaround. 267 */ 268 callout_schedule(&tls_ofld->handshake_timer, TLS_SRV_HELLO_RD_TM * hz); 269 270 CTR2(KTR_CXGBE, "%s: tid %d sending RX_DATA_ACK", __func__, toep->tid); 271 send_rx_modulate(sc, toep); 272 } 273 274 static void 275 tls_start_handshake_timer(struct toepcb *toep) 276 { 277 struct tls_ofld_info *tls_ofld = &toep->tls; 278 279 INP_WLOCK_ASSERT(toep->inp); 280 callout_reset(&tls_ofld->handshake_timer, TLS_SRV_HELLO_BKOFF_TM * hz, 281 tls_send_handshake_ack, toep); 282 } 283 284 void 285 tls_stop_handshake_timer(struct toepcb *toep) 286 { 287 struct tls_ofld_info *tls_ofld = &toep->tls; 288 289 INP_WLOCK_ASSERT(toep->inp); 290 callout_stop(&tls_ofld->handshake_timer); 291 } 292 293 int 294 tls_alloc_ktls(struct toepcb *toep, struct ktls_session *tls, int direction) 295 { 296 struct adapter *sc = td_adapter(toep->td); 297 int error, explicit_iv_size, key_offset, mac_first; 298 299 if (!can_tls_offload(td_adapter(toep->td))) 300 return (EINVAL); 301 switch (ulp_mode(toep)) { 302 case ULP_MODE_TLS: 303 break; 304 case ULP_MODE_NONE: 305 case ULP_MODE_TCPDDP: 306 if (direction != KTLS_TX) 307 return (EINVAL); 308 break; 309 default: 310 return (EINVAL); 311 } 312 313 switch (tls->params.cipher_algorithm) { 314 case CRYPTO_AES_CBC: 315 /* XXX: Explicitly ignore any provided IV. */ 316 switch (tls->params.cipher_key_len) { 317 case 128 / 8: 318 case 192 / 8: 319 case 256 / 8: 320 break; 321 default: 322 error = EINVAL; 323 goto clr_ofld; 324 } 325 switch (tls->params.auth_algorithm) { 326 case CRYPTO_SHA1_HMAC: 327 case CRYPTO_SHA2_256_HMAC: 328 case CRYPTO_SHA2_384_HMAC: 329 break; 330 default: 331 error = EPROTONOSUPPORT; 332 goto clr_ofld; 333 } 334 explicit_iv_size = AES_BLOCK_LEN; 335 mac_first = 1; 336 break; 337 case CRYPTO_AES_NIST_GCM_16: 338 if (tls->params.iv_len != SALT_SIZE) { 339 error = EINVAL; 340 goto clr_ofld; 341 } 342 switch (tls->params.cipher_key_len) { 343 case 128 / 8: 344 case 192 / 8: 345 case 256 / 8: 346 break; 347 default: 348 error = EINVAL; 349 goto clr_ofld; 350 } 351 explicit_iv_size = 8; 352 mac_first = 0; 353 break; 354 default: 355 error = EPROTONOSUPPORT; 356 goto clr_ofld; 357 } 358 359 /* Only TLS 1.1 and TLS 1.2 are currently supported. */ 360 if (tls->params.tls_vmajor != TLS_MAJOR_VER_ONE || 361 tls->params.tls_vminor < TLS_MINOR_VER_ONE || 362 tls->params.tls_vminor > TLS_MINOR_VER_TWO) { 363 error = EPROTONOSUPPORT; 364 goto clr_ofld; 365 } 366 367 /* Bail if we already have a key. */ 368 if (direction == KTLS_TX) { 369 if (toep->tls.tx_key_addr != -1) 370 return (EOPNOTSUPP); 371 } else { 372 if (toep->tls.rx_key_addr != -1) 373 return (EOPNOTSUPP); 374 } 375 376 error = tls_program_key_id(toep, tls, direction); 377 if (error) { 378 if (direction == KTLS_RX) 379 goto clr_ofld; 380 return (error); 381 } 382 383 if (direction == KTLS_TX) { 384 toep->tls.scmd0.seqno_numivs = 385 (V_SCMD_SEQ_NO_CTRL(3) | 386 V_SCMD_PROTO_VERSION(t4_tls_proto_ver(tls)) | 387 V_SCMD_ENC_DEC_CTRL(SCMD_ENCDECCTRL_ENCRYPT) | 388 V_SCMD_CIPH_AUTH_SEQ_CTRL((mac_first == 0)) | 389 V_SCMD_CIPH_MODE(t4_tls_cipher_mode(tls)) | 390 V_SCMD_AUTH_MODE(t4_tls_auth_mode(tls)) | 391 V_SCMD_HMAC_CTRL(t4_tls_hmac_ctrl(tls)) | 392 V_SCMD_IV_SIZE(explicit_iv_size / 2)); 393 394 toep->tls.scmd0.ivgen_hdrlen = 395 (V_SCMD_IV_GEN_CTRL(1) | 396 V_SCMD_KEY_CTX_INLINE(0) | 397 V_SCMD_TLS_FRAG_ENABLE(1)); 398 399 toep->tls.iv_len = explicit_iv_size; 400 toep->tls.frag_size = tls->params.max_frame_len; 401 toep->tls.fcplenmax = get_tp_plen_max(tls); 402 toep->tls.expn_per_ulp = tls->params.tls_hlen + 403 tls->params.tls_tlen; 404 toep->tls.pdus_per_ulp = 1; 405 toep->tls.adjusted_plen = toep->tls.expn_per_ulp + 406 tls->params.max_frame_len; 407 toep->tls.tx_key_info_size = t4_tls_key_info_size(tls); 408 } else { 409 /* Stop timer on handshake completion */ 410 tls_stop_handshake_timer(toep); 411 412 toep->flags &= ~TPF_FORCE_CREDITS; 413 toep->flags |= TPF_TLS_RECEIVE; 414 toep->tls.rx_version = tls->params.tls_vmajor << 8 | 415 tls->params.tls_vminor; 416 417 /* 418 * RX key tags are an index into the key portion of MA 419 * memory stored as an offset from the base address in 420 * units of 64 bytes. 421 */ 422 key_offset = toep->tls.rx_key_addr - sc->vres.key.start; 423 t4_set_tls_keyid(toep, key_offset / 64); 424 t4_set_tls_tcb_field(toep, W_TCB_ULP_RAW, 425 V_TCB_ULP_RAW(M_TCB_ULP_RAW), 426 V_TCB_ULP_RAW((V_TF_TLS_KEY_SIZE(3) | 427 V_TF_TLS_CONTROL(1) | 428 V_TF_TLS_ACTIVE(1) | 429 V_TF_TLS_ENABLE(1)))); 430 t4_set_tls_tcb_field(toep, W_TCB_TLS_SEQ, 431 V_TCB_TLS_SEQ(M_TCB_TLS_SEQ), 432 V_TCB_TLS_SEQ(0)); 433 t4_clear_rx_quiesce(toep); 434 } 435 436 return (0); 437 438 clr_ofld: 439 if (ulp_mode(toep) == ULP_MODE_TLS) { 440 CTR2(KTR_CXGBE, "%s: tid %d clr_ofld_mode", __func__, 441 toep->tid); 442 tls_clr_ofld_mode(toep); 443 } 444 return (error); 445 } 446 447 void 448 tls_init_toep(struct toepcb *toep) 449 { 450 struct tls_ofld_info *tls_ofld = &toep->tls; 451 452 tls_ofld->rx_key_addr = -1; 453 tls_ofld->tx_key_addr = -1; 454 } 455 456 void 457 tls_establish(struct toepcb *toep) 458 { 459 460 /* 461 * Enable PDU extraction. 462 * 463 * XXX: Supposedly this should be done by the firmware when 464 * the ULP_MODE FLOWC parameter is set in send_flowc_wr(), but 465 * in practice this seems to be required. 466 */ 467 CTR2(KTR_CXGBE, "%s: tid %d setting TLS_ENABLE", __func__, toep->tid); 468 t4_set_tls_tcb_field(toep, W_TCB_ULP_RAW, V_TCB_ULP_RAW(M_TCB_ULP_RAW), 469 V_TCB_ULP_RAW(V_TF_TLS_ENABLE(1))); 470 471 toep->flags |= TPF_FORCE_CREDITS | TPF_TLS_ESTABLISHED; 472 473 callout_init_rw(&toep->tls.handshake_timer, &toep->inp->inp_lock, 0); 474 tls_start_handshake_timer(toep); 475 } 476 477 void 478 tls_detach(struct toepcb *toep) 479 { 480 481 if (toep->flags & TPF_TLS_ESTABLISHED) { 482 tls_stop_handshake_timer(toep); 483 toep->flags &= ~TPF_TLS_ESTABLISHED; 484 } 485 } 486 487 void 488 tls_uninit_toep(struct toepcb *toep) 489 { 490 491 MPASS((toep->flags & TPF_TLS_ESTABLISHED) == 0); 492 clear_tls_keyid(toep); 493 } 494 495 #define MAX_OFLD_TX_CREDITS (SGE_MAX_WR_LEN / 16) 496 #define MIN_OFLD_TLSTX_CREDITS(toep) \ 497 (howmany(sizeof(struct fw_tlstx_data_wr) + \ 498 sizeof(struct cpl_tx_tls_sfo) + sizeof(struct ulptx_idata) + \ 499 sizeof(struct ulptx_sc_memrd) + \ 500 AES_BLOCK_LEN + 1, 16)) 501 502 static void 503 write_tlstx_wr(struct fw_tlstx_data_wr *txwr, struct toepcb *toep, 504 unsigned int plen, unsigned int expn, uint8_t credits, int shove) 505 { 506 struct tls_ofld_info *tls_ofld = &toep->tls; 507 unsigned int len = plen + expn; 508 509 txwr->op_to_immdlen = htobe32(V_WR_OP(FW_TLSTX_DATA_WR) | 510 V_FW_TLSTX_DATA_WR_COMPL(1) | 511 V_FW_TLSTX_DATA_WR_IMMDLEN(0)); 512 txwr->flowid_len16 = htobe32(V_FW_TLSTX_DATA_WR_FLOWID(toep->tid) | 513 V_FW_TLSTX_DATA_WR_LEN16(credits)); 514 txwr->plen = htobe32(len); 515 txwr->lsodisable_to_flags = htobe32(V_TX_ULP_MODE(ULP_MODE_TLS) | 516 V_TX_URG(0) | /* F_T6_TX_FORCE | */ V_TX_SHOVE(shove)); 517 txwr->ctxloc_to_exp = htobe32(V_FW_TLSTX_DATA_WR_NUMIVS(1) | 518 V_FW_TLSTX_DATA_WR_EXP(expn) | 519 V_FW_TLSTX_DATA_WR_CTXLOC(TLS_SFO_WR_CONTEXTLOC_DDR) | 520 V_FW_TLSTX_DATA_WR_IVDSGL(0) | 521 V_FW_TLSTX_DATA_WR_KEYSIZE(tls_ofld->tx_key_info_size >> 4)); 522 txwr->mfs = htobe16(tls_ofld->frag_size); 523 txwr->adjustedplen_pkd = htobe16( 524 V_FW_TLSTX_DATA_WR_ADJUSTEDPLEN(tls_ofld->adjusted_plen)); 525 txwr->expinplenmax_pkd = htobe16( 526 V_FW_TLSTX_DATA_WR_EXPINPLENMAX(tls_ofld->expn_per_ulp)); 527 txwr->pdusinplenmax_pkd = 528 V_FW_TLSTX_DATA_WR_PDUSINPLENMAX(tls_ofld->pdus_per_ulp); 529 } 530 531 static void 532 write_tlstx_cpl(struct cpl_tx_tls_sfo *cpl, struct toepcb *toep, 533 struct tls_hdr *tls_hdr, unsigned int plen, uint64_t seqno) 534 { 535 struct tls_ofld_info *tls_ofld = &toep->tls; 536 int data_type, seglen; 537 538 seglen = plen; 539 data_type = tls_content_type(tls_hdr->type); 540 cpl->op_to_seg_len = htobe32(V_CPL_TX_TLS_SFO_OPCODE(CPL_TX_TLS_SFO) | 541 V_CPL_TX_TLS_SFO_DATA_TYPE(data_type) | 542 V_CPL_TX_TLS_SFO_CPL_LEN(2) | V_CPL_TX_TLS_SFO_SEG_LEN(seglen)); 543 cpl->pld_len = htobe32(plen); 544 if (data_type == CPL_TX_TLS_SFO_TYPE_CUSTOM) 545 cpl->type_protover = htobe32( 546 V_CPL_TX_TLS_SFO_TYPE(tls_hdr->type)); 547 cpl->seqno_numivs = htobe32(tls_ofld->scmd0.seqno_numivs | 548 V_SCMD_NUM_IVS(1)); 549 cpl->ivgen_hdrlen = htobe32(tls_ofld->scmd0.ivgen_hdrlen); 550 cpl->scmd1 = htobe64(seqno); 551 } 552 553 static int 554 count_ext_pgs_segs(struct mbuf *m) 555 { 556 vm_paddr_t nextpa; 557 u_int i, nsegs; 558 559 MPASS(m->m_epg_npgs > 0); 560 nsegs = 1; 561 nextpa = m->m_epg_pa[0] + PAGE_SIZE; 562 for (i = 1; i < m->m_epg_npgs; i++) { 563 if (nextpa != m->m_epg_pa[i]) 564 nsegs++; 565 nextpa = m->m_epg_pa[i] + PAGE_SIZE; 566 } 567 return (nsegs); 568 } 569 570 static void 571 write_ktlstx_sgl(void *dst, struct mbuf *m, int nsegs) 572 { 573 struct ulptx_sgl *usgl = dst; 574 vm_paddr_t pa; 575 uint32_t len; 576 int i, j; 577 578 KASSERT(nsegs > 0, ("%s: nsegs 0", __func__)); 579 580 usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) | 581 V_ULPTX_NSGE(nsegs)); 582 583 /* Figure out the first S/G length. */ 584 pa = m->m_epg_pa[0] + m->m_epg_1st_off; 585 usgl->addr0 = htobe64(pa); 586 len = m_epg_pagelen(m, 0, m->m_epg_1st_off); 587 pa += len; 588 for (i = 1; i < m->m_epg_npgs; i++) { 589 if (m->m_epg_pa[i] != pa) 590 break; 591 len += m_epg_pagelen(m, i, 0); 592 pa += m_epg_pagelen(m, i, 0); 593 } 594 usgl->len0 = htobe32(len); 595 #ifdef INVARIANTS 596 nsegs--; 597 #endif 598 599 j = -1; 600 for (; i < m->m_epg_npgs; i++) { 601 if (j == -1 || m->m_epg_pa[i] != pa) { 602 if (j >= 0) 603 usgl->sge[j / 2].len[j & 1] = htobe32(len); 604 j++; 605 #ifdef INVARIANTS 606 nsegs--; 607 #endif 608 pa = m->m_epg_pa[i]; 609 usgl->sge[j / 2].addr[j & 1] = htobe64(pa); 610 len = m_epg_pagelen(m, i, 0); 611 pa += len; 612 } else { 613 len += m_epg_pagelen(m, i, 0); 614 pa += m_epg_pagelen(m, i, 0); 615 } 616 } 617 if (j >= 0) { 618 usgl->sge[j / 2].len[j & 1] = htobe32(len); 619 620 if ((j & 1) == 0) 621 usgl->sge[j / 2].len[1] = htobe32(0); 622 } 623 KASSERT(nsegs == 0, ("%s: nsegs %d, m %p", __func__, nsegs, m)); 624 } 625 626 /* 627 * Similar to t4_push_frames() but handles sockets that contain TLS 628 * record mbufs. 629 */ 630 void 631 t4_push_ktls(struct adapter *sc, struct toepcb *toep, int drop) 632 { 633 struct tls_hdr *thdr; 634 struct fw_tlstx_data_wr *txwr; 635 struct cpl_tx_tls_sfo *cpl; 636 struct ulptx_idata *idata; 637 struct ulptx_sc_memrd *memrd; 638 struct wrqe *wr; 639 struct mbuf *m; 640 u_int nsegs, credits, wr_len; 641 u_int expn_size; 642 struct inpcb *inp = toep->inp; 643 struct tcpcb *tp = intotcpcb(inp); 644 struct socket *so = inp->inp_socket; 645 struct sockbuf *sb = &so->so_snd; 646 int tls_size, tx_credits, shove, sowwakeup; 647 struct ofld_tx_sdesc *txsd; 648 char *buf; 649 650 INP_WLOCK_ASSERT(inp); 651 KASSERT(toep->flags & TPF_FLOWC_WR_SENT, 652 ("%s: flowc_wr not sent for tid %u.", __func__, toep->tid)); 653 654 KASSERT(ulp_mode(toep) == ULP_MODE_NONE || 655 ulp_mode(toep) == ULP_MODE_TCPDDP || ulp_mode(toep) == ULP_MODE_TLS, 656 ("%s: ulp_mode %u for toep %p", __func__, ulp_mode(toep), toep)); 657 KASSERT(tls_tx_key(toep), 658 ("%s: TX key not set for toep %p", __func__, toep)); 659 660 #ifdef VERBOSE_TRACES 661 CTR4(KTR_CXGBE, "%s: tid %d toep flags %#x tp flags %#x drop %d", 662 __func__, toep->tid, toep->flags, tp->t_flags); 663 #endif 664 if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN)) 665 return; 666 667 #ifdef RATELIMIT 668 if (__predict_false(inp->inp_flags2 & INP_RATE_LIMIT_CHANGED) && 669 (update_tx_rate_limit(sc, toep, so->so_max_pacing_rate) == 0)) { 670 inp->inp_flags2 &= ~INP_RATE_LIMIT_CHANGED; 671 } 672 #endif 673 674 /* 675 * This function doesn't resume by itself. Someone else must clear the 676 * flag and call this function. 677 */ 678 if (__predict_false(toep->flags & TPF_TX_SUSPENDED)) { 679 KASSERT(drop == 0, 680 ("%s: drop (%d) != 0 but tx is suspended", __func__, drop)); 681 return; 682 } 683 684 txsd = &toep->txsd[toep->txsd_pidx]; 685 for (;;) { 686 tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS); 687 688 SOCKBUF_LOCK(sb); 689 sowwakeup = drop; 690 if (drop) { 691 sbdrop_locked(sb, drop); 692 drop = 0; 693 } 694 695 m = sb->sb_sndptr != NULL ? sb->sb_sndptr->m_next : sb->sb_mb; 696 697 /* 698 * Send a FIN if requested, but only if there's no 699 * more data to send. 700 */ 701 if (m == NULL && toep->flags & TPF_SEND_FIN) { 702 if (sowwakeup) 703 sowwakeup_locked(so); 704 else 705 SOCKBUF_UNLOCK(sb); 706 SOCKBUF_UNLOCK_ASSERT(sb); 707 t4_close_conn(sc, toep); 708 return; 709 } 710 711 /* 712 * If there is no ready data to send, wait until more 713 * data arrives. 714 */ 715 if (m == NULL || (m->m_flags & M_NOTAVAIL) != 0) { 716 if (sowwakeup) 717 sowwakeup_locked(so); 718 else 719 SOCKBUF_UNLOCK(sb); 720 SOCKBUF_UNLOCK_ASSERT(sb); 721 #ifdef VERBOSE_TRACES 722 CTR2(KTR_CXGBE, "%s: tid %d no ready data to send", 723 __func__, toep->tid); 724 #endif 725 return; 726 } 727 728 KASSERT(m->m_flags & M_EXTPG, ("%s: mbuf %p is not NOMAP", 729 __func__, m)); 730 KASSERT(m->m_epg_tls != NULL, 731 ("%s: mbuf %p doesn't have TLS session", __func__, m)); 732 733 /* Calculate WR length. */ 734 wr_len = sizeof(struct fw_tlstx_data_wr) + 735 sizeof(struct cpl_tx_tls_sfo) + 736 sizeof(struct ulptx_idata) + sizeof(struct ulptx_sc_memrd); 737 738 /* Explicit IVs for AES-CBC and AES-GCM are <= 16. */ 739 MPASS(toep->tls.iv_len <= AES_BLOCK_LEN); 740 wr_len += AES_BLOCK_LEN; 741 742 /* Account for SGL in work request length. */ 743 nsegs = count_ext_pgs_segs(m); 744 wr_len += sizeof(struct ulptx_sgl) + 745 ((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8; 746 747 /* Not enough credits for this work request. */ 748 if (howmany(wr_len, 16) > tx_credits) { 749 if (sowwakeup) 750 sowwakeup_locked(so); 751 else 752 SOCKBUF_UNLOCK(sb); 753 SOCKBUF_UNLOCK_ASSERT(sb); 754 #ifdef VERBOSE_TRACES 755 CTR5(KTR_CXGBE, 756 "%s: tid %d mbuf %p requires %d credits, but only %d available", 757 __func__, toep->tid, m, howmany(wr_len, 16), 758 tx_credits); 759 #endif 760 toep->flags |= TPF_TX_SUSPENDED; 761 return; 762 } 763 764 /* Shove if there is no additional data pending. */ 765 shove = ((m->m_next == NULL || 766 (m->m_next->m_flags & M_NOTAVAIL) != 0)) && 767 (tp->t_flags & TF_MORETOCOME) == 0; 768 769 if (sb->sb_flags & SB_AUTOSIZE && 770 V_tcp_do_autosndbuf && 771 sb->sb_hiwat < V_tcp_autosndbuf_max && 772 sbused(sb) >= sb->sb_hiwat * 7 / 8) { 773 int newsize = min(sb->sb_hiwat + V_tcp_autosndbuf_inc, 774 V_tcp_autosndbuf_max); 775 776 if (!sbreserve_locked(so, SO_SND, newsize, NULL)) 777 sb->sb_flags &= ~SB_AUTOSIZE; 778 else 779 sowwakeup = 1; /* room available */ 780 } 781 if (sowwakeup) 782 sowwakeup_locked(so); 783 else 784 SOCKBUF_UNLOCK(sb); 785 SOCKBUF_UNLOCK_ASSERT(sb); 786 787 if (__predict_false(toep->flags & TPF_FIN_SENT)) 788 panic("%s: excess tx.", __func__); 789 790 wr = alloc_wrqe(roundup2(wr_len, 16), &toep->ofld_txq->wrq); 791 if (wr == NULL) { 792 /* XXX: how will we recover from this? */ 793 toep->flags |= TPF_TX_SUSPENDED; 794 return; 795 } 796 797 thdr = (struct tls_hdr *)&m->m_epg_hdr; 798 #ifdef VERBOSE_TRACES 799 CTR5(KTR_CXGBE, "%s: tid %d TLS record %ju type %d len %#x", 800 __func__, toep->tid, m->m_epg_seqno, thdr->type, 801 m->m_len); 802 #endif 803 txwr = wrtod(wr); 804 cpl = (struct cpl_tx_tls_sfo *)(txwr + 1); 805 memset(txwr, 0, roundup2(wr_len, 16)); 806 credits = howmany(wr_len, 16); 807 expn_size = m->m_epg_hdrlen + 808 m->m_epg_trllen; 809 tls_size = m->m_len - expn_size; 810 write_tlstx_wr(txwr, toep, tls_size, expn_size, credits, shove); 811 write_tlstx_cpl(cpl, toep, thdr, tls_size, m->m_epg_seqno); 812 813 idata = (struct ulptx_idata *)(cpl + 1); 814 idata->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP)); 815 idata->len = htobe32(0); 816 memrd = (struct ulptx_sc_memrd *)(idata + 1); 817 memrd->cmd_to_len = htobe32(V_ULPTX_CMD(ULP_TX_SC_MEMRD) | 818 V_ULP_TX_SC_MORE(1) | 819 V_ULPTX_LEN16(toep->tls.tx_key_info_size >> 4)); 820 memrd->addr = htobe32(toep->tls.tx_key_addr >> 5); 821 822 /* Copy IV. */ 823 buf = (char *)(memrd + 1); 824 memcpy(buf, thdr + 1, toep->tls.iv_len); 825 buf += AES_BLOCK_LEN; 826 827 write_ktlstx_sgl(buf, m, nsegs); 828 829 KASSERT(toep->tx_credits >= credits, 830 ("%s: not enough credits", __func__)); 831 832 toep->tx_credits -= credits; 833 834 tp->snd_nxt += m->m_len; 835 tp->snd_max += m->m_len; 836 837 SOCKBUF_LOCK(sb); 838 sb->sb_sndptr = m; 839 SOCKBUF_UNLOCK(sb); 840 841 toep->flags |= TPF_TX_DATA_SENT; 842 if (toep->tx_credits < MIN_OFLD_TLSTX_CREDITS(toep)) 843 toep->flags |= TPF_TX_SUSPENDED; 844 845 KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__)); 846 txsd->plen = m->m_len; 847 txsd->tx_credits = credits; 848 txsd++; 849 if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) { 850 toep->txsd_pidx = 0; 851 txsd = &toep->txsd[0]; 852 } 853 toep->txsd_avail--; 854 855 counter_u64_add(toep->ofld_txq->tx_toe_tls_records, 1); 856 counter_u64_add(toep->ofld_txq->tx_toe_tls_octets, m->m_len); 857 858 t4_l2t_send(sc, wr, toep->l2te); 859 } 860 } 861 862 /* 863 * For TLS data we place received mbufs received via CPL_TLS_DATA into 864 * an mbufq in the TLS offload state. When CPL_RX_TLS_CMP is 865 * received, the completed PDUs are placed into the socket receive 866 * buffer. 867 * 868 * The TLS code reuses the ulp_pdu_reclaimq to hold the pending mbufs. 869 */ 870 static int 871 do_tls_data(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 872 { 873 struct adapter *sc = iq->adapter; 874 const struct cpl_tls_data *cpl = mtod(m, const void *); 875 unsigned int tid = GET_TID(cpl); 876 struct toepcb *toep = lookup_tid(sc, tid); 877 struct inpcb *inp = toep->inp; 878 struct tcpcb *tp; 879 int len; 880 881 /* XXX: Should this match do_rx_data instead? */ 882 KASSERT(!(toep->flags & TPF_SYNQE), 883 ("%s: toep %p claims to be a synq entry", __func__, toep)); 884 885 KASSERT(toep->tid == tid, ("%s: toep tid/atid mismatch", __func__)); 886 887 /* strip off CPL header */ 888 m_adj(m, sizeof(*cpl)); 889 len = m->m_pkthdr.len; 890 891 toep->ofld_rxq->rx_toe_tls_octets += len; 892 893 KASSERT(len == G_CPL_TLS_DATA_LENGTH(be32toh(cpl->length_pkd)), 894 ("%s: payload length mismatch", __func__)); 895 896 INP_WLOCK(inp); 897 if (inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) { 898 CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x", 899 __func__, tid, len, inp->inp_flags); 900 INP_WUNLOCK(inp); 901 m_freem(m); 902 return (0); 903 } 904 905 /* Save TCP sequence number. */ 906 m->m_pkthdr.tls_tcp_seq = be32toh(cpl->seq); 907 908 if (mbufq_enqueue(&toep->ulp_pdu_reclaimq, m)) { 909 #ifdef INVARIANTS 910 panic("Failed to queue TLS data packet"); 911 #else 912 printf("%s: Failed to queue TLS data packet\n", __func__); 913 INP_WUNLOCK(inp); 914 m_freem(m); 915 return (0); 916 #endif 917 } 918 919 tp = intotcpcb(inp); 920 tp->t_rcvtime = ticks; 921 922 #ifdef VERBOSE_TRACES 923 CTR4(KTR_CXGBE, "%s: tid %u len %d seq %u", __func__, tid, len, 924 be32toh(cpl->seq)); 925 #endif 926 927 INP_WUNLOCK(inp); 928 return (0); 929 } 930 931 static int 932 do_rx_tls_cmp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 933 { 934 struct adapter *sc = iq->adapter; 935 const struct cpl_rx_tls_cmp *cpl = mtod(m, const void *); 936 struct tlsrx_hdr_pkt *tls_hdr_pkt; 937 unsigned int tid = GET_TID(cpl); 938 struct toepcb *toep = lookup_tid(sc, tid); 939 struct inpcb *inp = toep->inp; 940 struct tcpcb *tp; 941 struct socket *so; 942 struct sockbuf *sb; 943 struct mbuf *tls_data; 944 struct tls_get_record *tgr; 945 struct mbuf *control; 946 int pdu_length, rx_credits; 947 #if defined(KTR) || defined(INVARIANTS) 948 int len; 949 #endif 950 951 KASSERT(toep->tid == tid, ("%s: toep tid/atid mismatch", __func__)); 952 KASSERT(!(toep->flags & TPF_SYNQE), 953 ("%s: toep %p claims to be a synq entry", __func__, toep)); 954 955 /* strip off CPL header */ 956 m_adj(m, sizeof(*cpl)); 957 #if defined(KTR) || defined(INVARIANTS) 958 len = m->m_pkthdr.len; 959 #endif 960 961 toep->ofld_rxq->rx_toe_tls_records++; 962 963 KASSERT(len == G_CPL_RX_TLS_CMP_LENGTH(be32toh(cpl->pdulength_length)), 964 ("%s: payload length mismatch", __func__)); 965 966 INP_WLOCK(inp); 967 if (inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) { 968 CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x", 969 __func__, tid, len, inp->inp_flags); 970 INP_WUNLOCK(inp); 971 m_freem(m); 972 return (0); 973 } 974 975 pdu_length = G_CPL_RX_TLS_CMP_PDULENGTH(be32toh(cpl->pdulength_length)); 976 977 so = inp_inpcbtosocket(inp); 978 tp = intotcpcb(inp); 979 980 #ifdef VERBOSE_TRACES 981 CTR6(KTR_CXGBE, "%s: tid %u PDU len %d len %d seq %u, rcv_nxt %u", 982 __func__, tid, pdu_length, len, be32toh(cpl->seq), tp->rcv_nxt); 983 #endif 984 985 tp->rcv_nxt += pdu_length; 986 KASSERT(tp->rcv_wnd >= pdu_length, 987 ("%s: negative window size", __func__)); 988 tp->rcv_wnd -= pdu_length; 989 990 /* XXX: Not sure what to do about urgent data. */ 991 992 /* 993 * The payload of this CPL is the TLS header followed by 994 * additional fields. 995 */ 996 KASSERT(m->m_len >= sizeof(*tls_hdr_pkt), 997 ("%s: payload too small", __func__)); 998 tls_hdr_pkt = mtod(m, void *); 999 1000 tls_data = mbufq_dequeue(&toep->ulp_pdu_reclaimq); 1001 if (tls_data != NULL) { 1002 KASSERT(be32toh(cpl->seq) == tls_data->m_pkthdr.tls_tcp_seq, 1003 ("%s: sequence mismatch", __func__)); 1004 } 1005 1006 /* Report decryption errors as EBADMSG. */ 1007 if ((tls_hdr_pkt->res_to_mac_error & M_TLSRX_HDR_PKT_ERROR) != 0) { 1008 m_freem(m); 1009 m_freem(tls_data); 1010 1011 CURVNET_SET(toep->vnet); 1012 so->so_error = EBADMSG; 1013 sorwakeup(so); 1014 1015 INP_WUNLOCK(inp); 1016 CURVNET_RESTORE(); 1017 1018 return (0); 1019 } 1020 1021 /* Allocate the control message mbuf. */ 1022 control = sbcreatecontrol(NULL, sizeof(*tgr), TLS_GET_RECORD, 1023 IPPROTO_TCP, M_NOWAIT); 1024 if (control == NULL) { 1025 m_freem(m); 1026 m_freem(tls_data); 1027 1028 CURVNET_SET(toep->vnet); 1029 so->so_error = ENOBUFS; 1030 sorwakeup(so); 1031 1032 INP_WUNLOCK(inp); 1033 CURVNET_RESTORE(); 1034 1035 return (0); 1036 } 1037 1038 tgr = (struct tls_get_record *) 1039 CMSG_DATA(mtod(control, struct cmsghdr *)); 1040 memset(tgr, 0, sizeof(*tgr)); 1041 tgr->tls_type = tls_hdr_pkt->type; 1042 tgr->tls_vmajor = be16toh(tls_hdr_pkt->version) >> 8; 1043 tgr->tls_vminor = be16toh(tls_hdr_pkt->version) & 0xff; 1044 1045 m_freem(m); 1046 1047 if (tls_data != NULL) { 1048 m_last(tls_data)->m_flags |= M_EOR; 1049 tgr->tls_length = htobe16(tls_data->m_pkthdr.len); 1050 } else 1051 tgr->tls_length = 0; 1052 m = tls_data; 1053 1054 sb = &so->so_rcv; 1055 SOCKBUF_LOCK(sb); 1056 1057 if (__predict_false(sb->sb_state & SBS_CANTRCVMORE)) { 1058 struct epoch_tracker et; 1059 1060 CTR3(KTR_CXGBE, "%s: tid %u, excess rx (%d bytes)", 1061 __func__, tid, pdu_length); 1062 m_freem(m); 1063 m_freem(control); 1064 SOCKBUF_UNLOCK(sb); 1065 INP_WUNLOCK(inp); 1066 1067 CURVNET_SET(toep->vnet); 1068 NET_EPOCH_ENTER(et); 1069 INP_WLOCK(inp); 1070 tp = tcp_drop(tp, ECONNRESET); 1071 if (tp) 1072 INP_WUNLOCK(inp); 1073 NET_EPOCH_EXIT(et); 1074 CURVNET_RESTORE(); 1075 1076 return (0); 1077 } 1078 1079 /* 1080 * Not all of the bytes on the wire are included in the socket buffer 1081 * (e.g. the MAC of the TLS record). However, those bytes are included 1082 * in the TCP sequence space. 1083 */ 1084 1085 /* receive buffer autosize */ 1086 MPASS(toep->vnet == so->so_vnet); 1087 CURVNET_SET(toep->vnet); 1088 if (sb->sb_flags & SB_AUTOSIZE && 1089 V_tcp_do_autorcvbuf && 1090 sb->sb_hiwat < V_tcp_autorcvbuf_max && 1091 m->m_pkthdr.len > (sbspace(sb) / 8 * 7)) { 1092 unsigned int hiwat = sb->sb_hiwat; 1093 unsigned int newsize = min(hiwat + sc->tt.autorcvbuf_inc, 1094 V_tcp_autorcvbuf_max); 1095 1096 if (!sbreserve_locked(so, SO_RCV, newsize, NULL)) 1097 sb->sb_flags &= ~SB_AUTOSIZE; 1098 } 1099 1100 sbappendcontrol_locked(sb, m, control, 0); 1101 rx_credits = sbspace(sb) > tp->rcv_wnd ? sbspace(sb) - tp->rcv_wnd : 0; 1102 #ifdef VERBOSE_TRACES 1103 CTR4(KTR_CXGBE, "%s: tid %u rx_credits %u rcv_wnd %u", 1104 __func__, tid, rx_credits, tp->rcv_wnd); 1105 #endif 1106 if (rx_credits > 0 && sbused(sb) + tp->rcv_wnd < sb->sb_lowat) { 1107 rx_credits = send_rx_credits(sc, toep, rx_credits); 1108 tp->rcv_wnd += rx_credits; 1109 tp->rcv_adv += rx_credits; 1110 } 1111 1112 sorwakeup_locked(so); 1113 SOCKBUF_UNLOCK_ASSERT(sb); 1114 1115 INP_WUNLOCK(inp); 1116 CURVNET_RESTORE(); 1117 return (0); 1118 } 1119 1120 void 1121 do_rx_data_tls(const struct cpl_rx_data *cpl, struct toepcb *toep, 1122 struct mbuf *m) 1123 { 1124 struct inpcb *inp = toep->inp; 1125 struct tls_ofld_info *tls_ofld = &toep->tls; 1126 struct tls_hdr *hdr; 1127 struct tcpcb *tp; 1128 struct socket *so; 1129 struct sockbuf *sb; 1130 int len, rx_credits; 1131 1132 len = m->m_pkthdr.len; 1133 1134 INP_WLOCK_ASSERT(inp); 1135 1136 so = inp_inpcbtosocket(inp); 1137 tp = intotcpcb(inp); 1138 sb = &so->so_rcv; 1139 SOCKBUF_LOCK(sb); 1140 CURVNET_SET(toep->vnet); 1141 1142 tp->rcv_nxt += len; 1143 KASSERT(tp->rcv_wnd >= len, ("%s: negative window size", __func__)); 1144 tp->rcv_wnd -= len; 1145 1146 /* Do we have a full TLS header? */ 1147 if (len < sizeof(*hdr)) { 1148 CTR3(KTR_CXGBE, "%s: tid %u len %d: too short for a TLS header", 1149 __func__, toep->tid, len); 1150 so->so_error = EMSGSIZE; 1151 goto out; 1152 } 1153 hdr = mtod(m, struct tls_hdr *); 1154 1155 /* Is the header valid? */ 1156 if (be16toh(hdr->version) != tls_ofld->rx_version) { 1157 CTR3(KTR_CXGBE, "%s: tid %u invalid version %04x", 1158 __func__, toep->tid, be16toh(hdr->version)); 1159 so->so_error = EINVAL; 1160 goto out; 1161 } 1162 if (be16toh(hdr->length) < sizeof(*hdr)) { 1163 CTR3(KTR_CXGBE, "%s: tid %u invalid length %u", 1164 __func__, toep->tid, be16toh(hdr->length)); 1165 so->so_error = EBADMSG; 1166 goto out; 1167 } 1168 1169 /* Did we get a truncated record? */ 1170 if (len < be16toh(hdr->length)) { 1171 CTR4(KTR_CXGBE, "%s: tid %u truncated TLS record (%d vs %u)", 1172 __func__, toep->tid, len, be16toh(hdr->length)); 1173 1174 so->so_error = EMSGSIZE; 1175 goto out; 1176 } 1177 1178 /* Is the header type unknown? */ 1179 switch (hdr->type) { 1180 case CONTENT_TYPE_CCS: 1181 case CONTENT_TYPE_ALERT: 1182 case CONTENT_TYPE_APP_DATA: 1183 case CONTENT_TYPE_HANDSHAKE: 1184 break; 1185 default: 1186 CTR3(KTR_CXGBE, "%s: tid %u invalid TLS record type %u", 1187 __func__, toep->tid, hdr->type); 1188 so->so_error = EBADMSG; 1189 goto out; 1190 } 1191 1192 /* 1193 * Just punt. Although this could fall back to software 1194 * decryption, this case should never really happen. 1195 */ 1196 CTR4(KTR_CXGBE, "%s: tid %u dropping TLS record type %u, length %u", 1197 __func__, toep->tid, hdr->type, be16toh(hdr->length)); 1198 so->so_error = EBADMSG; 1199 1200 out: 1201 /* 1202 * This connection is going to die anyway, so probably don't 1203 * need to bother with returning credits. 1204 */ 1205 rx_credits = sbspace(sb) > tp->rcv_wnd ? sbspace(sb) - tp->rcv_wnd : 0; 1206 #ifdef VERBOSE_TRACES 1207 CTR4(KTR_CXGBE, "%s: tid %u rx_credits %u rcv_wnd %u", 1208 __func__, toep->tid, rx_credits, tp->rcv_wnd); 1209 #endif 1210 if (rx_credits > 0 && sbused(sb) + tp->rcv_wnd < sb->sb_lowat) { 1211 rx_credits = send_rx_credits(toep->vi->adapter, toep, 1212 rx_credits); 1213 tp->rcv_wnd += rx_credits; 1214 tp->rcv_adv += rx_credits; 1215 } 1216 1217 sorwakeup_locked(so); 1218 SOCKBUF_UNLOCK_ASSERT(sb); 1219 1220 INP_WUNLOCK(inp); 1221 CURVNET_RESTORE(); 1222 1223 m_freem(m); 1224 } 1225 1226 void 1227 t4_tls_mod_load(void) 1228 { 1229 1230 t4_register_cpl_handler(CPL_TLS_DATA, do_tls_data); 1231 t4_register_cpl_handler(CPL_RX_TLS_CMP, do_rx_tls_cmp); 1232 } 1233 1234 void 1235 t4_tls_mod_unload(void) 1236 { 1237 1238 t4_register_cpl_handler(CPL_TLS_DATA, NULL); 1239 t4_register_cpl_handler(CPL_RX_TLS_CMP, NULL); 1240 } 1241 #endif /* TCP_OFFLOAD */ 1242 #endif /* KERN_TLS */ 1243