1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2017-2018 Chelsio Communications, Inc. 5 * All rights reserved. 6 * Written by: John Baldwin <jhb@FreeBSD.org> 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include "opt_inet.h" 31 #include "opt_kern_tls.h" 32 33 #include <sys/cdefs.h> 34 __FBSDID("$FreeBSD$"); 35 36 #ifdef KERN_TLS 37 #include <sys/param.h> 38 #include <sys/ktr.h> 39 #include <sys/ktls.h> 40 #include <sys/sglist.h> 41 #include <sys/socket.h> 42 #include <sys/socketvar.h> 43 #include <sys/systm.h> 44 #include <netinet/in.h> 45 #include <netinet/in_pcb.h> 46 #include <netinet/tcp_var.h> 47 #include <netinet/toecore.h> 48 #include <opencrypto/cryptodev.h> 49 #include <opencrypto/xform.h> 50 51 #ifdef TCP_OFFLOAD 52 #include "common/common.h" 53 #include "common/t4_tcb.h" 54 #include "crypto/t4_crypto.h" 55 #include "tom/t4_tom_l2t.h" 56 #include "tom/t4_tom.h" 57 58 /* 59 * The TCP sequence number of a CPL_TLS_DATA mbuf is saved here while 60 * the mbuf is in the ulp_pdu_reclaimq. 61 */ 62 #define tls_tcp_seq PH_loc.thirtytwo[0] 63 64 static void 65 t4_set_tls_tcb_field(struct toepcb *toep, uint16_t word, uint64_t mask, 66 uint64_t val) 67 { 68 struct adapter *sc = td_adapter(toep->td); 69 70 t4_set_tcb_field(sc, &toep->ofld_txq->wrq, toep, word, mask, val, 0, 0); 71 } 72 73 /* TLS and DTLS common routines */ 74 bool 75 can_tls_offload(struct adapter *sc) 76 { 77 78 return (sc->tt.tls && sc->cryptocaps & FW_CAPS_CONFIG_TLSKEYS); 79 } 80 81 int 82 tls_tx_key(struct toepcb *toep) 83 { 84 struct tls_ofld_info *tls_ofld = &toep->tls; 85 86 return (tls_ofld->tx_key_addr >= 0); 87 } 88 89 /* Set TLS Key-Id in TCB */ 90 static void 91 t4_set_tls_keyid(struct toepcb *toep, unsigned int key_id) 92 { 93 94 t4_set_tls_tcb_field(toep, W_TCB_RX_TLS_KEY_TAG, 95 V_TCB_RX_TLS_KEY_TAG(M_TCB_RX_TLS_BUF_TAG), 96 V_TCB_RX_TLS_KEY_TAG(key_id)); 97 } 98 99 /* Clear TF_RX_QUIESCE to re-enable receive. */ 100 static void 101 t4_clear_rx_quiesce(struct toepcb *toep) 102 { 103 104 t4_set_tls_tcb_field(toep, W_TCB_T_FLAGS, V_TF_RX_QUIESCE(1), 0); 105 } 106 107 static void 108 tls_clr_ofld_mode(struct toepcb *toep) 109 { 110 111 tls_stop_handshake_timer(toep); 112 113 KASSERT(toep->tls.rx_key_addr == -1, 114 ("%s: tid %d has RX key", __func__, toep->tid)); 115 116 /* Switch to plain TOE mode. */ 117 t4_set_tls_tcb_field(toep, W_TCB_ULP_RAW, 118 V_TCB_ULP_RAW(V_TF_TLS_ENABLE(1)), 119 V_TCB_ULP_RAW(V_TF_TLS_ENABLE(0))); 120 t4_set_tls_tcb_field(toep, W_TCB_ULP_TYPE, 121 V_TCB_ULP_TYPE(M_TCB_ULP_TYPE), V_TCB_ULP_TYPE(ULP_MODE_NONE)); 122 t4_clear_rx_quiesce(toep); 123 124 toep->flags &= ~(TPF_FORCE_CREDITS | TPF_TLS_ESTABLISHED); 125 toep->params.ulp_mode = ULP_MODE_NONE; 126 } 127 128 /* TLS/DTLS content type for CPL SFO */ 129 static inline unsigned char 130 tls_content_type(unsigned char content_type) 131 { 132 /* 133 * XXX: Shouldn't this map CONTENT_TYPE_APP_DATA to DATA and 134 * default to "CUSTOM" for all other types including 135 * heartbeat? 136 */ 137 switch (content_type) { 138 case CONTENT_TYPE_CCS: 139 return CPL_TX_TLS_SFO_TYPE_CCS; 140 case CONTENT_TYPE_ALERT: 141 return CPL_TX_TLS_SFO_TYPE_ALERT; 142 case CONTENT_TYPE_HANDSHAKE: 143 return CPL_TX_TLS_SFO_TYPE_HANDSHAKE; 144 case CONTENT_TYPE_HEARTBEAT: 145 return CPL_TX_TLS_SFO_TYPE_HEARTBEAT; 146 } 147 return CPL_TX_TLS_SFO_TYPE_DATA; 148 } 149 150 /* TLS Key memory management */ 151 static void 152 clear_tls_keyid(struct toepcb *toep) 153 { 154 struct tls_ofld_info *tls_ofld = &toep->tls; 155 struct adapter *sc = td_adapter(toep->td); 156 157 if (tls_ofld->rx_key_addr >= 0) { 158 t4_free_tls_keyid(sc, tls_ofld->rx_key_addr); 159 tls_ofld->rx_key_addr = -1; 160 } 161 if (tls_ofld->tx_key_addr >= 0) { 162 t4_free_tls_keyid(sc, tls_ofld->tx_key_addr); 163 tls_ofld->tx_key_addr = -1; 164 } 165 } 166 167 static int 168 get_tp_plen_max(struct ktls_session *tls) 169 { 170 int plen = ((min(3*4096, TP_TX_PG_SZ))/1448) * 1448; 171 172 return (tls->params.max_frame_len <= 8192 ? plen : FC_TP_PLEN_MAX); 173 } 174 175 /* Send request to get the key-id */ 176 static int 177 tls_program_key_id(struct toepcb *toep, struct ktls_session *tls, 178 int direction) 179 { 180 struct tls_ofld_info *tls_ofld = &toep->tls; 181 struct adapter *sc = td_adapter(toep->td); 182 struct ofld_tx_sdesc *txsd; 183 int keyid; 184 struct wrqe *wr; 185 struct tls_key_req *kwr; 186 struct tls_keyctx *kctx; 187 188 #ifdef INVARIANTS 189 int kwrlen, kctxlen, len; 190 191 kwrlen = sizeof(*kwr); 192 kctxlen = roundup2(sizeof(*kctx), 32); 193 len = roundup2(kwrlen + kctxlen, 16); 194 MPASS(TLS_KEY_WR_SZ == len); 195 #endif 196 if (toep->txsd_avail == 0) 197 return (EAGAIN); 198 199 if ((keyid = t4_alloc_tls_keyid(sc)) < 0) { 200 return (ENOSPC); 201 } 202 203 wr = alloc_wrqe(TLS_KEY_WR_SZ, &toep->ofld_txq->wrq); 204 if (wr == NULL) { 205 t4_free_tls_keyid(sc, keyid); 206 return (ENOMEM); 207 } 208 kwr = wrtod(wr); 209 memset(kwr, 0, TLS_KEY_WR_SZ); 210 211 t4_write_tlskey_wr(tls, direction, toep->tid, F_FW_WR_COMPL, keyid, 212 kwr); 213 kctx = (struct tls_keyctx *)(kwr + 1); 214 if (direction == KTLS_TX) 215 tls_ofld->tx_key_addr = keyid; 216 else 217 tls_ofld->rx_key_addr = keyid; 218 t4_tls_key_ctx(tls, direction, kctx); 219 220 txsd = &toep->txsd[toep->txsd_pidx]; 221 txsd->tx_credits = DIV_ROUND_UP(TLS_KEY_WR_SZ, 16); 222 txsd->plen = 0; 223 toep->tx_credits -= txsd->tx_credits; 224 if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) 225 toep->txsd_pidx = 0; 226 toep->txsd_avail--; 227 228 t4_wrq_tx(sc, wr); 229 230 return (0); 231 } 232 233 /* 234 * In some cases a client connection can hang without sending the 235 * ServerHelloDone message from the NIC to the host. Send a dummy 236 * RX_DATA_ACK with RX_MODULATE to unstick the connection. 237 */ 238 static void 239 tls_send_handshake_ack(void *arg) 240 { 241 struct toepcb *toep = arg; 242 struct tls_ofld_info *tls_ofld = &toep->tls; 243 struct adapter *sc = td_adapter(toep->td); 244 245 /* Bail without rescheduling if the connection has closed. */ 246 if ((toep->flags & (TPF_FIN_SENT | TPF_ABORT_SHUTDOWN)) != 0) 247 return; 248 249 /* 250 * If this connection has timed out without receiving more 251 * data, downgrade to plain TOE mode and don't re-arm the 252 * timer. 253 */ 254 if (sc->tt.tls_rx_timeout != 0) { 255 struct inpcb *inp; 256 struct tcpcb *tp; 257 258 inp = toep->inp; 259 tp = intotcpcb(inp); 260 if ((ticks - tp->t_rcvtime) >= sc->tt.tls_rx_timeout) { 261 CTR2(KTR_CXGBE, "%s: tid %d clr_ofld_mode", __func__, 262 toep->tid); 263 tls_clr_ofld_mode(toep); 264 return; 265 } 266 } 267 268 /* 269 * XXX: Does not have the t4_get_tcb() checks to refine the 270 * workaround. 271 */ 272 callout_schedule(&tls_ofld->handshake_timer, TLS_SRV_HELLO_RD_TM * hz); 273 274 CTR2(KTR_CXGBE, "%s: tid %d sending RX_DATA_ACK", __func__, toep->tid); 275 send_rx_modulate(sc, toep); 276 } 277 278 static void 279 tls_start_handshake_timer(struct toepcb *toep) 280 { 281 struct tls_ofld_info *tls_ofld = &toep->tls; 282 283 INP_WLOCK_ASSERT(toep->inp); 284 callout_reset(&tls_ofld->handshake_timer, TLS_SRV_HELLO_BKOFF_TM * hz, 285 tls_send_handshake_ack, toep); 286 } 287 288 void 289 tls_stop_handshake_timer(struct toepcb *toep) 290 { 291 struct tls_ofld_info *tls_ofld = &toep->tls; 292 293 INP_WLOCK_ASSERT(toep->inp); 294 callout_stop(&tls_ofld->handshake_timer); 295 } 296 297 int 298 tls_alloc_ktls(struct toepcb *toep, struct ktls_session *tls, int direction) 299 { 300 struct adapter *sc = td_adapter(toep->td); 301 int error, explicit_iv_size, key_offset, mac_first; 302 303 if (!can_tls_offload(td_adapter(toep->td))) 304 return (EINVAL); 305 switch (ulp_mode(toep)) { 306 case ULP_MODE_TLS: 307 break; 308 case ULP_MODE_NONE: 309 case ULP_MODE_TCPDDP: 310 if (direction != KTLS_TX) 311 return (EINVAL); 312 break; 313 default: 314 return (EINVAL); 315 } 316 317 switch (tls->params.cipher_algorithm) { 318 case CRYPTO_AES_CBC: 319 /* XXX: Explicitly ignore any provided IV. */ 320 switch (tls->params.cipher_key_len) { 321 case 128 / 8: 322 case 192 / 8: 323 case 256 / 8: 324 break; 325 default: 326 error = EINVAL; 327 goto clr_ofld; 328 } 329 switch (tls->params.auth_algorithm) { 330 case CRYPTO_SHA1_HMAC: 331 case CRYPTO_SHA2_256_HMAC: 332 case CRYPTO_SHA2_384_HMAC: 333 break; 334 default: 335 error = EPROTONOSUPPORT; 336 goto clr_ofld; 337 } 338 explicit_iv_size = AES_BLOCK_LEN; 339 mac_first = 1; 340 break; 341 case CRYPTO_AES_NIST_GCM_16: 342 if (tls->params.iv_len != SALT_SIZE) { 343 error = EINVAL; 344 goto clr_ofld; 345 } 346 switch (tls->params.cipher_key_len) { 347 case 128 / 8: 348 case 192 / 8: 349 case 256 / 8: 350 break; 351 default: 352 error = EINVAL; 353 goto clr_ofld; 354 } 355 explicit_iv_size = 8; 356 mac_first = 0; 357 break; 358 default: 359 error = EPROTONOSUPPORT; 360 goto clr_ofld; 361 } 362 363 /* Only TLS 1.1 and TLS 1.2 are currently supported. */ 364 if (tls->params.tls_vmajor != TLS_MAJOR_VER_ONE || 365 tls->params.tls_vminor < TLS_MINOR_VER_ONE || 366 tls->params.tls_vminor > TLS_MINOR_VER_TWO) { 367 error = EPROTONOSUPPORT; 368 goto clr_ofld; 369 } 370 371 /* Bail if we already have a key. */ 372 if (direction == KTLS_TX) { 373 if (toep->tls.tx_key_addr != -1) 374 return (EOPNOTSUPP); 375 } else { 376 if (toep->tls.rx_key_addr != -1) 377 return (EOPNOTSUPP); 378 } 379 380 error = tls_program_key_id(toep, tls, direction); 381 if (error) { 382 if (direction == KTLS_RX) 383 goto clr_ofld; 384 return (error); 385 } 386 387 if (direction == KTLS_TX) { 388 toep->tls.scmd0.seqno_numivs = 389 (V_SCMD_SEQ_NO_CTRL(3) | 390 V_SCMD_PROTO_VERSION(t4_tls_proto_ver(tls)) | 391 V_SCMD_ENC_DEC_CTRL(SCMD_ENCDECCTRL_ENCRYPT) | 392 V_SCMD_CIPH_AUTH_SEQ_CTRL((mac_first == 0)) | 393 V_SCMD_CIPH_MODE(t4_tls_cipher_mode(tls)) | 394 V_SCMD_AUTH_MODE(t4_tls_auth_mode(tls)) | 395 V_SCMD_HMAC_CTRL(t4_tls_hmac_ctrl(tls)) | 396 V_SCMD_IV_SIZE(explicit_iv_size / 2)); 397 398 toep->tls.scmd0.ivgen_hdrlen = 399 (V_SCMD_IV_GEN_CTRL(1) | 400 V_SCMD_KEY_CTX_INLINE(0) | 401 V_SCMD_TLS_FRAG_ENABLE(1)); 402 403 if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) 404 toep->tls.iv_len = 8; 405 else 406 toep->tls.iv_len = AES_BLOCK_LEN; 407 408 toep->tls.frag_size = tls->params.max_frame_len; 409 toep->tls.fcplenmax = get_tp_plen_max(tls); 410 toep->tls.expn_per_ulp = tls->params.tls_hlen + 411 tls->params.tls_tlen; 412 toep->tls.pdus_per_ulp = 1; 413 toep->tls.adjusted_plen = toep->tls.expn_per_ulp + 414 tls->params.max_frame_len; 415 toep->tls.tx_key_info_size = t4_tls_key_info_size(tls); 416 } else { 417 /* Stop timer on handshake completion */ 418 tls_stop_handshake_timer(toep); 419 420 toep->flags &= ~TPF_FORCE_CREDITS; 421 toep->flags |= TPF_TLS_RECEIVE; 422 toep->tls.rx_version = tls->params.tls_vmajor << 8 | 423 tls->params.tls_vminor; 424 425 /* 426 * RX key tags are an index into the key portion of MA 427 * memory stored as an offset from the base address in 428 * units of 64 bytes. 429 */ 430 key_offset = toep->tls.rx_key_addr - sc->vres.key.start; 431 t4_set_tls_keyid(toep, key_offset / 64); 432 t4_set_tls_tcb_field(toep, W_TCB_ULP_RAW, 433 V_TCB_ULP_RAW(M_TCB_ULP_RAW), 434 V_TCB_ULP_RAW((V_TF_TLS_KEY_SIZE(3) | 435 V_TF_TLS_CONTROL(1) | 436 V_TF_TLS_ACTIVE(1) | 437 V_TF_TLS_ENABLE(1)))); 438 t4_set_tls_tcb_field(toep, W_TCB_TLS_SEQ, 439 V_TCB_TLS_SEQ(M_TCB_TLS_SEQ), 440 V_TCB_TLS_SEQ(0)); 441 t4_clear_rx_quiesce(toep); 442 } 443 444 return (0); 445 446 clr_ofld: 447 if (ulp_mode(toep) == ULP_MODE_TLS) { 448 CTR2(KTR_CXGBE, "%s: tid %d clr_ofld_mode", __func__, 449 toep->tid); 450 tls_clr_ofld_mode(toep); 451 } 452 return (error); 453 } 454 455 void 456 tls_init_toep(struct toepcb *toep) 457 { 458 struct tls_ofld_info *tls_ofld = &toep->tls; 459 460 tls_ofld->rx_key_addr = -1; 461 tls_ofld->tx_key_addr = -1; 462 } 463 464 void 465 tls_establish(struct toepcb *toep) 466 { 467 468 /* 469 * Enable PDU extraction. 470 * 471 * XXX: Supposedly this should be done by the firmware when 472 * the ULP_MODE FLOWC parameter is set in send_flowc_wr(), but 473 * in practice this seems to be required. 474 */ 475 CTR2(KTR_CXGBE, "%s: tid %d setting TLS_ENABLE", __func__, toep->tid); 476 t4_set_tls_tcb_field(toep, W_TCB_ULP_RAW, V_TCB_ULP_RAW(M_TCB_ULP_RAW), 477 V_TCB_ULP_RAW(V_TF_TLS_ENABLE(1))); 478 479 toep->flags |= TPF_FORCE_CREDITS | TPF_TLS_ESTABLISHED; 480 481 callout_init_rw(&toep->tls.handshake_timer, &toep->inp->inp_lock, 0); 482 tls_start_handshake_timer(toep); 483 } 484 485 void 486 tls_detach(struct toepcb *toep) 487 { 488 489 if (toep->flags & TPF_TLS_ESTABLISHED) { 490 tls_stop_handshake_timer(toep); 491 toep->flags &= ~TPF_TLS_ESTABLISHED; 492 } 493 } 494 495 void 496 tls_uninit_toep(struct toepcb *toep) 497 { 498 499 MPASS((toep->flags & TPF_TLS_ESTABLISHED) == 0); 500 clear_tls_keyid(toep); 501 } 502 503 #define MAX_OFLD_TX_CREDITS (SGE_MAX_WR_LEN / 16) 504 #define MIN_OFLD_TLSTX_CREDITS(toep) \ 505 (howmany(sizeof(struct fw_tlstx_data_wr) + \ 506 sizeof(struct cpl_tx_tls_sfo) + sizeof(struct ulptx_idata) + \ 507 sizeof(struct ulptx_sc_memrd) + \ 508 AES_BLOCK_LEN + 1, 16)) 509 510 static inline u_int 511 max_imm_tls_space(int tx_credits) 512 { 513 const int n = 2; /* Use only up to 2 desc for imm. data WR */ 514 int space; 515 516 KASSERT(tx_credits >= 0 && 517 tx_credits <= MAX_OFLD_TX_CREDITS, 518 ("%s: %d credits", __func__, tx_credits)); 519 520 if (tx_credits >= (n * EQ_ESIZE) / 16) 521 space = (n * EQ_ESIZE); 522 else 523 space = tx_credits * 16; 524 return (space); 525 } 526 527 static void 528 write_tlstx_wr(struct fw_tlstx_data_wr *txwr, struct toepcb *toep, 529 unsigned int immdlen, unsigned int plen, unsigned int expn, 530 unsigned int pdus, uint8_t credits, int shove, int imm_ivs) 531 { 532 struct tls_ofld_info *tls_ofld = &toep->tls; 533 unsigned int len = plen + expn; 534 535 txwr->op_to_immdlen = htobe32(V_WR_OP(FW_TLSTX_DATA_WR) | 536 V_FW_TLSTX_DATA_WR_COMPL(1) | 537 V_FW_TLSTX_DATA_WR_IMMDLEN(immdlen)); 538 txwr->flowid_len16 = htobe32(V_FW_TLSTX_DATA_WR_FLOWID(toep->tid) | 539 V_FW_TLSTX_DATA_WR_LEN16(credits)); 540 txwr->plen = htobe32(len); 541 txwr->lsodisable_to_flags = htobe32(V_TX_ULP_MODE(ULP_MODE_TLS) | 542 V_TX_URG(0) | /* F_T6_TX_FORCE | */ V_TX_SHOVE(shove)); 543 txwr->ctxloc_to_exp = htobe32(V_FW_TLSTX_DATA_WR_NUMIVS(pdus) | 544 V_FW_TLSTX_DATA_WR_EXP(expn) | 545 V_FW_TLSTX_DATA_WR_CTXLOC(TLS_SFO_WR_CONTEXTLOC_DDR) | 546 V_FW_TLSTX_DATA_WR_IVDSGL(!imm_ivs) | 547 V_FW_TLSTX_DATA_WR_KEYSIZE(tls_ofld->tx_key_info_size >> 4)); 548 txwr->mfs = htobe16(tls_ofld->frag_size); 549 txwr->adjustedplen_pkd = htobe16( 550 V_FW_TLSTX_DATA_WR_ADJUSTEDPLEN(tls_ofld->adjusted_plen)); 551 txwr->expinplenmax_pkd = htobe16( 552 V_FW_TLSTX_DATA_WR_EXPINPLENMAX(tls_ofld->expn_per_ulp)); 553 txwr->pdusinplenmax_pkd = 554 V_FW_TLSTX_DATA_WR_PDUSINPLENMAX(tls_ofld->pdus_per_ulp); 555 } 556 557 static void 558 write_tlstx_cpl(struct cpl_tx_tls_sfo *cpl, struct toepcb *toep, 559 struct tls_hdr *tls_hdr, unsigned int plen, unsigned int pdus) 560 { 561 struct tls_ofld_info *tls_ofld = &toep->tls; 562 int data_type, seglen; 563 564 if (plen < tls_ofld->frag_size) 565 seglen = plen; 566 else 567 seglen = tls_ofld->frag_size; 568 data_type = tls_content_type(tls_hdr->type); 569 cpl->op_to_seg_len = htobe32(V_CPL_TX_TLS_SFO_OPCODE(CPL_TX_TLS_SFO) | 570 V_CPL_TX_TLS_SFO_DATA_TYPE(data_type) | 571 V_CPL_TX_TLS_SFO_CPL_LEN(2) | V_CPL_TX_TLS_SFO_SEG_LEN(seglen)); 572 cpl->pld_len = htobe32(plen); 573 if (data_type == CPL_TX_TLS_SFO_TYPE_HEARTBEAT) 574 cpl->type_protover = htobe32( 575 V_CPL_TX_TLS_SFO_TYPE(tls_hdr->type)); 576 cpl->seqno_numivs = htobe32(tls_ofld->scmd0.seqno_numivs | 577 V_SCMD_NUM_IVS(pdus)); 578 cpl->ivgen_hdrlen = htobe32(tls_ofld->scmd0.ivgen_hdrlen); 579 cpl->scmd1 = htobe64(tls_ofld->tx_seq_no); 580 tls_ofld->tx_seq_no += pdus; 581 } 582 583 static int 584 count_ext_pgs_segs(struct mbuf *m) 585 { 586 vm_paddr_t nextpa; 587 u_int i, nsegs; 588 589 MPASS(m->m_epg_npgs > 0); 590 nsegs = 1; 591 nextpa = m->m_epg_pa[0] + PAGE_SIZE; 592 for (i = 1; i < m->m_epg_npgs; i++) { 593 if (nextpa != m->m_epg_pa[i]) 594 nsegs++; 595 nextpa = m->m_epg_pa[i] + PAGE_SIZE; 596 } 597 return (nsegs); 598 } 599 600 static void 601 write_ktlstx_sgl(void *dst, struct mbuf *m, int nsegs) 602 { 603 struct ulptx_sgl *usgl = dst; 604 vm_paddr_t pa; 605 uint32_t len; 606 int i, j; 607 608 KASSERT(nsegs > 0, ("%s: nsegs 0", __func__)); 609 610 usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) | 611 V_ULPTX_NSGE(nsegs)); 612 613 /* Figure out the first S/G length. */ 614 pa = m->m_epg_pa[0] + m->m_epg_1st_off; 615 usgl->addr0 = htobe64(pa); 616 len = m_epg_pagelen(m, 0, m->m_epg_1st_off); 617 pa += len; 618 for (i = 1; i < m->m_epg_npgs; i++) { 619 if (m->m_epg_pa[i] != pa) 620 break; 621 len += m_epg_pagelen(m, i, 0); 622 pa += m_epg_pagelen(m, i, 0); 623 } 624 usgl->len0 = htobe32(len); 625 #ifdef INVARIANTS 626 nsegs--; 627 #endif 628 629 j = -1; 630 for (; i < m->m_epg_npgs; i++) { 631 if (j == -1 || m->m_epg_pa[i] != pa) { 632 if (j >= 0) 633 usgl->sge[j / 2].len[j & 1] = htobe32(len); 634 j++; 635 #ifdef INVARIANTS 636 nsegs--; 637 #endif 638 pa = m->m_epg_pa[i]; 639 usgl->sge[j / 2].addr[j & 1] = htobe64(pa); 640 len = m_epg_pagelen(m, i, 0); 641 pa += len; 642 } else { 643 len += m_epg_pagelen(m, i, 0); 644 pa += m_epg_pagelen(m, i, 0); 645 } 646 } 647 if (j >= 0) { 648 usgl->sge[j / 2].len[j & 1] = htobe32(len); 649 650 if ((j & 1) == 0) 651 usgl->sge[j / 2].len[1] = htobe32(0); 652 } 653 KASSERT(nsegs == 0, ("%s: nsegs %d, m %p", __func__, nsegs, m)); 654 } 655 656 /* 657 * Similar to t4_push_frames() but handles sockets that contain TLS 658 * record mbufs. 659 */ 660 void 661 t4_push_ktls(struct adapter *sc, struct toepcb *toep, int drop) 662 { 663 struct tls_hdr *thdr; 664 struct fw_tlstx_data_wr *txwr; 665 struct cpl_tx_tls_sfo *cpl; 666 struct ulptx_idata *idata; 667 struct ulptx_sc_memrd *memrd; 668 struct wrqe *wr; 669 struct mbuf *m; 670 u_int nsegs, credits, wr_len; 671 u_int expn_size; 672 struct inpcb *inp = toep->inp; 673 struct tcpcb *tp = intotcpcb(inp); 674 struct socket *so = inp->inp_socket; 675 struct sockbuf *sb = &so->so_snd; 676 int tls_size, tx_credits, shove, sowwakeup; 677 struct ofld_tx_sdesc *txsd; 678 char *buf; 679 680 INP_WLOCK_ASSERT(inp); 681 KASSERT(toep->flags & TPF_FLOWC_WR_SENT, 682 ("%s: flowc_wr not sent for tid %u.", __func__, toep->tid)); 683 684 KASSERT(ulp_mode(toep) == ULP_MODE_NONE || 685 ulp_mode(toep) == ULP_MODE_TCPDDP || ulp_mode(toep) == ULP_MODE_TLS, 686 ("%s: ulp_mode %u for toep %p", __func__, ulp_mode(toep), toep)); 687 KASSERT(tls_tx_key(toep), 688 ("%s: TX key not set for toep %p", __func__, toep)); 689 690 #ifdef VERBOSE_TRACES 691 CTR4(KTR_CXGBE, "%s: tid %d toep flags %#x tp flags %#x drop %d", 692 __func__, toep->tid, toep->flags, tp->t_flags); 693 #endif 694 if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN)) 695 return; 696 697 #ifdef RATELIMIT 698 if (__predict_false(inp->inp_flags2 & INP_RATE_LIMIT_CHANGED) && 699 (update_tx_rate_limit(sc, toep, so->so_max_pacing_rate) == 0)) { 700 inp->inp_flags2 &= ~INP_RATE_LIMIT_CHANGED; 701 } 702 #endif 703 704 /* 705 * This function doesn't resume by itself. Someone else must clear the 706 * flag and call this function. 707 */ 708 if (__predict_false(toep->flags & TPF_TX_SUSPENDED)) { 709 KASSERT(drop == 0, 710 ("%s: drop (%d) != 0 but tx is suspended", __func__, drop)); 711 return; 712 } 713 714 txsd = &toep->txsd[toep->txsd_pidx]; 715 for (;;) { 716 tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS); 717 718 SOCKBUF_LOCK(sb); 719 sowwakeup = drop; 720 if (drop) { 721 sbdrop_locked(sb, drop); 722 drop = 0; 723 } 724 725 m = sb->sb_sndptr != NULL ? sb->sb_sndptr->m_next : sb->sb_mb; 726 727 /* 728 * Send a FIN if requested, but only if there's no 729 * more data to send. 730 */ 731 if (m == NULL && toep->flags & TPF_SEND_FIN) { 732 if (sowwakeup) 733 sowwakeup_locked(so); 734 else 735 SOCKBUF_UNLOCK(sb); 736 SOCKBUF_UNLOCK_ASSERT(sb); 737 t4_close_conn(sc, toep); 738 return; 739 } 740 741 /* 742 * If there is no ready data to send, wait until more 743 * data arrives. 744 */ 745 if (m == NULL || (m->m_flags & M_NOTAVAIL) != 0) { 746 if (sowwakeup) 747 sowwakeup_locked(so); 748 else 749 SOCKBUF_UNLOCK(sb); 750 SOCKBUF_UNLOCK_ASSERT(sb); 751 #ifdef VERBOSE_TRACES 752 CTR2(KTR_CXGBE, "%s: tid %d no ready data to send", 753 __func__, toep->tid); 754 #endif 755 return; 756 } 757 758 KASSERT(m->m_flags & M_EXTPG, ("%s: mbuf %p is not NOMAP", 759 __func__, m)); 760 KASSERT(m->m_epg_tls != NULL, 761 ("%s: mbuf %p doesn't have TLS session", __func__, m)); 762 763 /* Calculate WR length. */ 764 wr_len = sizeof(struct fw_tlstx_data_wr) + 765 sizeof(struct cpl_tx_tls_sfo) + 766 sizeof(struct ulptx_idata) + sizeof(struct ulptx_sc_memrd); 767 768 /* Explicit IVs for AES-CBC and AES-GCM are <= 16. */ 769 MPASS(toep->tls.iv_len <= AES_BLOCK_LEN); 770 wr_len += AES_BLOCK_LEN; 771 772 /* Account for SGL in work request length. */ 773 nsegs = count_ext_pgs_segs(m); 774 wr_len += sizeof(struct ulptx_sgl) + 775 ((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8; 776 777 /* Not enough credits for this work request. */ 778 if (howmany(wr_len, 16) > tx_credits) { 779 if (sowwakeup) 780 sowwakeup_locked(so); 781 else 782 SOCKBUF_UNLOCK(sb); 783 SOCKBUF_UNLOCK_ASSERT(sb); 784 #ifdef VERBOSE_TRACES 785 CTR5(KTR_CXGBE, 786 "%s: tid %d mbuf %p requires %d credits, but only %d available", 787 __func__, toep->tid, m, howmany(wr_len, 16), 788 tx_credits); 789 #endif 790 toep->flags |= TPF_TX_SUSPENDED; 791 return; 792 } 793 794 /* Shove if there is no additional data pending. */ 795 shove = ((m->m_next == NULL || 796 (m->m_next->m_flags & M_NOTAVAIL) != 0)) && 797 (tp->t_flags & TF_MORETOCOME) == 0; 798 799 if (sb->sb_flags & SB_AUTOSIZE && 800 V_tcp_do_autosndbuf && 801 sb->sb_hiwat < V_tcp_autosndbuf_max && 802 sbused(sb) >= sb->sb_hiwat * 7 / 8) { 803 int newsize = min(sb->sb_hiwat + V_tcp_autosndbuf_inc, 804 V_tcp_autosndbuf_max); 805 806 if (!sbreserve_locked(sb, newsize, so, NULL)) 807 sb->sb_flags &= ~SB_AUTOSIZE; 808 else 809 sowwakeup = 1; /* room available */ 810 } 811 if (sowwakeup) 812 sowwakeup_locked(so); 813 else 814 SOCKBUF_UNLOCK(sb); 815 SOCKBUF_UNLOCK_ASSERT(sb); 816 817 if (__predict_false(toep->flags & TPF_FIN_SENT)) 818 panic("%s: excess tx.", __func__); 819 820 wr = alloc_wrqe(roundup2(wr_len, 16), &toep->ofld_txq->wrq); 821 if (wr == NULL) { 822 /* XXX: how will we recover from this? */ 823 toep->flags |= TPF_TX_SUSPENDED; 824 return; 825 } 826 827 thdr = (struct tls_hdr *)&m->m_epg_hdr; 828 #ifdef VERBOSE_TRACES 829 CTR5(KTR_CXGBE, "%s: tid %d TLS record %ju type %d len %#x", 830 __func__, toep->tid, m->m_epg_seqno, thdr->type, 831 m->m_len); 832 #endif 833 txwr = wrtod(wr); 834 cpl = (struct cpl_tx_tls_sfo *)(txwr + 1); 835 memset(txwr, 0, roundup2(wr_len, 16)); 836 credits = howmany(wr_len, 16); 837 expn_size = m->m_epg_hdrlen + 838 m->m_epg_trllen; 839 tls_size = m->m_len - expn_size; 840 write_tlstx_wr(txwr, toep, 0, 841 tls_size, expn_size, 1, credits, shove, 1); 842 toep->tls.tx_seq_no = m->m_epg_seqno; 843 write_tlstx_cpl(cpl, toep, thdr, tls_size, 1); 844 845 idata = (struct ulptx_idata *)(cpl + 1); 846 idata->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP)); 847 idata->len = htobe32(0); 848 memrd = (struct ulptx_sc_memrd *)(idata + 1); 849 memrd->cmd_to_len = htobe32(V_ULPTX_CMD(ULP_TX_SC_MEMRD) | 850 V_ULP_TX_SC_MORE(1) | 851 V_ULPTX_LEN16(toep->tls.tx_key_info_size >> 4)); 852 memrd->addr = htobe32(toep->tls.tx_key_addr >> 5); 853 854 /* Copy IV. */ 855 buf = (char *)(memrd + 1); 856 memcpy(buf, thdr + 1, toep->tls.iv_len); 857 buf += AES_BLOCK_LEN; 858 859 write_ktlstx_sgl(buf, m, nsegs); 860 861 KASSERT(toep->tx_credits >= credits, 862 ("%s: not enough credits", __func__)); 863 864 toep->tx_credits -= credits; 865 866 tp->snd_nxt += m->m_len; 867 tp->snd_max += m->m_len; 868 869 SOCKBUF_LOCK(sb); 870 sb->sb_sndptr = m; 871 SOCKBUF_UNLOCK(sb); 872 873 toep->flags |= TPF_TX_DATA_SENT; 874 if (toep->tx_credits < MIN_OFLD_TLSTX_CREDITS(toep)) 875 toep->flags |= TPF_TX_SUSPENDED; 876 877 KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__)); 878 txsd->plen = m->m_len; 879 txsd->tx_credits = credits; 880 txsd++; 881 if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) { 882 toep->txsd_pidx = 0; 883 txsd = &toep->txsd[0]; 884 } 885 toep->txsd_avail--; 886 887 counter_u64_add(toep->ofld_txq->tx_toe_tls_records, 1); 888 counter_u64_add(toep->ofld_txq->tx_toe_tls_octets, m->m_len); 889 890 t4_l2t_send(sc, wr, toep->l2te); 891 } 892 } 893 894 /* 895 * For TLS data we place received mbufs received via CPL_TLS_DATA into 896 * an mbufq in the TLS offload state. When CPL_RX_TLS_CMP is 897 * received, the completed PDUs are placed into the socket receive 898 * buffer. 899 * 900 * The TLS code reuses the ulp_pdu_reclaimq to hold the pending mbufs. 901 */ 902 static int 903 do_tls_data(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 904 { 905 struct adapter *sc = iq->adapter; 906 const struct cpl_tls_data *cpl = mtod(m, const void *); 907 unsigned int tid = GET_TID(cpl); 908 struct toepcb *toep = lookup_tid(sc, tid); 909 struct inpcb *inp = toep->inp; 910 struct tcpcb *tp; 911 int len; 912 913 /* XXX: Should this match do_rx_data instead? */ 914 KASSERT(!(toep->flags & TPF_SYNQE), 915 ("%s: toep %p claims to be a synq entry", __func__, toep)); 916 917 KASSERT(toep->tid == tid, ("%s: toep tid/atid mismatch", __func__)); 918 919 /* strip off CPL header */ 920 m_adj(m, sizeof(*cpl)); 921 len = m->m_pkthdr.len; 922 923 toep->ofld_rxq->rx_toe_tls_octets += len; 924 925 KASSERT(len == G_CPL_TLS_DATA_LENGTH(be32toh(cpl->length_pkd)), 926 ("%s: payload length mismatch", __func__)); 927 928 INP_WLOCK(inp); 929 if (inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) { 930 CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x", 931 __func__, tid, len, inp->inp_flags); 932 INP_WUNLOCK(inp); 933 m_freem(m); 934 return (0); 935 } 936 937 /* Save TCP sequence number. */ 938 m->m_pkthdr.tls_tcp_seq = be32toh(cpl->seq); 939 940 if (mbufq_enqueue(&toep->ulp_pdu_reclaimq, m)) { 941 #ifdef INVARIANTS 942 panic("Failed to queue TLS data packet"); 943 #else 944 printf("%s: Failed to queue TLS data packet\n", __func__); 945 INP_WUNLOCK(inp); 946 m_freem(m); 947 return (0); 948 #endif 949 } 950 951 tp = intotcpcb(inp); 952 tp->t_rcvtime = ticks; 953 954 #ifdef VERBOSE_TRACES 955 CTR4(KTR_CXGBE, "%s: tid %u len %d seq %u", __func__, tid, len, 956 be32toh(cpl->seq)); 957 #endif 958 959 INP_WUNLOCK(inp); 960 return (0); 961 } 962 963 static int 964 do_rx_tls_cmp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 965 { 966 struct adapter *sc = iq->adapter; 967 const struct cpl_rx_tls_cmp *cpl = mtod(m, const void *); 968 struct tlsrx_hdr_pkt *tls_hdr_pkt; 969 unsigned int tid = GET_TID(cpl); 970 struct toepcb *toep = lookup_tid(sc, tid); 971 struct inpcb *inp = toep->inp; 972 struct tcpcb *tp; 973 struct socket *so; 974 struct sockbuf *sb; 975 struct mbuf *tls_data; 976 struct tls_get_record *tgr; 977 struct mbuf *control; 978 int len, pdu_length, rx_credits; 979 980 KASSERT(toep->tid == tid, ("%s: toep tid/atid mismatch", __func__)); 981 KASSERT(!(toep->flags & TPF_SYNQE), 982 ("%s: toep %p claims to be a synq entry", __func__, toep)); 983 984 /* strip off CPL header */ 985 m_adj(m, sizeof(*cpl)); 986 len = m->m_pkthdr.len; 987 988 toep->ofld_rxq->rx_toe_tls_records++; 989 990 KASSERT(len == G_CPL_RX_TLS_CMP_LENGTH(be32toh(cpl->pdulength_length)), 991 ("%s: payload length mismatch", __func__)); 992 993 INP_WLOCK(inp); 994 if (inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) { 995 CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x", 996 __func__, tid, len, inp->inp_flags); 997 INP_WUNLOCK(inp); 998 m_freem(m); 999 return (0); 1000 } 1001 1002 pdu_length = G_CPL_RX_TLS_CMP_PDULENGTH(be32toh(cpl->pdulength_length)); 1003 1004 so = inp_inpcbtosocket(inp); 1005 tp = intotcpcb(inp); 1006 1007 #ifdef VERBOSE_TRACES 1008 CTR6(KTR_CXGBE, "%s: tid %u PDU len %d len %d seq %u, rcv_nxt %u", 1009 __func__, tid, pdu_length, len, be32toh(cpl->seq), tp->rcv_nxt); 1010 #endif 1011 1012 tp->rcv_nxt += pdu_length; 1013 KASSERT(tp->rcv_wnd >= pdu_length, 1014 ("%s: negative window size", __func__)); 1015 tp->rcv_wnd -= pdu_length; 1016 1017 /* XXX: Not sure what to do about urgent data. */ 1018 1019 /* 1020 * The payload of this CPL is the TLS header followed by 1021 * additional fields. 1022 */ 1023 KASSERT(m->m_len >= sizeof(*tls_hdr_pkt), 1024 ("%s: payload too small", __func__)); 1025 tls_hdr_pkt = mtod(m, void *); 1026 1027 tls_data = mbufq_dequeue(&toep->ulp_pdu_reclaimq); 1028 if (tls_data != NULL) { 1029 KASSERT(be32toh(cpl->seq) == tls_data->m_pkthdr.tls_tcp_seq, 1030 ("%s: sequence mismatch", __func__)); 1031 } 1032 1033 /* Report decryption errors as EBADMSG. */ 1034 if ((tls_hdr_pkt->res_to_mac_error & M_TLSRX_HDR_PKT_ERROR) != 0) { 1035 m_freem(m); 1036 m_freem(tls_data); 1037 1038 CURVNET_SET(toep->vnet); 1039 so->so_error = EBADMSG; 1040 sorwakeup(so); 1041 1042 INP_WUNLOCK(inp); 1043 CURVNET_RESTORE(); 1044 1045 return (0); 1046 } 1047 1048 /* Allocate the control message mbuf. */ 1049 control = sbcreatecontrol(NULL, sizeof(*tgr), TLS_GET_RECORD, 1050 IPPROTO_TCP); 1051 if (control == NULL) { 1052 m_freem(m); 1053 m_freem(tls_data); 1054 1055 CURVNET_SET(toep->vnet); 1056 so->so_error = ENOBUFS; 1057 sorwakeup(so); 1058 1059 INP_WUNLOCK(inp); 1060 CURVNET_RESTORE(); 1061 1062 return (0); 1063 } 1064 1065 tgr = (struct tls_get_record *) 1066 CMSG_DATA(mtod(control, struct cmsghdr *)); 1067 tgr->tls_type = tls_hdr_pkt->type; 1068 tgr->tls_vmajor = be16toh(tls_hdr_pkt->version) >> 8; 1069 tgr->tls_vminor = be16toh(tls_hdr_pkt->version) & 0xff; 1070 1071 m_freem(m); 1072 1073 if (tls_data != NULL) { 1074 m_last(tls_data)->m_flags |= M_EOR; 1075 tgr->tls_length = htobe16(tls_data->m_pkthdr.len); 1076 } else 1077 tgr->tls_length = 0; 1078 m = tls_data; 1079 1080 sb = &so->so_rcv; 1081 SOCKBUF_LOCK(sb); 1082 1083 if (__predict_false(sb->sb_state & SBS_CANTRCVMORE)) { 1084 struct epoch_tracker et; 1085 1086 CTR3(KTR_CXGBE, "%s: tid %u, excess rx (%d bytes)", 1087 __func__, tid, pdu_length); 1088 m_freem(m); 1089 m_freem(control); 1090 SOCKBUF_UNLOCK(sb); 1091 INP_WUNLOCK(inp); 1092 1093 CURVNET_SET(toep->vnet); 1094 NET_EPOCH_ENTER(et); 1095 INP_WLOCK(inp); 1096 tp = tcp_drop(tp, ECONNRESET); 1097 if (tp) 1098 INP_WUNLOCK(inp); 1099 NET_EPOCH_EXIT(et); 1100 CURVNET_RESTORE(); 1101 1102 return (0); 1103 } 1104 1105 /* 1106 * Not all of the bytes on the wire are included in the socket buffer 1107 * (e.g. the MAC of the TLS record). However, those bytes are included 1108 * in the TCP sequence space. 1109 */ 1110 1111 /* receive buffer autosize */ 1112 MPASS(toep->vnet == so->so_vnet); 1113 CURVNET_SET(toep->vnet); 1114 if (sb->sb_flags & SB_AUTOSIZE && 1115 V_tcp_do_autorcvbuf && 1116 sb->sb_hiwat < V_tcp_autorcvbuf_max && 1117 m->m_pkthdr.len > (sbspace(sb) / 8 * 7)) { 1118 unsigned int hiwat = sb->sb_hiwat; 1119 unsigned int newsize = min(hiwat + sc->tt.autorcvbuf_inc, 1120 V_tcp_autorcvbuf_max); 1121 1122 if (!sbreserve_locked(sb, newsize, so, NULL)) 1123 sb->sb_flags &= ~SB_AUTOSIZE; 1124 } 1125 1126 sbappendcontrol_locked(sb, m, control, 0); 1127 rx_credits = sbspace(sb) > tp->rcv_wnd ? sbspace(sb) - tp->rcv_wnd : 0; 1128 #ifdef VERBOSE_TRACES 1129 CTR4(KTR_CXGBE, "%s: tid %u rx_credits %u rcv_wnd %u", 1130 __func__, tid, rx_credits, tp->rcv_wnd); 1131 #endif 1132 if (rx_credits > 0 && sbused(sb) + tp->rcv_wnd < sb->sb_lowat) { 1133 rx_credits = send_rx_credits(sc, toep, rx_credits); 1134 tp->rcv_wnd += rx_credits; 1135 tp->rcv_adv += rx_credits; 1136 } 1137 1138 sorwakeup_locked(so); 1139 SOCKBUF_UNLOCK_ASSERT(sb); 1140 1141 INP_WUNLOCK(inp); 1142 CURVNET_RESTORE(); 1143 return (0); 1144 } 1145 1146 void 1147 do_rx_data_tls(const struct cpl_rx_data *cpl, struct toepcb *toep, 1148 struct mbuf *m) 1149 { 1150 struct inpcb *inp = toep->inp; 1151 struct tls_ofld_info *tls_ofld = &toep->tls; 1152 struct tls_hdr *hdr; 1153 struct tcpcb *tp; 1154 struct socket *so; 1155 struct sockbuf *sb; 1156 int len, rx_credits; 1157 1158 len = m->m_pkthdr.len; 1159 1160 INP_WLOCK_ASSERT(inp); 1161 1162 so = inp_inpcbtosocket(inp); 1163 tp = intotcpcb(inp); 1164 sb = &so->so_rcv; 1165 SOCKBUF_LOCK(sb); 1166 CURVNET_SET(toep->vnet); 1167 1168 tp->rcv_nxt += len; 1169 KASSERT(tp->rcv_wnd >= len, ("%s: negative window size", __func__)); 1170 tp->rcv_wnd -= len; 1171 1172 /* Do we have a full TLS header? */ 1173 if (len < sizeof(*hdr)) { 1174 CTR3(KTR_CXGBE, "%s: tid %u len %d: too short for a TLS header", 1175 __func__, toep->tid, len); 1176 so->so_error = EMSGSIZE; 1177 goto out; 1178 } 1179 hdr = mtod(m, struct tls_hdr *); 1180 1181 /* Is the header valid? */ 1182 if (be16toh(hdr->version) != tls_ofld->rx_version) { 1183 CTR3(KTR_CXGBE, "%s: tid %u invalid version %04x", 1184 __func__, toep->tid, be16toh(hdr->version)); 1185 so->so_error = EINVAL; 1186 goto out; 1187 } 1188 if (be16toh(hdr->length) < sizeof(*hdr)) { 1189 CTR3(KTR_CXGBE, "%s: tid %u invalid length %u", 1190 __func__, toep->tid, be16toh(hdr->length)); 1191 so->so_error = EBADMSG; 1192 goto out; 1193 } 1194 1195 /* Did we get a truncated record? */ 1196 if (len < be16toh(hdr->length)) { 1197 CTR4(KTR_CXGBE, "%s: tid %u truncated TLS record (%d vs %u)", 1198 __func__, toep->tid, len, be16toh(hdr->length)); 1199 1200 so->so_error = EMSGSIZE; 1201 goto out; 1202 } 1203 1204 /* Is the header type unknown? */ 1205 switch (hdr->type) { 1206 case CONTENT_TYPE_CCS: 1207 case CONTENT_TYPE_ALERT: 1208 case CONTENT_TYPE_APP_DATA: 1209 case CONTENT_TYPE_HANDSHAKE: 1210 break; 1211 default: 1212 CTR3(KTR_CXGBE, "%s: tid %u invalid TLS record type %u", 1213 __func__, toep->tid, hdr->type); 1214 so->so_error = EBADMSG; 1215 goto out; 1216 } 1217 1218 /* 1219 * Just punt. Although this could fall back to software 1220 * decryption, this case should never really happen. 1221 */ 1222 CTR4(KTR_CXGBE, "%s: tid %u dropping TLS record type %u, length %u", 1223 __func__, toep->tid, hdr->type, be16toh(hdr->length)); 1224 so->so_error = EBADMSG; 1225 1226 out: 1227 /* 1228 * This connection is going to die anyway, so probably don't 1229 * need to bother with returning credits. 1230 */ 1231 rx_credits = sbspace(sb) > tp->rcv_wnd ? sbspace(sb) - tp->rcv_wnd : 0; 1232 #ifdef VERBOSE_TRACES 1233 CTR4(KTR_CXGBE, "%s: tid %u rx_credits %u rcv_wnd %u", 1234 __func__, toep->tid, rx_credits, tp->rcv_wnd); 1235 #endif 1236 if (rx_credits > 0 && sbused(sb) + tp->rcv_wnd < sb->sb_lowat) { 1237 rx_credits = send_rx_credits(toep->vi->adapter, toep, 1238 rx_credits); 1239 tp->rcv_wnd += rx_credits; 1240 tp->rcv_adv += rx_credits; 1241 } 1242 1243 sorwakeup_locked(so); 1244 SOCKBUF_UNLOCK_ASSERT(sb); 1245 1246 INP_WUNLOCK(inp); 1247 CURVNET_RESTORE(); 1248 1249 m_freem(m); 1250 } 1251 1252 void 1253 t4_tls_mod_load(void) 1254 { 1255 1256 t4_register_cpl_handler(CPL_TLS_DATA, do_tls_data); 1257 t4_register_cpl_handler(CPL_RX_TLS_CMP, do_rx_tls_cmp); 1258 } 1259 1260 void 1261 t4_tls_mod_unload(void) 1262 { 1263 1264 t4_register_cpl_handler(CPL_TLS_DATA, NULL); 1265 t4_register_cpl_handler(CPL_RX_TLS_CMP, NULL); 1266 } 1267 #endif /* TCP_OFFLOAD */ 1268 #endif /* KERN_TLS */ 1269