1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2017-2018 Chelsio Communications, Inc. 5 * All rights reserved. 6 * Written by: John Baldwin <jhb@FreeBSD.org> 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include "opt_inet.h" 31 #include "opt_kern_tls.h" 32 33 #include <sys/cdefs.h> 34 __FBSDID("$FreeBSD$"); 35 36 #include <sys/param.h> 37 #include <sys/ktr.h> 38 #ifdef KERN_TLS 39 #include <sys/ktls.h> 40 #endif 41 #include <sys/sglist.h> 42 #include <sys/socket.h> 43 #include <sys/socketvar.h> 44 #include <sys/systm.h> 45 #include <netinet/in.h> 46 #include <netinet/in_pcb.h> 47 #include <netinet/tcp_var.h> 48 #include <netinet/toecore.h> 49 #ifdef KERN_TLS 50 #include <opencrypto/cryptodev.h> 51 #include <opencrypto/xform.h> 52 #endif 53 54 #ifdef TCP_OFFLOAD 55 #include "common/common.h" 56 #include "common/t4_tcb.h" 57 #include "crypto/t4_crypto.h" 58 #include "tom/t4_tom_l2t.h" 59 #include "tom/t4_tom.h" 60 61 /* 62 * The TCP sequence number of a CPL_TLS_DATA mbuf is saved here while 63 * the mbuf is in the ulp_pdu_reclaimq. 64 */ 65 #define tls_tcp_seq PH_loc.thirtytwo[0] 66 67 /* 68 * Handshake lock used for the handshake timer. Having a global lock 69 * is perhaps not ideal, but it avoids having to use callout_drain() 70 * in tls_uninit_toep() which can't block. Also, the timer shouldn't 71 * actually fire for most connections. 72 */ 73 static struct mtx tls_handshake_lock; 74 75 static void 76 t4_set_tls_tcb_field(struct toepcb *toep, uint16_t word, uint64_t mask, 77 uint64_t val) 78 { 79 struct adapter *sc = td_adapter(toep->td); 80 81 t4_set_tcb_field(sc, toep->ofld_txq, toep, word, mask, val, 0, 0); 82 } 83 84 /* TLS and DTLS common routines */ 85 bool 86 can_tls_offload(struct adapter *sc) 87 { 88 89 return (sc->tt.tls && sc->cryptocaps & FW_CAPS_CONFIG_TLSKEYS); 90 } 91 92 int 93 tls_tx_key(struct toepcb *toep) 94 { 95 struct tls_ofld_info *tls_ofld = &toep->tls; 96 97 return (tls_ofld->tx_key_addr >= 0); 98 } 99 100 int 101 tls_rx_key(struct toepcb *toep) 102 { 103 struct tls_ofld_info *tls_ofld = &toep->tls; 104 105 return (tls_ofld->rx_key_addr >= 0); 106 } 107 108 static int 109 key_size(struct toepcb *toep) 110 { 111 struct tls_ofld_info *tls_ofld = &toep->tls; 112 113 return ((tls_ofld->key_location == TLS_SFO_WR_CONTEXTLOC_IMMEDIATE) ? 114 tls_ofld->k_ctx.tx_key_info_size : KEY_IN_DDR_SIZE); 115 } 116 117 /* Set TLS Key-Id in TCB */ 118 static void 119 t4_set_tls_keyid(struct toepcb *toep, unsigned int key_id) 120 { 121 122 t4_set_tls_tcb_field(toep, W_TCB_RX_TLS_KEY_TAG, 123 V_TCB_RX_TLS_KEY_TAG(M_TCB_RX_TLS_BUF_TAG), 124 V_TCB_RX_TLS_KEY_TAG(key_id)); 125 } 126 127 /* Clear TF_RX_QUIESCE to re-enable receive. */ 128 static void 129 t4_clear_rx_quiesce(struct toepcb *toep) 130 { 131 132 t4_set_tls_tcb_field(toep, W_TCB_T_FLAGS, V_TF_RX_QUIESCE(1), 0); 133 } 134 135 static void 136 tls_clr_ofld_mode(struct toepcb *toep) 137 { 138 139 tls_stop_handshake_timer(toep); 140 141 KASSERT(toep->tls.rx_key_addr == -1, 142 ("%s: tid %d has RX key", __func__, toep->tid)); 143 144 /* Switch to plain TOE mode. */ 145 t4_set_tls_tcb_field(toep, W_TCB_ULP_RAW, 146 V_TCB_ULP_RAW(V_TF_TLS_ENABLE(1)), 147 V_TCB_ULP_RAW(V_TF_TLS_ENABLE(0))); 148 t4_set_tls_tcb_field(toep, W_TCB_ULP_TYPE, 149 V_TCB_ULP_TYPE(M_TCB_ULP_TYPE), V_TCB_ULP_TYPE(ULP_MODE_NONE)); 150 t4_clear_rx_quiesce(toep); 151 152 toep->flags &= ~TPF_FORCE_CREDITS; 153 toep->params.ulp_mode = ULP_MODE_NONE; 154 } 155 156 static void 157 tls_clr_quiesce(struct toepcb *toep) 158 { 159 160 tls_stop_handshake_timer(toep); 161 t4_clear_rx_quiesce(toep); 162 } 163 164 /* 165 * Calculate the TLS data expansion size 166 */ 167 static int 168 tls_expansion_size(struct toepcb *toep, int data_len, int full_pdus_only, 169 unsigned short *pdus_per_ulp) 170 { 171 struct tls_ofld_info *tls_ofld = &toep->tls; 172 struct tls_scmd *scmd = &tls_ofld->scmd0; 173 int expn_size = 0, frag_count = 0, pad_per_pdu = 0, 174 pad_last_pdu = 0, last_frag_size = 0, max_frag_size = 0; 175 int exp_per_pdu = 0; 176 int hdr_len = TLS_HEADER_LENGTH; 177 178 do { 179 max_frag_size = tls_ofld->k_ctx.frag_size; 180 if (G_SCMD_CIPH_MODE(scmd->seqno_numivs) == 181 SCMD_CIPH_MODE_AES_GCM) { 182 frag_count = (data_len / max_frag_size); 183 exp_per_pdu = GCM_TAG_SIZE + AEAD_EXPLICIT_DATA_SIZE + 184 hdr_len; 185 expn_size = frag_count * exp_per_pdu; 186 if (full_pdus_only) { 187 *pdus_per_ulp = data_len / (exp_per_pdu + 188 max_frag_size); 189 if (*pdus_per_ulp > 32) 190 *pdus_per_ulp = 32; 191 else if(!*pdus_per_ulp) 192 *pdus_per_ulp = 1; 193 expn_size = (*pdus_per_ulp) * exp_per_pdu; 194 break; 195 } 196 if ((last_frag_size = data_len % max_frag_size) > 0) { 197 frag_count += 1; 198 expn_size += exp_per_pdu; 199 } 200 break; 201 } else if (G_SCMD_CIPH_MODE(scmd->seqno_numivs) != 202 SCMD_CIPH_MODE_NOP) { 203 /* Calculate the number of fragments we can make */ 204 frag_count = (data_len / max_frag_size); 205 if (frag_count > 0) { 206 pad_per_pdu = (((howmany((max_frag_size + 207 tls_ofld->mac_length), 208 CIPHER_BLOCK_SIZE)) * 209 CIPHER_BLOCK_SIZE) - 210 (max_frag_size + 211 tls_ofld->mac_length)); 212 if (!pad_per_pdu) 213 pad_per_pdu = CIPHER_BLOCK_SIZE; 214 exp_per_pdu = pad_per_pdu + 215 tls_ofld->mac_length + 216 hdr_len + CIPHER_BLOCK_SIZE; 217 expn_size = frag_count * exp_per_pdu; 218 } 219 if (full_pdus_only) { 220 *pdus_per_ulp = data_len / (exp_per_pdu + 221 max_frag_size); 222 if (*pdus_per_ulp > 32) 223 *pdus_per_ulp = 32; 224 else if (!*pdus_per_ulp) 225 *pdus_per_ulp = 1; 226 expn_size = (*pdus_per_ulp) * exp_per_pdu; 227 break; 228 } 229 /* Consider the last fragment */ 230 if ((last_frag_size = data_len % max_frag_size) > 0) { 231 pad_last_pdu = (((howmany((last_frag_size + 232 tls_ofld->mac_length), 233 CIPHER_BLOCK_SIZE)) * 234 CIPHER_BLOCK_SIZE) - 235 (last_frag_size + 236 tls_ofld->mac_length)); 237 if (!pad_last_pdu) 238 pad_last_pdu = CIPHER_BLOCK_SIZE; 239 expn_size += (pad_last_pdu + 240 tls_ofld->mac_length + hdr_len + 241 CIPHER_BLOCK_SIZE); 242 } 243 } 244 } while (0); 245 246 return (expn_size); 247 } 248 249 /* Copy Key to WR */ 250 static void 251 tls_copy_tx_key(struct toepcb *toep, void *dst) 252 { 253 struct tls_ofld_info *tls_ofld = &toep->tls; 254 struct ulptx_sc_memrd *sc_memrd; 255 struct ulptx_idata *sc; 256 257 if (tls_ofld->k_ctx.tx_key_info_size <= 0) 258 return; 259 260 if (tls_ofld->key_location == TLS_SFO_WR_CONTEXTLOC_DDR) { 261 sc = dst; 262 sc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP)); 263 sc->len = htobe32(0); 264 sc_memrd = (struct ulptx_sc_memrd *)(sc + 1); 265 sc_memrd->cmd_to_len = htobe32(V_ULPTX_CMD(ULP_TX_SC_MEMRD) | 266 V_ULP_TX_SC_MORE(1) | 267 V_ULPTX_LEN16(tls_ofld->k_ctx.tx_key_info_size >> 4)); 268 sc_memrd->addr = htobe32(tls_ofld->tx_key_addr >> 5); 269 } else if (tls_ofld->key_location == TLS_SFO_WR_CONTEXTLOC_IMMEDIATE) { 270 memcpy(dst, &tls_ofld->k_ctx.tx, 271 tls_ofld->k_ctx.tx_key_info_size); 272 } 273 } 274 275 /* TLS/DTLS content type for CPL SFO */ 276 static inline unsigned char 277 tls_content_type(unsigned char content_type) 278 { 279 /* 280 * XXX: Shouldn't this map CONTENT_TYPE_APP_DATA to DATA and 281 * default to "CUSTOM" for all other types including 282 * heartbeat? 283 */ 284 switch (content_type) { 285 case CONTENT_TYPE_CCS: 286 return CPL_TX_TLS_SFO_TYPE_CCS; 287 case CONTENT_TYPE_ALERT: 288 return CPL_TX_TLS_SFO_TYPE_ALERT; 289 case CONTENT_TYPE_HANDSHAKE: 290 return CPL_TX_TLS_SFO_TYPE_HANDSHAKE; 291 case CONTENT_TYPE_HEARTBEAT: 292 return CPL_TX_TLS_SFO_TYPE_HEARTBEAT; 293 } 294 return CPL_TX_TLS_SFO_TYPE_DATA; 295 } 296 297 static unsigned char 298 get_cipher_key_size(unsigned int ck_size) 299 { 300 switch (ck_size) { 301 case AES_NOP: /* NOP */ 302 return 15; 303 case AES_128: /* AES128 */ 304 return CH_CK_SIZE_128; 305 case AES_192: /* AES192 */ 306 return CH_CK_SIZE_192; 307 case AES_256: /* AES256 */ 308 return CH_CK_SIZE_256; 309 default: 310 return CH_CK_SIZE_256; 311 } 312 } 313 314 static unsigned char 315 get_mac_key_size(unsigned int mk_size) 316 { 317 switch (mk_size) { 318 case SHA_NOP: /* NOP */ 319 return CH_MK_SIZE_128; 320 case SHA_GHASH: /* GHASH */ 321 case SHA_512: /* SHA512 */ 322 return CH_MK_SIZE_512; 323 case SHA_224: /* SHA2-224 */ 324 return CH_MK_SIZE_192; 325 case SHA_256: /* SHA2-256*/ 326 return CH_MK_SIZE_256; 327 case SHA_384: /* SHA384 */ 328 return CH_MK_SIZE_512; 329 case SHA1: /* SHA1 */ 330 default: 331 return CH_MK_SIZE_160; 332 } 333 } 334 335 static unsigned int 336 get_proto_ver(int proto_ver) 337 { 338 switch (proto_ver) { 339 case TLS1_2_VERSION: 340 return TLS_1_2_VERSION; 341 case TLS1_1_VERSION: 342 return TLS_1_1_VERSION; 343 case DTLS1_2_VERSION: 344 return DTLS_1_2_VERSION; 345 default: 346 return TLS_VERSION_MAX; 347 } 348 } 349 350 static void 351 tls_rxkey_flit1(struct tls_keyctx *kwr, struct tls_key_context *kctx) 352 { 353 354 if (kctx->state.enc_mode == CH_EVP_CIPH_GCM_MODE) { 355 kwr->u.rxhdr.ivinsert_to_authinsrt = 356 htobe64(V_TLS_KEYCTX_TX_WR_IVINSERT(6ULL) | 357 V_TLS_KEYCTX_TX_WR_AADSTRTOFST(1ULL) | 358 V_TLS_KEYCTX_TX_WR_AADSTOPOFST(5ULL) | 359 V_TLS_KEYCTX_TX_WR_AUTHSRTOFST(14ULL) | 360 V_TLS_KEYCTX_TX_WR_AUTHSTOPOFST(16ULL) | 361 V_TLS_KEYCTX_TX_WR_CIPHERSRTOFST(14ULL) | 362 V_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST(0ULL) | 363 V_TLS_KEYCTX_TX_WR_AUTHINSRT(16ULL)); 364 kwr->u.rxhdr.ivpresent_to_rxmk_size &= 365 ~(V_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT(1)); 366 kwr->u.rxhdr.authmode_to_rxvalid &= 367 ~(V_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL(1)); 368 } else { 369 kwr->u.rxhdr.ivinsert_to_authinsrt = 370 htobe64(V_TLS_KEYCTX_TX_WR_IVINSERT(6ULL) | 371 V_TLS_KEYCTX_TX_WR_AADSTRTOFST(1ULL) | 372 V_TLS_KEYCTX_TX_WR_AADSTOPOFST(5ULL) | 373 V_TLS_KEYCTX_TX_WR_AUTHSRTOFST(22ULL) | 374 V_TLS_KEYCTX_TX_WR_AUTHSTOPOFST(0ULL) | 375 V_TLS_KEYCTX_TX_WR_CIPHERSRTOFST(22ULL) | 376 V_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST(0ULL) | 377 V_TLS_KEYCTX_TX_WR_AUTHINSRT(0ULL)); 378 } 379 } 380 381 /* Rx key */ 382 static void 383 prepare_rxkey_wr(struct tls_keyctx *kwr, struct tls_key_context *kctx) 384 { 385 unsigned int ck_size = kctx->cipher_secret_size; 386 unsigned int mk_size = kctx->mac_secret_size; 387 int proto_ver = kctx->proto_ver; 388 389 kwr->u.rxhdr.flitcnt_hmacctrl = 390 ((kctx->rx_key_info_size >> 4) << 3) | kctx->hmac_ctrl; 391 392 kwr->u.rxhdr.protover_ciphmode = 393 V_TLS_KEYCTX_TX_WR_PROTOVER(get_proto_ver(proto_ver)) | 394 V_TLS_KEYCTX_TX_WR_CIPHMODE(kctx->state.enc_mode); 395 396 kwr->u.rxhdr.authmode_to_rxvalid = 397 V_TLS_KEYCTX_TX_WR_AUTHMODE(kctx->state.auth_mode) | 398 V_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL(1) | 399 V_TLS_KEYCTX_TX_WR_SEQNUMCTRL(3) | 400 V_TLS_KEYCTX_TX_WR_RXVALID(1); 401 402 kwr->u.rxhdr.ivpresent_to_rxmk_size = 403 V_TLS_KEYCTX_TX_WR_IVPRESENT(0) | 404 V_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT(1) | 405 V_TLS_KEYCTX_TX_WR_RXCK_SIZE(get_cipher_key_size(ck_size)) | 406 V_TLS_KEYCTX_TX_WR_RXMK_SIZE(get_mac_key_size(mk_size)); 407 408 tls_rxkey_flit1(kwr, kctx); 409 410 /* No key reversal for GCM */ 411 if (kctx->state.enc_mode != CH_EVP_CIPH_GCM_MODE) { 412 t4_aes_getdeckey(kwr->keys.edkey, kctx->rx.key, 413 (kctx->cipher_secret_size << 3)); 414 memcpy(kwr->keys.edkey + kctx->cipher_secret_size, 415 kctx->rx.key + kctx->cipher_secret_size, 416 (IPAD_SIZE + OPAD_SIZE)); 417 } else { 418 memcpy(kwr->keys.edkey, kctx->rx.key, 419 (kctx->rx_key_info_size - SALT_SIZE)); 420 memcpy(kwr->u.rxhdr.rxsalt, kctx->rx.salt, SALT_SIZE); 421 } 422 } 423 424 /* Tx key */ 425 static void 426 prepare_txkey_wr(struct tls_keyctx *kwr, struct tls_key_context *kctx) 427 { 428 unsigned int ck_size = kctx->cipher_secret_size; 429 unsigned int mk_size = kctx->mac_secret_size; 430 431 kwr->u.txhdr.ctxlen = 432 (kctx->tx_key_info_size >> 4); 433 kwr->u.txhdr.dualck_to_txvalid = 434 V_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(1) | 435 V_TLS_KEYCTX_TX_WR_SALT_PRESENT(1) | 436 V_TLS_KEYCTX_TX_WR_TXCK_SIZE(get_cipher_key_size(ck_size)) | 437 V_TLS_KEYCTX_TX_WR_TXMK_SIZE(get_mac_key_size(mk_size)) | 438 V_TLS_KEYCTX_TX_WR_TXVALID(1); 439 440 memcpy(kwr->keys.edkey, kctx->tx.key, HDR_KCTX_SIZE); 441 if (kctx->state.enc_mode == CH_EVP_CIPH_GCM_MODE) { 442 memcpy(kwr->u.txhdr.txsalt, kctx->tx.salt, SALT_SIZE); 443 kwr->u.txhdr.dualck_to_txvalid &= 444 ~(V_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(1)); 445 } 446 kwr->u.txhdr.dualck_to_txvalid = htons(kwr->u.txhdr.dualck_to_txvalid); 447 } 448 449 /* TLS Key memory management */ 450 static int 451 get_new_keyid(struct toepcb *toep) 452 { 453 struct adapter *sc = td_adapter(toep->td); 454 vmem_addr_t addr; 455 456 if (vmem_alloc(sc->key_map, TLS_KEY_CONTEXT_SZ, M_NOWAIT | M_FIRSTFIT, 457 &addr) != 0) 458 return (-1); 459 460 return (addr); 461 } 462 463 static void 464 free_keyid(struct toepcb *toep, int keyid) 465 { 466 struct adapter *sc = td_adapter(toep->td); 467 468 vmem_free(sc->key_map, keyid, TLS_KEY_CONTEXT_SZ); 469 } 470 471 static void 472 clear_tls_keyid(struct toepcb *toep) 473 { 474 struct tls_ofld_info *tls_ofld = &toep->tls; 475 476 if (tls_ofld->rx_key_addr >= 0) { 477 free_keyid(toep, tls_ofld->rx_key_addr); 478 tls_ofld->rx_key_addr = -1; 479 } 480 if (tls_ofld->tx_key_addr >= 0) { 481 free_keyid(toep, tls_ofld->tx_key_addr); 482 tls_ofld->tx_key_addr = -1; 483 } 484 } 485 486 static int 487 get_keyid(struct tls_ofld_info *tls_ofld, unsigned int ops) 488 { 489 return (ops & KEY_WRITE_RX ? tls_ofld->rx_key_addr : 490 ((ops & KEY_WRITE_TX) ? tls_ofld->tx_key_addr : -1)); 491 } 492 493 static int 494 get_tp_plen_max(struct tls_ofld_info *tls_ofld) 495 { 496 int plen = ((min(3*4096, TP_TX_PG_SZ))/1448) * 1448; 497 498 return (tls_ofld->k_ctx.frag_size <= 8192 ? plen : FC_TP_PLEN_MAX); 499 } 500 501 /* Send request to get the key-id */ 502 static int 503 tls_program_key_id(struct toepcb *toep, struct tls_key_context *k_ctx) 504 { 505 struct tls_ofld_info *tls_ofld = &toep->tls; 506 struct adapter *sc = td_adapter(toep->td); 507 struct ofld_tx_sdesc *txsd; 508 int kwrlen, kctxlen, keyid, len; 509 struct wrqe *wr; 510 struct tls_key_req *kwr; 511 struct tls_keyctx *kctx; 512 513 kwrlen = sizeof(*kwr); 514 kctxlen = roundup2(sizeof(*kctx), 32); 515 len = roundup2(kwrlen + kctxlen, 16); 516 517 if (toep->txsd_avail == 0) 518 return (EAGAIN); 519 520 /* Dont initialize key for re-neg */ 521 if (!G_KEY_CLR_LOC(k_ctx->l_p_key)) { 522 if ((keyid = get_new_keyid(toep)) < 0) { 523 return (ENOSPC); 524 } 525 } else { 526 keyid = get_keyid(tls_ofld, k_ctx->l_p_key); 527 } 528 529 wr = alloc_wrqe(len, toep->ofld_txq); 530 if (wr == NULL) { 531 free_keyid(toep, keyid); 532 return (ENOMEM); 533 } 534 kwr = wrtod(wr); 535 memset(kwr, 0, kwrlen); 536 537 kwr->wr_hi = htobe32(V_FW_WR_OP(FW_ULPTX_WR) | F_FW_WR_COMPL | 538 F_FW_WR_ATOMIC); 539 kwr->wr_mid = htobe32(V_FW_WR_LEN16(DIV_ROUND_UP(len, 16)) | 540 V_FW_WR_FLOWID(toep->tid)); 541 kwr->protocol = get_proto_ver(k_ctx->proto_ver); 542 kwr->mfs = htons(k_ctx->frag_size); 543 kwr->reneg_to_write_rx = k_ctx->l_p_key; 544 545 /* master command */ 546 kwr->cmd = htobe32(V_ULPTX_CMD(ULP_TX_MEM_WRITE) | 547 V_T5_ULP_MEMIO_ORDER(1) | V_T5_ULP_MEMIO_IMM(1)); 548 kwr->dlen = htobe32(V_ULP_MEMIO_DATA_LEN(kctxlen >> 5)); 549 kwr->len16 = htobe32((toep->tid << 8) | 550 DIV_ROUND_UP(len - sizeof(struct work_request_hdr), 16)); 551 kwr->kaddr = htobe32(V_ULP_MEMIO_ADDR(keyid >> 5)); 552 553 /* sub command */ 554 kwr->sc_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM)); 555 kwr->sc_len = htobe32(kctxlen); 556 557 kctx = (struct tls_keyctx *)(kwr + 1); 558 memset(kctx, 0, kctxlen); 559 560 if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_TX) { 561 tls_ofld->tx_key_addr = keyid; 562 prepare_txkey_wr(kctx, k_ctx); 563 } else if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_RX) { 564 tls_ofld->rx_key_addr = keyid; 565 prepare_rxkey_wr(kctx, k_ctx); 566 } 567 568 txsd = &toep->txsd[toep->txsd_pidx]; 569 txsd->tx_credits = DIV_ROUND_UP(len, 16); 570 txsd->plen = 0; 571 toep->tx_credits -= txsd->tx_credits; 572 if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) 573 toep->txsd_pidx = 0; 574 toep->txsd_avail--; 575 576 t4_wrq_tx(sc, wr); 577 578 return (0); 579 } 580 581 /* Store a key received from SSL in DDR. */ 582 static int 583 program_key_context(struct tcpcb *tp, struct toepcb *toep, 584 struct tls_key_context *uk_ctx) 585 { 586 struct adapter *sc = td_adapter(toep->td); 587 struct tls_ofld_info *tls_ofld = &toep->tls; 588 struct tls_key_context *k_ctx; 589 int error, key_offset; 590 591 if (tp->t_state != TCPS_ESTABLISHED) { 592 /* 593 * XXX: Matches Linux driver, but not sure this is a 594 * very appropriate error. 595 */ 596 return (ENOENT); 597 } 598 599 /* Stop timer on handshake completion */ 600 tls_stop_handshake_timer(toep); 601 602 toep->flags &= ~TPF_FORCE_CREDITS; 603 604 CTR4(KTR_CXGBE, "%s: tid %d %s proto_ver %#x", __func__, toep->tid, 605 G_KEY_GET_LOC(uk_ctx->l_p_key) == KEY_WRITE_RX ? "KEY_WRITE_RX" : 606 "KEY_WRITE_TX", uk_ctx->proto_ver); 607 608 if (G_KEY_GET_LOC(uk_ctx->l_p_key) == KEY_WRITE_RX && 609 ulp_mode(toep) != ULP_MODE_TLS) 610 return (EOPNOTSUPP); 611 612 /* Don't copy the 'tx' and 'rx' fields. */ 613 k_ctx = &tls_ofld->k_ctx; 614 memcpy(&k_ctx->l_p_key, &uk_ctx->l_p_key, 615 sizeof(*k_ctx) - offsetof(struct tls_key_context, l_p_key)); 616 617 /* TLS version != 1.1 and !1.2 OR DTLS != 1.2 */ 618 if (get_proto_ver(k_ctx->proto_ver) > DTLS_1_2_VERSION) { 619 if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_RX) { 620 tls_ofld->rx_key_addr = -1; 621 t4_clear_rx_quiesce(toep); 622 } else { 623 tls_ofld->tx_key_addr = -1; 624 } 625 return (0); 626 } 627 628 if (k_ctx->state.enc_mode == CH_EVP_CIPH_GCM_MODE) { 629 k_ctx->iv_size = 4; 630 k_ctx->mac_first = 0; 631 k_ctx->hmac_ctrl = 0; 632 } else { 633 k_ctx->iv_size = 8; /* for CBC, iv is 16B, unit of 2B */ 634 k_ctx->mac_first = 1; 635 } 636 637 tls_ofld->scmd0.seqno_numivs = 638 (V_SCMD_SEQ_NO_CTRL(3) | 639 V_SCMD_PROTO_VERSION(get_proto_ver(k_ctx->proto_ver)) | 640 V_SCMD_ENC_DEC_CTRL(SCMD_ENCDECCTRL_ENCRYPT) | 641 V_SCMD_CIPH_AUTH_SEQ_CTRL((k_ctx->mac_first == 0)) | 642 V_SCMD_CIPH_MODE(k_ctx->state.enc_mode) | 643 V_SCMD_AUTH_MODE(k_ctx->state.auth_mode) | 644 V_SCMD_HMAC_CTRL(k_ctx->hmac_ctrl) | 645 V_SCMD_IV_SIZE(k_ctx->iv_size)); 646 647 tls_ofld->scmd0.ivgen_hdrlen = 648 (V_SCMD_IV_GEN_CTRL(k_ctx->iv_ctrl) | 649 V_SCMD_KEY_CTX_INLINE(0) | 650 V_SCMD_TLS_FRAG_ENABLE(1)); 651 652 tls_ofld->mac_length = k_ctx->mac_secret_size; 653 654 if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_RX) { 655 k_ctx->rx = uk_ctx->rx; 656 /* Dont initialize key for re-neg */ 657 if (!G_KEY_CLR_LOC(k_ctx->l_p_key)) 658 tls_ofld->rx_key_addr = -1; 659 } else { 660 k_ctx->tx = uk_ctx->tx; 661 /* Dont initialize key for re-neg */ 662 if (!G_KEY_CLR_LOC(k_ctx->l_p_key)) 663 tls_ofld->tx_key_addr = -1; 664 } 665 666 /* Flush pending data before new Tx key becomes active */ 667 if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_TX) { 668 struct sockbuf *sb; 669 670 /* XXX: This might not drain everything. */ 671 t4_push_frames(sc, toep, 0); 672 sb = &toep->inp->inp_socket->so_snd; 673 SOCKBUF_LOCK(sb); 674 675 /* XXX: This asserts that everything has been pushed. */ 676 MPASS(sb->sb_sndptr == NULL || sb->sb_sndptr->m_next == NULL); 677 sb->sb_sndptr = NULL; 678 tls_ofld->sb_off = sbavail(sb); 679 SOCKBUF_UNLOCK(sb); 680 tls_ofld->tx_seq_no = 0; 681 } 682 683 if ((G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_RX) || 684 (tls_ofld->key_location == TLS_SFO_WR_CONTEXTLOC_DDR)) { 685 686 /* 687 * XXX: The userland library sets tx_key_info_size, not 688 * rx_key_info_size. 689 */ 690 k_ctx->rx_key_info_size = k_ctx->tx_key_info_size; 691 692 error = tls_program_key_id(toep, k_ctx); 693 if (error) { 694 /* XXX: Only clear quiesce for KEY_WRITE_RX? */ 695 t4_clear_rx_quiesce(toep); 696 return (error); 697 } 698 } 699 700 if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_RX) { 701 /* 702 * RX key tags are an index into the key portion of MA 703 * memory stored as an offset from the base address in 704 * units of 64 bytes. 705 */ 706 key_offset = tls_ofld->rx_key_addr - sc->vres.key.start; 707 t4_set_tls_keyid(toep, key_offset / 64); 708 t4_set_tls_tcb_field(toep, W_TCB_ULP_RAW, 709 V_TCB_ULP_RAW(M_TCB_ULP_RAW), 710 V_TCB_ULP_RAW((V_TF_TLS_KEY_SIZE(3) | 711 V_TF_TLS_CONTROL(1) | 712 V_TF_TLS_ACTIVE(1) | 713 V_TF_TLS_ENABLE(1)))); 714 t4_set_tls_tcb_field(toep, W_TCB_TLS_SEQ, 715 V_TCB_TLS_SEQ(M_TCB_TLS_SEQ), 716 V_TCB_TLS_SEQ(0)); 717 t4_clear_rx_quiesce(toep); 718 719 toep->flags |= TPF_TLS_RECEIVE; 720 } else { 721 unsigned short pdus_per_ulp; 722 723 if (tls_ofld->key_location == TLS_SFO_WR_CONTEXTLOC_IMMEDIATE) 724 tls_ofld->tx_key_addr = 1; 725 726 tls_ofld->fcplenmax = get_tp_plen_max(tls_ofld); 727 tls_ofld->expn_per_ulp = tls_expansion_size(toep, 728 tls_ofld->fcplenmax, 1, &pdus_per_ulp); 729 tls_ofld->pdus_per_ulp = pdus_per_ulp; 730 tls_ofld->adjusted_plen = tls_ofld->pdus_per_ulp * 731 ((tls_ofld->expn_per_ulp/tls_ofld->pdus_per_ulp) + 732 tls_ofld->k_ctx.frag_size); 733 } 734 735 return (0); 736 } 737 738 /* 739 * In some cases a client connection can hang without sending the 740 * ServerHelloDone message from the NIC to the host. Send a dummy 741 * RX_DATA_ACK with RX_MODULATE to unstick the connection. 742 */ 743 static void 744 tls_send_handshake_ack(void *arg) 745 { 746 struct toepcb *toep = arg; 747 struct tls_ofld_info *tls_ofld = &toep->tls; 748 struct adapter *sc = td_adapter(toep->td); 749 750 /* 751 * XXX: Does not have the t4_get_tcb() checks to refine the 752 * workaround. 753 */ 754 callout_schedule(&tls_ofld->handshake_timer, TLS_SRV_HELLO_RD_TM * hz); 755 756 CTR2(KTR_CXGBE, "%s: tid %d sending RX_DATA_ACK", __func__, toep->tid); 757 send_rx_modulate(sc, toep); 758 } 759 760 static void 761 tls_start_handshake_timer(struct toepcb *toep) 762 { 763 struct tls_ofld_info *tls_ofld = &toep->tls; 764 765 mtx_lock(&tls_handshake_lock); 766 callout_reset(&tls_ofld->handshake_timer, TLS_SRV_HELLO_BKOFF_TM * hz, 767 tls_send_handshake_ack, toep); 768 mtx_unlock(&tls_handshake_lock); 769 } 770 771 void 772 tls_stop_handshake_timer(struct toepcb *toep) 773 { 774 struct tls_ofld_info *tls_ofld = &toep->tls; 775 776 mtx_lock(&tls_handshake_lock); 777 callout_stop(&tls_ofld->handshake_timer); 778 mtx_unlock(&tls_handshake_lock); 779 } 780 781 int 782 t4_ctloutput_tls(struct socket *so, struct sockopt *sopt) 783 { 784 struct tls_key_context uk_ctx; 785 struct inpcb *inp; 786 struct tcpcb *tp; 787 struct toepcb *toep; 788 int error, optval; 789 790 error = 0; 791 if (sopt->sopt_dir == SOPT_SET && 792 sopt->sopt_name == TCP_TLSOM_SET_TLS_CONTEXT) { 793 error = sooptcopyin(sopt, &uk_ctx, sizeof(uk_ctx), 794 sizeof(uk_ctx)); 795 if (error) 796 return (error); 797 } 798 799 inp = sotoinpcb(so); 800 KASSERT(inp != NULL, ("tcp_ctloutput: inp == NULL")); 801 INP_WLOCK(inp); 802 if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) { 803 INP_WUNLOCK(inp); 804 return (ECONNRESET); 805 } 806 tp = intotcpcb(inp); 807 toep = tp->t_toe; 808 switch (sopt->sopt_dir) { 809 case SOPT_SET: 810 switch (sopt->sopt_name) { 811 case TCP_TLSOM_SET_TLS_CONTEXT: 812 if (toep->tls.mode == TLS_MODE_KTLS) 813 error = EINVAL; 814 else { 815 error = program_key_context(tp, toep, &uk_ctx); 816 if (error == 0) 817 toep->tls.mode = TLS_MODE_TLSOM; 818 } 819 INP_WUNLOCK(inp); 820 break; 821 case TCP_TLSOM_CLR_TLS_TOM: 822 if (toep->tls.mode == TLS_MODE_KTLS) 823 error = EINVAL; 824 else if (ulp_mode(toep) == ULP_MODE_TLS) { 825 CTR2(KTR_CXGBE, "%s: tid %d CLR_TLS_TOM", 826 __func__, toep->tid); 827 tls_clr_ofld_mode(toep); 828 } else 829 error = EOPNOTSUPP; 830 INP_WUNLOCK(inp); 831 break; 832 case TCP_TLSOM_CLR_QUIES: 833 if (toep->tls.mode == TLS_MODE_KTLS) 834 error = EINVAL; 835 else if (ulp_mode(toep) == ULP_MODE_TLS) { 836 CTR2(KTR_CXGBE, "%s: tid %d CLR_QUIES", 837 __func__, toep->tid); 838 tls_clr_quiesce(toep); 839 } else 840 error = EOPNOTSUPP; 841 INP_WUNLOCK(inp); 842 break; 843 default: 844 INP_WUNLOCK(inp); 845 error = EOPNOTSUPP; 846 break; 847 } 848 break; 849 case SOPT_GET: 850 switch (sopt->sopt_name) { 851 case TCP_TLSOM_GET_TLS_TOM: 852 /* 853 * TLS TX is permitted on any TOE socket, but 854 * TLS RX requires a TLS ULP mode. 855 */ 856 optval = TLS_TOM_NONE; 857 if (can_tls_offload(td_adapter(toep->td)) && 858 toep->tls.mode != TLS_MODE_KTLS) { 859 switch (ulp_mode(toep)) { 860 case ULP_MODE_NONE: 861 case ULP_MODE_TCPDDP: 862 optval = TLS_TOM_TXONLY; 863 break; 864 case ULP_MODE_TLS: 865 optval = TLS_TOM_BOTH; 866 break; 867 } 868 } 869 CTR3(KTR_CXGBE, "%s: tid %d GET_TLS_TOM = %d", 870 __func__, toep->tid, optval); 871 INP_WUNLOCK(inp); 872 error = sooptcopyout(sopt, &optval, sizeof(optval)); 873 break; 874 default: 875 INP_WUNLOCK(inp); 876 error = EOPNOTSUPP; 877 break; 878 } 879 break; 880 } 881 return (error); 882 } 883 884 #ifdef KERN_TLS 885 static void 886 init_ktls_key_context(struct ktls_session *tls, struct tls_key_context *k_ctx, 887 int direction) 888 { 889 struct auth_hash *axf; 890 u_int key_info_size, mac_key_size; 891 char *hash, *key; 892 893 k_ctx->l_p_key = V_KEY_GET_LOC(direction == KTLS_TX ? KEY_WRITE_TX : 894 KEY_WRITE_RX); 895 k_ctx->proto_ver = tls->params.tls_vmajor << 8 | tls->params.tls_vminor; 896 k_ctx->cipher_secret_size = tls->params.cipher_key_len; 897 key_info_size = sizeof(struct tx_keyctx_hdr) + 898 k_ctx->cipher_secret_size; 899 if (direction == KTLS_TX) 900 key = k_ctx->tx.key; 901 else 902 key = k_ctx->rx.key; 903 memcpy(key, tls->params.cipher_key, tls->params.cipher_key_len); 904 hash = key + tls->params.cipher_key_len; 905 if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) { 906 k_ctx->state.auth_mode = SCMD_AUTH_MODE_GHASH; 907 k_ctx->state.enc_mode = SCMD_CIPH_MODE_AES_GCM; 908 k_ctx->iv_size = 4; 909 k_ctx->mac_first = 0; 910 k_ctx->hmac_ctrl = SCMD_HMAC_CTRL_NOP; 911 key_info_size += GMAC_BLOCK_LEN; 912 k_ctx->mac_secret_size = 0; 913 if (direction == KTLS_TX) 914 memcpy(k_ctx->tx.salt, tls->params.iv, SALT_SIZE); 915 else 916 memcpy(k_ctx->rx.salt, tls->params.iv, SALT_SIZE); 917 t4_init_gmac_hash(tls->params.cipher_key, 918 tls->params.cipher_key_len, hash); 919 } else { 920 switch (tls->params.auth_algorithm) { 921 case CRYPTO_SHA1_HMAC: 922 axf = &auth_hash_hmac_sha1; 923 mac_key_size = SHA1_HASH_LEN; 924 k_ctx->state.auth_mode = SCMD_AUTH_MODE_SHA1; 925 break; 926 case CRYPTO_SHA2_256_HMAC: 927 axf = &auth_hash_hmac_sha2_256; 928 mac_key_size = SHA2_256_HASH_LEN; 929 k_ctx->state.auth_mode = SCMD_AUTH_MODE_SHA256; 930 break; 931 case CRYPTO_SHA2_384_HMAC: 932 axf = &auth_hash_hmac_sha2_384; 933 mac_key_size = SHA2_512_HASH_LEN; 934 k_ctx->state.auth_mode = SCMD_AUTH_MODE_SHA512_384; 935 break; 936 default: 937 panic("bad auth mode"); 938 } 939 k_ctx->state.enc_mode = SCMD_CIPH_MODE_AES_CBC; 940 k_ctx->iv_size = 8; /* for CBC, iv is 16B, unit of 2B */ 941 k_ctx->mac_first = 1; 942 k_ctx->hmac_ctrl = SCMD_HMAC_CTRL_NO_TRUNC; 943 key_info_size += roundup2(mac_key_size, 16) * 2; 944 k_ctx->mac_secret_size = mac_key_size; 945 t4_init_hmac_digest(axf, mac_key_size, tls->params.auth_key, 946 tls->params.auth_key_len, hash); 947 } 948 949 if (direction == KTLS_TX) 950 k_ctx->tx_key_info_size = key_info_size; 951 else 952 k_ctx->rx_key_info_size = key_info_size; 953 k_ctx->frag_size = tls->params.max_frame_len; 954 k_ctx->iv_ctrl = 1; 955 } 956 957 int 958 tls_alloc_ktls(struct toepcb *toep, struct ktls_session *tls, int direction) 959 { 960 struct adapter *sc = td_adapter(toep->td); 961 struct tls_key_context *k_ctx; 962 int error, key_offset; 963 964 if (toep->tls.mode == TLS_MODE_TLSOM) 965 return (EINVAL); 966 if (!can_tls_offload(td_adapter(toep->td))) 967 return (EINVAL); 968 switch (ulp_mode(toep)) { 969 case ULP_MODE_TLS: 970 break; 971 case ULP_MODE_NONE: 972 case ULP_MODE_TCPDDP: 973 if (direction != KTLS_TX) 974 return (EINVAL); 975 break; 976 default: 977 return (EINVAL); 978 } 979 980 switch (tls->params.cipher_algorithm) { 981 case CRYPTO_AES_CBC: 982 /* XXX: Explicitly ignore any provided IV. */ 983 switch (tls->params.cipher_key_len) { 984 case 128 / 8: 985 case 192 / 8: 986 case 256 / 8: 987 break; 988 default: 989 return (EINVAL); 990 } 991 switch (tls->params.auth_algorithm) { 992 case CRYPTO_SHA1_HMAC: 993 case CRYPTO_SHA2_256_HMAC: 994 case CRYPTO_SHA2_384_HMAC: 995 break; 996 default: 997 return (EPROTONOSUPPORT); 998 } 999 break; 1000 case CRYPTO_AES_NIST_GCM_16: 1001 if (tls->params.iv_len != SALT_SIZE) 1002 return (EINVAL); 1003 switch (tls->params.cipher_key_len) { 1004 case 128 / 8: 1005 case 192 / 8: 1006 case 256 / 8: 1007 break; 1008 default: 1009 return (EINVAL); 1010 } 1011 break; 1012 default: 1013 return (EPROTONOSUPPORT); 1014 } 1015 1016 /* Only TLS 1.1 and TLS 1.2 are currently supported. */ 1017 if (tls->params.tls_vmajor != TLS_MAJOR_VER_ONE || 1018 tls->params.tls_vminor < TLS_MINOR_VER_ONE || 1019 tls->params.tls_vminor > TLS_MINOR_VER_TWO) 1020 return (EPROTONOSUPPORT); 1021 1022 /* Bail if we already have a key. */ 1023 if (direction == KTLS_TX) { 1024 if (toep->tls.tx_key_addr != -1) 1025 return (EOPNOTSUPP); 1026 } else { 1027 if (toep->tls.rx_key_addr != -1) 1028 return (EOPNOTSUPP); 1029 } 1030 1031 /* 1032 * XXX: This assumes no key renegotation. If KTLS ever supports 1033 * that we will want to allocate TLS sessions dynamically rather 1034 * than as a static member of toep. 1035 */ 1036 k_ctx = &toep->tls.k_ctx; 1037 init_ktls_key_context(tls, k_ctx, direction); 1038 1039 error = tls_program_key_id(toep, k_ctx); 1040 if (error) 1041 return (error); 1042 1043 if (direction == KTLS_TX) { 1044 toep->tls.scmd0.seqno_numivs = 1045 (V_SCMD_SEQ_NO_CTRL(3) | 1046 V_SCMD_PROTO_VERSION(get_proto_ver(k_ctx->proto_ver)) | 1047 V_SCMD_ENC_DEC_CTRL(SCMD_ENCDECCTRL_ENCRYPT) | 1048 V_SCMD_CIPH_AUTH_SEQ_CTRL((k_ctx->mac_first == 0)) | 1049 V_SCMD_CIPH_MODE(k_ctx->state.enc_mode) | 1050 V_SCMD_AUTH_MODE(k_ctx->state.auth_mode) | 1051 V_SCMD_HMAC_CTRL(k_ctx->hmac_ctrl) | 1052 V_SCMD_IV_SIZE(k_ctx->iv_size)); 1053 1054 toep->tls.scmd0.ivgen_hdrlen = 1055 (V_SCMD_IV_GEN_CTRL(k_ctx->iv_ctrl) | 1056 V_SCMD_KEY_CTX_INLINE(0) | 1057 V_SCMD_TLS_FRAG_ENABLE(1)); 1058 1059 if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) 1060 toep->tls.iv_len = 8; 1061 else 1062 toep->tls.iv_len = AES_BLOCK_LEN; 1063 1064 toep->tls.mac_length = k_ctx->mac_secret_size; 1065 1066 toep->tls.fcplenmax = get_tp_plen_max(&toep->tls); 1067 toep->tls.expn_per_ulp = tls->params.tls_hlen + 1068 tls->params.tls_tlen; 1069 toep->tls.pdus_per_ulp = 1; 1070 toep->tls.adjusted_plen = toep->tls.expn_per_ulp + 1071 toep->tls.k_ctx.frag_size; 1072 } else { 1073 /* Stop timer on handshake completion */ 1074 tls_stop_handshake_timer(toep); 1075 1076 toep->flags &= ~TPF_FORCE_CREDITS; 1077 toep->flags |= TPF_TLS_RECEIVE; 1078 1079 /* 1080 * RX key tags are an index into the key portion of MA 1081 * memory stored as an offset from the base address in 1082 * units of 64 bytes. 1083 */ 1084 key_offset = toep->tls.rx_key_addr - sc->vres.key.start; 1085 t4_set_tls_keyid(toep, key_offset / 64); 1086 t4_set_tls_tcb_field(toep, W_TCB_ULP_RAW, 1087 V_TCB_ULP_RAW(M_TCB_ULP_RAW), 1088 V_TCB_ULP_RAW((V_TF_TLS_KEY_SIZE(3) | 1089 V_TF_TLS_CONTROL(1) | 1090 V_TF_TLS_ACTIVE(1) | 1091 V_TF_TLS_ENABLE(1)))); 1092 t4_set_tls_tcb_field(toep, W_TCB_TLS_SEQ, 1093 V_TCB_TLS_SEQ(M_TCB_TLS_SEQ), 1094 V_TCB_TLS_SEQ(0)); 1095 t4_clear_rx_quiesce(toep); 1096 } 1097 1098 toep->tls.mode = TLS_MODE_KTLS; 1099 1100 return (0); 1101 } 1102 #endif 1103 1104 void 1105 tls_init_toep(struct toepcb *toep) 1106 { 1107 struct tls_ofld_info *tls_ofld = &toep->tls; 1108 1109 tls_ofld->mode = TLS_MODE_OFF; 1110 tls_ofld->key_location = TLS_SFO_WR_CONTEXTLOC_DDR; 1111 tls_ofld->rx_key_addr = -1; 1112 tls_ofld->tx_key_addr = -1; 1113 if (ulp_mode(toep) == ULP_MODE_TLS) 1114 callout_init_mtx(&tls_ofld->handshake_timer, 1115 &tls_handshake_lock, 0); 1116 } 1117 1118 void 1119 tls_establish(struct toepcb *toep) 1120 { 1121 1122 /* 1123 * Enable PDU extraction. 1124 * 1125 * XXX: Supposedly this should be done by the firmware when 1126 * the ULP_MODE FLOWC parameter is set in send_flowc_wr(), but 1127 * in practice this seems to be required. 1128 */ 1129 CTR2(KTR_CXGBE, "%s: tid %d setting TLS_ENABLE", __func__, toep->tid); 1130 t4_set_tls_tcb_field(toep, W_TCB_ULP_RAW, V_TCB_ULP_RAW(M_TCB_ULP_RAW), 1131 V_TCB_ULP_RAW(V_TF_TLS_ENABLE(1))); 1132 1133 toep->flags |= TPF_FORCE_CREDITS; 1134 1135 tls_start_handshake_timer(toep); 1136 } 1137 1138 void 1139 tls_uninit_toep(struct toepcb *toep) 1140 { 1141 1142 if (ulp_mode(toep) == ULP_MODE_TLS) 1143 tls_stop_handshake_timer(toep); 1144 clear_tls_keyid(toep); 1145 } 1146 1147 #define MAX_OFLD_TX_CREDITS (SGE_MAX_WR_LEN / 16) 1148 #define MIN_OFLD_TLSTX_CREDITS(toep) \ 1149 (howmany(sizeof(struct fw_tlstx_data_wr) + \ 1150 sizeof(struct cpl_tx_tls_sfo) + key_size((toep)) + \ 1151 CIPHER_BLOCK_SIZE + 1, 16)) 1152 1153 static inline u_int 1154 max_imm_tls_space(int tx_credits) 1155 { 1156 const int n = 2; /* Use only up to 2 desc for imm. data WR */ 1157 int space; 1158 1159 KASSERT(tx_credits >= 0 && 1160 tx_credits <= MAX_OFLD_TX_CREDITS, 1161 ("%s: %d credits", __func__, tx_credits)); 1162 1163 if (tx_credits >= (n * EQ_ESIZE) / 16) 1164 space = (n * EQ_ESIZE); 1165 else 1166 space = tx_credits * 16; 1167 return (space); 1168 } 1169 1170 static int 1171 count_mbuf_segs(struct mbuf *m, int skip, int len, int *max_nsegs_1mbufp) 1172 { 1173 int max_nsegs_1mbuf, n, nsegs; 1174 1175 while (skip >= m->m_len) { 1176 skip -= m->m_len; 1177 m = m->m_next; 1178 } 1179 1180 nsegs = 0; 1181 max_nsegs_1mbuf = 0; 1182 while (len > 0) { 1183 n = sglist_count(mtod(m, char *) + skip, m->m_len - skip); 1184 if (n > max_nsegs_1mbuf) 1185 max_nsegs_1mbuf = n; 1186 nsegs += n; 1187 len -= m->m_len - skip; 1188 skip = 0; 1189 m = m->m_next; 1190 } 1191 *max_nsegs_1mbufp = max_nsegs_1mbuf; 1192 return (nsegs); 1193 } 1194 1195 static void 1196 write_tlstx_wr(struct fw_tlstx_data_wr *txwr, struct toepcb *toep, 1197 unsigned int immdlen, unsigned int plen, unsigned int expn, 1198 unsigned int pdus, uint8_t credits, int shove, int imm_ivs) 1199 { 1200 struct tls_ofld_info *tls_ofld = &toep->tls; 1201 unsigned int len = plen + expn; 1202 1203 txwr->op_to_immdlen = htobe32(V_WR_OP(FW_TLSTX_DATA_WR) | 1204 V_FW_TLSTX_DATA_WR_COMPL(1) | 1205 V_FW_TLSTX_DATA_WR_IMMDLEN(immdlen)); 1206 txwr->flowid_len16 = htobe32(V_FW_TLSTX_DATA_WR_FLOWID(toep->tid) | 1207 V_FW_TLSTX_DATA_WR_LEN16(credits)); 1208 txwr->plen = htobe32(len); 1209 txwr->lsodisable_to_flags = htobe32(V_TX_ULP_MODE(ULP_MODE_TLS) | 1210 V_TX_URG(0) | /* F_T6_TX_FORCE | */ V_TX_SHOVE(shove)); 1211 txwr->ctxloc_to_exp = htobe32(V_FW_TLSTX_DATA_WR_NUMIVS(pdus) | 1212 V_FW_TLSTX_DATA_WR_EXP(expn) | 1213 V_FW_TLSTX_DATA_WR_CTXLOC(tls_ofld->key_location) | 1214 V_FW_TLSTX_DATA_WR_IVDSGL(!imm_ivs) | 1215 V_FW_TLSTX_DATA_WR_KEYSIZE(tls_ofld->k_ctx.tx_key_info_size >> 4)); 1216 txwr->mfs = htobe16(tls_ofld->k_ctx.frag_size); 1217 txwr->adjustedplen_pkd = htobe16( 1218 V_FW_TLSTX_DATA_WR_ADJUSTEDPLEN(tls_ofld->adjusted_plen)); 1219 txwr->expinplenmax_pkd = htobe16( 1220 V_FW_TLSTX_DATA_WR_EXPINPLENMAX(tls_ofld->expn_per_ulp)); 1221 txwr->pdusinplenmax_pkd = 1222 V_FW_TLSTX_DATA_WR_PDUSINPLENMAX(tls_ofld->pdus_per_ulp); 1223 } 1224 1225 static void 1226 write_tlstx_cpl(struct cpl_tx_tls_sfo *cpl, struct toepcb *toep, 1227 struct tls_hdr *tls_hdr, unsigned int plen, unsigned int pdus) 1228 { 1229 struct tls_ofld_info *tls_ofld = &toep->tls; 1230 int data_type, seglen; 1231 1232 if (plen < tls_ofld->k_ctx.frag_size) 1233 seglen = plen; 1234 else 1235 seglen = tls_ofld->k_ctx.frag_size; 1236 data_type = tls_content_type(tls_hdr->type); 1237 cpl->op_to_seg_len = htobe32(V_CPL_TX_TLS_SFO_OPCODE(CPL_TX_TLS_SFO) | 1238 V_CPL_TX_TLS_SFO_DATA_TYPE(data_type) | 1239 V_CPL_TX_TLS_SFO_CPL_LEN(2) | V_CPL_TX_TLS_SFO_SEG_LEN(seglen)); 1240 cpl->pld_len = htobe32(plen); 1241 if (data_type == CPL_TX_TLS_SFO_TYPE_HEARTBEAT) 1242 cpl->type_protover = htobe32( 1243 V_CPL_TX_TLS_SFO_TYPE(tls_hdr->type)); 1244 cpl->seqno_numivs = htobe32(tls_ofld->scmd0.seqno_numivs | 1245 V_SCMD_NUM_IVS(pdus)); 1246 cpl->ivgen_hdrlen = htobe32(tls_ofld->scmd0.ivgen_hdrlen); 1247 cpl->scmd1 = htobe64(tls_ofld->tx_seq_no); 1248 tls_ofld->tx_seq_no += pdus; 1249 } 1250 1251 /* 1252 * Similar to write_tx_sgl() except that it accepts an optional 1253 * trailer buffer for IVs. 1254 */ 1255 static void 1256 write_tlstx_sgl(void *dst, struct mbuf *start, int skip, int plen, 1257 void *iv_buffer, int iv_len, int nsegs, int n) 1258 { 1259 struct mbuf *m; 1260 struct ulptx_sgl *usgl = dst; 1261 int i, j, rc; 1262 struct sglist sg; 1263 struct sglist_seg segs[n]; 1264 1265 KASSERT(nsegs > 0, ("%s: nsegs 0", __func__)); 1266 1267 sglist_init(&sg, n, segs); 1268 usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) | 1269 V_ULPTX_NSGE(nsegs)); 1270 1271 for (m = start; skip >= m->m_len; m = m->m_next) 1272 skip -= m->m_len; 1273 1274 i = -1; 1275 for (m = start; plen > 0; m = m->m_next) { 1276 rc = sglist_append(&sg, mtod(m, char *) + skip, 1277 m->m_len - skip); 1278 if (__predict_false(rc != 0)) 1279 panic("%s: sglist_append %d", __func__, rc); 1280 plen -= m->m_len - skip; 1281 skip = 0; 1282 1283 for (j = 0; j < sg.sg_nseg; i++, j++) { 1284 if (i < 0) { 1285 usgl->len0 = htobe32(segs[j].ss_len); 1286 usgl->addr0 = htobe64(segs[j].ss_paddr); 1287 } else { 1288 usgl->sge[i / 2].len[i & 1] = 1289 htobe32(segs[j].ss_len); 1290 usgl->sge[i / 2].addr[i & 1] = 1291 htobe64(segs[j].ss_paddr); 1292 } 1293 #ifdef INVARIANTS 1294 nsegs--; 1295 #endif 1296 } 1297 sglist_reset(&sg); 1298 } 1299 if (iv_buffer != NULL) { 1300 rc = sglist_append(&sg, iv_buffer, iv_len); 1301 if (__predict_false(rc != 0)) 1302 panic("%s: sglist_append %d", __func__, rc); 1303 1304 for (j = 0; j < sg.sg_nseg; i++, j++) { 1305 if (i < 0) { 1306 usgl->len0 = htobe32(segs[j].ss_len); 1307 usgl->addr0 = htobe64(segs[j].ss_paddr); 1308 } else { 1309 usgl->sge[i / 2].len[i & 1] = 1310 htobe32(segs[j].ss_len); 1311 usgl->sge[i / 2].addr[i & 1] = 1312 htobe64(segs[j].ss_paddr); 1313 } 1314 #ifdef INVARIANTS 1315 nsegs--; 1316 #endif 1317 } 1318 } 1319 if (i & 1) 1320 usgl->sge[i / 2].len[1] = htobe32(0); 1321 KASSERT(nsegs == 0, ("%s: nsegs %d, start %p, iv_buffer %p", 1322 __func__, nsegs, start, iv_buffer)); 1323 } 1324 1325 /* 1326 * Similar to t4_push_frames() but handles TLS sockets when TLS offload 1327 * is enabled. Rather than transmitting bulk data, the socket buffer 1328 * contains TLS records. The work request requires a full TLS record, 1329 * so batch mbufs up until a full TLS record is seen. This requires 1330 * reading the TLS header out of the start of each record to determine 1331 * its length. 1332 */ 1333 void 1334 t4_push_tls_records(struct adapter *sc, struct toepcb *toep, int drop) 1335 { 1336 struct tls_hdr thdr; 1337 struct mbuf *sndptr; 1338 struct fw_tlstx_data_wr *txwr; 1339 struct cpl_tx_tls_sfo *cpl; 1340 struct wrqe *wr; 1341 u_int plen, nsegs, credits, space, max_nsegs_1mbuf, wr_len; 1342 u_int expn_size, iv_len, pdus, sndptroff; 1343 struct tls_ofld_info *tls_ofld = &toep->tls; 1344 struct inpcb *inp = toep->inp; 1345 struct tcpcb *tp = intotcpcb(inp); 1346 struct socket *so = inp->inp_socket; 1347 struct sockbuf *sb = &so->so_snd; 1348 int tls_size, tx_credits, shove, /* compl,*/ sowwakeup; 1349 struct ofld_tx_sdesc *txsd; 1350 bool imm_ivs, imm_payload; 1351 void *iv_buffer, *iv_dst, *buf; 1352 1353 INP_WLOCK_ASSERT(inp); 1354 KASSERT(toep->flags & TPF_FLOWC_WR_SENT, 1355 ("%s: flowc_wr not sent for tid %u.", __func__, toep->tid)); 1356 1357 KASSERT(ulp_mode(toep) == ULP_MODE_NONE || 1358 ulp_mode(toep) == ULP_MODE_TCPDDP || ulp_mode(toep) == ULP_MODE_TLS, 1359 ("%s: ulp_mode %u for toep %p", __func__, ulp_mode(toep), toep)); 1360 KASSERT(tls_tx_key(toep), 1361 ("%s: TX key not set for toep %p", __func__, toep)); 1362 1363 #ifdef VERBOSE_TRACES 1364 CTR4(KTR_CXGBE, "%s: tid %d toep flags %#x tp flags %#x drop %d", 1365 __func__, toep->tid, toep->flags, tp->t_flags); 1366 #endif 1367 if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN)) 1368 return; 1369 1370 #ifdef RATELIMIT 1371 if (__predict_false(inp->inp_flags2 & INP_RATE_LIMIT_CHANGED) && 1372 (update_tx_rate_limit(sc, toep, so->so_max_pacing_rate) == 0)) { 1373 inp->inp_flags2 &= ~INP_RATE_LIMIT_CHANGED; 1374 } 1375 #endif 1376 1377 /* 1378 * This function doesn't resume by itself. Someone else must clear the 1379 * flag and call this function. 1380 */ 1381 if (__predict_false(toep->flags & TPF_TX_SUSPENDED)) { 1382 KASSERT(drop == 0, 1383 ("%s: drop (%d) != 0 but tx is suspended", __func__, drop)); 1384 return; 1385 } 1386 1387 txsd = &toep->txsd[toep->txsd_pidx]; 1388 for (;;) { 1389 tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS); 1390 space = max_imm_tls_space(tx_credits); 1391 wr_len = sizeof(struct fw_tlstx_data_wr) + 1392 sizeof(struct cpl_tx_tls_sfo) + key_size(toep); 1393 if (wr_len + CIPHER_BLOCK_SIZE + 1 > space) { 1394 #ifdef VERBOSE_TRACES 1395 CTR5(KTR_CXGBE, 1396 "%s: tid %d tx_credits %d min_wr %d space %d", 1397 __func__, toep->tid, tx_credits, wr_len + 1398 CIPHER_BLOCK_SIZE + 1, space); 1399 #endif 1400 return; 1401 } 1402 1403 SOCKBUF_LOCK(sb); 1404 sowwakeup = drop; 1405 if (drop) { 1406 sbdrop_locked(sb, drop); 1407 MPASS(tls_ofld->sb_off >= drop); 1408 tls_ofld->sb_off -= drop; 1409 drop = 0; 1410 } 1411 1412 /* 1413 * Send a FIN if requested, but only if there's no 1414 * more data to send. 1415 */ 1416 if (sbavail(sb) == tls_ofld->sb_off && 1417 toep->flags & TPF_SEND_FIN) { 1418 if (sowwakeup) 1419 sowwakeup_locked(so); 1420 else 1421 SOCKBUF_UNLOCK(sb); 1422 SOCKBUF_UNLOCK_ASSERT(sb); 1423 t4_close_conn(sc, toep); 1424 return; 1425 } 1426 1427 if (sbavail(sb) < tls_ofld->sb_off + TLS_HEADER_LENGTH) { 1428 /* 1429 * A full TLS header is not yet queued, stop 1430 * for now until more data is added to the 1431 * socket buffer. However, if the connection 1432 * has been closed, we will never get the rest 1433 * of the header so just discard the partial 1434 * header and close the connection. 1435 */ 1436 #ifdef VERBOSE_TRACES 1437 CTR5(KTR_CXGBE, "%s: tid %d sbavail %d sb_off %d%s", 1438 __func__, toep->tid, sbavail(sb), tls_ofld->sb_off, 1439 toep->flags & TPF_SEND_FIN ? "" : " SEND_FIN"); 1440 #endif 1441 if (sowwakeup) 1442 sowwakeup_locked(so); 1443 else 1444 SOCKBUF_UNLOCK(sb); 1445 SOCKBUF_UNLOCK_ASSERT(sb); 1446 if (toep->flags & TPF_SEND_FIN) 1447 t4_close_conn(sc, toep); 1448 return; 1449 } 1450 1451 /* Read the header of the next TLS record. */ 1452 sndptr = sbsndmbuf(sb, tls_ofld->sb_off, &sndptroff); 1453 m_copydata(sndptr, sndptroff, sizeof(thdr), (caddr_t)&thdr); 1454 tls_size = htons(thdr.length); 1455 plen = TLS_HEADER_LENGTH + tls_size; 1456 pdus = howmany(tls_size, tls_ofld->k_ctx.frag_size); 1457 iv_len = pdus * CIPHER_BLOCK_SIZE; 1458 1459 if (sbavail(sb) < tls_ofld->sb_off + plen) { 1460 /* 1461 * The full TLS record is not yet queued, stop 1462 * for now until more data is added to the 1463 * socket buffer. However, if the connection 1464 * has been closed, we will never get the rest 1465 * of the record so just discard the partial 1466 * record and close the connection. 1467 */ 1468 #ifdef VERBOSE_TRACES 1469 CTR6(KTR_CXGBE, 1470 "%s: tid %d sbavail %d sb_off %d plen %d%s", 1471 __func__, toep->tid, sbavail(sb), tls_ofld->sb_off, 1472 plen, toep->flags & TPF_SEND_FIN ? "" : 1473 " SEND_FIN"); 1474 #endif 1475 if (sowwakeup) 1476 sowwakeup_locked(so); 1477 else 1478 SOCKBUF_UNLOCK(sb); 1479 SOCKBUF_UNLOCK_ASSERT(sb); 1480 if (toep->flags & TPF_SEND_FIN) 1481 t4_close_conn(sc, toep); 1482 return; 1483 } 1484 1485 /* Shove if there is no additional data pending. */ 1486 shove = (sbavail(sb) == tls_ofld->sb_off + plen) && 1487 !(tp->t_flags & TF_MORETOCOME); 1488 1489 if (sb->sb_flags & SB_AUTOSIZE && 1490 V_tcp_do_autosndbuf && 1491 sb->sb_hiwat < V_tcp_autosndbuf_max && 1492 sbused(sb) >= sb->sb_hiwat * 7 / 8) { 1493 int newsize = min(sb->sb_hiwat + V_tcp_autosndbuf_inc, 1494 V_tcp_autosndbuf_max); 1495 1496 if (!sbreserve_locked(sb, newsize, so, NULL)) 1497 sb->sb_flags &= ~SB_AUTOSIZE; 1498 else 1499 sowwakeup = 1; /* room available */ 1500 } 1501 if (sowwakeup) 1502 sowwakeup_locked(so); 1503 else 1504 SOCKBUF_UNLOCK(sb); 1505 SOCKBUF_UNLOCK_ASSERT(sb); 1506 1507 if (__predict_false(toep->flags & TPF_FIN_SENT)) 1508 panic("%s: excess tx.", __func__); 1509 1510 /* Determine whether to use immediate vs SGL. */ 1511 imm_payload = false; 1512 imm_ivs = false; 1513 if (wr_len + iv_len <= space) { 1514 imm_ivs = true; 1515 wr_len += iv_len; 1516 if (wr_len + tls_size <= space) { 1517 wr_len += tls_size; 1518 imm_payload = true; 1519 } 1520 } 1521 1522 /* Allocate space for IVs if needed. */ 1523 if (!imm_ivs) { 1524 iv_buffer = malloc(iv_len, M_CXGBE, M_NOWAIT); 1525 if (iv_buffer == NULL) { 1526 /* 1527 * XXX: How to restart this? 1528 */ 1529 if (sowwakeup) 1530 sowwakeup_locked(so); 1531 else 1532 SOCKBUF_UNLOCK(sb); 1533 SOCKBUF_UNLOCK_ASSERT(sb); 1534 CTR3(KTR_CXGBE, 1535 "%s: tid %d failed to alloc IV space len %d", 1536 __func__, toep->tid, iv_len); 1537 return; 1538 } 1539 } else 1540 iv_buffer = NULL; 1541 1542 /* Determine size of SGL. */ 1543 nsegs = 0; 1544 max_nsegs_1mbuf = 0; /* max # of SGL segments in any one mbuf */ 1545 if (!imm_payload) { 1546 nsegs = count_mbuf_segs(sndptr, sndptroff + 1547 TLS_HEADER_LENGTH, tls_size, &max_nsegs_1mbuf); 1548 if (!imm_ivs) { 1549 int n = sglist_count(iv_buffer, iv_len); 1550 nsegs += n; 1551 if (n > max_nsegs_1mbuf) 1552 max_nsegs_1mbuf = n; 1553 } 1554 1555 /* Account for SGL in work request length. */ 1556 wr_len += sizeof(struct ulptx_sgl) + 1557 ((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8; 1558 } 1559 1560 wr = alloc_wrqe(roundup2(wr_len, 16), toep->ofld_txq); 1561 if (wr == NULL) { 1562 /* XXX: how will we recover from this? */ 1563 toep->flags |= TPF_TX_SUSPENDED; 1564 return; 1565 } 1566 1567 #ifdef VERBOSE_TRACES 1568 CTR5(KTR_CXGBE, "%s: tid %d TLS record %d len %#x pdus %d", 1569 __func__, toep->tid, thdr.type, tls_size, pdus); 1570 #endif 1571 txwr = wrtod(wr); 1572 cpl = (struct cpl_tx_tls_sfo *)(txwr + 1); 1573 memset(txwr, 0, roundup2(wr_len, 16)); 1574 credits = howmany(wr_len, 16); 1575 expn_size = tls_expansion_size(toep, tls_size, 0, NULL); 1576 write_tlstx_wr(txwr, toep, imm_payload ? tls_size : 0, 1577 tls_size, expn_size, pdus, credits, shove, imm_ivs ? 1 : 0); 1578 write_tlstx_cpl(cpl, toep, &thdr, tls_size, pdus); 1579 tls_copy_tx_key(toep, cpl + 1); 1580 1581 /* Generate random IVs */ 1582 buf = (char *)(cpl + 1) + key_size(toep); 1583 if (imm_ivs) { 1584 MPASS(iv_buffer == NULL); 1585 iv_dst = buf; 1586 buf = (char *)iv_dst + iv_len; 1587 } else 1588 iv_dst = iv_buffer; 1589 arc4rand(iv_dst, iv_len, 0); 1590 1591 if (imm_payload) { 1592 m_copydata(sndptr, sndptroff + TLS_HEADER_LENGTH, 1593 tls_size, buf); 1594 } else { 1595 write_tlstx_sgl(buf, sndptr, 1596 sndptroff + TLS_HEADER_LENGTH, tls_size, iv_buffer, 1597 iv_len, nsegs, max_nsegs_1mbuf); 1598 } 1599 1600 KASSERT(toep->tx_credits >= credits, 1601 ("%s: not enough credits", __func__)); 1602 1603 toep->tx_credits -= credits; 1604 1605 tp->snd_nxt += plen; 1606 tp->snd_max += plen; 1607 1608 SOCKBUF_LOCK(sb); 1609 sbsndptr_adv(sb, sb->sb_sndptr, plen); 1610 tls_ofld->sb_off += plen; 1611 SOCKBUF_UNLOCK(sb); 1612 1613 toep->flags |= TPF_TX_DATA_SENT; 1614 if (toep->tx_credits < MIN_OFLD_TLSTX_CREDITS(toep)) 1615 toep->flags |= TPF_TX_SUSPENDED; 1616 1617 KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__)); 1618 txsd->plen = plen; 1619 txsd->tx_credits = credits; 1620 txsd->iv_buffer = iv_buffer; 1621 txsd++; 1622 if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) { 1623 toep->txsd_pidx = 0; 1624 txsd = &toep->txsd[0]; 1625 } 1626 toep->txsd_avail--; 1627 1628 atomic_add_long(&toep->vi->pi->tx_toe_tls_records, 1); 1629 atomic_add_long(&toep->vi->pi->tx_toe_tls_octets, plen); 1630 1631 t4_l2t_send(sc, wr, toep->l2te); 1632 } 1633 } 1634 1635 #ifdef KERN_TLS 1636 static int 1637 count_ext_pgs_segs(struct mbuf *m) 1638 { 1639 vm_paddr_t nextpa; 1640 u_int i, nsegs; 1641 1642 MPASS(m->m_epg_npgs > 0); 1643 nsegs = 1; 1644 nextpa = m->m_epg_pa[0] + PAGE_SIZE; 1645 for (i = 1; i < m->m_epg_npgs; i++) { 1646 if (nextpa != m->m_epg_pa[i]) 1647 nsegs++; 1648 nextpa = m->m_epg_pa[i] + PAGE_SIZE; 1649 } 1650 return (nsegs); 1651 } 1652 1653 static void 1654 write_ktlstx_sgl(void *dst, struct mbuf *m, int nsegs) 1655 { 1656 struct ulptx_sgl *usgl = dst; 1657 vm_paddr_t pa; 1658 uint32_t len; 1659 int i, j; 1660 1661 KASSERT(nsegs > 0, ("%s: nsegs 0", __func__)); 1662 1663 usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) | 1664 V_ULPTX_NSGE(nsegs)); 1665 1666 /* Figure out the first S/G length. */ 1667 pa = m->m_epg_pa[0] + m->m_epg_1st_off; 1668 usgl->addr0 = htobe64(pa); 1669 len = m_epg_pagelen(m, 0, m->m_epg_1st_off); 1670 pa += len; 1671 for (i = 1; i < m->m_epg_npgs; i++) { 1672 if (m->m_epg_pa[i] != pa) 1673 break; 1674 len += m_epg_pagelen(m, i, 0); 1675 pa += m_epg_pagelen(m, i, 0); 1676 } 1677 usgl->len0 = htobe32(len); 1678 #ifdef INVARIANTS 1679 nsegs--; 1680 #endif 1681 1682 j = -1; 1683 for (; i < m->m_epg_npgs; i++) { 1684 if (j == -1 || m->m_epg_pa[i] != pa) { 1685 if (j >= 0) 1686 usgl->sge[j / 2].len[j & 1] = htobe32(len); 1687 j++; 1688 #ifdef INVARIANTS 1689 nsegs--; 1690 #endif 1691 pa = m->m_epg_pa[i]; 1692 usgl->sge[j / 2].addr[j & 1] = htobe64(pa); 1693 len = m_epg_pagelen(m, i, 0); 1694 pa += len; 1695 } else { 1696 len += m_epg_pagelen(m, i, 0); 1697 pa += m_epg_pagelen(m, i, 0); 1698 } 1699 } 1700 if (j >= 0) { 1701 usgl->sge[j / 2].len[j & 1] = htobe32(len); 1702 1703 if ((j & 1) == 0) 1704 usgl->sge[j / 2].len[1] = htobe32(0); 1705 } 1706 KASSERT(nsegs == 0, ("%s: nsegs %d, m %p", __func__, nsegs, m)); 1707 } 1708 1709 /* 1710 * Similar to t4_push_frames() but handles sockets that contain TLS 1711 * record mbufs. Unlike TLSOM, each mbuf is a complete TLS record and 1712 * corresponds to a single work request. 1713 */ 1714 void 1715 t4_push_ktls(struct adapter *sc, struct toepcb *toep, int drop) 1716 { 1717 struct tls_hdr *thdr; 1718 struct fw_tlstx_data_wr *txwr; 1719 struct cpl_tx_tls_sfo *cpl; 1720 struct wrqe *wr; 1721 struct mbuf *m; 1722 u_int nsegs, credits, wr_len; 1723 u_int expn_size; 1724 struct inpcb *inp = toep->inp; 1725 struct tcpcb *tp = intotcpcb(inp); 1726 struct socket *so = inp->inp_socket; 1727 struct sockbuf *sb = &so->so_snd; 1728 int tls_size, tx_credits, shove, sowwakeup; 1729 struct ofld_tx_sdesc *txsd; 1730 char *buf; 1731 1732 INP_WLOCK_ASSERT(inp); 1733 KASSERT(toep->flags & TPF_FLOWC_WR_SENT, 1734 ("%s: flowc_wr not sent for tid %u.", __func__, toep->tid)); 1735 1736 KASSERT(ulp_mode(toep) == ULP_MODE_NONE || 1737 ulp_mode(toep) == ULP_MODE_TCPDDP || ulp_mode(toep) == ULP_MODE_TLS, 1738 ("%s: ulp_mode %u for toep %p", __func__, ulp_mode(toep), toep)); 1739 KASSERT(tls_tx_key(toep), 1740 ("%s: TX key not set for toep %p", __func__, toep)); 1741 1742 #ifdef VERBOSE_TRACES 1743 CTR4(KTR_CXGBE, "%s: tid %d toep flags %#x tp flags %#x drop %d", 1744 __func__, toep->tid, toep->flags, tp->t_flags); 1745 #endif 1746 if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN)) 1747 return; 1748 1749 #ifdef RATELIMIT 1750 if (__predict_false(inp->inp_flags2 & INP_RATE_LIMIT_CHANGED) && 1751 (update_tx_rate_limit(sc, toep, so->so_max_pacing_rate) == 0)) { 1752 inp->inp_flags2 &= ~INP_RATE_LIMIT_CHANGED; 1753 } 1754 #endif 1755 1756 /* 1757 * This function doesn't resume by itself. Someone else must clear the 1758 * flag and call this function. 1759 */ 1760 if (__predict_false(toep->flags & TPF_TX_SUSPENDED)) { 1761 KASSERT(drop == 0, 1762 ("%s: drop (%d) != 0 but tx is suspended", __func__, drop)); 1763 return; 1764 } 1765 1766 txsd = &toep->txsd[toep->txsd_pidx]; 1767 for (;;) { 1768 tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS); 1769 1770 SOCKBUF_LOCK(sb); 1771 sowwakeup = drop; 1772 if (drop) { 1773 sbdrop_locked(sb, drop); 1774 drop = 0; 1775 } 1776 1777 m = sb->sb_sndptr != NULL ? sb->sb_sndptr->m_next : sb->sb_mb; 1778 1779 /* 1780 * Send a FIN if requested, but only if there's no 1781 * more data to send. 1782 */ 1783 if (m == NULL && toep->flags & TPF_SEND_FIN) { 1784 if (sowwakeup) 1785 sowwakeup_locked(so); 1786 else 1787 SOCKBUF_UNLOCK(sb); 1788 SOCKBUF_UNLOCK_ASSERT(sb); 1789 t4_close_conn(sc, toep); 1790 return; 1791 } 1792 1793 /* 1794 * If there is no ready data to send, wait until more 1795 * data arrives. 1796 */ 1797 if (m == NULL || (m->m_flags & M_NOTAVAIL) != 0) { 1798 if (sowwakeup) 1799 sowwakeup_locked(so); 1800 else 1801 SOCKBUF_UNLOCK(sb); 1802 SOCKBUF_UNLOCK_ASSERT(sb); 1803 #ifdef VERBOSE_TRACES 1804 CTR2(KTR_CXGBE, "%s: tid %d no ready data to send", 1805 __func__, toep->tid); 1806 #endif 1807 return; 1808 } 1809 1810 KASSERT(m->m_flags & M_EXTPG, ("%s: mbuf %p is not NOMAP", 1811 __func__, m)); 1812 KASSERT(m->m_epg_tls != NULL, 1813 ("%s: mbuf %p doesn't have TLS session", __func__, m)); 1814 1815 /* Calculate WR length. */ 1816 wr_len = sizeof(struct fw_tlstx_data_wr) + 1817 sizeof(struct cpl_tx_tls_sfo) + key_size(toep); 1818 1819 /* Explicit IVs for AES-CBC and AES-GCM are <= 16. */ 1820 MPASS(toep->tls.iv_len <= AES_BLOCK_LEN); 1821 wr_len += AES_BLOCK_LEN; 1822 1823 /* Account for SGL in work request length. */ 1824 nsegs = count_ext_pgs_segs(m); 1825 wr_len += sizeof(struct ulptx_sgl) + 1826 ((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8; 1827 1828 /* Not enough credits for this work request. */ 1829 if (howmany(wr_len, 16) > tx_credits) { 1830 if (sowwakeup) 1831 sowwakeup_locked(so); 1832 else 1833 SOCKBUF_UNLOCK(sb); 1834 SOCKBUF_UNLOCK_ASSERT(sb); 1835 #ifdef VERBOSE_TRACES 1836 CTR5(KTR_CXGBE, 1837 "%s: tid %d mbuf %p requires %d credits, but only %d available", 1838 __func__, toep->tid, m, howmany(wr_len, 16), 1839 tx_credits); 1840 #endif 1841 toep->flags |= TPF_TX_SUSPENDED; 1842 return; 1843 } 1844 1845 /* Shove if there is no additional data pending. */ 1846 shove = ((m->m_next == NULL || 1847 (m->m_next->m_flags & M_NOTAVAIL) != 0)) && 1848 (tp->t_flags & TF_MORETOCOME) == 0; 1849 1850 if (sb->sb_flags & SB_AUTOSIZE && 1851 V_tcp_do_autosndbuf && 1852 sb->sb_hiwat < V_tcp_autosndbuf_max && 1853 sbused(sb) >= sb->sb_hiwat * 7 / 8) { 1854 int newsize = min(sb->sb_hiwat + V_tcp_autosndbuf_inc, 1855 V_tcp_autosndbuf_max); 1856 1857 if (!sbreserve_locked(sb, newsize, so, NULL)) 1858 sb->sb_flags &= ~SB_AUTOSIZE; 1859 else 1860 sowwakeup = 1; /* room available */ 1861 } 1862 if (sowwakeup) 1863 sowwakeup_locked(so); 1864 else 1865 SOCKBUF_UNLOCK(sb); 1866 SOCKBUF_UNLOCK_ASSERT(sb); 1867 1868 if (__predict_false(toep->flags & TPF_FIN_SENT)) 1869 panic("%s: excess tx.", __func__); 1870 1871 wr = alloc_wrqe(roundup2(wr_len, 16), toep->ofld_txq); 1872 if (wr == NULL) { 1873 /* XXX: how will we recover from this? */ 1874 toep->flags |= TPF_TX_SUSPENDED; 1875 return; 1876 } 1877 1878 thdr = (struct tls_hdr *)&m->m_epg_hdr; 1879 #ifdef VERBOSE_TRACES 1880 CTR5(KTR_CXGBE, "%s: tid %d TLS record %ju type %d len %#x", 1881 __func__, toep->tid, m->m_epg_seqno, thdr->type, 1882 m->m_len); 1883 #endif 1884 txwr = wrtod(wr); 1885 cpl = (struct cpl_tx_tls_sfo *)(txwr + 1); 1886 memset(txwr, 0, roundup2(wr_len, 16)); 1887 credits = howmany(wr_len, 16); 1888 expn_size = m->m_epg_hdrlen + 1889 m->m_epg_trllen; 1890 tls_size = m->m_len - expn_size; 1891 write_tlstx_wr(txwr, toep, 0, 1892 tls_size, expn_size, 1, credits, shove, 1); 1893 toep->tls.tx_seq_no = m->m_epg_seqno; 1894 write_tlstx_cpl(cpl, toep, thdr, tls_size, 1); 1895 tls_copy_tx_key(toep, cpl + 1); 1896 1897 /* Copy IV. */ 1898 buf = (char *)(cpl + 1) + key_size(toep); 1899 memcpy(buf, thdr + 1, toep->tls.iv_len); 1900 buf += AES_BLOCK_LEN; 1901 1902 write_ktlstx_sgl(buf, m, nsegs); 1903 1904 KASSERT(toep->tx_credits >= credits, 1905 ("%s: not enough credits", __func__)); 1906 1907 toep->tx_credits -= credits; 1908 1909 tp->snd_nxt += m->m_len; 1910 tp->snd_max += m->m_len; 1911 1912 SOCKBUF_LOCK(sb); 1913 sb->sb_sndptr = m; 1914 SOCKBUF_UNLOCK(sb); 1915 1916 toep->flags |= TPF_TX_DATA_SENT; 1917 if (toep->tx_credits < MIN_OFLD_TLSTX_CREDITS(toep)) 1918 toep->flags |= TPF_TX_SUSPENDED; 1919 1920 KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__)); 1921 txsd->plen = m->m_len; 1922 txsd->tx_credits = credits; 1923 txsd++; 1924 if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) { 1925 toep->txsd_pidx = 0; 1926 txsd = &toep->txsd[0]; 1927 } 1928 toep->txsd_avail--; 1929 1930 atomic_add_long(&toep->vi->pi->tx_toe_tls_records, 1); 1931 atomic_add_long(&toep->vi->pi->tx_toe_tls_octets, m->m_len); 1932 1933 t4_l2t_send(sc, wr, toep->l2te); 1934 } 1935 } 1936 #endif 1937 1938 /* 1939 * For TLS data we place received mbufs received via CPL_TLS_DATA into 1940 * an mbufq in the TLS offload state. When CPL_RX_TLS_CMP is 1941 * received, the completed PDUs are placed into the socket receive 1942 * buffer. 1943 * 1944 * The TLS code reuses the ulp_pdu_reclaimq to hold the pending mbufs. 1945 */ 1946 static int 1947 do_tls_data(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 1948 { 1949 struct adapter *sc = iq->adapter; 1950 const struct cpl_tls_data *cpl = mtod(m, const void *); 1951 unsigned int tid = GET_TID(cpl); 1952 struct toepcb *toep = lookup_tid(sc, tid); 1953 struct inpcb *inp = toep->inp; 1954 struct tcpcb *tp; 1955 int len; 1956 1957 /* XXX: Should this match do_rx_data instead? */ 1958 KASSERT(!(toep->flags & TPF_SYNQE), 1959 ("%s: toep %p claims to be a synq entry", __func__, toep)); 1960 1961 KASSERT(toep->tid == tid, ("%s: toep tid/atid mismatch", __func__)); 1962 1963 /* strip off CPL header */ 1964 m_adj(m, sizeof(*cpl)); 1965 len = m->m_pkthdr.len; 1966 1967 atomic_add_long(&toep->vi->pi->rx_toe_tls_octets, len); 1968 1969 KASSERT(len == G_CPL_TLS_DATA_LENGTH(be32toh(cpl->length_pkd)), 1970 ("%s: payload length mismatch", __func__)); 1971 1972 INP_WLOCK(inp); 1973 if (inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) { 1974 CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x", 1975 __func__, tid, len, inp->inp_flags); 1976 INP_WUNLOCK(inp); 1977 m_freem(m); 1978 return (0); 1979 } 1980 1981 /* Save TCP sequence number. */ 1982 m->m_pkthdr.tls_tcp_seq = be32toh(cpl->seq); 1983 1984 if (mbufq_enqueue(&toep->ulp_pdu_reclaimq, m)) { 1985 #ifdef INVARIANTS 1986 panic("Failed to queue TLS data packet"); 1987 #else 1988 printf("%s: Failed to queue TLS data packet\n", __func__); 1989 INP_WUNLOCK(inp); 1990 m_freem(m); 1991 return (0); 1992 #endif 1993 } 1994 1995 tp = intotcpcb(inp); 1996 tp->t_rcvtime = ticks; 1997 1998 #ifdef VERBOSE_TRACES 1999 CTR4(KTR_CXGBE, "%s: tid %u len %d seq %u", __func__, tid, len, 2000 be32toh(cpl->seq)); 2001 #endif 2002 2003 INP_WUNLOCK(inp); 2004 return (0); 2005 } 2006 2007 static int 2008 do_rx_tls_cmp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 2009 { 2010 struct adapter *sc = iq->adapter; 2011 const struct cpl_rx_tls_cmp *cpl = mtod(m, const void *); 2012 struct tlsrx_hdr_pkt *tls_hdr_pkt; 2013 unsigned int tid = GET_TID(cpl); 2014 struct toepcb *toep = lookup_tid(sc, tid); 2015 struct inpcb *inp = toep->inp; 2016 struct tcpcb *tp; 2017 struct socket *so; 2018 struct sockbuf *sb; 2019 struct mbuf *tls_data; 2020 #ifdef KERN_TLS 2021 struct tls_get_record *tgr; 2022 struct mbuf *control; 2023 #endif 2024 int len, pdu_length, rx_credits; 2025 2026 KASSERT(toep->tid == tid, ("%s: toep tid/atid mismatch", __func__)); 2027 KASSERT(!(toep->flags & TPF_SYNQE), 2028 ("%s: toep %p claims to be a synq entry", __func__, toep)); 2029 2030 /* strip off CPL header */ 2031 m_adj(m, sizeof(*cpl)); 2032 len = m->m_pkthdr.len; 2033 2034 atomic_add_long(&toep->vi->pi->rx_toe_tls_records, 1); 2035 2036 KASSERT(len == G_CPL_RX_TLS_CMP_LENGTH(be32toh(cpl->pdulength_length)), 2037 ("%s: payload length mismatch", __func__)); 2038 2039 INP_WLOCK(inp); 2040 if (inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) { 2041 CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x", 2042 __func__, tid, len, inp->inp_flags); 2043 INP_WUNLOCK(inp); 2044 m_freem(m); 2045 return (0); 2046 } 2047 2048 pdu_length = G_CPL_RX_TLS_CMP_PDULENGTH(be32toh(cpl->pdulength_length)); 2049 2050 so = inp_inpcbtosocket(inp); 2051 tp = intotcpcb(inp); 2052 2053 #ifdef VERBOSE_TRACES 2054 CTR6(KTR_CXGBE, "%s: tid %u PDU len %d len %d seq %u, rcv_nxt %u", 2055 __func__, tid, pdu_length, len, be32toh(cpl->seq), tp->rcv_nxt); 2056 #endif 2057 2058 tp->rcv_nxt += pdu_length; 2059 KASSERT(tp->rcv_wnd >= pdu_length, 2060 ("%s: negative window size", __func__)); 2061 tp->rcv_wnd -= pdu_length; 2062 2063 /* XXX: Not sure what to do about urgent data. */ 2064 2065 /* 2066 * The payload of this CPL is the TLS header followed by 2067 * additional fields. 2068 */ 2069 KASSERT(m->m_len >= sizeof(*tls_hdr_pkt), 2070 ("%s: payload too small", __func__)); 2071 tls_hdr_pkt = mtod(m, void *); 2072 2073 tls_data = mbufq_dequeue(&toep->ulp_pdu_reclaimq); 2074 if (tls_data != NULL) { 2075 KASSERT(be32toh(cpl->seq) == tls_data->m_pkthdr.tls_tcp_seq, 2076 ("%s: sequence mismatch", __func__)); 2077 } 2078 2079 #ifdef KERN_TLS 2080 if (toep->tls.mode == TLS_MODE_KTLS) { 2081 /* Report decryption errors as EBADMSG. */ 2082 if ((tls_hdr_pkt->res_to_mac_error & M_TLSRX_HDR_PKT_ERROR) != 2083 0) { 2084 m_freem(m); 2085 m_freem(tls_data); 2086 2087 CURVNET_SET(toep->vnet); 2088 so->so_error = EBADMSG; 2089 sorwakeup(so); 2090 2091 INP_WUNLOCK(inp); 2092 CURVNET_RESTORE(); 2093 2094 return (0); 2095 } 2096 2097 /* Allocate the control message mbuf. */ 2098 control = sbcreatecontrol(NULL, sizeof(*tgr), TLS_GET_RECORD, 2099 IPPROTO_TCP); 2100 if (control == NULL) { 2101 m_freem(m); 2102 m_freem(tls_data); 2103 2104 CURVNET_SET(toep->vnet); 2105 so->so_error = ENOBUFS; 2106 sorwakeup(so); 2107 2108 INP_WUNLOCK(inp); 2109 CURVNET_RESTORE(); 2110 2111 return (0); 2112 } 2113 2114 tgr = (struct tls_get_record *) 2115 CMSG_DATA(mtod(control, struct cmsghdr *)); 2116 tgr->tls_type = tls_hdr_pkt->type; 2117 tgr->tls_vmajor = be16toh(tls_hdr_pkt->version) >> 8; 2118 tgr->tls_vminor = be16toh(tls_hdr_pkt->version) & 0xff; 2119 2120 m_freem(m); 2121 2122 if (tls_data != NULL) { 2123 m_last(tls_data)->m_flags |= M_EOR; 2124 tgr->tls_length = htobe16(tls_data->m_pkthdr.len); 2125 } else 2126 tgr->tls_length = 0; 2127 m = tls_data; 2128 } else 2129 #endif 2130 { 2131 /* 2132 * Only the TLS header is sent to OpenSSL, so report 2133 * errors by altering the record type. 2134 */ 2135 if ((tls_hdr_pkt->res_to_mac_error & M_TLSRX_HDR_PKT_ERROR) != 2136 0) 2137 tls_hdr_pkt->type = CONTENT_TYPE_ERROR; 2138 2139 /* Trim this CPL's mbuf to only include the TLS header. */ 2140 KASSERT(m->m_len == len && m->m_next == NULL, 2141 ("%s: CPL spans multiple mbufs", __func__)); 2142 m->m_len = TLS_HEADER_LENGTH; 2143 m->m_pkthdr.len = TLS_HEADER_LENGTH; 2144 2145 if (tls_data != NULL) { 2146 /* 2147 * Update the TLS header length to be the length of 2148 * the payload data. 2149 */ 2150 tls_hdr_pkt->length = htobe16(tls_data->m_pkthdr.len); 2151 2152 m->m_next = tls_data; 2153 m->m_pkthdr.len += tls_data->m_len; 2154 } 2155 2156 #ifdef KERN_TLS 2157 control = NULL; 2158 #endif 2159 } 2160 2161 sb = &so->so_rcv; 2162 SOCKBUF_LOCK(sb); 2163 2164 if (__predict_false(sb->sb_state & SBS_CANTRCVMORE)) { 2165 struct epoch_tracker et; 2166 2167 CTR3(KTR_CXGBE, "%s: tid %u, excess rx (%d bytes)", 2168 __func__, tid, pdu_length); 2169 m_freem(m); 2170 #ifdef KERN_TLS 2171 m_freem(control); 2172 #endif 2173 SOCKBUF_UNLOCK(sb); 2174 INP_WUNLOCK(inp); 2175 2176 CURVNET_SET(toep->vnet); 2177 NET_EPOCH_ENTER(et); 2178 INP_WLOCK(inp); 2179 tp = tcp_drop(tp, ECONNRESET); 2180 if (tp) 2181 INP_WUNLOCK(inp); 2182 NET_EPOCH_EXIT(et); 2183 CURVNET_RESTORE(); 2184 2185 return (0); 2186 } 2187 2188 /* 2189 * Not all of the bytes on the wire are included in the socket buffer 2190 * (e.g. the MAC of the TLS record). However, those bytes are included 2191 * in the TCP sequence space. 2192 */ 2193 2194 /* receive buffer autosize */ 2195 MPASS(toep->vnet == so->so_vnet); 2196 CURVNET_SET(toep->vnet); 2197 if (sb->sb_flags & SB_AUTOSIZE && 2198 V_tcp_do_autorcvbuf && 2199 sb->sb_hiwat < V_tcp_autorcvbuf_max && 2200 m->m_pkthdr.len > (sbspace(sb) / 8 * 7)) { 2201 unsigned int hiwat = sb->sb_hiwat; 2202 unsigned int newsize = min(hiwat + sc->tt.autorcvbuf_inc, 2203 V_tcp_autorcvbuf_max); 2204 2205 if (!sbreserve_locked(sb, newsize, so, NULL)) 2206 sb->sb_flags &= ~SB_AUTOSIZE; 2207 } 2208 2209 #ifdef KERN_TLS 2210 if (control != NULL) 2211 sbappendcontrol_locked(sb, m, control, 0); 2212 else 2213 #endif 2214 sbappendstream_locked(sb, m, 0); 2215 rx_credits = sbspace(sb) > tp->rcv_wnd ? sbspace(sb) - tp->rcv_wnd : 0; 2216 #ifdef VERBOSE_TRACES 2217 CTR4(KTR_CXGBE, "%s: tid %u rx_credits %u rcv_wnd %u", 2218 __func__, tid, rx_credits, tp->rcv_wnd); 2219 #endif 2220 if (rx_credits > 0 && sbused(sb) + tp->rcv_wnd < sb->sb_lowat) { 2221 rx_credits = send_rx_credits(sc, toep, rx_credits); 2222 tp->rcv_wnd += rx_credits; 2223 tp->rcv_adv += rx_credits; 2224 } 2225 2226 sorwakeup_locked(so); 2227 SOCKBUF_UNLOCK_ASSERT(sb); 2228 2229 INP_WUNLOCK(inp); 2230 CURVNET_RESTORE(); 2231 return (0); 2232 } 2233 2234 void 2235 do_rx_data_tls(const struct cpl_rx_data *cpl, struct toepcb *toep, 2236 struct mbuf *m) 2237 { 2238 struct inpcb *inp = toep->inp; 2239 struct tls_ofld_info *tls_ofld = &toep->tls; 2240 struct tls_hdr *hdr; 2241 struct tcpcb *tp; 2242 struct socket *so; 2243 struct sockbuf *sb; 2244 int error, len, rx_credits; 2245 2246 len = m->m_pkthdr.len; 2247 2248 INP_WLOCK_ASSERT(inp); 2249 2250 so = inp_inpcbtosocket(inp); 2251 tp = intotcpcb(inp); 2252 sb = &so->so_rcv; 2253 SOCKBUF_LOCK(sb); 2254 CURVNET_SET(toep->vnet); 2255 2256 tp->rcv_nxt += len; 2257 KASSERT(tp->rcv_wnd >= len, ("%s: negative window size", __func__)); 2258 tp->rcv_wnd -= len; 2259 2260 /* Do we have a full TLS header? */ 2261 if (len < sizeof(*hdr)) { 2262 CTR3(KTR_CXGBE, "%s: tid %u len %d: too short for a TLS header", 2263 __func__, toep->tid, len); 2264 so->so_error = EMSGSIZE; 2265 goto out; 2266 } 2267 hdr = mtod(m, struct tls_hdr *); 2268 2269 /* Is the header valid? */ 2270 if (be16toh(hdr->version) != tls_ofld->k_ctx.proto_ver) { 2271 CTR3(KTR_CXGBE, "%s: tid %u invalid version %04x", 2272 __func__, toep->tid, be16toh(hdr->version)); 2273 error = EINVAL; 2274 goto report_error; 2275 } 2276 if (be16toh(hdr->length) < sizeof(*hdr)) { 2277 CTR3(KTR_CXGBE, "%s: tid %u invalid length %u", 2278 __func__, toep->tid, be16toh(hdr->length)); 2279 error = EBADMSG; 2280 goto report_error; 2281 } 2282 2283 /* Did we get a truncated record? */ 2284 if (len < be16toh(hdr->length)) { 2285 CTR4(KTR_CXGBE, "%s: tid %u truncated TLS record (%d vs %u)", 2286 __func__, toep->tid, len, be16toh(hdr->length)); 2287 2288 error = EMSGSIZE; 2289 goto report_error; 2290 } 2291 2292 /* Is the header type unknown? */ 2293 switch (hdr->type) { 2294 case CONTENT_TYPE_CCS: 2295 case CONTENT_TYPE_ALERT: 2296 case CONTENT_TYPE_APP_DATA: 2297 case CONTENT_TYPE_HANDSHAKE: 2298 break; 2299 default: 2300 CTR3(KTR_CXGBE, "%s: tid %u invalid TLS record type %u", 2301 __func__, toep->tid, hdr->type); 2302 error = EBADMSG; 2303 goto report_error; 2304 } 2305 2306 /* 2307 * Just punt. Although this could fall back to software 2308 * decryption, this case should never really happen. 2309 */ 2310 CTR4(KTR_CXGBE, "%s: tid %u dropping TLS record type %u, length %u", 2311 __func__, toep->tid, hdr->type, be16toh(hdr->length)); 2312 error = EBADMSG; 2313 2314 report_error: 2315 #ifdef KERN_TLS 2316 if (toep->tls.mode == TLS_MODE_KTLS) 2317 so->so_error = error; 2318 else 2319 #endif 2320 { 2321 /* 2322 * Report errors by sending an empty TLS record 2323 * with an error record type. 2324 */ 2325 hdr->type = CONTENT_TYPE_ERROR; 2326 2327 /* Trim this CPL's mbuf to only include the TLS header. */ 2328 KASSERT(m->m_len == len && m->m_next == NULL, 2329 ("%s: CPL spans multiple mbufs", __func__)); 2330 m->m_len = TLS_HEADER_LENGTH; 2331 m->m_pkthdr.len = TLS_HEADER_LENGTH; 2332 2333 sbappendstream_locked(sb, m, 0); 2334 m = NULL; 2335 } 2336 2337 out: 2338 /* 2339 * This connection is going to die anyway, so probably don't 2340 * need to bother with returning credits. 2341 */ 2342 rx_credits = sbspace(sb) > tp->rcv_wnd ? sbspace(sb) - tp->rcv_wnd : 0; 2343 #ifdef VERBOSE_TRACES 2344 CTR4(KTR_CXGBE, "%s: tid %u rx_credits %u rcv_wnd %u", 2345 __func__, toep->tid, rx_credits, tp->rcv_wnd); 2346 #endif 2347 if (rx_credits > 0 && sbused(sb) + tp->rcv_wnd < sb->sb_lowat) { 2348 rx_credits = send_rx_credits(toep->vi->adapter, toep, 2349 rx_credits); 2350 tp->rcv_wnd += rx_credits; 2351 tp->rcv_adv += rx_credits; 2352 } 2353 2354 sorwakeup_locked(so); 2355 SOCKBUF_UNLOCK_ASSERT(sb); 2356 2357 INP_WUNLOCK(inp); 2358 CURVNET_RESTORE(); 2359 2360 m_freem(m); 2361 } 2362 2363 void 2364 t4_tls_mod_load(void) 2365 { 2366 2367 mtx_init(&tls_handshake_lock, "t4tls handshake", NULL, MTX_DEF); 2368 t4_register_cpl_handler(CPL_TLS_DATA, do_tls_data); 2369 t4_register_cpl_handler(CPL_RX_TLS_CMP, do_rx_tls_cmp); 2370 } 2371 2372 void 2373 t4_tls_mod_unload(void) 2374 { 2375 2376 t4_register_cpl_handler(CPL_TLS_DATA, NULL); 2377 t4_register_cpl_handler(CPL_RX_TLS_CMP, NULL); 2378 mtx_destroy(&tls_handshake_lock); 2379 } 2380 #endif /* TCP_OFFLOAD */ 2381