1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2017-2018 Chelsio Communications, Inc. 5 * All rights reserved. 6 * Written by: John Baldwin <jhb@FreeBSD.org> 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include "opt_inet.h" 31 32 #include <sys/cdefs.h> 33 __FBSDID("$FreeBSD$"); 34 35 #include <sys/param.h> 36 #include <sys/sglist.h> 37 #include <sys/socket.h> 38 #include <sys/socketvar.h> 39 #include <sys/systm.h> 40 #include <netinet/in.h> 41 #include <netinet/in_pcb.h> 42 #include <netinet/tcp_var.h> 43 #include <netinet/toecore.h> 44 45 #ifdef TCP_OFFLOAD 46 #include "common/common.h" 47 #include "common/t4_tcb.h" 48 #include "tom/t4_tom_l2t.h" 49 #include "tom/t4_tom.h" 50 51 /* 52 * The TCP sequence number of a CPL_TLS_DATA mbuf is saved here while 53 * the mbuf is in the ulp_pdu_reclaimq. 54 */ 55 #define tls_tcp_seq PH_loc.thirtytwo[0] 56 57 /* 58 * Handshake lock used for the handshake timer. Having a global lock 59 * is perhaps not ideal, but it avoids having to use callout_drain() 60 * in tls_uninit_toep() which can't block. Also, the timer shouldn't 61 * actually fire for most connections. 62 */ 63 static struct mtx tls_handshake_lock; 64 65 static void 66 t4_set_tls_tcb_field(struct toepcb *toep, uint16_t word, uint64_t mask, 67 uint64_t val) 68 { 69 struct adapter *sc = td_adapter(toep->td); 70 71 t4_set_tcb_field(sc, toep->ofld_txq, toep, word, mask, val, 0, 0); 72 } 73 74 /* TLS and DTLS common routines */ 75 bool 76 can_tls_offload(struct adapter *sc) 77 { 78 79 return (sc->tt.tls && sc->cryptocaps & FW_CAPS_CONFIG_TLSKEYS); 80 } 81 82 int 83 tls_tx_key(struct toepcb *toep) 84 { 85 struct tls_ofld_info *tls_ofld = &toep->tls; 86 87 return (tls_ofld->tx_key_addr >= 0); 88 } 89 90 int 91 tls_rx_key(struct toepcb *toep) 92 { 93 struct tls_ofld_info *tls_ofld = &toep->tls; 94 95 return (tls_ofld->rx_key_addr >= 0); 96 } 97 98 static int 99 key_size(struct toepcb *toep) 100 { 101 struct tls_ofld_info *tls_ofld = &toep->tls; 102 103 return ((tls_ofld->key_location == TLS_SFO_WR_CONTEXTLOC_IMMEDIATE) ? 104 tls_ofld->k_ctx.tx_key_info_size : KEY_IN_DDR_SIZE); 105 } 106 107 /* Set TLS Key-Id in TCB */ 108 static void 109 t4_set_tls_keyid(struct toepcb *toep, unsigned int key_id) 110 { 111 112 t4_set_tls_tcb_field(toep, W_TCB_RX_TLS_KEY_TAG, 113 V_TCB_RX_TLS_KEY_TAG(M_TCB_RX_TLS_BUF_TAG), 114 V_TCB_RX_TLS_KEY_TAG(key_id)); 115 } 116 117 /* Clear TF_RX_QUIESCE to re-enable receive. */ 118 static void 119 t4_clear_rx_quiesce(struct toepcb *toep) 120 { 121 122 t4_set_tls_tcb_field(toep, W_TCB_T_FLAGS, V_TF_RX_QUIESCE(1), 0); 123 } 124 125 static void 126 tls_clr_ofld_mode(struct toepcb *toep) 127 { 128 129 tls_stop_handshake_timer(toep); 130 131 /* Operate in PDU extraction mode only. */ 132 t4_set_tls_tcb_field(toep, W_TCB_ULP_RAW, 133 V_TCB_ULP_RAW(M_TCB_ULP_RAW), 134 V_TCB_ULP_RAW(V_TF_TLS_ENABLE(1))); 135 t4_clear_rx_quiesce(toep); 136 } 137 138 static void 139 tls_clr_quiesce(struct toepcb *toep) 140 { 141 142 tls_stop_handshake_timer(toep); 143 t4_clear_rx_quiesce(toep); 144 } 145 146 /* 147 * Calculate the TLS data expansion size 148 */ 149 static int 150 tls_expansion_size(struct toepcb *toep, int data_len, int full_pdus_only, 151 unsigned short *pdus_per_ulp) 152 { 153 struct tls_ofld_info *tls_ofld = &toep->tls; 154 struct tls_scmd *scmd = &tls_ofld->scmd0; 155 int expn_size = 0, frag_count = 0, pad_per_pdu = 0, 156 pad_last_pdu = 0, last_frag_size = 0, max_frag_size = 0; 157 int exp_per_pdu = 0; 158 int hdr_len = TLS_HEADER_LENGTH; 159 160 do { 161 max_frag_size = tls_ofld->k_ctx.frag_size; 162 if (G_SCMD_CIPH_MODE(scmd->seqno_numivs) == 163 SCMD_CIPH_MODE_AES_GCM) { 164 frag_count = (data_len / max_frag_size); 165 exp_per_pdu = GCM_TAG_SIZE + AEAD_EXPLICIT_DATA_SIZE + 166 hdr_len; 167 expn_size = frag_count * exp_per_pdu; 168 if (full_pdus_only) { 169 *pdus_per_ulp = data_len / (exp_per_pdu + 170 max_frag_size); 171 if (*pdus_per_ulp > 32) 172 *pdus_per_ulp = 32; 173 else if(!*pdus_per_ulp) 174 *pdus_per_ulp = 1; 175 expn_size = (*pdus_per_ulp) * exp_per_pdu; 176 break; 177 } 178 if ((last_frag_size = data_len % max_frag_size) > 0) { 179 frag_count += 1; 180 expn_size += exp_per_pdu; 181 } 182 break; 183 } else if (G_SCMD_CIPH_MODE(scmd->seqno_numivs) != 184 SCMD_CIPH_MODE_NOP) { 185 /* Calculate the number of fragments we can make */ 186 frag_count = (data_len / max_frag_size); 187 if (frag_count > 0) { 188 pad_per_pdu = (((howmany((max_frag_size + 189 tls_ofld->mac_length), 190 CIPHER_BLOCK_SIZE)) * 191 CIPHER_BLOCK_SIZE) - 192 (max_frag_size + 193 tls_ofld->mac_length)); 194 if (!pad_per_pdu) 195 pad_per_pdu = CIPHER_BLOCK_SIZE; 196 exp_per_pdu = pad_per_pdu + 197 tls_ofld->mac_length + 198 hdr_len + CIPHER_BLOCK_SIZE; 199 expn_size = frag_count * exp_per_pdu; 200 } 201 if (full_pdus_only) { 202 *pdus_per_ulp = data_len / (exp_per_pdu + 203 max_frag_size); 204 if (*pdus_per_ulp > 32) 205 *pdus_per_ulp = 32; 206 else if (!*pdus_per_ulp) 207 *pdus_per_ulp = 1; 208 expn_size = (*pdus_per_ulp) * exp_per_pdu; 209 break; 210 } 211 /* Consider the last fragment */ 212 if ((last_frag_size = data_len % max_frag_size) > 0) { 213 pad_last_pdu = (((howmany((last_frag_size + 214 tls_ofld->mac_length), 215 CIPHER_BLOCK_SIZE)) * 216 CIPHER_BLOCK_SIZE) - 217 (last_frag_size + 218 tls_ofld->mac_length)); 219 if (!pad_last_pdu) 220 pad_last_pdu = CIPHER_BLOCK_SIZE; 221 expn_size += (pad_last_pdu + 222 tls_ofld->mac_length + hdr_len + 223 CIPHER_BLOCK_SIZE); 224 } 225 } 226 } while (0); 227 228 return (expn_size); 229 } 230 231 /* Copy Key to WR */ 232 static void 233 tls_copy_tx_key(struct toepcb *toep, void *dst) 234 { 235 struct tls_ofld_info *tls_ofld = &toep->tls; 236 struct ulptx_sc_memrd *sc_memrd; 237 struct ulptx_idata *sc; 238 239 if (tls_ofld->k_ctx.tx_key_info_size <= 0) 240 return; 241 242 if (tls_ofld->key_location == TLS_SFO_WR_CONTEXTLOC_DDR) { 243 sc = dst; 244 sc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP)); 245 sc->len = htobe32(0); 246 sc_memrd = (struct ulptx_sc_memrd *)(sc + 1); 247 sc_memrd->cmd_to_len = htobe32(V_ULPTX_CMD(ULP_TX_SC_MEMRD) | 248 V_ULP_TX_SC_MORE(1) | 249 V_ULPTX_LEN16(tls_ofld->k_ctx.tx_key_info_size >> 4)); 250 sc_memrd->addr = htobe32(tls_ofld->tx_key_addr >> 5); 251 } else if (tls_ofld->key_location == TLS_SFO_WR_CONTEXTLOC_IMMEDIATE) { 252 memcpy(dst, &tls_ofld->k_ctx.tx, 253 tls_ofld->k_ctx.tx_key_info_size); 254 } 255 } 256 257 /* TLS/DTLS content type for CPL SFO */ 258 static inline unsigned char 259 tls_content_type(unsigned char content_type) 260 { 261 /* 262 * XXX: Shouldn't this map CONTENT_TYPE_APP_DATA to DATA and 263 * default to "CUSTOM" for all other types including 264 * heartbeat? 265 */ 266 switch (content_type) { 267 case CONTENT_TYPE_CCS: 268 return CPL_TX_TLS_SFO_TYPE_CCS; 269 case CONTENT_TYPE_ALERT: 270 return CPL_TX_TLS_SFO_TYPE_ALERT; 271 case CONTENT_TYPE_HANDSHAKE: 272 return CPL_TX_TLS_SFO_TYPE_HANDSHAKE; 273 case CONTENT_TYPE_HEARTBEAT: 274 return CPL_TX_TLS_SFO_TYPE_HEARTBEAT; 275 } 276 return CPL_TX_TLS_SFO_TYPE_DATA; 277 } 278 279 static unsigned char 280 get_cipher_key_size(unsigned int ck_size) 281 { 282 switch (ck_size) { 283 case AES_NOP: /* NOP */ 284 return 15; 285 case AES_128: /* AES128 */ 286 return CH_CK_SIZE_128; 287 case AES_192: /* AES192 */ 288 return CH_CK_SIZE_192; 289 case AES_256: /* AES256 */ 290 return CH_CK_SIZE_256; 291 default: 292 return CH_CK_SIZE_256; 293 } 294 } 295 296 static unsigned char 297 get_mac_key_size(unsigned int mk_size) 298 { 299 switch (mk_size) { 300 case SHA_NOP: /* NOP */ 301 return CH_MK_SIZE_128; 302 case SHA_GHASH: /* GHASH */ 303 case SHA_512: /* SHA512 */ 304 return CH_MK_SIZE_512; 305 case SHA_224: /* SHA2-224 */ 306 return CH_MK_SIZE_192; 307 case SHA_256: /* SHA2-256*/ 308 return CH_MK_SIZE_256; 309 case SHA_384: /* SHA384 */ 310 return CH_MK_SIZE_512; 311 case SHA1: /* SHA1 */ 312 default: 313 return CH_MK_SIZE_160; 314 } 315 } 316 317 static unsigned int 318 get_proto_ver(int proto_ver) 319 { 320 switch (proto_ver) { 321 case TLS1_2_VERSION: 322 return TLS_1_2_VERSION; 323 case TLS1_1_VERSION: 324 return TLS_1_1_VERSION; 325 case DTLS1_2_VERSION: 326 return DTLS_1_2_VERSION; 327 default: 328 return TLS_VERSION_MAX; 329 } 330 } 331 332 static void 333 tls_rxkey_flit1(struct tls_keyctx *kwr, struct tls_key_context *kctx) 334 { 335 336 if (kctx->state.enc_mode == CH_EVP_CIPH_GCM_MODE) { 337 kwr->u.rxhdr.ivinsert_to_authinsrt = 338 htobe64(V_TLS_KEYCTX_TX_WR_IVINSERT(6ULL) | 339 V_TLS_KEYCTX_TX_WR_AADSTRTOFST(1ULL) | 340 V_TLS_KEYCTX_TX_WR_AADSTOPOFST(5ULL) | 341 V_TLS_KEYCTX_TX_WR_AUTHSRTOFST(14ULL) | 342 V_TLS_KEYCTX_TX_WR_AUTHSTOPOFST(16ULL) | 343 V_TLS_KEYCTX_TX_WR_CIPHERSRTOFST(14ULL) | 344 V_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST(0ULL) | 345 V_TLS_KEYCTX_TX_WR_AUTHINSRT(16ULL)); 346 kwr->u.rxhdr.ivpresent_to_rxmk_size &= 347 ~(V_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT(1)); 348 kwr->u.rxhdr.authmode_to_rxvalid &= 349 ~(V_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL(1)); 350 } else { 351 kwr->u.rxhdr.ivinsert_to_authinsrt = 352 htobe64(V_TLS_KEYCTX_TX_WR_IVINSERT(6ULL) | 353 V_TLS_KEYCTX_TX_WR_AADSTRTOFST(1ULL) | 354 V_TLS_KEYCTX_TX_WR_AADSTOPOFST(5ULL) | 355 V_TLS_KEYCTX_TX_WR_AUTHSRTOFST(22ULL) | 356 V_TLS_KEYCTX_TX_WR_AUTHSTOPOFST(0ULL) | 357 V_TLS_KEYCTX_TX_WR_CIPHERSRTOFST(22ULL) | 358 V_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST(0ULL) | 359 V_TLS_KEYCTX_TX_WR_AUTHINSRT(0ULL)); 360 } 361 } 362 363 /* Rx key */ 364 static void 365 prepare_rxkey_wr(struct tls_keyctx *kwr, struct tls_key_context *kctx) 366 { 367 unsigned int ck_size = kctx->cipher_secret_size; 368 unsigned int mk_size = kctx->mac_secret_size; 369 int proto_ver = kctx->proto_ver; 370 371 kwr->u.rxhdr.flitcnt_hmacctrl = 372 ((kctx->tx_key_info_size >> 4) << 3) | kctx->hmac_ctrl; 373 374 kwr->u.rxhdr.protover_ciphmode = 375 V_TLS_KEYCTX_TX_WR_PROTOVER(get_proto_ver(proto_ver)) | 376 V_TLS_KEYCTX_TX_WR_CIPHMODE(kctx->state.enc_mode); 377 378 kwr->u.rxhdr.authmode_to_rxvalid = 379 V_TLS_KEYCTX_TX_WR_AUTHMODE(kctx->state.auth_mode) | 380 V_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL(1) | 381 V_TLS_KEYCTX_TX_WR_SEQNUMCTRL(3) | 382 V_TLS_KEYCTX_TX_WR_RXVALID(1); 383 384 kwr->u.rxhdr.ivpresent_to_rxmk_size = 385 V_TLS_KEYCTX_TX_WR_IVPRESENT(0) | 386 V_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT(1) | 387 V_TLS_KEYCTX_TX_WR_RXCK_SIZE(get_cipher_key_size(ck_size)) | 388 V_TLS_KEYCTX_TX_WR_RXMK_SIZE(get_mac_key_size(mk_size)); 389 390 tls_rxkey_flit1(kwr, kctx); 391 392 /* No key reversal for GCM */ 393 if (kctx->state.enc_mode != CH_EVP_CIPH_GCM_MODE) { 394 t4_aes_getdeckey(kwr->keys.edkey, kctx->rx.key, 395 (kctx->cipher_secret_size << 3)); 396 memcpy(kwr->keys.edkey + kctx->cipher_secret_size, 397 kctx->rx.key + kctx->cipher_secret_size, 398 (IPAD_SIZE + OPAD_SIZE)); 399 } else { 400 memcpy(kwr->keys.edkey, kctx->rx.key, 401 (kctx->tx_key_info_size - SALT_SIZE)); 402 memcpy(kwr->u.rxhdr.rxsalt, kctx->rx.salt, SALT_SIZE); 403 } 404 } 405 406 /* Tx key */ 407 static void 408 prepare_txkey_wr(struct tls_keyctx *kwr, struct tls_key_context *kctx) 409 { 410 unsigned int ck_size = kctx->cipher_secret_size; 411 unsigned int mk_size = kctx->mac_secret_size; 412 413 kwr->u.txhdr.ctxlen = 414 (kctx->tx_key_info_size >> 4); 415 kwr->u.txhdr.dualck_to_txvalid = 416 V_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(1) | 417 V_TLS_KEYCTX_TX_WR_SALT_PRESENT(1) | 418 V_TLS_KEYCTX_TX_WR_TXCK_SIZE(get_cipher_key_size(ck_size)) | 419 V_TLS_KEYCTX_TX_WR_TXMK_SIZE(get_mac_key_size(mk_size)) | 420 V_TLS_KEYCTX_TX_WR_TXVALID(1); 421 422 memcpy(kwr->keys.edkey, kctx->tx.key, HDR_KCTX_SIZE); 423 if (kctx->state.enc_mode == CH_EVP_CIPH_GCM_MODE) { 424 memcpy(kwr->u.txhdr.txsalt, kctx->tx.salt, SALT_SIZE); 425 kwr->u.txhdr.dualck_to_txvalid &= 426 ~(V_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(1)); 427 } 428 kwr->u.txhdr.dualck_to_txvalid = htons(kwr->u.txhdr.dualck_to_txvalid); 429 } 430 431 /* TLS Key memory management */ 432 int 433 tls_init_kmap(struct adapter *sc, struct tom_data *td) 434 { 435 436 td->key_map = vmem_create("T4TLS key map", sc->vres.key.start, 437 sc->vres.key.size, 8, 0, M_FIRSTFIT | M_NOWAIT); 438 if (td->key_map == NULL) 439 return (ENOMEM); 440 return (0); 441 } 442 443 void 444 tls_free_kmap(struct tom_data *td) 445 { 446 447 if (td->key_map != NULL) 448 vmem_destroy(td->key_map); 449 } 450 451 static int 452 get_new_keyid(struct toepcb *toep, struct tls_key_context *k_ctx) 453 { 454 struct tom_data *td = toep->td; 455 vmem_addr_t addr; 456 457 if (vmem_alloc(td->key_map, TLS_KEY_CONTEXT_SZ, M_NOWAIT | M_FIRSTFIT, 458 &addr) != 0) 459 return (-1); 460 461 return (addr); 462 } 463 464 static void 465 free_keyid(struct toepcb *toep, int keyid) 466 { 467 struct tom_data *td = toep->td; 468 469 vmem_free(td->key_map, keyid, TLS_KEY_CONTEXT_SZ); 470 } 471 472 static void 473 clear_tls_keyid(struct toepcb *toep) 474 { 475 struct tls_ofld_info *tls_ofld = &toep->tls; 476 477 if (tls_ofld->rx_key_addr >= 0) { 478 free_keyid(toep, tls_ofld->rx_key_addr); 479 tls_ofld->rx_key_addr = -1; 480 } 481 if (tls_ofld->tx_key_addr >= 0) { 482 free_keyid(toep, tls_ofld->tx_key_addr); 483 tls_ofld->tx_key_addr = -1; 484 } 485 } 486 487 static int 488 get_keyid(struct tls_ofld_info *tls_ofld, unsigned int ops) 489 { 490 return (ops & KEY_WRITE_RX ? tls_ofld->rx_key_addr : 491 ((ops & KEY_WRITE_TX) ? tls_ofld->tx_key_addr : -1)); 492 } 493 494 static int 495 get_tp_plen_max(struct tls_ofld_info *tls_ofld) 496 { 497 int plen = ((min(3*4096, TP_TX_PG_SZ))/1448) * 1448; 498 499 return (tls_ofld->k_ctx.frag_size <= 8192 ? plen : FC_TP_PLEN_MAX); 500 } 501 502 /* Send request to get the key-id */ 503 static int 504 tls_program_key_id(struct toepcb *toep, struct tls_key_context *k_ctx) 505 { 506 struct tls_ofld_info *tls_ofld = &toep->tls; 507 struct adapter *sc = td_adapter(toep->td); 508 struct ofld_tx_sdesc *txsd; 509 int kwrlen, kctxlen, keyid, len; 510 struct wrqe *wr; 511 struct tls_key_req *kwr; 512 struct tls_keyctx *kctx; 513 514 kwrlen = roundup2(sizeof(*kwr), 16); 515 kctxlen = roundup2(sizeof(*kctx), 32); 516 len = kwrlen + kctxlen; 517 518 if (toep->txsd_avail == 0) 519 return (EAGAIN); 520 521 /* Dont initialize key for re-neg */ 522 if (!G_KEY_CLR_LOC(k_ctx->l_p_key)) { 523 if ((keyid = get_new_keyid(toep, k_ctx)) < 0) { 524 return (ENOSPC); 525 } 526 } else { 527 keyid = get_keyid(tls_ofld, k_ctx->l_p_key); 528 } 529 530 wr = alloc_wrqe(len, toep->ofld_txq); 531 if (wr == NULL) { 532 free_keyid(toep, keyid); 533 return (ENOMEM); 534 } 535 kwr = wrtod(wr); 536 memset(kwr, 0, kwrlen); 537 538 kwr->wr_hi = htobe32(V_FW_WR_OP(FW_ULPTX_WR) | F_FW_WR_COMPL | 539 F_FW_WR_ATOMIC); 540 kwr->wr_mid = htobe32(V_FW_WR_LEN16(DIV_ROUND_UP(len, 16)) | 541 V_FW_WR_FLOWID(toep->tid)); 542 kwr->protocol = get_proto_ver(k_ctx->proto_ver); 543 kwr->mfs = htons(k_ctx->frag_size); 544 kwr->reneg_to_write_rx = k_ctx->l_p_key; 545 546 /* master command */ 547 kwr->cmd = htobe32(V_ULPTX_CMD(ULP_TX_MEM_WRITE) | 548 V_T5_ULP_MEMIO_ORDER(1) | V_T5_ULP_MEMIO_IMM(1)); 549 kwr->dlen = htobe32(V_ULP_MEMIO_DATA_LEN(kctxlen >> 5)); 550 kwr->len16 = htobe32((toep->tid << 8) | 551 DIV_ROUND_UP(len - sizeof(struct work_request_hdr), 16)); 552 kwr->kaddr = htobe32(V_ULP_MEMIO_ADDR(keyid >> 5)); 553 554 /* sub command */ 555 kwr->sc_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM)); 556 kwr->sc_len = htobe32(kctxlen); 557 558 /* XXX: This assumes that kwrlen == sizeof(*kwr). */ 559 kctx = (struct tls_keyctx *)(kwr + 1); 560 memset(kctx, 0, kctxlen); 561 562 if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_TX) { 563 tls_ofld->tx_key_addr = keyid; 564 prepare_txkey_wr(kctx, k_ctx); 565 } else if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_RX) { 566 tls_ofld->rx_key_addr = keyid; 567 prepare_rxkey_wr(kctx, k_ctx); 568 } 569 570 txsd = &toep->txsd[toep->txsd_pidx]; 571 txsd->tx_credits = DIV_ROUND_UP(len, 16); 572 txsd->plen = 0; 573 toep->tx_credits -= txsd->tx_credits; 574 if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) 575 toep->txsd_pidx = 0; 576 toep->txsd_avail--; 577 578 t4_wrq_tx(sc, wr); 579 580 return (0); 581 } 582 583 /* Store a key received from SSL in DDR. */ 584 static int 585 program_key_context(struct tcpcb *tp, struct toepcb *toep, 586 struct tls_key_context *uk_ctx) 587 { 588 struct adapter *sc = td_adapter(toep->td); 589 struct tls_ofld_info *tls_ofld = &toep->tls; 590 struct tls_key_context *k_ctx; 591 int error, key_offset; 592 593 if (tp->t_state != TCPS_ESTABLISHED) { 594 /* 595 * XXX: Matches Linux driver, but not sure this is a 596 * very appropriate error. 597 */ 598 return (ENOENT); 599 } 600 601 /* Stop timer on handshake completion */ 602 tls_stop_handshake_timer(toep); 603 604 toep->flags &= ~TPF_FORCE_CREDITS; 605 606 CTR4(KTR_CXGBE, "%s: tid %d %s proto_ver %#x", __func__, toep->tid, 607 G_KEY_GET_LOC(uk_ctx->l_p_key) == KEY_WRITE_RX ? "KEY_WRITE_RX" : 608 "KEY_WRITE_TX", uk_ctx->proto_ver); 609 610 if (G_KEY_GET_LOC(uk_ctx->l_p_key) == KEY_WRITE_RX && 611 toep->ulp_mode != ULP_MODE_TLS) 612 return (EOPNOTSUPP); 613 614 /* Don't copy the 'tx' and 'rx' fields. */ 615 k_ctx = &tls_ofld->k_ctx; 616 memcpy(&k_ctx->l_p_key, &uk_ctx->l_p_key, 617 sizeof(*k_ctx) - offsetof(struct tls_key_context, l_p_key)); 618 619 /* TLS version != 1.1 and !1.2 OR DTLS != 1.2 */ 620 if (get_proto_ver(k_ctx->proto_ver) > DTLS_1_2_VERSION) { 621 if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_RX) { 622 tls_ofld->rx_key_addr = -1; 623 t4_clear_rx_quiesce(toep); 624 } else { 625 tls_ofld->tx_key_addr = -1; 626 } 627 return (0); 628 } 629 630 if (k_ctx->state.enc_mode == CH_EVP_CIPH_GCM_MODE) { 631 k_ctx->iv_size = 4; 632 k_ctx->mac_first = 0; 633 k_ctx->hmac_ctrl = 0; 634 } else { 635 k_ctx->iv_size = 8; /* for CBC, iv is 16B, unit of 2B */ 636 k_ctx->mac_first = 1; 637 } 638 639 tls_ofld->scmd0.seqno_numivs = 640 (V_SCMD_SEQ_NO_CTRL(3) | 641 V_SCMD_PROTO_VERSION(get_proto_ver(k_ctx->proto_ver)) | 642 V_SCMD_ENC_DEC_CTRL(SCMD_ENCDECCTRL_ENCRYPT) | 643 V_SCMD_CIPH_AUTH_SEQ_CTRL((k_ctx->mac_first == 0)) | 644 V_SCMD_CIPH_MODE(k_ctx->state.enc_mode) | 645 V_SCMD_AUTH_MODE(k_ctx->state.auth_mode) | 646 V_SCMD_HMAC_CTRL(k_ctx->hmac_ctrl) | 647 V_SCMD_IV_SIZE(k_ctx->iv_size)); 648 649 tls_ofld->scmd0.ivgen_hdrlen = 650 (V_SCMD_IV_GEN_CTRL(k_ctx->iv_ctrl) | 651 V_SCMD_KEY_CTX_INLINE(0) | 652 V_SCMD_TLS_FRAG_ENABLE(1)); 653 654 tls_ofld->mac_length = k_ctx->mac_secret_size; 655 656 if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_RX) { 657 k_ctx->rx = uk_ctx->rx; 658 /* Dont initialize key for re-neg */ 659 if (!G_KEY_CLR_LOC(k_ctx->l_p_key)) 660 tls_ofld->rx_key_addr = -1; 661 } else { 662 k_ctx->tx = uk_ctx->tx; 663 /* Dont initialize key for re-neg */ 664 if (!G_KEY_CLR_LOC(k_ctx->l_p_key)) 665 tls_ofld->tx_key_addr = -1; 666 } 667 668 /* Flush pending data before new Tx key becomes active */ 669 if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_TX) { 670 struct sockbuf *sb; 671 672 /* XXX: This might not drain everything. */ 673 t4_push_frames(sc, toep, 0); 674 sb = &toep->inp->inp_socket->so_snd; 675 SOCKBUF_LOCK(sb); 676 677 /* XXX: This asserts that everything has been pushed. */ 678 MPASS(sb->sb_sndptr == NULL || sb->sb_sndptr->m_next == NULL); 679 sb->sb_sndptr = NULL; 680 tls_ofld->sb_off = sbavail(sb); 681 SOCKBUF_UNLOCK(sb); 682 tls_ofld->tx_seq_no = 0; 683 } 684 685 if ((G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_RX) || 686 (tls_ofld->key_location == TLS_SFO_WR_CONTEXTLOC_DDR)) { 687 error = tls_program_key_id(toep, k_ctx); 688 if (error) { 689 /* XXX: Only clear quiesce for KEY_WRITE_RX? */ 690 t4_clear_rx_quiesce(toep); 691 return (error); 692 } 693 } 694 695 if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_RX) { 696 /* 697 * RX key tags are an index into the key portion of MA 698 * memory stored as an offset from the base address in 699 * units of 64 bytes. 700 */ 701 key_offset = tls_ofld->rx_key_addr - sc->vres.key.start; 702 t4_set_tls_keyid(toep, key_offset / 64); 703 t4_set_tls_tcb_field(toep, W_TCB_ULP_RAW, 704 V_TCB_ULP_RAW(M_TCB_ULP_RAW), 705 V_TCB_ULP_RAW((V_TF_TLS_KEY_SIZE(3) | 706 V_TF_TLS_CONTROL(1) | 707 V_TF_TLS_ACTIVE(1) | 708 V_TF_TLS_ENABLE(1)))); 709 t4_set_tls_tcb_field(toep, W_TCB_TLS_SEQ, 710 V_TCB_TLS_SEQ(M_TCB_TLS_SEQ), 711 V_TCB_TLS_SEQ(0)); 712 t4_clear_rx_quiesce(toep); 713 } else { 714 unsigned short pdus_per_ulp; 715 716 if (tls_ofld->key_location == TLS_SFO_WR_CONTEXTLOC_IMMEDIATE) 717 tls_ofld->tx_key_addr = 1; 718 719 tls_ofld->fcplenmax = get_tp_plen_max(tls_ofld); 720 tls_ofld->expn_per_ulp = tls_expansion_size(toep, 721 tls_ofld->fcplenmax, 1, &pdus_per_ulp); 722 tls_ofld->pdus_per_ulp = pdus_per_ulp; 723 tls_ofld->adjusted_plen = tls_ofld->pdus_per_ulp * 724 ((tls_ofld->expn_per_ulp/tls_ofld->pdus_per_ulp) + 725 tls_ofld->k_ctx.frag_size); 726 } 727 728 return (0); 729 } 730 731 /* 732 * In some cases a client connection can hang without sending the 733 * ServerHelloDone message from the NIC to the host. Send a dummy 734 * RX_DATA_ACK with RX_MODULATE to unstick the connection. 735 */ 736 static void 737 tls_send_handshake_ack(void *arg) 738 { 739 struct toepcb *toep = arg; 740 struct tls_ofld_info *tls_ofld = &toep->tls; 741 struct adapter *sc = td_adapter(toep->td); 742 743 /* 744 * XXX: Does not have the t4_get_tcb() checks to refine the 745 * workaround. 746 */ 747 callout_schedule(&tls_ofld->handshake_timer, TLS_SRV_HELLO_RD_TM * hz); 748 749 CTR2(KTR_CXGBE, "%s: tid %d sending RX_DATA_ACK", __func__, toep->tid); 750 send_rx_modulate(sc, toep); 751 } 752 753 static void 754 tls_start_handshake_timer(struct toepcb *toep) 755 { 756 struct tls_ofld_info *tls_ofld = &toep->tls; 757 758 mtx_lock(&tls_handshake_lock); 759 callout_reset(&tls_ofld->handshake_timer, TLS_SRV_HELLO_BKOFF_TM * hz, 760 tls_send_handshake_ack, toep); 761 mtx_unlock(&tls_handshake_lock); 762 } 763 764 void 765 tls_stop_handshake_timer(struct toepcb *toep) 766 { 767 struct tls_ofld_info *tls_ofld = &toep->tls; 768 769 mtx_lock(&tls_handshake_lock); 770 callout_stop(&tls_ofld->handshake_timer); 771 mtx_unlock(&tls_handshake_lock); 772 } 773 774 int 775 t4_ctloutput_tls(struct socket *so, struct sockopt *sopt) 776 { 777 struct tls_key_context uk_ctx; 778 struct inpcb *inp; 779 struct tcpcb *tp; 780 struct toepcb *toep; 781 int error, optval; 782 783 error = 0; 784 if (sopt->sopt_dir == SOPT_SET && 785 sopt->sopt_name == TCP_TLSOM_SET_TLS_CONTEXT) { 786 error = sooptcopyin(sopt, &uk_ctx, sizeof(uk_ctx), 787 sizeof(uk_ctx)); 788 if (error) 789 return (error); 790 } 791 792 inp = sotoinpcb(so); 793 KASSERT(inp != NULL, ("tcp_ctloutput: inp == NULL")); 794 INP_WLOCK(inp); 795 if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) { 796 INP_WUNLOCK(inp); 797 return (ECONNRESET); 798 } 799 tp = intotcpcb(inp); 800 toep = tp->t_toe; 801 switch (sopt->sopt_dir) { 802 case SOPT_SET: 803 switch (sopt->sopt_name) { 804 case TCP_TLSOM_SET_TLS_CONTEXT: 805 error = program_key_context(tp, toep, &uk_ctx); 806 INP_WUNLOCK(inp); 807 break; 808 case TCP_TLSOM_CLR_TLS_TOM: 809 if (toep->ulp_mode == ULP_MODE_TLS) { 810 CTR2(KTR_CXGBE, "%s: tid %d CLR_TLS_TOM", 811 __func__, toep->tid); 812 tls_clr_ofld_mode(toep); 813 } else 814 error = EOPNOTSUPP; 815 INP_WUNLOCK(inp); 816 break; 817 case TCP_TLSOM_CLR_QUIES: 818 if (toep->ulp_mode == ULP_MODE_TLS) { 819 CTR2(KTR_CXGBE, "%s: tid %d CLR_QUIES", 820 __func__, toep->tid); 821 tls_clr_quiesce(toep); 822 } else 823 error = EOPNOTSUPP; 824 INP_WUNLOCK(inp); 825 break; 826 default: 827 INP_WUNLOCK(inp); 828 error = EOPNOTSUPP; 829 break; 830 } 831 break; 832 case SOPT_GET: 833 switch (sopt->sopt_name) { 834 case TCP_TLSOM_GET_TLS_TOM: 835 /* 836 * TLS TX is permitted on any TOE socket, but 837 * TLS RX requires a TLS ULP mode. 838 */ 839 optval = TLS_TOM_NONE; 840 if (can_tls_offload(td_adapter(toep->td))) { 841 switch (toep->ulp_mode) { 842 case ULP_MODE_NONE: 843 case ULP_MODE_TCPDDP: 844 optval = TLS_TOM_TXONLY; 845 break; 846 case ULP_MODE_TLS: 847 optval = TLS_TOM_BOTH; 848 break; 849 } 850 } 851 CTR3(KTR_CXGBE, "%s: tid %d GET_TLS_TOM = %d", 852 __func__, toep->tid, optval); 853 INP_WUNLOCK(inp); 854 error = sooptcopyout(sopt, &optval, sizeof(optval)); 855 break; 856 default: 857 INP_WUNLOCK(inp); 858 error = EOPNOTSUPP; 859 break; 860 } 861 break; 862 } 863 return (error); 864 } 865 866 void 867 tls_init_toep(struct toepcb *toep) 868 { 869 struct tls_ofld_info *tls_ofld = &toep->tls; 870 871 tls_ofld->key_location = TLS_SFO_WR_CONTEXTLOC_DDR; 872 tls_ofld->rx_key_addr = -1; 873 tls_ofld->tx_key_addr = -1; 874 if (toep->ulp_mode == ULP_MODE_TLS) 875 callout_init_mtx(&tls_ofld->handshake_timer, 876 &tls_handshake_lock, 0); 877 } 878 879 void 880 tls_establish(struct toepcb *toep) 881 { 882 883 /* 884 * Enable PDU extraction. 885 * 886 * XXX: Supposedly this should be done by the firmware when 887 * the ULP_MODE FLOWC parameter is set in send_flowc_wr(), but 888 * in practice this seems to be required. 889 */ 890 CTR2(KTR_CXGBE, "%s: tid %d setting TLS_ENABLE", __func__, toep->tid); 891 t4_set_tls_tcb_field(toep, W_TCB_ULP_RAW, V_TCB_ULP_RAW(M_TCB_ULP_RAW), 892 V_TCB_ULP_RAW(V_TF_TLS_ENABLE(1))); 893 894 toep->flags |= TPF_FORCE_CREDITS; 895 896 tls_start_handshake_timer(toep); 897 } 898 899 void 900 tls_uninit_toep(struct toepcb *toep) 901 { 902 903 if (toep->ulp_mode == ULP_MODE_TLS) 904 tls_stop_handshake_timer(toep); 905 clear_tls_keyid(toep); 906 } 907 908 #define MAX_OFLD_TX_CREDITS (SGE_MAX_WR_LEN / 16) 909 #define MIN_OFLD_TLSTX_CREDITS(toep) \ 910 (howmany(sizeof(struct fw_tlstx_data_wr) + \ 911 sizeof(struct cpl_tx_tls_sfo) + key_size((toep)) + \ 912 CIPHER_BLOCK_SIZE + 1, 16)) 913 914 static inline u_int 915 max_imm_tls_space(int tx_credits) 916 { 917 const int n = 2; /* Use only up to 2 desc for imm. data WR */ 918 int space; 919 920 KASSERT(tx_credits >= 0 && 921 tx_credits <= MAX_OFLD_TX_CREDITS, 922 ("%s: %d credits", __func__, tx_credits)); 923 924 if (tx_credits >= (n * EQ_ESIZE) / 16) 925 space = (n * EQ_ESIZE); 926 else 927 space = tx_credits * 16; 928 return (space); 929 } 930 931 static int 932 count_mbuf_segs(struct mbuf *m, int skip, int len, int *max_nsegs_1mbufp) 933 { 934 int max_nsegs_1mbuf, n, nsegs; 935 936 while (skip >= m->m_len) { 937 skip -= m->m_len; 938 m = m->m_next; 939 } 940 941 nsegs = 0; 942 max_nsegs_1mbuf = 0; 943 while (len > 0) { 944 n = sglist_count(mtod(m, char *) + skip, m->m_len - skip); 945 if (n > max_nsegs_1mbuf) 946 max_nsegs_1mbuf = n; 947 nsegs += n; 948 len -= m->m_len - skip; 949 skip = 0; 950 m = m->m_next; 951 } 952 *max_nsegs_1mbufp = max_nsegs_1mbuf; 953 return (nsegs); 954 } 955 956 static void 957 write_tlstx_wr(struct fw_tlstx_data_wr *txwr, struct toepcb *toep, 958 unsigned int immdlen, unsigned int plen, unsigned int expn, 959 unsigned int pdus, uint8_t credits, int shove, int imm_ivs) 960 { 961 struct tls_ofld_info *tls_ofld = &toep->tls; 962 unsigned int len = plen + expn; 963 964 txwr->op_to_immdlen = htobe32(V_WR_OP(FW_TLSTX_DATA_WR) | 965 V_FW_TLSTX_DATA_WR_COMPL(1) | 966 V_FW_TLSTX_DATA_WR_IMMDLEN(immdlen)); 967 txwr->flowid_len16 = htobe32(V_FW_TLSTX_DATA_WR_FLOWID(toep->tid) | 968 V_FW_TLSTX_DATA_WR_LEN16(credits)); 969 txwr->plen = htobe32(len); 970 txwr->lsodisable_to_flags = htobe32(V_TX_ULP_MODE(ULP_MODE_TLS) | 971 V_TX_URG(0) | /* F_T6_TX_FORCE | */ V_TX_SHOVE(shove)); 972 txwr->ctxloc_to_exp = htobe32(V_FW_TLSTX_DATA_WR_NUMIVS(pdus) | 973 V_FW_TLSTX_DATA_WR_EXP(expn) | 974 V_FW_TLSTX_DATA_WR_CTXLOC(tls_ofld->key_location) | 975 V_FW_TLSTX_DATA_WR_IVDSGL(!imm_ivs) | 976 V_FW_TLSTX_DATA_WR_KEYSIZE(tls_ofld->k_ctx.tx_key_info_size >> 4)); 977 txwr->mfs = htobe16(tls_ofld->k_ctx.frag_size); 978 txwr->adjustedplen_pkd = htobe16( 979 V_FW_TLSTX_DATA_WR_ADJUSTEDPLEN(tls_ofld->adjusted_plen)); 980 txwr->expinplenmax_pkd = htobe16( 981 V_FW_TLSTX_DATA_WR_EXPINPLENMAX(tls_ofld->expn_per_ulp)); 982 txwr->pdusinplenmax_pkd = htobe16( 983 V_FW_TLSTX_DATA_WR_PDUSINPLENMAX(tls_ofld->pdus_per_ulp)); 984 } 985 986 static void 987 write_tlstx_cpl(struct cpl_tx_tls_sfo *cpl, struct toepcb *toep, 988 struct tls_hdr *tls_hdr, unsigned int plen, unsigned int pdus) 989 { 990 struct tls_ofld_info *tls_ofld = &toep->tls; 991 int data_type, seglen; 992 993 if (plen < tls_ofld->k_ctx.frag_size) 994 seglen = plen; 995 else 996 seglen = tls_ofld->k_ctx.frag_size; 997 data_type = tls_content_type(tls_hdr->type); 998 cpl->op_to_seg_len = htobe32(V_CPL_TX_TLS_SFO_OPCODE(CPL_TX_TLS_SFO) | 999 V_CPL_TX_TLS_SFO_DATA_TYPE(data_type) | 1000 V_CPL_TX_TLS_SFO_CPL_LEN(2) | V_CPL_TX_TLS_SFO_SEG_LEN(seglen)); 1001 cpl->pld_len = htobe32(plen); 1002 if (data_type == CPL_TX_TLS_SFO_TYPE_HEARTBEAT) 1003 cpl->type_protover = htobe32( 1004 V_CPL_TX_TLS_SFO_TYPE(tls_hdr->type)); 1005 cpl->seqno_numivs = htobe32(tls_ofld->scmd0.seqno_numivs | 1006 V_SCMD_NUM_IVS(pdus)); 1007 cpl->ivgen_hdrlen = htobe32(tls_ofld->scmd0.ivgen_hdrlen); 1008 cpl->scmd1 = htobe64(tls_ofld->tx_seq_no); 1009 tls_ofld->tx_seq_no += pdus; 1010 } 1011 1012 /* 1013 * Similar to write_tx_sgl() except that it accepts an optional 1014 * trailer buffer for IVs. 1015 */ 1016 static void 1017 write_tlstx_sgl(void *dst, struct mbuf *start, int skip, int plen, 1018 void *iv_buffer, int iv_len, int nsegs, int n) 1019 { 1020 struct mbuf *m; 1021 struct ulptx_sgl *usgl = dst; 1022 int i, j, rc; 1023 struct sglist sg; 1024 struct sglist_seg segs[n]; 1025 1026 KASSERT(nsegs > 0, ("%s: nsegs 0", __func__)); 1027 1028 sglist_init(&sg, n, segs); 1029 usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) | 1030 V_ULPTX_NSGE(nsegs)); 1031 1032 for (m = start; skip >= m->m_len; m = m->m_next) 1033 skip -= m->m_len; 1034 1035 i = -1; 1036 for (m = start; plen > 0; m = m->m_next) { 1037 rc = sglist_append(&sg, mtod(m, char *) + skip, 1038 m->m_len - skip); 1039 if (__predict_false(rc != 0)) 1040 panic("%s: sglist_append %d", __func__, rc); 1041 plen -= m->m_len - skip; 1042 skip = 0; 1043 1044 for (j = 0; j < sg.sg_nseg; i++, j++) { 1045 if (i < 0) { 1046 usgl->len0 = htobe32(segs[j].ss_len); 1047 usgl->addr0 = htobe64(segs[j].ss_paddr); 1048 } else { 1049 usgl->sge[i / 2].len[i & 1] = 1050 htobe32(segs[j].ss_len); 1051 usgl->sge[i / 2].addr[i & 1] = 1052 htobe64(segs[j].ss_paddr); 1053 } 1054 #ifdef INVARIANTS 1055 nsegs--; 1056 #endif 1057 } 1058 sglist_reset(&sg); 1059 } 1060 if (iv_buffer != NULL) { 1061 rc = sglist_append(&sg, iv_buffer, iv_len); 1062 if (__predict_false(rc != 0)) 1063 panic("%s: sglist_append %d", __func__, rc); 1064 1065 for (j = 0; j < sg.sg_nseg; i++, j++) { 1066 if (i < 0) { 1067 usgl->len0 = htobe32(segs[j].ss_len); 1068 usgl->addr0 = htobe64(segs[j].ss_paddr); 1069 } else { 1070 usgl->sge[i / 2].len[i & 1] = 1071 htobe32(segs[j].ss_len); 1072 usgl->sge[i / 2].addr[i & 1] = 1073 htobe64(segs[j].ss_paddr); 1074 } 1075 #ifdef INVARIANTS 1076 nsegs--; 1077 #endif 1078 } 1079 } 1080 if (i & 1) 1081 usgl->sge[i / 2].len[1] = htobe32(0); 1082 KASSERT(nsegs == 0, ("%s: nsegs %d, start %p, iv_buffer %p", 1083 __func__, nsegs, start, iv_buffer)); 1084 } 1085 1086 /* 1087 * Similar to t4_push_frames() but handles TLS sockets when TLS offload 1088 * is enabled. Rather than transmitting bulk data, the socket buffer 1089 * contains TLS records. The work request requires a full TLS record, 1090 * so batch mbufs up until a full TLS record is seen. This requires 1091 * reading the TLS header out of the start of each record to determine 1092 * its length. 1093 */ 1094 void 1095 t4_push_tls_records(struct adapter *sc, struct toepcb *toep, int drop) 1096 { 1097 struct tls_hdr thdr; 1098 struct mbuf *sndptr; 1099 struct fw_tlstx_data_wr *txwr; 1100 struct cpl_tx_tls_sfo *cpl; 1101 struct wrqe *wr; 1102 u_int plen, nsegs, credits, space, max_nsegs_1mbuf, wr_len; 1103 u_int expn_size, iv_len, pdus, sndptroff; 1104 struct tls_ofld_info *tls_ofld = &toep->tls; 1105 struct inpcb *inp = toep->inp; 1106 struct tcpcb *tp = intotcpcb(inp); 1107 struct socket *so = inp->inp_socket; 1108 struct sockbuf *sb = &so->so_snd; 1109 int tls_size, tx_credits, shove, /* compl,*/ sowwakeup; 1110 struct ofld_tx_sdesc *txsd; 1111 bool imm_ivs, imm_payload; 1112 void *iv_buffer, *iv_dst, *buf; 1113 1114 INP_WLOCK_ASSERT(inp); 1115 KASSERT(toep->flags & TPF_FLOWC_WR_SENT, 1116 ("%s: flowc_wr not sent for tid %u.", __func__, toep->tid)); 1117 1118 KASSERT(toep->ulp_mode == ULP_MODE_NONE || 1119 toep->ulp_mode == ULP_MODE_TCPDDP || toep->ulp_mode == ULP_MODE_TLS, 1120 ("%s: ulp_mode %u for toep %p", __func__, toep->ulp_mode, toep)); 1121 KASSERT(tls_tx_key(toep), 1122 ("%s: TX key not set for toep %p", __func__, toep)); 1123 1124 #ifdef VERBOSE_TRACES 1125 CTR4(KTR_CXGBE, "%s: tid %d toep flags %#x tp flags %#x drop %d", 1126 __func__, toep->tid, toep->flags, tp->t_flags); 1127 #endif 1128 if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN)) 1129 return; 1130 1131 #ifdef RATELIMIT 1132 if (__predict_false(inp->inp_flags2 & INP_RATE_LIMIT_CHANGED) && 1133 (update_tx_rate_limit(sc, toep, so->so_max_pacing_rate) == 0)) { 1134 inp->inp_flags2 &= ~INP_RATE_LIMIT_CHANGED; 1135 } 1136 #endif 1137 1138 /* 1139 * This function doesn't resume by itself. Someone else must clear the 1140 * flag and call this function. 1141 */ 1142 if (__predict_false(toep->flags & TPF_TX_SUSPENDED)) { 1143 KASSERT(drop == 0, 1144 ("%s: drop (%d) != 0 but tx is suspended", __func__, drop)); 1145 return; 1146 } 1147 1148 txsd = &toep->txsd[toep->txsd_pidx]; 1149 for (;;) { 1150 tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS); 1151 space = max_imm_tls_space(tx_credits); 1152 wr_len = sizeof(struct fw_tlstx_data_wr) + 1153 sizeof(struct cpl_tx_tls_sfo) + key_size(toep); 1154 if (wr_len + CIPHER_BLOCK_SIZE + 1 > space) { 1155 #ifdef VERBOSE_TRACES 1156 CTR5(KTR_CXGBE, 1157 "%s: tid %d tx_credits %d min_wr %d space %d", 1158 __func__, toep->tid, tx_credits, wr_len + 1159 CIPHER_BLOCK_SIZE + 1, space); 1160 #endif 1161 return; 1162 } 1163 1164 SOCKBUF_LOCK(sb); 1165 sowwakeup = drop; 1166 if (drop) { 1167 sbdrop_locked(sb, drop); 1168 MPASS(tls_ofld->sb_off >= drop); 1169 tls_ofld->sb_off -= drop; 1170 drop = 0; 1171 } 1172 1173 /* 1174 * Send a FIN if requested, but only if there's no 1175 * more data to send. 1176 */ 1177 if (sbavail(sb) == tls_ofld->sb_off && 1178 toep->flags & TPF_SEND_FIN) { 1179 if (sowwakeup) 1180 sowwakeup_locked(so); 1181 else 1182 SOCKBUF_UNLOCK(sb); 1183 SOCKBUF_UNLOCK_ASSERT(sb); 1184 t4_close_conn(sc, toep); 1185 return; 1186 } 1187 1188 if (sbavail(sb) < tls_ofld->sb_off + TLS_HEADER_LENGTH) { 1189 /* 1190 * A full TLS header is not yet queued, stop 1191 * for now until more data is added to the 1192 * socket buffer. 1193 */ 1194 #ifdef VERBOSE_TRACES 1195 CTR4(KTR_CXGBE, "%s: tid %d sbavail %d sb_off %d", 1196 __func__, toep->tid, sbavail(sb), tls_ofld->sb_off); 1197 #endif 1198 if (sowwakeup) 1199 sowwakeup_locked(so); 1200 else 1201 SOCKBUF_UNLOCK(sb); 1202 SOCKBUF_UNLOCK_ASSERT(sb); 1203 return; 1204 } 1205 1206 /* Read the header of the next TLS record. */ 1207 sndptr = sbsndmbuf(sb, tls_ofld->sb_off, &sndptroff); 1208 MPASS(!IS_AIOTX_MBUF(sndptr)); 1209 m_copydata(sndptr, sndptroff, sizeof(thdr), (caddr_t)&thdr); 1210 tls_size = htons(thdr.length); 1211 plen = TLS_HEADER_LENGTH + tls_size; 1212 pdus = howmany(tls_size, tls_ofld->k_ctx.frag_size); 1213 iv_len = pdus * CIPHER_BLOCK_SIZE; 1214 1215 if (sbavail(sb) < tls_ofld->sb_off + plen) { 1216 /* 1217 * The full TLS record is not yet queued, stop 1218 * for now until more data is added to the 1219 * socket buffer. 1220 */ 1221 #ifdef VERBOSE_TRACES 1222 CTR5(KTR_CXGBE, 1223 "%s: tid %d sbavail %d sb_off %d plen %d", 1224 __func__, toep->tid, sbavail(sb), tls_ofld->sb_off, 1225 plen); 1226 #endif 1227 if (sowwakeup) 1228 sowwakeup_locked(so); 1229 else 1230 SOCKBUF_UNLOCK(sb); 1231 SOCKBUF_UNLOCK_ASSERT(sb); 1232 return; 1233 } 1234 1235 /* Shove if there is no additional data pending. */ 1236 shove = (sbavail(sb) == tls_ofld->sb_off + plen) && 1237 !(tp->t_flags & TF_MORETOCOME); 1238 1239 if (sb->sb_flags & SB_AUTOSIZE && 1240 V_tcp_do_autosndbuf && 1241 sb->sb_hiwat < V_tcp_autosndbuf_max && 1242 sbused(sb) >= sb->sb_hiwat * 7 / 8) { 1243 int newsize = min(sb->sb_hiwat + V_tcp_autosndbuf_inc, 1244 V_tcp_autosndbuf_max); 1245 1246 if (!sbreserve_locked(sb, newsize, so, NULL)) 1247 sb->sb_flags &= ~SB_AUTOSIZE; 1248 else 1249 sowwakeup = 1; /* room available */ 1250 } 1251 if (sowwakeup) 1252 sowwakeup_locked(so); 1253 else 1254 SOCKBUF_UNLOCK(sb); 1255 SOCKBUF_UNLOCK_ASSERT(sb); 1256 1257 if (__predict_false(toep->flags & TPF_FIN_SENT)) 1258 panic("%s: excess tx.", __func__); 1259 1260 /* Determine whether to use immediate vs SGL. */ 1261 imm_payload = false; 1262 imm_ivs = false; 1263 if (wr_len + iv_len <= space) { 1264 imm_ivs = true; 1265 wr_len += iv_len; 1266 if (wr_len + tls_size <= space) { 1267 wr_len += tls_size; 1268 imm_payload = true; 1269 } 1270 } 1271 1272 /* Allocate space for IVs if needed. */ 1273 if (!imm_ivs) { 1274 iv_buffer = malloc(iv_len, M_CXGBE, M_NOWAIT); 1275 if (iv_buffer == NULL) { 1276 /* 1277 * XXX: How to restart this? 1278 */ 1279 if (sowwakeup) 1280 sowwakeup_locked(so); 1281 else 1282 SOCKBUF_UNLOCK(sb); 1283 SOCKBUF_UNLOCK_ASSERT(sb); 1284 CTR3(KTR_CXGBE, 1285 "%s: tid %d failed to alloc IV space len %d", 1286 __func__, toep->tid, iv_len); 1287 return; 1288 } 1289 } else 1290 iv_buffer = NULL; 1291 1292 /* Determine size of SGL. */ 1293 nsegs = 0; 1294 max_nsegs_1mbuf = 0; /* max # of SGL segments in any one mbuf */ 1295 if (!imm_payload) { 1296 nsegs = count_mbuf_segs(sndptr, sndptroff + 1297 TLS_HEADER_LENGTH, tls_size, &max_nsegs_1mbuf); 1298 if (!imm_ivs) { 1299 int n = sglist_count(iv_buffer, iv_len); 1300 nsegs += n; 1301 if (n > max_nsegs_1mbuf) 1302 max_nsegs_1mbuf = n; 1303 } 1304 1305 /* Account for SGL in work request length. */ 1306 wr_len += sizeof(struct ulptx_sgl) + 1307 ((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8; 1308 } 1309 1310 wr = alloc_wrqe(roundup2(wr_len, 16), toep->ofld_txq); 1311 if (wr == NULL) { 1312 /* XXX: how will we recover from this? */ 1313 toep->flags |= TPF_TX_SUSPENDED; 1314 return; 1315 } 1316 1317 #ifdef VERBOSE_TRACES 1318 CTR5(KTR_CXGBE, "%s: tid %d TLS record %d len %#x pdus %d", 1319 __func__, toep->tid, thdr.type, tls_size, pdus); 1320 #endif 1321 txwr = wrtod(wr); 1322 cpl = (struct cpl_tx_tls_sfo *)(txwr + 1); 1323 memset(txwr, 0, roundup2(wr_len, 16)); 1324 credits = howmany(wr_len, 16); 1325 expn_size = tls_expansion_size(toep, tls_size, 0, NULL); 1326 write_tlstx_wr(txwr, toep, imm_payload ? tls_size : 0, 1327 tls_size, expn_size, pdus, credits, shove, imm_ivs ? 1 : 0); 1328 write_tlstx_cpl(cpl, toep, &thdr, tls_size, pdus); 1329 tls_copy_tx_key(toep, cpl + 1); 1330 1331 /* Generate random IVs */ 1332 buf = (char *)(cpl + 1) + key_size(toep); 1333 if (imm_ivs) { 1334 MPASS(iv_buffer == NULL); 1335 iv_dst = buf; 1336 buf = (char *)iv_dst + iv_len; 1337 } else 1338 iv_dst = iv_buffer; 1339 arc4rand(iv_dst, iv_len, 0); 1340 1341 if (imm_payload) { 1342 m_copydata(sndptr, sndptroff + TLS_HEADER_LENGTH, 1343 tls_size, buf); 1344 } else { 1345 write_tlstx_sgl(buf, sndptr, 1346 sndptroff + TLS_HEADER_LENGTH, tls_size, iv_buffer, 1347 iv_len, nsegs, max_nsegs_1mbuf); 1348 } 1349 1350 KASSERT(toep->tx_credits >= credits, 1351 ("%s: not enough credits", __func__)); 1352 1353 toep->tx_credits -= credits; 1354 1355 tp->snd_nxt += plen; 1356 tp->snd_max += plen; 1357 1358 SOCKBUF_LOCK(sb); 1359 sbsndptr(sb, tls_ofld->sb_off, plen, &sndptroff); 1360 tls_ofld->sb_off += plen; 1361 SOCKBUF_UNLOCK(sb); 1362 1363 toep->flags |= TPF_TX_DATA_SENT; 1364 if (toep->tx_credits < MIN_OFLD_TLSTX_CREDITS(toep)) 1365 toep->flags |= TPF_TX_SUSPENDED; 1366 1367 KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__)); 1368 txsd->plen = plen; 1369 txsd->tx_credits = credits; 1370 txsd->iv_buffer = iv_buffer; 1371 txsd++; 1372 if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) { 1373 toep->txsd_pidx = 0; 1374 txsd = &toep->txsd[0]; 1375 } 1376 toep->txsd_avail--; 1377 1378 atomic_add_long(&toep->vi->pi->tx_tls_records, 1); 1379 atomic_add_long(&toep->vi->pi->tx_tls_octets, plen); 1380 1381 t4_l2t_send(sc, wr, toep->l2te); 1382 } 1383 } 1384 1385 /* 1386 * For TLS data we place received mbufs received via CPL_TLS_DATA into 1387 * an mbufq in the TLS offload state. When CPL_RX_TLS_CMP is 1388 * received, the completed PDUs are placed into the socket receive 1389 * buffer. 1390 * 1391 * The TLS code reuses the ulp_pdu_reclaimq to hold the pending mbufs. 1392 */ 1393 static int 1394 do_tls_data(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 1395 { 1396 struct adapter *sc = iq->adapter; 1397 const struct cpl_tls_data *cpl = mtod(m, const void *); 1398 unsigned int tid = GET_TID(cpl); 1399 struct toepcb *toep = lookup_tid(sc, tid); 1400 struct inpcb *inp = toep->inp; 1401 struct tcpcb *tp; 1402 int len; 1403 1404 /* XXX: Should this match do_rx_data instead? */ 1405 KASSERT(!(toep->flags & TPF_SYNQE), 1406 ("%s: toep %p claims to be a synq entry", __func__, toep)); 1407 1408 KASSERT(toep->tid == tid, ("%s: toep tid/atid mismatch", __func__)); 1409 1410 /* strip off CPL header */ 1411 m_adj(m, sizeof(*cpl)); 1412 len = m->m_pkthdr.len; 1413 1414 atomic_add_long(&toep->vi->pi->rx_tls_octets, len); 1415 1416 KASSERT(len == G_CPL_TLS_DATA_LENGTH(be32toh(cpl->length_pkd)), 1417 ("%s: payload length mismatch", __func__)); 1418 1419 INP_WLOCK(inp); 1420 if (inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) { 1421 CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x", 1422 __func__, tid, len, inp->inp_flags); 1423 INP_WUNLOCK(inp); 1424 m_freem(m); 1425 return (0); 1426 } 1427 1428 /* Save TCP sequence number. */ 1429 m->m_pkthdr.tls_tcp_seq = be32toh(cpl->seq); 1430 1431 if (mbufq_enqueue(&toep->ulp_pdu_reclaimq, m)) { 1432 #ifdef INVARIANTS 1433 panic("Failed to queue TLS data packet"); 1434 #else 1435 printf("%s: Failed to queue TLS data packet\n", __func__); 1436 INP_WUNLOCK(inp); 1437 m_freem(m); 1438 return (0); 1439 #endif 1440 } 1441 1442 tp = intotcpcb(inp); 1443 tp->t_rcvtime = ticks; 1444 1445 #ifdef VERBOSE_TRACES 1446 CTR4(KTR_CXGBE, "%s: tid %u len %d seq %u", __func__, tid, len, 1447 be32toh(cpl->seq)); 1448 #endif 1449 1450 INP_WUNLOCK(inp); 1451 return (0); 1452 } 1453 1454 static int 1455 do_rx_tls_cmp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 1456 { 1457 struct adapter *sc = iq->adapter; 1458 const struct cpl_rx_tls_cmp *cpl = mtod(m, const void *); 1459 struct tlsrx_hdr_pkt *tls_hdr_pkt; 1460 unsigned int tid = GET_TID(cpl); 1461 struct toepcb *toep = lookup_tid(sc, tid); 1462 struct inpcb *inp = toep->inp; 1463 struct tcpcb *tp; 1464 struct socket *so; 1465 struct sockbuf *sb; 1466 struct mbuf *tls_data; 1467 int len, pdu_length, pdu_overhead, sb_length; 1468 1469 KASSERT(toep->tid == tid, ("%s: toep tid/atid mismatch", __func__)); 1470 KASSERT(!(toep->flags & TPF_SYNQE), 1471 ("%s: toep %p claims to be a synq entry", __func__, toep)); 1472 1473 /* strip off CPL header */ 1474 m_adj(m, sizeof(*cpl)); 1475 len = m->m_pkthdr.len; 1476 1477 atomic_add_long(&toep->vi->pi->rx_tls_records, 1); 1478 1479 KASSERT(len == G_CPL_RX_TLS_CMP_LENGTH(be32toh(cpl->pdulength_length)), 1480 ("%s: payload length mismatch", __func__)); 1481 1482 INP_WLOCK(inp); 1483 if (inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) { 1484 CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x", 1485 __func__, tid, len, inp->inp_flags); 1486 INP_WUNLOCK(inp); 1487 m_freem(m); 1488 return (0); 1489 } 1490 1491 pdu_length = G_CPL_RX_TLS_CMP_PDULENGTH(be32toh(cpl->pdulength_length)); 1492 1493 tp = intotcpcb(inp); 1494 1495 #ifdef VERBOSE_TRACES 1496 CTR6(KTR_CXGBE, "%s: tid %u PDU len %d len %d seq %u, rcv_nxt %u", 1497 __func__, tid, pdu_length, len, be32toh(cpl->seq), tp->rcv_nxt); 1498 #endif 1499 1500 tp->rcv_nxt += pdu_length; 1501 if (tp->rcv_wnd < pdu_length) { 1502 toep->tls.rcv_over += pdu_length - tp->rcv_wnd; 1503 tp->rcv_wnd = 0; 1504 } else 1505 tp->rcv_wnd -= pdu_length; 1506 1507 /* XXX: Not sure what to do about urgent data. */ 1508 1509 /* 1510 * The payload of this CPL is the TLS header followed by 1511 * additional fields. 1512 */ 1513 KASSERT(m->m_len >= sizeof(*tls_hdr_pkt), 1514 ("%s: payload too small", __func__)); 1515 tls_hdr_pkt = mtod(m, void *); 1516 1517 /* 1518 * Only the TLS header is sent to OpenSSL, so report errors by 1519 * altering the record type. 1520 */ 1521 if ((tls_hdr_pkt->res_to_mac_error & M_TLSRX_HDR_PKT_ERROR) != 0) 1522 tls_hdr_pkt->type = CONTENT_TYPE_ERROR; 1523 1524 /* Trim this CPL's mbuf to only include the TLS header. */ 1525 KASSERT(m->m_len == len && m->m_next == NULL, 1526 ("%s: CPL spans multiple mbufs", __func__)); 1527 m->m_len = TLS_HEADER_LENGTH; 1528 m->m_pkthdr.len = TLS_HEADER_LENGTH; 1529 1530 tls_data = mbufq_dequeue(&toep->ulp_pdu_reclaimq); 1531 if (tls_data != NULL) { 1532 KASSERT(be32toh(cpl->seq) == tls_data->m_pkthdr.tls_tcp_seq, 1533 ("%s: sequence mismatch", __func__)); 1534 1535 /* 1536 * Update the TLS header length to be the length of 1537 * the payload data. 1538 */ 1539 tls_hdr_pkt->length = htobe16(tls_data->m_pkthdr.len); 1540 1541 m->m_next = tls_data; 1542 m->m_pkthdr.len += tls_data->m_len; 1543 } 1544 1545 so = inp_inpcbtosocket(inp); 1546 sb = &so->so_rcv; 1547 SOCKBUF_LOCK(sb); 1548 1549 if (__predict_false(sb->sb_state & SBS_CANTRCVMORE)) { 1550 CTR3(KTR_CXGBE, "%s: tid %u, excess rx (%d bytes)", 1551 __func__, tid, pdu_length); 1552 m_freem(m); 1553 SOCKBUF_UNLOCK(sb); 1554 INP_WUNLOCK(inp); 1555 1556 CURVNET_SET(toep->vnet); 1557 INP_INFO_RLOCK(&V_tcbinfo); 1558 INP_WLOCK(inp); 1559 tp = tcp_drop(tp, ECONNRESET); 1560 if (tp) 1561 INP_WUNLOCK(inp); 1562 INP_INFO_RUNLOCK(&V_tcbinfo); 1563 CURVNET_RESTORE(); 1564 1565 return (0); 1566 } 1567 1568 /* 1569 * Not all of the bytes on the wire are included in the socket 1570 * buffer (e.g. the MAC of the TLS record). However, those 1571 * bytes are included in the TCP sequence space. To handle 1572 * this, compute the delta for this TLS record in 1573 * 'pdu_overhead' and treat those bytes as having already been 1574 * "read" by the application for the purposes of expanding the 1575 * window. The meat of the TLS record passed to the 1576 * application ('sb_length') will still not be counted as 1577 * "read" until userland actually reads the bytes. 1578 * 1579 * XXX: Some of the calculations below are probably still not 1580 * really correct. 1581 */ 1582 sb_length = m->m_pkthdr.len; 1583 pdu_overhead = pdu_length - sb_length; 1584 toep->rx_credits += pdu_overhead; 1585 tp->rcv_wnd += pdu_overhead; 1586 tp->rcv_adv += pdu_overhead; 1587 1588 /* receive buffer autosize */ 1589 MPASS(toep->vnet == so->so_vnet); 1590 CURVNET_SET(toep->vnet); 1591 if (sb->sb_flags & SB_AUTOSIZE && 1592 V_tcp_do_autorcvbuf && 1593 sb->sb_hiwat < V_tcp_autorcvbuf_max && 1594 sb_length > (sbspace(sb) / 8 * 7)) { 1595 unsigned int hiwat = sb->sb_hiwat; 1596 unsigned int newsize = min(hiwat + V_tcp_autorcvbuf_inc, 1597 V_tcp_autorcvbuf_max); 1598 1599 if (!sbreserve_locked(sb, newsize, so, NULL)) 1600 sb->sb_flags &= ~SB_AUTOSIZE; 1601 else 1602 toep->rx_credits += newsize - hiwat; 1603 } 1604 1605 KASSERT(toep->sb_cc >= sbused(sb), 1606 ("%s: sb %p has more data (%d) than last time (%d).", 1607 __func__, sb, sbused(sb), toep->sb_cc)); 1608 toep->rx_credits += toep->sb_cc - sbused(sb); 1609 sbappendstream_locked(sb, m, 0); 1610 toep->sb_cc = sbused(sb); 1611 #ifdef VERBOSE_TRACES 1612 CTR5(KTR_CXGBE, "%s: tid %u PDU overhead %d rx_credits %u rcv_wnd %u", 1613 __func__, tid, pdu_overhead, toep->rx_credits, tp->rcv_wnd); 1614 #endif 1615 if (toep->rx_credits > 0 && toep->sb_cc + tp->rcv_wnd < sb->sb_lowat) { 1616 int credits; 1617 1618 credits = send_rx_credits(sc, toep, toep->rx_credits); 1619 toep->rx_credits -= credits; 1620 tp->rcv_wnd += credits; 1621 tp->rcv_adv += credits; 1622 } 1623 1624 sorwakeup_locked(so); 1625 SOCKBUF_UNLOCK_ASSERT(sb); 1626 1627 INP_WUNLOCK(inp); 1628 CURVNET_RESTORE(); 1629 return (0); 1630 } 1631 1632 void 1633 t4_tls_mod_load(void) 1634 { 1635 1636 mtx_init(&tls_handshake_lock, "t4tls handshake", NULL, MTX_DEF); 1637 t4_register_cpl_handler(CPL_TLS_DATA, do_tls_data); 1638 t4_register_cpl_handler(CPL_RX_TLS_CMP, do_rx_tls_cmp); 1639 } 1640 1641 void 1642 t4_tls_mod_unload(void) 1643 { 1644 1645 t4_register_cpl_handler(CPL_TLS_DATA, NULL); 1646 t4_register_cpl_handler(CPL_RX_TLS_CMP, NULL); 1647 mtx_destroy(&tls_handshake_lock); 1648 } 1649 #endif /* TCP_OFFLOAD */ 1650