1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2017-2018 Chelsio Communications, Inc. 5 * All rights reserved. 6 * Written by: John Baldwin <jhb@FreeBSD.org> 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include "opt_inet.h" 31 #include "opt_kern_tls.h" 32 33 #include <sys/cdefs.h> 34 __FBSDID("$FreeBSD$"); 35 36 #ifdef KERN_TLS 37 #include <sys/param.h> 38 #include <sys/ktr.h> 39 #include <sys/ktls.h> 40 #include <sys/sglist.h> 41 #include <sys/socket.h> 42 #include <sys/socketvar.h> 43 #include <sys/systm.h> 44 #include <netinet/in.h> 45 #include <netinet/in_pcb.h> 46 #include <netinet/tcp_var.h> 47 #include <netinet/toecore.h> 48 #include <opencrypto/cryptodev.h> 49 #include <opencrypto/xform.h> 50 51 #ifdef TCP_OFFLOAD 52 #include "common/common.h" 53 #include "common/t4_tcb.h" 54 #include "crypto/t4_crypto.h" 55 #include "tom/t4_tom_l2t.h" 56 #include "tom/t4_tom.h" 57 58 /* 59 * The TCP sequence number of a CPL_TLS_DATA mbuf is saved here while 60 * the mbuf is in the ulp_pdu_reclaimq. 61 */ 62 #define tls_tcp_seq PH_loc.thirtytwo[0] 63 64 static void 65 t4_set_tls_tcb_field(struct toepcb *toep, uint16_t word, uint64_t mask, 66 uint64_t val) 67 { 68 struct adapter *sc = td_adapter(toep->td); 69 70 t4_set_tcb_field(sc, &toep->ofld_txq->wrq, toep, word, mask, val, 0, 0); 71 } 72 73 /* TLS and DTLS common routines */ 74 bool 75 can_tls_offload(struct adapter *sc) 76 { 77 78 return (sc->tt.tls && sc->cryptocaps & FW_CAPS_CONFIG_TLSKEYS); 79 } 80 81 int 82 tls_tx_key(struct toepcb *toep) 83 { 84 struct tls_ofld_info *tls_ofld = &toep->tls; 85 86 return (tls_ofld->tx_key_addr >= 0); 87 } 88 89 /* Set TLS Key-Id in TCB */ 90 static void 91 t4_set_tls_keyid(struct toepcb *toep, unsigned int key_id) 92 { 93 94 t4_set_tls_tcb_field(toep, W_TCB_RX_TLS_KEY_TAG, 95 V_TCB_RX_TLS_KEY_TAG(M_TCB_RX_TLS_BUF_TAG), 96 V_TCB_RX_TLS_KEY_TAG(key_id)); 97 } 98 99 /* Clear TF_RX_QUIESCE to re-enable receive. */ 100 static void 101 t4_clear_rx_quiesce(struct toepcb *toep) 102 { 103 104 t4_set_tls_tcb_field(toep, W_TCB_T_FLAGS, V_TF_RX_QUIESCE(1), 0); 105 } 106 107 static void 108 tls_clr_ofld_mode(struct toepcb *toep) 109 { 110 111 tls_stop_handshake_timer(toep); 112 113 KASSERT(toep->tls.rx_key_addr == -1, 114 ("%s: tid %d has RX key", __func__, toep->tid)); 115 116 /* Switch to plain TOE mode. */ 117 t4_set_tls_tcb_field(toep, W_TCB_ULP_RAW, 118 V_TCB_ULP_RAW(V_TF_TLS_ENABLE(1)), 119 V_TCB_ULP_RAW(V_TF_TLS_ENABLE(0))); 120 t4_set_tls_tcb_field(toep, W_TCB_ULP_TYPE, 121 V_TCB_ULP_TYPE(M_TCB_ULP_TYPE), V_TCB_ULP_TYPE(ULP_MODE_NONE)); 122 t4_clear_rx_quiesce(toep); 123 124 toep->flags &= ~(TPF_FORCE_CREDITS | TPF_TLS_ESTABLISHED); 125 toep->params.ulp_mode = ULP_MODE_NONE; 126 } 127 128 /* TLS/DTLS content type for CPL SFO */ 129 static inline unsigned char 130 tls_content_type(unsigned char content_type) 131 { 132 /* 133 * XXX: Shouldn't this map CONTENT_TYPE_APP_DATA to DATA and 134 * default to "CUSTOM" for all other types including 135 * heartbeat? 136 */ 137 switch (content_type) { 138 case CONTENT_TYPE_CCS: 139 return CPL_TX_TLS_SFO_TYPE_CCS; 140 case CONTENT_TYPE_ALERT: 141 return CPL_TX_TLS_SFO_TYPE_ALERT; 142 case CONTENT_TYPE_HANDSHAKE: 143 return CPL_TX_TLS_SFO_TYPE_HANDSHAKE; 144 case CONTENT_TYPE_HEARTBEAT: 145 return CPL_TX_TLS_SFO_TYPE_HEARTBEAT; 146 } 147 return CPL_TX_TLS_SFO_TYPE_DATA; 148 } 149 150 static int 151 tls_key_info_size(struct ktls_session *tls) 152 { 153 u_int key_info_size, mac_key_size; 154 155 key_info_size = sizeof(struct tx_keyctx_hdr) + 156 tls->params.cipher_key_len; 157 if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) { 158 key_info_size += GMAC_BLOCK_LEN; 159 } else { 160 switch (tls->params.auth_algorithm) { 161 case CRYPTO_SHA1_HMAC: 162 mac_key_size = SHA1_HASH_LEN; 163 break; 164 case CRYPTO_SHA2_256_HMAC: 165 mac_key_size = SHA2_256_HASH_LEN; 166 break; 167 case CRYPTO_SHA2_384_HMAC: 168 mac_key_size = SHA2_512_HASH_LEN; 169 break; 170 default: 171 __assert_unreachable(); 172 } 173 key_info_size += roundup2(mac_key_size, 16) * 2; 174 } 175 return (key_info_size); 176 } 177 178 static int 179 tls_proto_ver(struct ktls_session *tls) 180 { 181 if (tls->params.tls_vminor == TLS_MINOR_VER_ONE) 182 return (SCMD_PROTO_VERSION_TLS_1_1); 183 else 184 return (SCMD_PROTO_VERSION_TLS_1_2); 185 } 186 187 static int 188 tls_cipher_mode(struct ktls_session *tls) 189 { 190 switch (tls->params.cipher_algorithm) { 191 case CRYPTO_AES_CBC: 192 return (SCMD_CIPH_MODE_AES_CBC); 193 case CRYPTO_AES_NIST_GCM_16: 194 return (SCMD_CIPH_MODE_AES_GCM); 195 default: 196 return (SCMD_CIPH_MODE_NOP); 197 } 198 } 199 200 static int 201 tls_auth_mode(struct ktls_session *tls) 202 { 203 switch (tls->params.cipher_algorithm) { 204 case CRYPTO_AES_CBC: 205 switch (tls->params.auth_algorithm) { 206 case CRYPTO_SHA1_HMAC: 207 return (SCMD_AUTH_MODE_SHA1); 208 case CRYPTO_SHA2_256_HMAC: 209 return (SCMD_AUTH_MODE_SHA256); 210 case CRYPTO_SHA2_384_HMAC: 211 return (SCMD_AUTH_MODE_SHA512_384); 212 default: 213 return (SCMD_AUTH_MODE_NOP); 214 } 215 case CRYPTO_AES_NIST_GCM_16: 216 return (SCMD_AUTH_MODE_GHASH); 217 default: 218 return (SCMD_AUTH_MODE_NOP); 219 } 220 } 221 222 static int 223 tls_hmac_ctrl(struct ktls_session *tls) 224 { 225 switch (tls->params.cipher_algorithm) { 226 case CRYPTO_AES_CBC: 227 return (SCMD_HMAC_CTRL_NO_TRUNC); 228 case CRYPTO_AES_NIST_GCM_16: 229 return (SCMD_HMAC_CTRL_NOP); 230 default: 231 return (SCMD_HMAC_CTRL_NOP); 232 } 233 } 234 235 static int 236 tls_cipher_key_size(struct ktls_session *tls) 237 { 238 switch (tls->params.cipher_key_len) { 239 case 128 / 8: 240 return (CHCR_KEYCTX_CIPHER_KEY_SIZE_128); 241 case 192 / 8: 242 return (CHCR_KEYCTX_CIPHER_KEY_SIZE_192); 243 case 256 / 8: 244 return (CHCR_KEYCTX_CIPHER_KEY_SIZE_256); 245 default: 246 __assert_unreachable(); 247 } 248 } 249 250 static int 251 tls_mac_key_size(struct ktls_session *tls) 252 { 253 if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) 254 /* 255 * XXX: This used to use 128 (SHA_NOP) for TOE, 256 * but NIC TLS has always used 512. 257 */ 258 return (CHCR_KEYCTX_MAC_KEY_SIZE_512); 259 else { 260 switch (tls->params.auth_algorithm) { 261 case CRYPTO_SHA1_HMAC: 262 return (CHCR_KEYCTX_MAC_KEY_SIZE_160); 263 case CRYPTO_SHA2_256_HMAC: 264 return (CHCR_KEYCTX_MAC_KEY_SIZE_256); 265 case CRYPTO_SHA2_384_HMAC: 266 return (CHCR_KEYCTX_MAC_KEY_SIZE_512); 267 default: 268 __assert_unreachable(); 269 } 270 } 271 } 272 273 static void 274 prepare_tls_keys(char *key, char *salt, struct ktls_session *tls, 275 int direction) 276 { 277 struct auth_hash *axf; 278 u_int mac_key_size; 279 char *hash; 280 281 if (direction == KTLS_RX && 282 tls->params.cipher_algorithm == CRYPTO_AES_CBC) 283 t4_aes_getdeckey(key, tls->params.cipher_key, 284 tls->params.cipher_key_len * 8); 285 else 286 memcpy(key, tls->params.cipher_key, 287 tls->params.cipher_key_len); 288 hash = key + tls->params.cipher_key_len; 289 if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) { 290 memcpy(salt, tls->params.iv, SALT_SIZE); 291 t4_init_gmac_hash(tls->params.cipher_key, 292 tls->params.cipher_key_len, hash); 293 } else { 294 switch (tls->params.auth_algorithm) { 295 case CRYPTO_SHA1_HMAC: 296 axf = &auth_hash_hmac_sha1; 297 mac_key_size = SHA1_HASH_LEN; 298 break; 299 case CRYPTO_SHA2_256_HMAC: 300 axf = &auth_hash_hmac_sha2_256; 301 mac_key_size = SHA2_256_HASH_LEN; 302 break; 303 case CRYPTO_SHA2_384_HMAC: 304 axf = &auth_hash_hmac_sha2_384; 305 mac_key_size = SHA2_512_HASH_LEN; 306 break; 307 default: 308 __assert_unreachable(); 309 } 310 t4_init_hmac_digest(axf, mac_key_size, tls->params.auth_key, 311 tls->params.auth_key_len, hash); 312 } 313 } 314 315 /* Rx key */ 316 static void 317 prepare_rxkey_wr(struct tls_keyctx *kwr, struct ktls_session *tls) 318 { 319 320 kwr->u.rxhdr.flitcnt_hmacctrl = 321 ((tls_key_info_size(tls) / 16) << 3) | tls_hmac_ctrl(tls); 322 323 kwr->u.rxhdr.protover_ciphmode = 324 V_TLS_KEYCTX_TX_WR_PROTOVER(tls_proto_ver(tls)) | 325 V_TLS_KEYCTX_TX_WR_CIPHMODE(tls_cipher_mode(tls)); 326 327 kwr->u.rxhdr.authmode_to_rxvalid = 328 V_TLS_KEYCTX_TX_WR_AUTHMODE(tls_auth_mode(tls)) | 329 V_TLS_KEYCTX_TX_WR_SEQNUMCTRL(3) | 330 V_TLS_KEYCTX_TX_WR_RXVALID(1); 331 332 kwr->u.rxhdr.ivpresent_to_rxmk_size = 333 V_TLS_KEYCTX_TX_WR_IVPRESENT(0) | 334 V_TLS_KEYCTX_TX_WR_RXCK_SIZE(tls_cipher_key_size(tls)) | 335 V_TLS_KEYCTX_TX_WR_RXMK_SIZE(tls_mac_key_size(tls)); 336 337 if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) { 338 kwr->u.rxhdr.ivinsert_to_authinsrt = 339 htobe64(V_TLS_KEYCTX_TX_WR_IVINSERT(6ULL) | 340 V_TLS_KEYCTX_TX_WR_AADSTRTOFST(1ULL) | 341 V_TLS_KEYCTX_TX_WR_AADSTOPOFST(5ULL) | 342 V_TLS_KEYCTX_TX_WR_AUTHSRTOFST(14ULL) | 343 V_TLS_KEYCTX_TX_WR_AUTHSTOPOFST(16ULL) | 344 V_TLS_KEYCTX_TX_WR_CIPHERSRTOFST(14ULL) | 345 V_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST(0ULL) | 346 V_TLS_KEYCTX_TX_WR_AUTHINSRT(16ULL)); 347 } else { 348 kwr->u.rxhdr.authmode_to_rxvalid |= 349 V_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL(1); 350 kwr->u.rxhdr.ivpresent_to_rxmk_size |= 351 V_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT(1); 352 kwr->u.rxhdr.ivinsert_to_authinsrt = 353 htobe64(V_TLS_KEYCTX_TX_WR_IVINSERT(6ULL) | 354 V_TLS_KEYCTX_TX_WR_AADSTRTOFST(1ULL) | 355 V_TLS_KEYCTX_TX_WR_AADSTOPOFST(5ULL) | 356 V_TLS_KEYCTX_TX_WR_AUTHSRTOFST(22ULL) | 357 V_TLS_KEYCTX_TX_WR_AUTHSTOPOFST(0ULL) | 358 V_TLS_KEYCTX_TX_WR_CIPHERSRTOFST(22ULL) | 359 V_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST(0ULL) | 360 V_TLS_KEYCTX_TX_WR_AUTHINSRT(0ULL)); 361 } 362 363 prepare_tls_keys(kwr->keys.edkey, kwr->u.rxhdr.rxsalt, tls, KTLS_RX); 364 } 365 366 /* Tx key */ 367 static void 368 prepare_txkey_wr(struct tls_keyctx *kwr, struct ktls_session *tls) 369 { 370 371 kwr->u.txhdr.ctxlen = tls_key_info_size(tls) / 16; 372 kwr->u.txhdr.dualck_to_txvalid = 373 V_TLS_KEYCTX_TX_WR_SALT_PRESENT(1) | 374 V_TLS_KEYCTX_TX_WR_TXCK_SIZE(tls_cipher_key_size(tls)) | 375 V_TLS_KEYCTX_TX_WR_TXMK_SIZE(tls_mac_key_size(tls)) | 376 V_TLS_KEYCTX_TX_WR_TXVALID(1); 377 if (tls->params.cipher_algorithm == CRYPTO_AES_CBC) 378 kwr->u.txhdr.dualck_to_txvalid |= 379 V_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(1); 380 kwr->u.txhdr.dualck_to_txvalid = htons(kwr->u.txhdr.dualck_to_txvalid); 381 382 prepare_tls_keys(kwr->keys.edkey, kwr->u.txhdr.txsalt, tls, KTLS_TX); 383 } 384 385 /* TLS Key memory management */ 386 static int 387 get_new_keyid(struct toepcb *toep) 388 { 389 struct adapter *sc = td_adapter(toep->td); 390 vmem_addr_t addr; 391 392 if (vmem_alloc(sc->key_map, TLS_KEY_CONTEXT_SZ, M_NOWAIT | M_FIRSTFIT, 393 &addr) != 0) 394 return (-1); 395 396 return (addr); 397 } 398 399 static void 400 free_keyid(struct toepcb *toep, int keyid) 401 { 402 struct adapter *sc = td_adapter(toep->td); 403 404 vmem_free(sc->key_map, keyid, TLS_KEY_CONTEXT_SZ); 405 } 406 407 static void 408 clear_tls_keyid(struct toepcb *toep) 409 { 410 struct tls_ofld_info *tls_ofld = &toep->tls; 411 412 if (tls_ofld->rx_key_addr >= 0) { 413 free_keyid(toep, tls_ofld->rx_key_addr); 414 tls_ofld->rx_key_addr = -1; 415 } 416 if (tls_ofld->tx_key_addr >= 0) { 417 free_keyid(toep, tls_ofld->tx_key_addr); 418 tls_ofld->tx_key_addr = -1; 419 } 420 } 421 422 static int 423 get_tp_plen_max(struct ktls_session *tls) 424 { 425 int plen = ((min(3*4096, TP_TX_PG_SZ))/1448) * 1448; 426 427 return (tls->params.max_frame_len <= 8192 ? plen : FC_TP_PLEN_MAX); 428 } 429 430 /* Send request to get the key-id */ 431 static int 432 tls_program_key_id(struct toepcb *toep, struct ktls_session *tls, 433 int direction) 434 { 435 struct tls_ofld_info *tls_ofld = &toep->tls; 436 struct adapter *sc = td_adapter(toep->td); 437 struct ofld_tx_sdesc *txsd; 438 int kwrlen, kctxlen, keyid, len; 439 struct wrqe *wr; 440 struct tls_key_req *kwr; 441 struct tls_keyctx *kctx; 442 443 kwrlen = sizeof(*kwr); 444 kctxlen = roundup2(sizeof(*kctx), 32); 445 len = roundup2(kwrlen + kctxlen, 16); 446 447 if (toep->txsd_avail == 0) 448 return (EAGAIN); 449 450 if ((keyid = get_new_keyid(toep)) < 0) { 451 return (ENOSPC); 452 } 453 454 wr = alloc_wrqe(len, &toep->ofld_txq->wrq); 455 if (wr == NULL) { 456 free_keyid(toep, keyid); 457 return (ENOMEM); 458 } 459 kwr = wrtod(wr); 460 memset(kwr, 0, kwrlen); 461 462 kwr->wr_hi = htobe32(V_FW_WR_OP(FW_ULPTX_WR) | F_FW_WR_COMPL | 463 F_FW_WR_ATOMIC); 464 kwr->wr_mid = htobe32(V_FW_WR_LEN16(DIV_ROUND_UP(len, 16)) | 465 V_FW_WR_FLOWID(toep->tid)); 466 kwr->protocol = tls_proto_ver(tls); 467 kwr->mfs = htons(tls->params.max_frame_len); 468 kwr->reneg_to_write_rx = V_KEY_GET_LOC(direction == KTLS_TX ? 469 KEY_WRITE_TX : KEY_WRITE_RX); 470 471 /* master command */ 472 kwr->cmd = htobe32(V_ULPTX_CMD(ULP_TX_MEM_WRITE) | 473 V_T5_ULP_MEMIO_ORDER(1) | V_T5_ULP_MEMIO_IMM(1)); 474 kwr->dlen = htobe32(V_ULP_MEMIO_DATA_LEN(kctxlen >> 5)); 475 kwr->len16 = htobe32((toep->tid << 8) | 476 DIV_ROUND_UP(len - sizeof(struct work_request_hdr), 16)); 477 kwr->kaddr = htobe32(V_ULP_MEMIO_ADDR(keyid >> 5)); 478 479 /* sub command */ 480 kwr->sc_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM)); 481 kwr->sc_len = htobe32(kctxlen); 482 483 kctx = (struct tls_keyctx *)(kwr + 1); 484 memset(kctx, 0, kctxlen); 485 486 if (direction == KTLS_TX) { 487 tls_ofld->tx_key_addr = keyid; 488 prepare_txkey_wr(kctx, tls); 489 } else { 490 tls_ofld->rx_key_addr = keyid; 491 prepare_rxkey_wr(kctx, tls); 492 } 493 494 txsd = &toep->txsd[toep->txsd_pidx]; 495 txsd->tx_credits = DIV_ROUND_UP(len, 16); 496 txsd->plen = 0; 497 toep->tx_credits -= txsd->tx_credits; 498 if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) 499 toep->txsd_pidx = 0; 500 toep->txsd_avail--; 501 502 t4_wrq_tx(sc, wr); 503 504 return (0); 505 } 506 507 /* 508 * In some cases a client connection can hang without sending the 509 * ServerHelloDone message from the NIC to the host. Send a dummy 510 * RX_DATA_ACK with RX_MODULATE to unstick the connection. 511 */ 512 static void 513 tls_send_handshake_ack(void *arg) 514 { 515 struct toepcb *toep = arg; 516 struct tls_ofld_info *tls_ofld = &toep->tls; 517 struct adapter *sc = td_adapter(toep->td); 518 519 /* Bail without rescheduling if the connection has closed. */ 520 if ((toep->flags & (TPF_FIN_SENT | TPF_ABORT_SHUTDOWN)) != 0) 521 return; 522 523 /* 524 * If this connection has timed out without receiving more 525 * data, downgrade to plain TOE mode and don't re-arm the 526 * timer. 527 */ 528 if (sc->tt.tls_rx_timeout != 0) { 529 struct inpcb *inp; 530 struct tcpcb *tp; 531 532 inp = toep->inp; 533 tp = intotcpcb(inp); 534 if ((ticks - tp->t_rcvtime) >= sc->tt.tls_rx_timeout) { 535 CTR2(KTR_CXGBE, "%s: tid %d clr_ofld_mode", __func__, 536 toep->tid); 537 tls_clr_ofld_mode(toep); 538 return; 539 } 540 } 541 542 /* 543 * XXX: Does not have the t4_get_tcb() checks to refine the 544 * workaround. 545 */ 546 callout_schedule(&tls_ofld->handshake_timer, TLS_SRV_HELLO_RD_TM * hz); 547 548 CTR2(KTR_CXGBE, "%s: tid %d sending RX_DATA_ACK", __func__, toep->tid); 549 send_rx_modulate(sc, toep); 550 } 551 552 static void 553 tls_start_handshake_timer(struct toepcb *toep) 554 { 555 struct tls_ofld_info *tls_ofld = &toep->tls; 556 557 INP_WLOCK_ASSERT(toep->inp); 558 callout_reset(&tls_ofld->handshake_timer, TLS_SRV_HELLO_BKOFF_TM * hz, 559 tls_send_handshake_ack, toep); 560 } 561 562 void 563 tls_stop_handshake_timer(struct toepcb *toep) 564 { 565 struct tls_ofld_info *tls_ofld = &toep->tls; 566 567 INP_WLOCK_ASSERT(toep->inp); 568 callout_stop(&tls_ofld->handshake_timer); 569 } 570 571 int 572 tls_alloc_ktls(struct toepcb *toep, struct ktls_session *tls, int direction) 573 { 574 struct adapter *sc = td_adapter(toep->td); 575 int error, explicit_iv_size, key_offset, mac_first; 576 577 if (!can_tls_offload(td_adapter(toep->td))) 578 return (EINVAL); 579 switch (ulp_mode(toep)) { 580 case ULP_MODE_TLS: 581 break; 582 case ULP_MODE_NONE: 583 case ULP_MODE_TCPDDP: 584 if (direction != KTLS_TX) 585 return (EINVAL); 586 break; 587 default: 588 return (EINVAL); 589 } 590 591 switch (tls->params.cipher_algorithm) { 592 case CRYPTO_AES_CBC: 593 /* XXX: Explicitly ignore any provided IV. */ 594 switch (tls->params.cipher_key_len) { 595 case 128 / 8: 596 case 192 / 8: 597 case 256 / 8: 598 break; 599 default: 600 error = EINVAL; 601 goto clr_ofld; 602 } 603 switch (tls->params.auth_algorithm) { 604 case CRYPTO_SHA1_HMAC: 605 case CRYPTO_SHA2_256_HMAC: 606 case CRYPTO_SHA2_384_HMAC: 607 break; 608 default: 609 error = EPROTONOSUPPORT; 610 goto clr_ofld; 611 } 612 explicit_iv_size = AES_BLOCK_LEN; 613 mac_first = 1; 614 break; 615 case CRYPTO_AES_NIST_GCM_16: 616 if (tls->params.iv_len != SALT_SIZE) { 617 error = EINVAL; 618 goto clr_ofld; 619 } 620 switch (tls->params.cipher_key_len) { 621 case 128 / 8: 622 case 192 / 8: 623 case 256 / 8: 624 break; 625 default: 626 error = EINVAL; 627 goto clr_ofld; 628 } 629 explicit_iv_size = 8; 630 mac_first = 0; 631 break; 632 default: 633 error = EPROTONOSUPPORT; 634 goto clr_ofld; 635 } 636 637 /* Only TLS 1.1 and TLS 1.2 are currently supported. */ 638 if (tls->params.tls_vmajor != TLS_MAJOR_VER_ONE || 639 tls->params.tls_vminor < TLS_MINOR_VER_ONE || 640 tls->params.tls_vminor > TLS_MINOR_VER_TWO) { 641 error = EPROTONOSUPPORT; 642 goto clr_ofld; 643 } 644 645 /* Bail if we already have a key. */ 646 if (direction == KTLS_TX) { 647 if (toep->tls.tx_key_addr != -1) 648 return (EOPNOTSUPP); 649 } else { 650 if (toep->tls.rx_key_addr != -1) 651 return (EOPNOTSUPP); 652 } 653 654 error = tls_program_key_id(toep, tls, direction); 655 if (error) { 656 if (direction == KTLS_RX) 657 goto clr_ofld; 658 return (error); 659 } 660 661 if (direction == KTLS_TX) { 662 toep->tls.scmd0.seqno_numivs = 663 (V_SCMD_SEQ_NO_CTRL(3) | 664 V_SCMD_PROTO_VERSION(tls_proto_ver(tls)) | 665 V_SCMD_ENC_DEC_CTRL(SCMD_ENCDECCTRL_ENCRYPT) | 666 V_SCMD_CIPH_AUTH_SEQ_CTRL((mac_first == 0)) | 667 V_SCMD_CIPH_MODE(tls_cipher_mode(tls)) | 668 V_SCMD_AUTH_MODE(tls_auth_mode(tls)) | 669 V_SCMD_HMAC_CTRL(tls_hmac_ctrl(tls)) | 670 V_SCMD_IV_SIZE(explicit_iv_size / 2)); 671 672 toep->tls.scmd0.ivgen_hdrlen = 673 (V_SCMD_IV_GEN_CTRL(1) | 674 V_SCMD_KEY_CTX_INLINE(0) | 675 V_SCMD_TLS_FRAG_ENABLE(1)); 676 677 if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) 678 toep->tls.iv_len = 8; 679 else 680 toep->tls.iv_len = AES_BLOCK_LEN; 681 682 toep->tls.frag_size = tls->params.max_frame_len; 683 toep->tls.fcplenmax = get_tp_plen_max(tls); 684 toep->tls.expn_per_ulp = tls->params.tls_hlen + 685 tls->params.tls_tlen; 686 toep->tls.pdus_per_ulp = 1; 687 toep->tls.adjusted_plen = toep->tls.expn_per_ulp + 688 tls->params.max_frame_len; 689 toep->tls.tx_key_info_size = tls_key_info_size(tls); 690 } else { 691 /* Stop timer on handshake completion */ 692 tls_stop_handshake_timer(toep); 693 694 toep->flags &= ~TPF_FORCE_CREDITS; 695 toep->flags |= TPF_TLS_RECEIVE; 696 toep->tls.rx_version = tls->params.tls_vmajor << 8 | 697 tls->params.tls_vminor; 698 699 /* 700 * RX key tags are an index into the key portion of MA 701 * memory stored as an offset from the base address in 702 * units of 64 bytes. 703 */ 704 key_offset = toep->tls.rx_key_addr - sc->vres.key.start; 705 t4_set_tls_keyid(toep, key_offset / 64); 706 t4_set_tls_tcb_field(toep, W_TCB_ULP_RAW, 707 V_TCB_ULP_RAW(M_TCB_ULP_RAW), 708 V_TCB_ULP_RAW((V_TF_TLS_KEY_SIZE(3) | 709 V_TF_TLS_CONTROL(1) | 710 V_TF_TLS_ACTIVE(1) | 711 V_TF_TLS_ENABLE(1)))); 712 t4_set_tls_tcb_field(toep, W_TCB_TLS_SEQ, 713 V_TCB_TLS_SEQ(M_TCB_TLS_SEQ), 714 V_TCB_TLS_SEQ(0)); 715 t4_clear_rx_quiesce(toep); 716 } 717 718 return (0); 719 720 clr_ofld: 721 if (ulp_mode(toep) == ULP_MODE_TLS) { 722 CTR2(KTR_CXGBE, "%s: tid %d clr_ofld_mode", __func__, 723 toep->tid); 724 tls_clr_ofld_mode(toep); 725 } 726 return (error); 727 } 728 729 void 730 tls_init_toep(struct toepcb *toep) 731 { 732 struct tls_ofld_info *tls_ofld = &toep->tls; 733 734 tls_ofld->rx_key_addr = -1; 735 tls_ofld->tx_key_addr = -1; 736 } 737 738 void 739 tls_establish(struct toepcb *toep) 740 { 741 742 /* 743 * Enable PDU extraction. 744 * 745 * XXX: Supposedly this should be done by the firmware when 746 * the ULP_MODE FLOWC parameter is set in send_flowc_wr(), but 747 * in practice this seems to be required. 748 */ 749 CTR2(KTR_CXGBE, "%s: tid %d setting TLS_ENABLE", __func__, toep->tid); 750 t4_set_tls_tcb_field(toep, W_TCB_ULP_RAW, V_TCB_ULP_RAW(M_TCB_ULP_RAW), 751 V_TCB_ULP_RAW(V_TF_TLS_ENABLE(1))); 752 753 toep->flags |= TPF_FORCE_CREDITS | TPF_TLS_ESTABLISHED; 754 755 callout_init_rw(&toep->tls.handshake_timer, &toep->inp->inp_lock, 0); 756 tls_start_handshake_timer(toep); 757 } 758 759 void 760 tls_detach(struct toepcb *toep) 761 { 762 763 if (toep->flags & TPF_TLS_ESTABLISHED) { 764 tls_stop_handshake_timer(toep); 765 toep->flags &= ~TPF_TLS_ESTABLISHED; 766 } 767 } 768 769 void 770 tls_uninit_toep(struct toepcb *toep) 771 { 772 773 MPASS((toep->flags & TPF_TLS_ESTABLISHED) == 0); 774 clear_tls_keyid(toep); 775 } 776 777 #define MAX_OFLD_TX_CREDITS (SGE_MAX_WR_LEN / 16) 778 #define MIN_OFLD_TLSTX_CREDITS(toep) \ 779 (howmany(sizeof(struct fw_tlstx_data_wr) + \ 780 sizeof(struct cpl_tx_tls_sfo) + sizeof(struct ulptx_idata) + \ 781 sizeof(struct ulptx_sc_memrd) + \ 782 AES_BLOCK_LEN + 1, 16)) 783 784 static inline u_int 785 max_imm_tls_space(int tx_credits) 786 { 787 const int n = 2; /* Use only up to 2 desc for imm. data WR */ 788 int space; 789 790 KASSERT(tx_credits >= 0 && 791 tx_credits <= MAX_OFLD_TX_CREDITS, 792 ("%s: %d credits", __func__, tx_credits)); 793 794 if (tx_credits >= (n * EQ_ESIZE) / 16) 795 space = (n * EQ_ESIZE); 796 else 797 space = tx_credits * 16; 798 return (space); 799 } 800 801 static void 802 write_tlstx_wr(struct fw_tlstx_data_wr *txwr, struct toepcb *toep, 803 unsigned int immdlen, unsigned int plen, unsigned int expn, 804 unsigned int pdus, uint8_t credits, int shove, int imm_ivs) 805 { 806 struct tls_ofld_info *tls_ofld = &toep->tls; 807 unsigned int len = plen + expn; 808 809 txwr->op_to_immdlen = htobe32(V_WR_OP(FW_TLSTX_DATA_WR) | 810 V_FW_TLSTX_DATA_WR_COMPL(1) | 811 V_FW_TLSTX_DATA_WR_IMMDLEN(immdlen)); 812 txwr->flowid_len16 = htobe32(V_FW_TLSTX_DATA_WR_FLOWID(toep->tid) | 813 V_FW_TLSTX_DATA_WR_LEN16(credits)); 814 txwr->plen = htobe32(len); 815 txwr->lsodisable_to_flags = htobe32(V_TX_ULP_MODE(ULP_MODE_TLS) | 816 V_TX_URG(0) | /* F_T6_TX_FORCE | */ V_TX_SHOVE(shove)); 817 txwr->ctxloc_to_exp = htobe32(V_FW_TLSTX_DATA_WR_NUMIVS(pdus) | 818 V_FW_TLSTX_DATA_WR_EXP(expn) | 819 V_FW_TLSTX_DATA_WR_CTXLOC(TLS_SFO_WR_CONTEXTLOC_DDR) | 820 V_FW_TLSTX_DATA_WR_IVDSGL(!imm_ivs) | 821 V_FW_TLSTX_DATA_WR_KEYSIZE(tls_ofld->tx_key_info_size >> 4)); 822 txwr->mfs = htobe16(tls_ofld->frag_size); 823 txwr->adjustedplen_pkd = htobe16( 824 V_FW_TLSTX_DATA_WR_ADJUSTEDPLEN(tls_ofld->adjusted_plen)); 825 txwr->expinplenmax_pkd = htobe16( 826 V_FW_TLSTX_DATA_WR_EXPINPLENMAX(tls_ofld->expn_per_ulp)); 827 txwr->pdusinplenmax_pkd = 828 V_FW_TLSTX_DATA_WR_PDUSINPLENMAX(tls_ofld->pdus_per_ulp); 829 } 830 831 static void 832 write_tlstx_cpl(struct cpl_tx_tls_sfo *cpl, struct toepcb *toep, 833 struct tls_hdr *tls_hdr, unsigned int plen, unsigned int pdus) 834 { 835 struct tls_ofld_info *tls_ofld = &toep->tls; 836 int data_type, seglen; 837 838 if (plen < tls_ofld->frag_size) 839 seglen = plen; 840 else 841 seglen = tls_ofld->frag_size; 842 data_type = tls_content_type(tls_hdr->type); 843 cpl->op_to_seg_len = htobe32(V_CPL_TX_TLS_SFO_OPCODE(CPL_TX_TLS_SFO) | 844 V_CPL_TX_TLS_SFO_DATA_TYPE(data_type) | 845 V_CPL_TX_TLS_SFO_CPL_LEN(2) | V_CPL_TX_TLS_SFO_SEG_LEN(seglen)); 846 cpl->pld_len = htobe32(plen); 847 if (data_type == CPL_TX_TLS_SFO_TYPE_HEARTBEAT) 848 cpl->type_protover = htobe32( 849 V_CPL_TX_TLS_SFO_TYPE(tls_hdr->type)); 850 cpl->seqno_numivs = htobe32(tls_ofld->scmd0.seqno_numivs | 851 V_SCMD_NUM_IVS(pdus)); 852 cpl->ivgen_hdrlen = htobe32(tls_ofld->scmd0.ivgen_hdrlen); 853 cpl->scmd1 = htobe64(tls_ofld->tx_seq_no); 854 tls_ofld->tx_seq_no += pdus; 855 } 856 857 static int 858 count_ext_pgs_segs(struct mbuf *m) 859 { 860 vm_paddr_t nextpa; 861 u_int i, nsegs; 862 863 MPASS(m->m_epg_npgs > 0); 864 nsegs = 1; 865 nextpa = m->m_epg_pa[0] + PAGE_SIZE; 866 for (i = 1; i < m->m_epg_npgs; i++) { 867 if (nextpa != m->m_epg_pa[i]) 868 nsegs++; 869 nextpa = m->m_epg_pa[i] + PAGE_SIZE; 870 } 871 return (nsegs); 872 } 873 874 static void 875 write_ktlstx_sgl(void *dst, struct mbuf *m, int nsegs) 876 { 877 struct ulptx_sgl *usgl = dst; 878 vm_paddr_t pa; 879 uint32_t len; 880 int i, j; 881 882 KASSERT(nsegs > 0, ("%s: nsegs 0", __func__)); 883 884 usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) | 885 V_ULPTX_NSGE(nsegs)); 886 887 /* Figure out the first S/G length. */ 888 pa = m->m_epg_pa[0] + m->m_epg_1st_off; 889 usgl->addr0 = htobe64(pa); 890 len = m_epg_pagelen(m, 0, m->m_epg_1st_off); 891 pa += len; 892 for (i = 1; i < m->m_epg_npgs; i++) { 893 if (m->m_epg_pa[i] != pa) 894 break; 895 len += m_epg_pagelen(m, i, 0); 896 pa += m_epg_pagelen(m, i, 0); 897 } 898 usgl->len0 = htobe32(len); 899 #ifdef INVARIANTS 900 nsegs--; 901 #endif 902 903 j = -1; 904 for (; i < m->m_epg_npgs; i++) { 905 if (j == -1 || m->m_epg_pa[i] != pa) { 906 if (j >= 0) 907 usgl->sge[j / 2].len[j & 1] = htobe32(len); 908 j++; 909 #ifdef INVARIANTS 910 nsegs--; 911 #endif 912 pa = m->m_epg_pa[i]; 913 usgl->sge[j / 2].addr[j & 1] = htobe64(pa); 914 len = m_epg_pagelen(m, i, 0); 915 pa += len; 916 } else { 917 len += m_epg_pagelen(m, i, 0); 918 pa += m_epg_pagelen(m, i, 0); 919 } 920 } 921 if (j >= 0) { 922 usgl->sge[j / 2].len[j & 1] = htobe32(len); 923 924 if ((j & 1) == 0) 925 usgl->sge[j / 2].len[1] = htobe32(0); 926 } 927 KASSERT(nsegs == 0, ("%s: nsegs %d, m %p", __func__, nsegs, m)); 928 } 929 930 /* 931 * Similar to t4_push_frames() but handles sockets that contain TLS 932 * record mbufs. 933 */ 934 void 935 t4_push_ktls(struct adapter *sc, struct toepcb *toep, int drop) 936 { 937 struct tls_hdr *thdr; 938 struct fw_tlstx_data_wr *txwr; 939 struct cpl_tx_tls_sfo *cpl; 940 struct ulptx_idata *idata; 941 struct ulptx_sc_memrd *memrd; 942 struct wrqe *wr; 943 struct mbuf *m; 944 u_int nsegs, credits, wr_len; 945 u_int expn_size; 946 struct inpcb *inp = toep->inp; 947 struct tcpcb *tp = intotcpcb(inp); 948 struct socket *so = inp->inp_socket; 949 struct sockbuf *sb = &so->so_snd; 950 int tls_size, tx_credits, shove, sowwakeup; 951 struct ofld_tx_sdesc *txsd; 952 char *buf; 953 954 INP_WLOCK_ASSERT(inp); 955 KASSERT(toep->flags & TPF_FLOWC_WR_SENT, 956 ("%s: flowc_wr not sent for tid %u.", __func__, toep->tid)); 957 958 KASSERT(ulp_mode(toep) == ULP_MODE_NONE || 959 ulp_mode(toep) == ULP_MODE_TCPDDP || ulp_mode(toep) == ULP_MODE_TLS, 960 ("%s: ulp_mode %u for toep %p", __func__, ulp_mode(toep), toep)); 961 KASSERT(tls_tx_key(toep), 962 ("%s: TX key not set for toep %p", __func__, toep)); 963 964 #ifdef VERBOSE_TRACES 965 CTR4(KTR_CXGBE, "%s: tid %d toep flags %#x tp flags %#x drop %d", 966 __func__, toep->tid, toep->flags, tp->t_flags); 967 #endif 968 if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN)) 969 return; 970 971 #ifdef RATELIMIT 972 if (__predict_false(inp->inp_flags2 & INP_RATE_LIMIT_CHANGED) && 973 (update_tx_rate_limit(sc, toep, so->so_max_pacing_rate) == 0)) { 974 inp->inp_flags2 &= ~INP_RATE_LIMIT_CHANGED; 975 } 976 #endif 977 978 /* 979 * This function doesn't resume by itself. Someone else must clear the 980 * flag and call this function. 981 */ 982 if (__predict_false(toep->flags & TPF_TX_SUSPENDED)) { 983 KASSERT(drop == 0, 984 ("%s: drop (%d) != 0 but tx is suspended", __func__, drop)); 985 return; 986 } 987 988 txsd = &toep->txsd[toep->txsd_pidx]; 989 for (;;) { 990 tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS); 991 992 SOCKBUF_LOCK(sb); 993 sowwakeup = drop; 994 if (drop) { 995 sbdrop_locked(sb, drop); 996 drop = 0; 997 } 998 999 m = sb->sb_sndptr != NULL ? sb->sb_sndptr->m_next : sb->sb_mb; 1000 1001 /* 1002 * Send a FIN if requested, but only if there's no 1003 * more data to send. 1004 */ 1005 if (m == NULL && toep->flags & TPF_SEND_FIN) { 1006 if (sowwakeup) 1007 sowwakeup_locked(so); 1008 else 1009 SOCKBUF_UNLOCK(sb); 1010 SOCKBUF_UNLOCK_ASSERT(sb); 1011 t4_close_conn(sc, toep); 1012 return; 1013 } 1014 1015 /* 1016 * If there is no ready data to send, wait until more 1017 * data arrives. 1018 */ 1019 if (m == NULL || (m->m_flags & M_NOTAVAIL) != 0) { 1020 if (sowwakeup) 1021 sowwakeup_locked(so); 1022 else 1023 SOCKBUF_UNLOCK(sb); 1024 SOCKBUF_UNLOCK_ASSERT(sb); 1025 #ifdef VERBOSE_TRACES 1026 CTR2(KTR_CXGBE, "%s: tid %d no ready data to send", 1027 __func__, toep->tid); 1028 #endif 1029 return; 1030 } 1031 1032 KASSERT(m->m_flags & M_EXTPG, ("%s: mbuf %p is not NOMAP", 1033 __func__, m)); 1034 KASSERT(m->m_epg_tls != NULL, 1035 ("%s: mbuf %p doesn't have TLS session", __func__, m)); 1036 1037 /* Calculate WR length. */ 1038 wr_len = sizeof(struct fw_tlstx_data_wr) + 1039 sizeof(struct cpl_tx_tls_sfo) + 1040 sizeof(struct ulptx_idata) + sizeof(struct ulptx_sc_memrd); 1041 1042 /* Explicit IVs for AES-CBC and AES-GCM are <= 16. */ 1043 MPASS(toep->tls.iv_len <= AES_BLOCK_LEN); 1044 wr_len += AES_BLOCK_LEN; 1045 1046 /* Account for SGL in work request length. */ 1047 nsegs = count_ext_pgs_segs(m); 1048 wr_len += sizeof(struct ulptx_sgl) + 1049 ((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8; 1050 1051 /* Not enough credits for this work request. */ 1052 if (howmany(wr_len, 16) > tx_credits) { 1053 if (sowwakeup) 1054 sowwakeup_locked(so); 1055 else 1056 SOCKBUF_UNLOCK(sb); 1057 SOCKBUF_UNLOCK_ASSERT(sb); 1058 #ifdef VERBOSE_TRACES 1059 CTR5(KTR_CXGBE, 1060 "%s: tid %d mbuf %p requires %d credits, but only %d available", 1061 __func__, toep->tid, m, howmany(wr_len, 16), 1062 tx_credits); 1063 #endif 1064 toep->flags |= TPF_TX_SUSPENDED; 1065 return; 1066 } 1067 1068 /* Shove if there is no additional data pending. */ 1069 shove = ((m->m_next == NULL || 1070 (m->m_next->m_flags & M_NOTAVAIL) != 0)) && 1071 (tp->t_flags & TF_MORETOCOME) == 0; 1072 1073 if (sb->sb_flags & SB_AUTOSIZE && 1074 V_tcp_do_autosndbuf && 1075 sb->sb_hiwat < V_tcp_autosndbuf_max && 1076 sbused(sb) >= sb->sb_hiwat * 7 / 8) { 1077 int newsize = min(sb->sb_hiwat + V_tcp_autosndbuf_inc, 1078 V_tcp_autosndbuf_max); 1079 1080 if (!sbreserve_locked(sb, newsize, so, NULL)) 1081 sb->sb_flags &= ~SB_AUTOSIZE; 1082 else 1083 sowwakeup = 1; /* room available */ 1084 } 1085 if (sowwakeup) 1086 sowwakeup_locked(so); 1087 else 1088 SOCKBUF_UNLOCK(sb); 1089 SOCKBUF_UNLOCK_ASSERT(sb); 1090 1091 if (__predict_false(toep->flags & TPF_FIN_SENT)) 1092 panic("%s: excess tx.", __func__); 1093 1094 wr = alloc_wrqe(roundup2(wr_len, 16), &toep->ofld_txq->wrq); 1095 if (wr == NULL) { 1096 /* XXX: how will we recover from this? */ 1097 toep->flags |= TPF_TX_SUSPENDED; 1098 return; 1099 } 1100 1101 thdr = (struct tls_hdr *)&m->m_epg_hdr; 1102 #ifdef VERBOSE_TRACES 1103 CTR5(KTR_CXGBE, "%s: tid %d TLS record %ju type %d len %#x", 1104 __func__, toep->tid, m->m_epg_seqno, thdr->type, 1105 m->m_len); 1106 #endif 1107 txwr = wrtod(wr); 1108 cpl = (struct cpl_tx_tls_sfo *)(txwr + 1); 1109 memset(txwr, 0, roundup2(wr_len, 16)); 1110 credits = howmany(wr_len, 16); 1111 expn_size = m->m_epg_hdrlen + 1112 m->m_epg_trllen; 1113 tls_size = m->m_len - expn_size; 1114 write_tlstx_wr(txwr, toep, 0, 1115 tls_size, expn_size, 1, credits, shove, 1); 1116 toep->tls.tx_seq_no = m->m_epg_seqno; 1117 write_tlstx_cpl(cpl, toep, thdr, tls_size, 1); 1118 1119 idata = (struct ulptx_idata *)(cpl + 1); 1120 idata->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP)); 1121 idata->len = htobe32(0); 1122 memrd = (struct ulptx_sc_memrd *)(idata + 1); 1123 memrd->cmd_to_len = htobe32(V_ULPTX_CMD(ULP_TX_SC_MEMRD) | 1124 V_ULP_TX_SC_MORE(1) | 1125 V_ULPTX_LEN16(toep->tls.tx_key_info_size >> 4)); 1126 memrd->addr = htobe32(toep->tls.tx_key_addr >> 5); 1127 1128 /* Copy IV. */ 1129 buf = (char *)(memrd + 1); 1130 memcpy(buf, thdr + 1, toep->tls.iv_len); 1131 buf += AES_BLOCK_LEN; 1132 1133 write_ktlstx_sgl(buf, m, nsegs); 1134 1135 KASSERT(toep->tx_credits >= credits, 1136 ("%s: not enough credits", __func__)); 1137 1138 toep->tx_credits -= credits; 1139 1140 tp->snd_nxt += m->m_len; 1141 tp->snd_max += m->m_len; 1142 1143 SOCKBUF_LOCK(sb); 1144 sb->sb_sndptr = m; 1145 SOCKBUF_UNLOCK(sb); 1146 1147 toep->flags |= TPF_TX_DATA_SENT; 1148 if (toep->tx_credits < MIN_OFLD_TLSTX_CREDITS(toep)) 1149 toep->flags |= TPF_TX_SUSPENDED; 1150 1151 KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__)); 1152 txsd->plen = m->m_len; 1153 txsd->tx_credits = credits; 1154 txsd++; 1155 if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) { 1156 toep->txsd_pidx = 0; 1157 txsd = &toep->txsd[0]; 1158 } 1159 toep->txsd_avail--; 1160 1161 counter_u64_add(toep->ofld_txq->tx_toe_tls_records, 1); 1162 counter_u64_add(toep->ofld_txq->tx_toe_tls_octets, m->m_len); 1163 1164 t4_l2t_send(sc, wr, toep->l2te); 1165 } 1166 } 1167 1168 /* 1169 * For TLS data we place received mbufs received via CPL_TLS_DATA into 1170 * an mbufq in the TLS offload state. When CPL_RX_TLS_CMP is 1171 * received, the completed PDUs are placed into the socket receive 1172 * buffer. 1173 * 1174 * The TLS code reuses the ulp_pdu_reclaimq to hold the pending mbufs. 1175 */ 1176 static int 1177 do_tls_data(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 1178 { 1179 struct adapter *sc = iq->adapter; 1180 const struct cpl_tls_data *cpl = mtod(m, const void *); 1181 unsigned int tid = GET_TID(cpl); 1182 struct toepcb *toep = lookup_tid(sc, tid); 1183 struct inpcb *inp = toep->inp; 1184 struct tcpcb *tp; 1185 int len; 1186 1187 /* XXX: Should this match do_rx_data instead? */ 1188 KASSERT(!(toep->flags & TPF_SYNQE), 1189 ("%s: toep %p claims to be a synq entry", __func__, toep)); 1190 1191 KASSERT(toep->tid == tid, ("%s: toep tid/atid mismatch", __func__)); 1192 1193 /* strip off CPL header */ 1194 m_adj(m, sizeof(*cpl)); 1195 len = m->m_pkthdr.len; 1196 1197 toep->ofld_rxq->rx_toe_tls_octets += len; 1198 1199 KASSERT(len == G_CPL_TLS_DATA_LENGTH(be32toh(cpl->length_pkd)), 1200 ("%s: payload length mismatch", __func__)); 1201 1202 INP_WLOCK(inp); 1203 if (inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) { 1204 CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x", 1205 __func__, tid, len, inp->inp_flags); 1206 INP_WUNLOCK(inp); 1207 m_freem(m); 1208 return (0); 1209 } 1210 1211 /* Save TCP sequence number. */ 1212 m->m_pkthdr.tls_tcp_seq = be32toh(cpl->seq); 1213 1214 if (mbufq_enqueue(&toep->ulp_pdu_reclaimq, m)) { 1215 #ifdef INVARIANTS 1216 panic("Failed to queue TLS data packet"); 1217 #else 1218 printf("%s: Failed to queue TLS data packet\n", __func__); 1219 INP_WUNLOCK(inp); 1220 m_freem(m); 1221 return (0); 1222 #endif 1223 } 1224 1225 tp = intotcpcb(inp); 1226 tp->t_rcvtime = ticks; 1227 1228 #ifdef VERBOSE_TRACES 1229 CTR4(KTR_CXGBE, "%s: tid %u len %d seq %u", __func__, tid, len, 1230 be32toh(cpl->seq)); 1231 #endif 1232 1233 INP_WUNLOCK(inp); 1234 return (0); 1235 } 1236 1237 static int 1238 do_rx_tls_cmp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 1239 { 1240 struct adapter *sc = iq->adapter; 1241 const struct cpl_rx_tls_cmp *cpl = mtod(m, const void *); 1242 struct tlsrx_hdr_pkt *tls_hdr_pkt; 1243 unsigned int tid = GET_TID(cpl); 1244 struct toepcb *toep = lookup_tid(sc, tid); 1245 struct inpcb *inp = toep->inp; 1246 struct tcpcb *tp; 1247 struct socket *so; 1248 struct sockbuf *sb; 1249 struct mbuf *tls_data; 1250 struct tls_get_record *tgr; 1251 struct mbuf *control; 1252 int len, pdu_length, rx_credits; 1253 1254 KASSERT(toep->tid == tid, ("%s: toep tid/atid mismatch", __func__)); 1255 KASSERT(!(toep->flags & TPF_SYNQE), 1256 ("%s: toep %p claims to be a synq entry", __func__, toep)); 1257 1258 /* strip off CPL header */ 1259 m_adj(m, sizeof(*cpl)); 1260 len = m->m_pkthdr.len; 1261 1262 toep->ofld_rxq->rx_toe_tls_records++; 1263 1264 KASSERT(len == G_CPL_RX_TLS_CMP_LENGTH(be32toh(cpl->pdulength_length)), 1265 ("%s: payload length mismatch", __func__)); 1266 1267 INP_WLOCK(inp); 1268 if (inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) { 1269 CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x", 1270 __func__, tid, len, inp->inp_flags); 1271 INP_WUNLOCK(inp); 1272 m_freem(m); 1273 return (0); 1274 } 1275 1276 pdu_length = G_CPL_RX_TLS_CMP_PDULENGTH(be32toh(cpl->pdulength_length)); 1277 1278 so = inp_inpcbtosocket(inp); 1279 tp = intotcpcb(inp); 1280 1281 #ifdef VERBOSE_TRACES 1282 CTR6(KTR_CXGBE, "%s: tid %u PDU len %d len %d seq %u, rcv_nxt %u", 1283 __func__, tid, pdu_length, len, be32toh(cpl->seq), tp->rcv_nxt); 1284 #endif 1285 1286 tp->rcv_nxt += pdu_length; 1287 KASSERT(tp->rcv_wnd >= pdu_length, 1288 ("%s: negative window size", __func__)); 1289 tp->rcv_wnd -= pdu_length; 1290 1291 /* XXX: Not sure what to do about urgent data. */ 1292 1293 /* 1294 * The payload of this CPL is the TLS header followed by 1295 * additional fields. 1296 */ 1297 KASSERT(m->m_len >= sizeof(*tls_hdr_pkt), 1298 ("%s: payload too small", __func__)); 1299 tls_hdr_pkt = mtod(m, void *); 1300 1301 tls_data = mbufq_dequeue(&toep->ulp_pdu_reclaimq); 1302 if (tls_data != NULL) { 1303 KASSERT(be32toh(cpl->seq) == tls_data->m_pkthdr.tls_tcp_seq, 1304 ("%s: sequence mismatch", __func__)); 1305 } 1306 1307 /* Report decryption errors as EBADMSG. */ 1308 if ((tls_hdr_pkt->res_to_mac_error & M_TLSRX_HDR_PKT_ERROR) != 0) { 1309 m_freem(m); 1310 m_freem(tls_data); 1311 1312 CURVNET_SET(toep->vnet); 1313 so->so_error = EBADMSG; 1314 sorwakeup(so); 1315 1316 INP_WUNLOCK(inp); 1317 CURVNET_RESTORE(); 1318 1319 return (0); 1320 } 1321 1322 /* Allocate the control message mbuf. */ 1323 control = sbcreatecontrol(NULL, sizeof(*tgr), TLS_GET_RECORD, 1324 IPPROTO_TCP); 1325 if (control == NULL) { 1326 m_freem(m); 1327 m_freem(tls_data); 1328 1329 CURVNET_SET(toep->vnet); 1330 so->so_error = ENOBUFS; 1331 sorwakeup(so); 1332 1333 INP_WUNLOCK(inp); 1334 CURVNET_RESTORE(); 1335 1336 return (0); 1337 } 1338 1339 tgr = (struct tls_get_record *) 1340 CMSG_DATA(mtod(control, struct cmsghdr *)); 1341 tgr->tls_type = tls_hdr_pkt->type; 1342 tgr->tls_vmajor = be16toh(tls_hdr_pkt->version) >> 8; 1343 tgr->tls_vminor = be16toh(tls_hdr_pkt->version) & 0xff; 1344 1345 m_freem(m); 1346 1347 if (tls_data != NULL) { 1348 m_last(tls_data)->m_flags |= M_EOR; 1349 tgr->tls_length = htobe16(tls_data->m_pkthdr.len); 1350 } else 1351 tgr->tls_length = 0; 1352 m = tls_data; 1353 1354 sb = &so->so_rcv; 1355 SOCKBUF_LOCK(sb); 1356 1357 if (__predict_false(sb->sb_state & SBS_CANTRCVMORE)) { 1358 struct epoch_tracker et; 1359 1360 CTR3(KTR_CXGBE, "%s: tid %u, excess rx (%d bytes)", 1361 __func__, tid, pdu_length); 1362 m_freem(m); 1363 m_freem(control); 1364 SOCKBUF_UNLOCK(sb); 1365 INP_WUNLOCK(inp); 1366 1367 CURVNET_SET(toep->vnet); 1368 NET_EPOCH_ENTER(et); 1369 INP_WLOCK(inp); 1370 tp = tcp_drop(tp, ECONNRESET); 1371 if (tp) 1372 INP_WUNLOCK(inp); 1373 NET_EPOCH_EXIT(et); 1374 CURVNET_RESTORE(); 1375 1376 return (0); 1377 } 1378 1379 /* 1380 * Not all of the bytes on the wire are included in the socket buffer 1381 * (e.g. the MAC of the TLS record). However, those bytes are included 1382 * in the TCP sequence space. 1383 */ 1384 1385 /* receive buffer autosize */ 1386 MPASS(toep->vnet == so->so_vnet); 1387 CURVNET_SET(toep->vnet); 1388 if (sb->sb_flags & SB_AUTOSIZE && 1389 V_tcp_do_autorcvbuf && 1390 sb->sb_hiwat < V_tcp_autorcvbuf_max && 1391 m->m_pkthdr.len > (sbspace(sb) / 8 * 7)) { 1392 unsigned int hiwat = sb->sb_hiwat; 1393 unsigned int newsize = min(hiwat + sc->tt.autorcvbuf_inc, 1394 V_tcp_autorcvbuf_max); 1395 1396 if (!sbreserve_locked(sb, newsize, so, NULL)) 1397 sb->sb_flags &= ~SB_AUTOSIZE; 1398 } 1399 1400 sbappendcontrol_locked(sb, m, control, 0); 1401 rx_credits = sbspace(sb) > tp->rcv_wnd ? sbspace(sb) - tp->rcv_wnd : 0; 1402 #ifdef VERBOSE_TRACES 1403 CTR4(KTR_CXGBE, "%s: tid %u rx_credits %u rcv_wnd %u", 1404 __func__, tid, rx_credits, tp->rcv_wnd); 1405 #endif 1406 if (rx_credits > 0 && sbused(sb) + tp->rcv_wnd < sb->sb_lowat) { 1407 rx_credits = send_rx_credits(sc, toep, rx_credits); 1408 tp->rcv_wnd += rx_credits; 1409 tp->rcv_adv += rx_credits; 1410 } 1411 1412 sorwakeup_locked(so); 1413 SOCKBUF_UNLOCK_ASSERT(sb); 1414 1415 INP_WUNLOCK(inp); 1416 CURVNET_RESTORE(); 1417 return (0); 1418 } 1419 1420 void 1421 do_rx_data_tls(const struct cpl_rx_data *cpl, struct toepcb *toep, 1422 struct mbuf *m) 1423 { 1424 struct inpcb *inp = toep->inp; 1425 struct tls_ofld_info *tls_ofld = &toep->tls; 1426 struct tls_hdr *hdr; 1427 struct tcpcb *tp; 1428 struct socket *so; 1429 struct sockbuf *sb; 1430 int len, rx_credits; 1431 1432 len = m->m_pkthdr.len; 1433 1434 INP_WLOCK_ASSERT(inp); 1435 1436 so = inp_inpcbtosocket(inp); 1437 tp = intotcpcb(inp); 1438 sb = &so->so_rcv; 1439 SOCKBUF_LOCK(sb); 1440 CURVNET_SET(toep->vnet); 1441 1442 tp->rcv_nxt += len; 1443 KASSERT(tp->rcv_wnd >= len, ("%s: negative window size", __func__)); 1444 tp->rcv_wnd -= len; 1445 1446 /* Do we have a full TLS header? */ 1447 if (len < sizeof(*hdr)) { 1448 CTR3(KTR_CXGBE, "%s: tid %u len %d: too short for a TLS header", 1449 __func__, toep->tid, len); 1450 so->so_error = EMSGSIZE; 1451 goto out; 1452 } 1453 hdr = mtod(m, struct tls_hdr *); 1454 1455 /* Is the header valid? */ 1456 if (be16toh(hdr->version) != tls_ofld->rx_version) { 1457 CTR3(KTR_CXGBE, "%s: tid %u invalid version %04x", 1458 __func__, toep->tid, be16toh(hdr->version)); 1459 so->so_error = EINVAL; 1460 goto out; 1461 } 1462 if (be16toh(hdr->length) < sizeof(*hdr)) { 1463 CTR3(KTR_CXGBE, "%s: tid %u invalid length %u", 1464 __func__, toep->tid, be16toh(hdr->length)); 1465 so->so_error = EBADMSG; 1466 goto out; 1467 } 1468 1469 /* Did we get a truncated record? */ 1470 if (len < be16toh(hdr->length)) { 1471 CTR4(KTR_CXGBE, "%s: tid %u truncated TLS record (%d vs %u)", 1472 __func__, toep->tid, len, be16toh(hdr->length)); 1473 1474 so->so_error = EMSGSIZE; 1475 goto out; 1476 } 1477 1478 /* Is the header type unknown? */ 1479 switch (hdr->type) { 1480 case CONTENT_TYPE_CCS: 1481 case CONTENT_TYPE_ALERT: 1482 case CONTENT_TYPE_APP_DATA: 1483 case CONTENT_TYPE_HANDSHAKE: 1484 break; 1485 default: 1486 CTR3(KTR_CXGBE, "%s: tid %u invalid TLS record type %u", 1487 __func__, toep->tid, hdr->type); 1488 so->so_error = EBADMSG; 1489 goto out; 1490 } 1491 1492 /* 1493 * Just punt. Although this could fall back to software 1494 * decryption, this case should never really happen. 1495 */ 1496 CTR4(KTR_CXGBE, "%s: tid %u dropping TLS record type %u, length %u", 1497 __func__, toep->tid, hdr->type, be16toh(hdr->length)); 1498 so->so_error = EBADMSG; 1499 1500 out: 1501 /* 1502 * This connection is going to die anyway, so probably don't 1503 * need to bother with returning credits. 1504 */ 1505 rx_credits = sbspace(sb) > tp->rcv_wnd ? sbspace(sb) - tp->rcv_wnd : 0; 1506 #ifdef VERBOSE_TRACES 1507 CTR4(KTR_CXGBE, "%s: tid %u rx_credits %u rcv_wnd %u", 1508 __func__, toep->tid, rx_credits, tp->rcv_wnd); 1509 #endif 1510 if (rx_credits > 0 && sbused(sb) + tp->rcv_wnd < sb->sb_lowat) { 1511 rx_credits = send_rx_credits(toep->vi->adapter, toep, 1512 rx_credits); 1513 tp->rcv_wnd += rx_credits; 1514 tp->rcv_adv += rx_credits; 1515 } 1516 1517 sorwakeup_locked(so); 1518 SOCKBUF_UNLOCK_ASSERT(sb); 1519 1520 INP_WUNLOCK(inp); 1521 CURVNET_RESTORE(); 1522 1523 m_freem(m); 1524 } 1525 1526 void 1527 t4_tls_mod_load(void) 1528 { 1529 1530 t4_register_cpl_handler(CPL_TLS_DATA, do_tls_data); 1531 t4_register_cpl_handler(CPL_RX_TLS_CMP, do_rx_tls_cmp); 1532 } 1533 1534 void 1535 t4_tls_mod_unload(void) 1536 { 1537 1538 t4_register_cpl_handler(CPL_TLS_DATA, NULL); 1539 t4_register_cpl_handler(CPL_RX_TLS_CMP, NULL); 1540 } 1541 #endif /* TCP_OFFLOAD */ 1542 #endif /* KERN_TLS */ 1543