1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2025 Chelsio Communications
5 * Written by: John Baldwin <jhb@FreeBSD.org>
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29 #include "opt_inet.h"
30 #include "opt_inet6.h"
31 #include "opt_kern_tls.h"
32
33 #include <sys/param.h>
34 #include <sys/ktr.h>
35 #include <sys/ktls.h>
36 #include <sys/sglist.h>
37 #include <sys/socket.h>
38 #include <sys/socketvar.h>
39 #include <sys/sockbuf.h>
40 #include <netinet/in.h>
41 #include <netinet/in_pcb.h>
42 #include <netinet/ip.h>
43 #include <netinet/ip6.h>
44 #include <netinet/tcp_var.h>
45 #include <opencrypto/cryptodev.h>
46 #include <opencrypto/xform.h>
47 #include <vm/vm.h>
48 #include <vm/pmap.h>
49
50 #include "common/common.h"
51 #include "common/t4_regs.h"
52 #include "common/t4_regs_values.h"
53 #include "common/t4_tcb.h"
54 #include "t4_l2t.h"
55 #include "t4_clip.h"
56 #include "t4_mp_ring.h"
57 #include "crypto/t4_crypto.h"
58
59 #if defined(INET) || defined(INET6)
60
61 #define TLS_HEADER_LENGTH 5
62
63 struct tls_scmd {
64 __be32 seqno_numivs;
65 __be32 ivgen_hdrlen;
66 };
67
68 struct tlspcb {
69 struct m_snd_tag com;
70 struct vi_info *vi; /* virtual interface */
71 struct adapter *sc;
72 struct sge_txq *txq;
73
74 int tx_key_addr;
75 bool inline_key;
76 bool tls13;
77 unsigned char enc_mode;
78
79 struct tls_scmd scmd0;
80 struct tls_scmd scmd0_partial;
81 struct tls_scmd scmd0_short;
82
83 unsigned int tx_key_info_size;
84
85 uint16_t prev_mss;
86
87 /* Fields used for GCM records using GHASH state. */
88 uint16_t ghash_offset;
89 uint64_t ghash_tls_seqno;
90 char ghash[AES_GMAC_HASH_LEN];
91 bool ghash_valid;
92 bool ghash_pending;
93 bool ghash_lcb;
94 bool queue_mbufs;
95 uint8_t rx_chid;
96 uint16_t rx_qid;
97 struct mbufq pending_mbufs;
98
99 /*
100 * Only used outside of setup and teardown when using inline
101 * keys or for partial GCM mode.
102 */
103 struct tls_keyctx keyctx;
104 };
105
106 static void t7_tls_tag_free(struct m_snd_tag *mst);
107 static int ktls_setup_keys(struct tlspcb *tlsp,
108 const struct ktls_session *tls, struct sge_txq *txq);
109
110 static void *zero_buffer;
111 static vm_paddr_t zero_buffer_pa;
112
113 static const struct if_snd_tag_sw t7_tls_tag_sw = {
114 .snd_tag_free = t7_tls_tag_free,
115 .type = IF_SND_TAG_TYPE_TLS
116 };
117
118 static inline struct tlspcb *
mst_to_tls(struct m_snd_tag * t)119 mst_to_tls(struct m_snd_tag *t)
120 {
121 return (__containerof(t, struct tlspcb, com));
122 }
123
124 static struct tlspcb *
alloc_tlspcb(struct ifnet * ifp,struct vi_info * vi,int flags)125 alloc_tlspcb(struct ifnet *ifp, struct vi_info *vi, int flags)
126 {
127 struct port_info *pi = vi->pi;
128 struct adapter *sc = pi->adapter;
129 struct tlspcb *tlsp;
130
131 tlsp = malloc(sizeof(*tlsp), M_CXGBE, M_ZERO | flags);
132 if (tlsp == NULL)
133 return (NULL);
134
135 m_snd_tag_init(&tlsp->com, ifp, &t7_tls_tag_sw);
136 tlsp->vi = vi;
137 tlsp->sc = sc;
138 tlsp->tx_key_addr = -1;
139 tlsp->ghash_offset = -1;
140 tlsp->rx_chid = pi->rx_chan;
141 tlsp->rx_qid = -1;
142 tlsp->txq = NULL;
143 mbufq_init(&tlsp->pending_mbufs, INT_MAX);
144
145 return (tlsp);
146 }
147
148 int
t7_tls_tag_alloc(struct ifnet * ifp,union if_snd_tag_alloc_params * params,struct m_snd_tag ** pt)149 t7_tls_tag_alloc(struct ifnet *ifp, union if_snd_tag_alloc_params *params,
150 struct m_snd_tag **pt)
151 {
152 const struct ktls_session *tls;
153 struct tlspcb *tlsp;
154 struct adapter *sc;
155 struct vi_info *vi;
156 struct inpcb *inp;
157 struct sge_txq *txq;
158 int error, iv_size, keyid, mac_first, qidx;
159 uint32_t flowid;
160
161 tls = params->tls.tls;
162
163 /* TLS 1.1 through TLS 1.3 are currently supported. */
164 if (tls->params.tls_vmajor != TLS_MAJOR_VER_ONE ||
165 tls->params.tls_vminor < TLS_MINOR_VER_ONE ||
166 tls->params.tls_vminor > TLS_MINOR_VER_THREE)
167 return (EPROTONOSUPPORT);
168
169 /* Sanity check values in *tls. */
170 switch (tls->params.cipher_algorithm) {
171 case CRYPTO_AES_CBC:
172 /* XXX: Explicitly ignore any provided IV. */
173 switch (tls->params.cipher_key_len) {
174 case 128 / 8:
175 case 192 / 8:
176 case 256 / 8:
177 break;
178 default:
179 return (EINVAL);
180 }
181 switch (tls->params.auth_algorithm) {
182 case CRYPTO_SHA1_HMAC:
183 case CRYPTO_SHA2_256_HMAC:
184 case CRYPTO_SHA2_384_HMAC:
185 break;
186 default:
187 return (EPROTONOSUPPORT);
188 }
189 iv_size = AES_BLOCK_LEN;
190 mac_first = 1;
191 break;
192 case CRYPTO_AES_NIST_GCM_16:
193 switch (tls->params.cipher_key_len) {
194 case 128 / 8:
195 case 192 / 8:
196 case 256 / 8:
197 break;
198 default:
199 return (EINVAL);
200 }
201
202 /*
203 * The IV size for TLS 1.2 is the explicit IV in the
204 * record header. For TLS 1.3 it is the size of the
205 * sequence number.
206 */
207 iv_size = 8;
208 mac_first = 0;
209 break;
210 default:
211 return (EPROTONOSUPPORT);
212 }
213
214 vi = if_getsoftc(ifp);
215 sc = vi->adapter;
216
217 tlsp = alloc_tlspcb(ifp, vi, M_WAITOK);
218
219 /*
220 * Pointers with the low bit set in the pointer can't
221 * be stored as the cookie in the CPL_FW6_PLD reply.
222 */
223 if (((uintptr_t)tlsp & CPL_FW6_COOKIE_MASK) != 0) {
224 error = EINVAL;
225 goto failed;
226 }
227
228 tlsp->tls13 = tls->params.tls_vminor == TLS_MINOR_VER_THREE;
229
230 if (sc->tlst.inline_keys)
231 keyid = -1;
232 else
233 keyid = t4_alloc_tls_keyid(sc);
234 if (keyid < 0) {
235 CTR(KTR_CXGBE, "%s: %p using immediate key ctx", __func__,
236 tlsp);
237 tlsp->inline_key = true;
238 } else {
239 tlsp->tx_key_addr = keyid;
240 CTR(KTR_CXGBE, "%s: %p allocated TX key addr %#x", __func__,
241 tlsp, tlsp->tx_key_addr);
242 }
243
244 inp = params->tls.inp;
245 INP_RLOCK(inp);
246 if (intotcpcb(inp)->t_flags & TF_DISCONNECTED) {
247 INP_RUNLOCK(inp);
248 error = ECONNRESET;
249 goto failed;
250 }
251
252 if (inp->inp_flowtype != M_HASHTYPE_NONE)
253 flowid = inp->inp_flowid;
254 else
255 flowid = arc4random();
256 qidx = flowid % vi->nrxq + vi->first_rxq;
257 tlsp->rx_qid = sc->sge.rxq[qidx].iq.abs_id;
258 qidx = (flowid % (vi->ntxq - vi->rsrv_noflowq)) + vi->rsrv_noflowq +
259 vi->first_txq;
260 tlsp->txq = txq = &sc->sge.txq[qidx];
261 INP_RUNLOCK(inp);
262
263 error = ktls_setup_keys(tlsp, tls, txq);
264 if (error)
265 goto failed;
266
267 tlsp->enc_mode = t4_tls_cipher_mode(tls);
268 tlsp->tx_key_info_size = t4_tls_key_info_size(tls);
269
270 /* The SCMD fields used when encrypting a full TLS record. */
271 if (tlsp->tls13)
272 tlsp->scmd0.seqno_numivs = V_SCMD_SEQ_NO_CTRL(0);
273 else
274 tlsp->scmd0.seqno_numivs = V_SCMD_SEQ_NO_CTRL(3);
275 tlsp->scmd0.seqno_numivs |=
276 V_SCMD_PROTO_VERSION(t4_tls_proto_ver(tls)) |
277 V_SCMD_ENC_DEC_CTRL(SCMD_ENCDECCTRL_ENCRYPT) |
278 V_SCMD_CIPH_AUTH_SEQ_CTRL((mac_first == 0)) |
279 V_SCMD_CIPH_MODE(tlsp->enc_mode) |
280 V_SCMD_AUTH_MODE(t4_tls_auth_mode(tls)) |
281 V_SCMD_HMAC_CTRL(t4_tls_hmac_ctrl(tls)) |
282 V_SCMD_IV_SIZE(iv_size / 2) | V_SCMD_NUM_IVS(1);
283 tlsp->scmd0.seqno_numivs = htobe32(tlsp->scmd0.seqno_numivs);
284
285 tlsp->scmd0.ivgen_hdrlen = V_SCMD_IV_GEN_CTRL(0) |
286 V_SCMD_TLS_FRAG_ENABLE(0);
287 if (tlsp->inline_key)
288 tlsp->scmd0.ivgen_hdrlen |= V_SCMD_KEY_CTX_INLINE(1);
289
290 /*
291 * The SCMD fields used when encrypting a short TLS record
292 * (no trailer and possibly a truncated payload).
293 */
294 tlsp->scmd0_short.seqno_numivs = V_SCMD_SEQ_NO_CTRL(0) |
295 V_SCMD_PROTO_VERSION(SCMD_PROTO_VERSION_GENERIC) |
296 V_SCMD_ENC_DEC_CTRL(SCMD_ENCDECCTRL_ENCRYPT) |
297 V_SCMD_CIPH_AUTH_SEQ_CTRL((mac_first == 0)) |
298 V_SCMD_AUTH_MODE(SCMD_AUTH_MODE_NOP) |
299 V_SCMD_HMAC_CTRL(SCMD_HMAC_CTRL_NOP) |
300 V_SCMD_IV_SIZE(AES_BLOCK_LEN / 2) | V_SCMD_NUM_IVS(0);
301 if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM)
302 tlsp->scmd0_short.seqno_numivs |=
303 V_SCMD_CIPH_MODE(SCMD_CIPH_MODE_AES_CTR);
304 else
305 tlsp->scmd0_short.seqno_numivs |=
306 V_SCMD_CIPH_MODE(tlsp->enc_mode);
307 tlsp->scmd0_short.seqno_numivs =
308 htobe32(tlsp->scmd0_short.seqno_numivs);
309
310 tlsp->scmd0_short.ivgen_hdrlen = V_SCMD_IV_GEN_CTRL(0) |
311 V_SCMD_TLS_FRAG_ENABLE(0) | V_SCMD_AADIVDROP(1);
312 if (tlsp->inline_key)
313 tlsp->scmd0_short.ivgen_hdrlen |= V_SCMD_KEY_CTX_INLINE(1);
314
315 /*
316 * The SCMD fields used when encrypting a short TLS record
317 * using a partial GHASH.
318 */
319 tlsp->scmd0_partial.seqno_numivs = V_SCMD_SEQ_NO_CTRL(0) |
320 V_SCMD_PROTO_VERSION(SCMD_PROTO_VERSION_GENERIC) |
321 V_SCMD_ENC_DEC_CTRL(SCMD_ENCDECCTRL_ENCRYPT) |
322 V_SCMD_CIPH_AUTH_SEQ_CTRL((mac_first == 0)) |
323 V_SCMD_CIPH_MODE(tlsp->enc_mode) |
324 V_SCMD_AUTH_MODE(t4_tls_auth_mode(tls)) |
325 V_SCMD_HMAC_CTRL(t4_tls_hmac_ctrl(tls)) |
326 V_SCMD_IV_SIZE(AES_BLOCK_LEN / 2) | V_SCMD_NUM_IVS(1);
327 tlsp->scmd0_partial.seqno_numivs =
328 htobe32(tlsp->scmd0_partial.seqno_numivs);
329
330 tlsp->scmd0_partial.ivgen_hdrlen = V_SCMD_IV_GEN_CTRL(0) |
331 V_SCMD_TLS_FRAG_ENABLE(0) | V_SCMD_AADIVDROP(1) |
332 V_SCMD_KEY_CTX_INLINE(1);
333
334 TXQ_LOCK(txq);
335 if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM)
336 txq->kern_tls_gcm++;
337 else
338 txq->kern_tls_cbc++;
339 TXQ_UNLOCK(txq);
340 *pt = &tlsp->com;
341 return (0);
342
343 failed:
344 m_snd_tag_rele(&tlsp->com);
345 return (error);
346 }
347
348 static int
ktls_setup_keys(struct tlspcb * tlsp,const struct ktls_session * tls,struct sge_txq * txq)349 ktls_setup_keys(struct tlspcb *tlsp, const struct ktls_session *tls,
350 struct sge_txq *txq)
351 {
352 struct tls_key_req *kwr;
353 struct tls_keyctx *kctx;
354 void *items[1];
355 struct mbuf *m;
356 int error;
357
358 /*
359 * Store the salt and keys in the key context. For
360 * connections with an inline key, this key context is passed
361 * as immediate data in each work request. For connections
362 * storing the key in DDR, a work request is used to store a
363 * copy of the key context in DDR.
364 */
365 t4_tls_key_ctx(tls, KTLS_TX, &tlsp->keyctx);
366 if (tlsp->inline_key)
367 return (0);
368
369 /* Populate key work request. */
370 m = alloc_wr_mbuf(TLS_KEY_WR_SZ, M_NOWAIT);
371 if (m == NULL) {
372 CTR(KTR_CXGBE, "%s: %p failed to alloc WR mbuf", __func__,
373 tlsp);
374 return (ENOMEM);
375 }
376 m->m_pkthdr.snd_tag = m_snd_tag_ref(&tlsp->com);
377 m->m_pkthdr.csum_flags |= CSUM_SND_TAG;
378 kwr = mtod(m, void *);
379 memset(kwr, 0, TLS_KEY_WR_SZ);
380
381 t4_write_tlskey_wr(tls, KTLS_TX, 0, 0, tlsp->tx_key_addr, kwr);
382 kctx = (struct tls_keyctx *)(kwr + 1);
383 memcpy(kctx, &tlsp->keyctx, sizeof(*kctx));
384
385 /*
386 * Place the key work request in the transmit queue. It
387 * should be sent to the NIC before any TLS packets using this
388 * session.
389 */
390 items[0] = m;
391 error = mp_ring_enqueue(txq->r, items, 1, 1);
392 if (error)
393 m_free(m);
394 else
395 CTR(KTR_CXGBE, "%s: %p sent key WR", __func__, tlsp);
396 return (error);
397 }
398
399 static u_int
ktls_base_wr_size(struct tlspcb * tlsp,bool inline_key)400 ktls_base_wr_size(struct tlspcb *tlsp, bool inline_key)
401 {
402 u_int wr_len;
403
404 wr_len = sizeof(struct fw_ulptx_wr); // 16
405 wr_len += sizeof(struct ulp_txpkt); // 8
406 wr_len += sizeof(struct ulptx_idata); // 8
407 wr_len += sizeof(struct cpl_tx_sec_pdu);// 32
408 if (inline_key)
409 wr_len += tlsp->tx_key_info_size;
410 else {
411 wr_len += sizeof(struct ulptx_sc_memrd);// 8
412 wr_len += sizeof(struct ulptx_idata); // 8
413 }
414 /* SplitMode CPL_RX_PHYS_DSGL here if needed. */
415 /* CPL_TX_*_LSO here if needed. */
416 wr_len += sizeof(struct cpl_tx_pkt_core);// 16
417 return (wr_len);
418 }
419
420 static u_int
ktls_sgl_size(u_int nsegs)421 ktls_sgl_size(u_int nsegs)
422 {
423 u_int wr_len;
424
425 /* First segment is part of ulptx_sgl. */
426 nsegs--;
427
428 wr_len = sizeof(struct ulptx_sgl);
429 wr_len += 8 * ((3 * nsegs) / 2 + (nsegs & 1));
430 return (wr_len);
431 }
432
433 /*
434 * A request that doesn't need to generate the TLS trailer is a short
435 * record. For these requests, part of the TLS record payload is
436 * encrypted without invoking the MAC.
437 *
438 * Returns true if this record should be sent as a short record. In
439 * either case, the remaining outputs describe the how much of the
440 * TLS record to send as input to the crypto block and the amount of
441 * crypto output to trim via SplitMode:
442 *
443 * *header_len - Number of bytes of TLS header to pass as immediate
444 * data
445 *
446 * *offset - Start offset of TLS record payload to pass as DSGL data
447 *
448 * *plen - Length of TLS record payload to pass as DSGL data
449 *
450 * *leading_waste - amount of non-packet-header bytes to drop at the
451 * start of the crypto output
452 *
453 * *trailing_waste - amount of crypto output to drop from the end
454 */
455 static bool
ktls_is_short_record(struct tlspcb * tlsp,struct mbuf * m_tls,u_int tlen,u_int rlen,u_int * header_len,u_int * offset,u_int * plen,u_int * leading_waste,u_int * trailing_waste,bool send_partial_ghash,bool request_ghash)456 ktls_is_short_record(struct tlspcb *tlsp, struct mbuf *m_tls, u_int tlen,
457 u_int rlen, u_int *header_len, u_int *offset, u_int *plen,
458 u_int *leading_waste, u_int *trailing_waste, bool send_partial_ghash,
459 bool request_ghash)
460 {
461 u_int new_tlen, trailer_len;
462
463 MPASS(tlen > m_tls->m_epg_hdrlen);
464
465 /*
466 * For TLS 1.3 treat the inner record type stored as the first
467 * byte of the trailer as part of the payload rather than part
468 * of the trailer.
469 */
470 trailer_len = m_tls->m_epg_trllen;
471 if (tlsp->tls13)
472 trailer_len--;
473
474 /*
475 * Default to sending the full record as input to the crypto
476 * engine and relying on SplitMode to drop any waste.
477 */
478 *header_len = m_tls->m_epg_hdrlen;
479 *offset = 0;
480 *plen = rlen - (m_tls->m_epg_hdrlen + trailer_len);
481 *leading_waste = mtod(m_tls, vm_offset_t);
482 *trailing_waste = rlen - tlen;
483 if (!tlsp->sc->tlst.short_records)
484 return (false);
485
486 if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_CBC) {
487 /*
488 * For AES-CBC we have to send input from the start of
489 * the TLS record payload that is a multiple of the
490 * block size. new_tlen rounds up tlen to the end of
491 * the containing AES block. If this last block
492 * overlaps with the trailer, send the full record to
493 * generate the MAC.
494 */
495 new_tlen = TLS_HEADER_LENGTH +
496 roundup2(tlen - TLS_HEADER_LENGTH, AES_BLOCK_LEN);
497 if (rlen - new_tlen < trailer_len)
498 return (false);
499
500 *trailing_waste = new_tlen - tlen;
501 *plen = new_tlen - m_tls->m_epg_hdrlen;
502 } else {
503 if (rlen - tlen < trailer_len ||
504 (rlen - tlen == trailer_len && request_ghash)) {
505 /*
506 * For AES-GCM we have to send the full record
507 * if the end overlaps with the trailer and a
508 * partial GHASH isn't being sent.
509 */
510 if (!send_partial_ghash)
511 return (false);
512
513 /*
514 * Will need to treat any excess trailer bytes as
515 * trailing waste. *trailing_waste is already
516 * correct.
517 */
518 } else {
519 /*
520 * We can use AES-CTR or AES-GCM in partial GHASH
521 * mode to encrypt a partial PDU.
522 *
523 * The last block can be partially encrypted
524 * without any trailing waste.
525 */
526 *trailing_waste = 0;
527 *plen = tlen - m_tls->m_epg_hdrlen;
528 }
529
530 /*
531 * If this request starts at the first byte of the
532 * payload (so the previous request sent the full TLS
533 * header as a tunnel packet) and a partial GHASH is
534 * being requested, the full TLS header must be sent
535 * as input for the GHASH.
536 */
537 if (mtod(m_tls, vm_offset_t) == m_tls->m_epg_hdrlen &&
538 request_ghash)
539 return (true);
540
541 /*
542 * In addition, we can minimize leading waste by
543 * starting encryption at the start of the closest AES
544 * block.
545 */
546 if (mtod(m_tls, vm_offset_t) >= m_tls->m_epg_hdrlen) {
547 *header_len = 0;
548 *offset = mtod(m_tls, vm_offset_t) -
549 m_tls->m_epg_hdrlen;
550 if (*offset >= *plen)
551 *offset = *plen;
552 else
553 *offset = rounddown2(*offset, AES_BLOCK_LEN);
554
555 /*
556 * If the request is just bytes from the trailer,
557 * trim the offset to the end of the payload.
558 */
559 *offset = min(*offset, *plen);
560 *plen -= *offset;
561 *leading_waste -= (m_tls->m_epg_hdrlen + *offset);
562 }
563 }
564 return (true);
565 }
566
567 /* Size of the AES-GCM TLS AAD for a given connection. */
568 static int
ktls_gcm_aad_len(struct tlspcb * tlsp)569 ktls_gcm_aad_len(struct tlspcb *tlsp)
570 {
571 return (tlsp->tls13 ? sizeof(struct tls_aead_data_13) :
572 sizeof(struct tls_aead_data));
573 }
574
575 static int
ktls_wr_len(struct tlspcb * tlsp,struct mbuf * m,struct mbuf * m_tls,int * nsegsp)576 ktls_wr_len(struct tlspcb *tlsp, struct mbuf *m, struct mbuf *m_tls,
577 int *nsegsp)
578 {
579 const struct tls_record_layer *hdr;
580 u_int header_len, imm_len, offset, plen, rlen, tlen, wr_len;
581 u_int leading_waste, trailing_waste;
582 bool inline_key, last_ghash_frag, request_ghash, send_partial_ghash;
583 bool short_record;
584
585 M_ASSERTEXTPG(m_tls);
586
587 /*
588 * The relative offset of the last byte to send from the TLS
589 * record.
590 */
591 tlen = mtod(m_tls, vm_offset_t) + m_tls->m_len;
592 if (tlen <= m_tls->m_epg_hdrlen) {
593 /*
594 * For requests that only want to send the TLS header,
595 * send a tunnelled packet as immediate data.
596 */
597 wr_len = sizeof(struct fw_eth_tx_pkt_wr) +
598 sizeof(struct cpl_tx_pkt_core) +
599 roundup2(m->m_len + m_tls->m_len, 16);
600 if (wr_len > SGE_MAX_WR_LEN) {
601 CTR(KTR_CXGBE,
602 "%s: %p TLS header-only packet too long (len %d)",
603 __func__, tlsp, m->m_len + m_tls->m_len);
604 }
605
606 /* This should always be the last TLS record in a chain. */
607 MPASS(m_tls->m_next == NULL);
608 *nsegsp = 0;
609 return (wr_len);
610 }
611
612 hdr = (void *)m_tls->m_epg_hdr;
613 rlen = TLS_HEADER_LENGTH + ntohs(hdr->tls_length);
614
615 /*
616 * See if this request might make use of GHASH state. This
617 * errs on the side of over-budgeting the WR size.
618 */
619 last_ghash_frag = false;
620 request_ghash = false;
621 send_partial_ghash = false;
622 if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM &&
623 tlsp->sc->tlst.partial_ghash && tlsp->sc->tlst.short_records) {
624 u_int trailer_len;
625
626 trailer_len = m_tls->m_epg_trllen;
627 if (tlsp->tls13)
628 trailer_len--;
629 KASSERT(trailer_len == AES_GMAC_HASH_LEN,
630 ("invalid trailer length for AES-GCM"));
631
632 /* Is this the start of a TLS record? */
633 if (mtod(m_tls, vm_offset_t) <= m_tls->m_epg_hdrlen) {
634 /*
635 * Might use partial GHASH if this doesn't
636 * send the full record.
637 */
638 if (tlen < rlen) {
639 if (tlen < (rlen - trailer_len))
640 send_partial_ghash = true;
641 request_ghash = true;
642 }
643 } else {
644 send_partial_ghash = true;
645 if (tlen < rlen)
646 request_ghash = true;
647 if (tlen >= (rlen - trailer_len))
648 last_ghash_frag = true;
649 }
650 }
651
652 /*
653 * Assume not sending partial GHASH for this call to get the
654 * larger size.
655 */
656 short_record = ktls_is_short_record(tlsp, m_tls, tlen, rlen,
657 &header_len, &offset, &plen, &leading_waste, &trailing_waste,
658 false, request_ghash);
659
660 inline_key = send_partial_ghash || tlsp->inline_key;
661
662 /* Calculate the size of the work request. */
663 wr_len = ktls_base_wr_size(tlsp, inline_key);
664
665 if (send_partial_ghash)
666 wr_len += AES_GMAC_HASH_LEN;
667
668 if (leading_waste != 0 || trailing_waste != 0) {
669 /*
670 * Partial records might require a SplitMode
671 * CPL_RX_PHYS_DSGL.
672 */
673 wr_len += sizeof(struct cpl_t7_rx_phys_dsgl);
674 }
675
676 /* Budget for an LSO header even if we don't use it. */
677 wr_len += sizeof(struct cpl_tx_pkt_lso_core);
678
679 /*
680 * Headers (including the TLS header) are always sent as
681 * immediate data. Short records include a raw AES IV as
682 * immediate data. TLS 1.3 non-short records include a
683 * placeholder for the sequence number as immediate data.
684 * Short records using a partial hash may also need to send
685 * TLS AAD. If a partial hash might be sent, assume a short
686 * record to get the larger size.
687 */
688 imm_len = m->m_len + header_len;
689 if (short_record || send_partial_ghash) {
690 imm_len += AES_BLOCK_LEN;
691 if (send_partial_ghash && header_len != 0)
692 imm_len += ktls_gcm_aad_len(tlsp);
693 } else if (tlsp->tls13)
694 imm_len += sizeof(uint64_t);
695 wr_len += roundup2(imm_len, 16);
696
697 /*
698 * TLS record payload via DSGL. For partial GCM mode we
699 * might need an extra SG entry for a placeholder.
700 */
701 *nsegsp = sglist_count_mbuf_epg(m_tls, m_tls->m_epg_hdrlen + offset,
702 plen);
703 wr_len += ktls_sgl_size(*nsegsp + (last_ghash_frag ? 1 : 0));
704
705 if (request_ghash) {
706 /* AES-GCM records might return a partial hash. */
707 wr_len += sizeof(struct ulp_txpkt);
708 wr_len += sizeof(struct ulptx_idata);
709 wr_len += sizeof(struct cpl_tx_tls_ack);
710 wr_len += sizeof(struct rss_header) +
711 sizeof(struct cpl_fw6_pld);
712 wr_len += AES_GMAC_HASH_LEN;
713 }
714
715 wr_len = roundup2(wr_len, 16);
716 return (wr_len);
717 }
718
719 /* Queue the next pending packet. */
720 static void
ktls_queue_next_packet(struct tlspcb * tlsp,bool enqueue_only)721 ktls_queue_next_packet(struct tlspcb *tlsp, bool enqueue_only)
722 {
723 #ifdef KTR
724 struct ether_header *eh;
725 struct tcphdr *tcp;
726 tcp_seq tcp_seqno;
727 #endif
728 struct mbuf *m;
729 void *items[1];
730 int rc;
731
732 TXQ_LOCK_ASSERT_OWNED(tlsp->txq);
733 KASSERT(tlsp->queue_mbufs, ("%s: mbufs not being queued for %p",
734 __func__, tlsp));
735 for (;;) {
736 m = mbufq_dequeue(&tlsp->pending_mbufs);
737 if (m == NULL) {
738 tlsp->queue_mbufs = false;
739 return;
740 }
741
742 #ifdef KTR
743 eh = mtod(m, struct ether_header *);
744 tcp = (struct tcphdr *)((char *)eh + m->m_pkthdr.l2hlen +
745 m->m_pkthdr.l3hlen);
746 tcp_seqno = ntohl(tcp->th_seq);
747 #ifdef VERBOSE_TRACES
748 CTR(KTR_CXGBE, "%s: pkt len %d TCP seq %u", __func__,
749 m->m_pkthdr.len, tcp_seqno);
750 #endif
751 #endif
752
753 items[0] = m;
754 if (enqueue_only)
755 rc = mp_ring_enqueue_only(tlsp->txq->r, items, 1);
756 else {
757 TXQ_UNLOCK(tlsp->txq);
758 rc = mp_ring_enqueue(tlsp->txq->r, items, 1, 256);
759 TXQ_LOCK(tlsp->txq);
760 }
761 if (__predict_true(rc == 0))
762 return;
763
764 CTR(KTR_CXGBE, "%s: pkt len %d TCP seq %u dropped", __func__,
765 m->m_pkthdr.len, tcp_seqno);
766 m_freem(m);
767 }
768 }
769
770 int
t7_ktls_parse_pkt(struct mbuf * m)771 t7_ktls_parse_pkt(struct mbuf *m)
772 {
773 struct tlspcb *tlsp;
774 struct ether_header *eh;
775 struct ip *ip;
776 struct ip6_hdr *ip6;
777 struct tcphdr *tcp;
778 struct mbuf *m_tls;
779 void *items[1];
780 int error, nsegs;
781 u_int wr_len, tot_len;
782 uint16_t eh_type;
783
784 /*
785 * Locate headers in initial mbuf.
786 *
787 * XXX: This assumes all of the headers are in the initial mbuf.
788 * Could perhaps use m_advance() like parse_pkt() if that turns
789 * out to not be true.
790 */
791 M_ASSERTPKTHDR(m);
792 MPASS(m->m_pkthdr.snd_tag != NULL);
793 tlsp = mst_to_tls(m->m_pkthdr.snd_tag);
794
795 if (m->m_len <= sizeof(*eh) + sizeof(*ip)) {
796 CTR(KTR_CXGBE, "%s: %p header mbuf too short", __func__, tlsp);
797 return (EINVAL);
798 }
799 eh = mtod(m, struct ether_header *);
800 eh_type = ntohs(eh->ether_type);
801 if (eh_type == ETHERTYPE_VLAN) {
802 struct ether_vlan_header *evh = (void *)eh;
803
804 eh_type = ntohs(evh->evl_proto);
805 m->m_pkthdr.l2hlen = sizeof(*evh);
806 } else
807 m->m_pkthdr.l2hlen = sizeof(*eh);
808
809 switch (eh_type) {
810 case ETHERTYPE_IP:
811 ip = (struct ip *)(eh + 1);
812 if (ip->ip_p != IPPROTO_TCP) {
813 CTR(KTR_CXGBE, "%s: %p mbuf not IPPROTO_TCP", __func__,
814 tlsp);
815 return (EINVAL);
816 }
817 m->m_pkthdr.l3hlen = ip->ip_hl * 4;
818 break;
819 case ETHERTYPE_IPV6:
820 ip6 = (struct ip6_hdr *)(eh + 1);
821 if (ip6->ip6_nxt != IPPROTO_TCP) {
822 CTR(KTR_CXGBE, "%s: %p, mbuf not IPPROTO_TCP (%u)",
823 __func__, tlsp, ip6->ip6_nxt);
824 return (EINVAL);
825 }
826 m->m_pkthdr.l3hlen = sizeof(struct ip6_hdr);
827 break;
828 default:
829 CTR(KTR_CXGBE, "%s: %p mbuf not ETHERTYPE_IP{,V6}", __func__,
830 tlsp);
831 return (EINVAL);
832 }
833 if (m->m_len < m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen +
834 sizeof(*tcp)) {
835 CTR(KTR_CXGBE, "%s: %p header mbuf too short (2)", __func__,
836 tlsp);
837 return (EINVAL);
838 }
839 tcp = (struct tcphdr *)((char *)(eh + 1) + m->m_pkthdr.l3hlen);
840 m->m_pkthdr.l4hlen = tcp->th_off * 4;
841
842 /* Bail if there is TCP payload before the TLS record. */
843 if (m->m_len != m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen +
844 m->m_pkthdr.l4hlen) {
845 CTR(KTR_CXGBE,
846 "%s: %p header mbuf bad length (%d + %d + %d != %d)",
847 __func__, tlsp, m->m_pkthdr.l2hlen, m->m_pkthdr.l3hlen,
848 m->m_pkthdr.l4hlen, m->m_len);
849 return (EINVAL);
850 }
851
852 /* Assume all headers are in 'm' for now. */
853 MPASS(m->m_next != NULL);
854 MPASS(m->m_next->m_flags & M_EXTPG);
855
856 tot_len = 0;
857
858 /*
859 * Each of the remaining mbufs in the chain should reference a
860 * TLS record.
861 */
862 for (m_tls = m->m_next; m_tls != NULL; m_tls = m_tls->m_next) {
863 MPASS(m_tls->m_flags & M_EXTPG);
864
865 wr_len = ktls_wr_len(tlsp, m, m_tls, &nsegs);
866 #ifdef VERBOSE_TRACES
867 CTR(KTR_CXGBE, "%s: %p wr_len %d nsegs %d", __func__, tlsp,
868 wr_len, nsegs);
869 #endif
870 if (wr_len > SGE_MAX_WR_LEN || nsegs > TX_SGL_SEGS)
871 return (EFBIG);
872 tot_len += roundup2(wr_len, EQ_ESIZE);
873
874 /*
875 * Store 'nsegs' for the first TLS record in the
876 * header mbuf's metadata.
877 */
878 if (m_tls == m->m_next)
879 set_mbuf_nsegs(m, nsegs);
880 }
881
882 MPASS(tot_len != 0);
883 set_mbuf_len16(m, tot_len / 16);
884
885 if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM) {
886 /* Defer packets beyond what has been sent so far. */
887 TXQ_LOCK(tlsp->txq);
888 if (tlsp->queue_mbufs) {
889 error = mbufq_enqueue(&tlsp->pending_mbufs, m);
890 if (error == 0) {
891 #ifdef VERBOSE_TRACES
892 CTR(KTR_CXGBE,
893 "%s: %p len16 %d nsegs %d TCP seq %u deferred",
894 __func__, tlsp, mbuf_len16(m),
895 mbuf_nsegs(m), ntohl(tcp->th_seq));
896 #endif
897 }
898 TXQ_UNLOCK(tlsp->txq);
899 return (error);
900 }
901 tlsp->queue_mbufs = true;
902 TXQ_UNLOCK(tlsp->txq);
903 }
904
905 #ifdef VERBOSE_TRACES
906 CTR(KTR_CXGBE, "%s: %p len16 %d nsegs %d", __func__, tlsp,
907 mbuf_len16(m), mbuf_nsegs(m));
908 #endif
909 items[0] = m;
910 error = mp_ring_enqueue(tlsp->txq->r, items, 1, 256);
911 if (__predict_false(error != 0)) {
912 if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM) {
913 TXQ_LOCK(tlsp->txq);
914 ktls_queue_next_packet(tlsp, false);
915 TXQ_UNLOCK(tlsp->txq);
916 }
917 }
918 return (error);
919 }
920
921 static inline bool
needs_vlan_insertion(struct mbuf * m)922 needs_vlan_insertion(struct mbuf *m)
923 {
924
925 M_ASSERTPKTHDR(m);
926
927 return (m->m_flags & M_VLANTAG);
928 }
929
930 static inline uint64_t
pkt_ctrl1(struct sge_txq * txq,struct mbuf * m,uint16_t eh_type)931 pkt_ctrl1(struct sge_txq *txq, struct mbuf *m, uint16_t eh_type)
932 {
933 uint64_t ctrl1;
934
935 /* Checksums are always offloaded */
936 if (eh_type == ETHERTYPE_IP) {
937 ctrl1 = V_TXPKT_CSUM_TYPE(TX_CSUM_TCPIP) |
938 V_T6_TXPKT_ETHHDR_LEN(m->m_pkthdr.l2hlen - ETHER_HDR_LEN) |
939 V_TXPKT_IPHDR_LEN(m->m_pkthdr.l3hlen);
940 } else {
941 MPASS(m->m_pkthdr.l3hlen == sizeof(struct ip6_hdr));
942 ctrl1 = V_TXPKT_CSUM_TYPE(TX_CSUM_TCPIP6) |
943 V_T6_TXPKT_ETHHDR_LEN(m->m_pkthdr.l2hlen - ETHER_HDR_LEN) |
944 V_TXPKT_IPHDR_LEN(m->m_pkthdr.l3hlen);
945 }
946 txq->txcsum++;
947
948 /* VLAN tag insertion */
949 if (needs_vlan_insertion(m)) {
950 ctrl1 |= F_TXPKT_VLAN_VLD |
951 V_TXPKT_VLAN(m->m_pkthdr.ether_vtag);
952 txq->vlan_insertion++;
953 }
954
955 return (ctrl1);
956 }
957
958 static inline void *
write_lso_cpl(void * cpl,struct mbuf * m0,uint16_t mss,uint16_t eh_type,int total_len)959 write_lso_cpl(void *cpl, struct mbuf *m0, uint16_t mss, uint16_t eh_type,
960 int total_len)
961 {
962 struct cpl_tx_pkt_lso_core *lso;
963 uint32_t ctrl;
964
965 KASSERT(m0->m_pkthdr.l2hlen > 0 && m0->m_pkthdr.l3hlen > 0 &&
966 m0->m_pkthdr.l4hlen > 0,
967 ("%s: mbuf %p needs TSO but missing header lengths",
968 __func__, m0));
969
970 ctrl = V_LSO_OPCODE(CPL_TX_PKT_LSO) |
971 F_LSO_FIRST_SLICE | F_LSO_LAST_SLICE |
972 V_LSO_ETHHDR_LEN((m0->m_pkthdr.l2hlen - ETHER_HDR_LEN) >> 2) |
973 V_LSO_IPHDR_LEN(m0->m_pkthdr.l3hlen >> 2) |
974 V_LSO_TCPHDR_LEN(m0->m_pkthdr.l4hlen >> 2);
975 if (eh_type == ETHERTYPE_IPV6)
976 ctrl |= F_LSO_IPV6;
977
978 lso = cpl;
979 lso->lso_ctrl = htobe32(ctrl);
980 lso->ipid_ofst = htobe16(0);
981 lso->mss = htobe16(mss);
982 lso->seqno_offset = htobe32(0);
983 lso->len = htobe32(total_len);
984
985 return (lso + 1);
986 }
987
988 static inline void *
write_tx_tls_ack(void * dst,u_int rx_chid,u_int hash_len,bool ghash_lcb)989 write_tx_tls_ack(void *dst, u_int rx_chid, u_int hash_len, bool ghash_lcb)
990 {
991 struct cpl_tx_tls_ack *cpl;
992 uint32_t flags;
993
994 flags = ghash_lcb ? F_CPL_TX_TLS_ACK_LCB : F_CPL_TX_TLS_ACK_PHASH;
995 cpl = dst;
996 cpl->op_to_Rsvd2 = htobe32(V_CPL_TX_TLS_ACK_OPCODE(CPL_TX_TLS_ACK) |
997 V_T7_CPL_TX_TLS_ACK_RXCHID(rx_chid) | F_CPL_TX_TLS_ACK_ULPTXLPBK |
998 flags);
999
1000 /* 32 == AckEncCpl, 16 == LCB */
1001 cpl->PldLen = htobe32(V_CPL_TX_TLS_ACK_PLDLEN(32 + 16 + hash_len));
1002 cpl->Rsvd3 = 0;
1003
1004 return (cpl + 1);
1005 }
1006
1007 static inline void *
write_fw6_pld(void * dst,u_int rx_chid,u_int rx_qid,u_int hash_len,uint64_t cookie)1008 write_fw6_pld(void *dst, u_int rx_chid, u_int rx_qid, u_int hash_len,
1009 uint64_t cookie)
1010 {
1011 struct rss_header *rss;
1012 struct cpl_fw6_pld *cpl;
1013
1014 rss = dst;
1015 memset(rss, 0, sizeof(*rss));
1016 rss->opcode = CPL_FW6_PLD;
1017 rss->qid = htobe16(rx_qid);
1018 rss->channel = rx_chid;
1019
1020 cpl = (void *)(rss + 1);
1021 memset(cpl, 0, sizeof(*cpl));
1022 cpl->opcode = CPL_FW6_PLD;
1023 cpl->len = htobe16(hash_len);
1024 cpl->data[1] = htobe64(cookie);
1025
1026 return (cpl + 1);
1027 }
1028
1029 static inline void *
write_split_mode_rx_phys(void * dst,struct mbuf * m,struct mbuf * m_tls,u_int crypto_hdr_len,u_int leading_waste,u_int trailing_waste)1030 write_split_mode_rx_phys(void *dst, struct mbuf *m, struct mbuf *m_tls,
1031 u_int crypto_hdr_len, u_int leading_waste, u_int trailing_waste)
1032 {
1033 struct cpl_t7_rx_phys_dsgl *cpl;
1034 uint16_t *len;
1035 uint8_t numsge;
1036
1037 /* Forward first (3) and third (1) segments. */
1038 numsge = 0xa;
1039
1040 cpl = dst;
1041 cpl->ot.opcode = CPL_RX_PHYS_DSGL;
1042 cpl->PhysAddrFields_lo_to_NumSGE =
1043 htobe32(F_CPL_T7_RX_PHYS_DSGL_SPLITMODE |
1044 V_CPL_T7_RX_PHYS_DSGL_NUMSGE(numsge));
1045
1046 len = (uint16_t *)(cpl->RSSCopy);
1047
1048 /*
1049 * First segment always contains packet headers as well as
1050 * transmit-related CPLs.
1051 */
1052 len[0] = htobe16(crypto_hdr_len);
1053
1054 /*
1055 * Second segment is "gap" of data to drop at the front of the
1056 * TLS record.
1057 */
1058 len[1] = htobe16(leading_waste);
1059
1060 /* Third segment is how much of the TLS record to send. */
1061 len[2] = htobe16(m_tls->m_len);
1062
1063 /* Fourth segment is how much data to drop at the end. */
1064 len[3] = htobe16(trailing_waste);
1065
1066 #ifdef VERBOSE_TRACES
1067 CTR(KTR_CXGBE, "%s: forward %u skip %u forward %u skip %u",
1068 __func__, be16toh(len[0]), be16toh(len[1]), be16toh(len[2]),
1069 be16toh(len[3]));
1070 #endif
1071 return (cpl + 1);
1072 }
1073
1074 /*
1075 * If the SGL ends on an address that is not 16 byte aligned, this function will
1076 * add a 0 filled flit at the end.
1077 */
1078 static void *
write_gl_to_buf(struct sglist * gl,caddr_t to)1079 write_gl_to_buf(struct sglist *gl, caddr_t to)
1080 {
1081 struct sglist_seg *seg;
1082 __be64 *flitp;
1083 struct ulptx_sgl *usgl;
1084 int i, nflits, nsegs;
1085
1086 KASSERT(((uintptr_t)to & 0xf) == 0,
1087 ("%s: SGL must start at a 16 byte boundary: %p", __func__, to));
1088
1089 nsegs = gl->sg_nseg;
1090 MPASS(nsegs > 0);
1091
1092 nflits = (3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1) + 2;
1093 flitp = (__be64 *)to;
1094 seg = &gl->sg_segs[0];
1095 usgl = (void *)flitp;
1096
1097 usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) |
1098 V_ULPTX_NSGE(nsegs));
1099 usgl->len0 = htobe32(seg->ss_len);
1100 usgl->addr0 = htobe64(seg->ss_paddr);
1101 seg++;
1102
1103 for (i = 0; i < nsegs - 1; i++, seg++) {
1104 usgl->sge[i / 2].len[i & 1] = htobe32(seg->ss_len);
1105 usgl->sge[i / 2].addr[i & 1] = htobe64(seg->ss_paddr);
1106 }
1107 if (i & 1)
1108 usgl->sge[i / 2].len[1] = htobe32(0);
1109 flitp += nflits;
1110
1111 if (nflits & 1) {
1112 MPASS(((uintptr_t)flitp) & 0xf);
1113 *flitp++ = 0;
1114 }
1115
1116 MPASS((((uintptr_t)flitp) & 0xf) == 0);
1117 return (flitp);
1118 }
1119
1120 static inline void
copy_to_txd(struct sge_eq * eq,const char * from,caddr_t * to,int len)1121 copy_to_txd(struct sge_eq *eq, const char *from, caddr_t *to, int len)
1122 {
1123
1124 MPASS((uintptr_t)(*to) >= (uintptr_t)&eq->desc[0]);
1125 MPASS((uintptr_t)(*to) < (uintptr_t)&eq->desc[eq->sidx]);
1126
1127 if (__predict_true((uintptr_t)(*to) + len <=
1128 (uintptr_t)&eq->desc[eq->sidx])) {
1129 bcopy(from, *to, len);
1130 (*to) += len;
1131 if ((uintptr_t)(*to) == (uintptr_t)&eq->desc[eq->sidx])
1132 (*to) = (caddr_t)eq->desc;
1133 } else {
1134 int portion = (uintptr_t)&eq->desc[eq->sidx] - (uintptr_t)(*to);
1135
1136 bcopy(from, *to, portion);
1137 from += portion;
1138 portion = len - portion; /* remaining */
1139 bcopy(from, (void *)eq->desc, portion);
1140 (*to) = (caddr_t)eq->desc + portion;
1141 }
1142 }
1143
1144 static int
ktls_write_tunnel_packet(struct sge_txq * txq,void * dst,struct mbuf * m,const void * src,u_int len,u_int available,tcp_seq tcp_seqno,u_int pidx,uint16_t eh_type,bool last_wr)1145 ktls_write_tunnel_packet(struct sge_txq *txq, void *dst, struct mbuf *m,
1146 const void *src, u_int len, u_int available, tcp_seq tcp_seqno, u_int pidx,
1147 uint16_t eh_type, bool last_wr)
1148 {
1149 struct tx_sdesc *txsd;
1150 struct fw_eth_tx_pkt_wr *wr;
1151 struct cpl_tx_pkt_core *cpl;
1152 uint32_t ctrl;
1153 int len16, ndesc, pktlen;
1154 struct ether_header *eh;
1155 struct ip *ip, newip;
1156 struct ip6_hdr *ip6, newip6;
1157 struct tcphdr *tcp, newtcp;
1158 caddr_t out;
1159
1160 TXQ_LOCK_ASSERT_OWNED(txq);
1161 M_ASSERTPKTHDR(m);
1162
1163 wr = dst;
1164 pktlen = m->m_len + len;
1165 ctrl = sizeof(struct cpl_tx_pkt_core) + pktlen;
1166 len16 = howmany(sizeof(struct fw_eth_tx_pkt_wr) + ctrl, 16);
1167 ndesc = tx_len16_to_desc(len16);
1168 MPASS(ndesc <= available);
1169
1170 /* Firmware work request header */
1171 /* TODO: Handle VF work request. */
1172 wr->op_immdlen = htobe32(V_FW_WR_OP(FW_ETH_TX_PKT_WR) |
1173 V_FW_ETH_TX_PKT_WR_IMMDLEN(ctrl));
1174
1175 ctrl = V_FW_WR_LEN16(len16);
1176 wr->equiq_to_len16 = htobe32(ctrl);
1177 wr->r3 = 0;
1178
1179 cpl = (void *)(wr + 1);
1180
1181 /* CPL header */
1182 cpl->ctrl0 = txq->cpl_ctrl0;
1183 cpl->pack = 0;
1184 cpl->len = htobe16(pktlen);
1185
1186 out = (void *)(cpl + 1);
1187
1188 /* Copy over Ethernet header. */
1189 eh = mtod(m, struct ether_header *);
1190 copy_to_txd(&txq->eq, (caddr_t)eh, &out, m->m_pkthdr.l2hlen);
1191
1192 /* Fixup length in IP header and copy out. */
1193 if (eh_type == ETHERTYPE_IP) {
1194 ip = (void *)((char *)eh + m->m_pkthdr.l2hlen);
1195 newip = *ip;
1196 newip.ip_len = htons(pktlen - m->m_pkthdr.l2hlen);
1197 copy_to_txd(&txq->eq, (caddr_t)&newip, &out, sizeof(newip));
1198 if (m->m_pkthdr.l3hlen > sizeof(*ip))
1199 copy_to_txd(&txq->eq, (caddr_t)(ip + 1), &out,
1200 m->m_pkthdr.l3hlen - sizeof(*ip));
1201 } else {
1202 ip6 = (void *)((char *)eh + m->m_pkthdr.l2hlen);
1203 newip6 = *ip6;
1204 newip6.ip6_plen = htons(pktlen - m->m_pkthdr.l2hlen -
1205 sizeof(*ip6));
1206 copy_to_txd(&txq->eq, (caddr_t)&newip6, &out, sizeof(newip6));
1207 MPASS(m->m_pkthdr.l3hlen == sizeof(*ip6));
1208 }
1209 cpl->ctrl1 = htobe64(pkt_ctrl1(txq, m, eh_type));
1210
1211 /* Set sequence number in TCP header. */
1212 tcp = (void *)((char *)eh + m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen);
1213 newtcp = *tcp;
1214 newtcp.th_seq = htonl(tcp_seqno);
1215 copy_to_txd(&txq->eq, (caddr_t)&newtcp, &out, sizeof(newtcp));
1216
1217 /* Copy rest of TCP header. */
1218 copy_to_txd(&txq->eq, (caddr_t)(tcp + 1), &out, m->m_len -
1219 (m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen + sizeof(*tcp)));
1220
1221 /* Copy the payload data. */
1222 copy_to_txd(&txq->eq, src, &out, len);
1223 txq->imm_wrs++;
1224
1225 txq->txpkt_wrs++;
1226
1227 txsd = &txq->sdesc[pidx];
1228 if (last_wr)
1229 txsd->m = m;
1230 else
1231 txsd->m = NULL;
1232 txsd->desc_used = ndesc;
1233
1234 return (ndesc);
1235 }
1236
1237 static int
ktls_write_tls_wr(struct tlspcb * tlsp,struct sge_txq * txq,void * dst,struct mbuf * m,struct tcphdr * tcp,struct mbuf * m_tls,u_int available,tcp_seq tcp_seqno,u_int pidx,uint16_t eh_type,uint16_t mss)1238 ktls_write_tls_wr(struct tlspcb *tlsp, struct sge_txq *txq,
1239 void *dst, struct mbuf *m, struct tcphdr *tcp, struct mbuf *m_tls,
1240 u_int available, tcp_seq tcp_seqno, u_int pidx, uint16_t eh_type,
1241 uint16_t mss)
1242 {
1243 struct sge_eq *eq = &txq->eq;
1244 struct tx_sdesc *txsd;
1245 struct fw_ulptx_wr *wr;
1246 struct ulp_txpkt *txpkt;
1247 struct ulptx_sc_memrd *memrd;
1248 struct ulptx_idata *idata;
1249 struct cpl_tx_sec_pdu *sec_pdu;
1250 struct cpl_tx_pkt_core *tx_pkt;
1251 const struct tls_record_layer *hdr;
1252 struct ip *ip;
1253 struct ip6_hdr *ip6;
1254 struct tcphdr *newtcp;
1255 char *iv, *out;
1256 u_int aad_start, aad_stop;
1257 u_int auth_start, auth_stop, auth_insert;
1258 u_int cipher_start, cipher_stop, iv_offset;
1259 u_int header_len, offset, plen, rlen, tlen;
1260 u_int imm_len, ndesc, nsegs, txpkt_lens[2], wr_len;
1261 u_int cpl_len, crypto_hdr_len, post_key_context_len;
1262 u_int leading_waste, trailing_waste;
1263 u_short ip_len;
1264 bool inline_key, ghash_lcb, last_ghash_frag, last_wr, need_lso;
1265 bool request_ghash, send_partial_ghash, short_record, split_mode;
1266 bool using_scratch;
1267
1268 MPASS(tlsp->txq == txq);
1269 M_ASSERTEXTPG(m_tls);
1270
1271 /* Final work request for this mbuf chain? */
1272 last_wr = (m_tls->m_next == NULL);
1273
1274 /*
1275 * The relative offset of the last byte to send from the TLS
1276 * record.
1277 */
1278 tlen = mtod(m_tls, vm_offset_t) + m_tls->m_len;
1279 if (tlen <= m_tls->m_epg_hdrlen) {
1280 /*
1281 * For requests that only want to send the TLS header,
1282 * send a tunnelled packet as immediate data.
1283 */
1284 #ifdef VERBOSE_TRACES
1285 CTR(KTR_CXGBE, "%s: %p header-only TLS record %u", __func__,
1286 tlsp, (u_int)m_tls->m_epg_seqno);
1287 #endif
1288 /* This should always be the last TLS record in a chain. */
1289 MPASS(last_wr);
1290
1291 txq->kern_tls_header++;
1292
1293 return (ktls_write_tunnel_packet(txq, dst, m,
1294 (char *)m_tls->m_epg_hdr + mtod(m_tls, vm_offset_t),
1295 m_tls->m_len, available, tcp_seqno, pidx, eh_type,
1296 last_wr));
1297 }
1298
1299 /* Locate the TLS header. */
1300 hdr = (void *)m_tls->m_epg_hdr;
1301 rlen = TLS_HEADER_LENGTH + ntohs(hdr->tls_length);
1302
1303 #ifdef VERBOSE_TRACES
1304 CTR(KTR_CXGBE, "%s: offset %lu len %u TCP seq %u TLS record %u",
1305 __func__, mtod(m_tls, vm_offset_t), m_tls->m_len, tcp_seqno,
1306 (u_int)m_tls->m_epg_seqno);
1307 #endif
1308
1309 /* Should this request make use of GHASH state? */
1310 ghash_lcb = false;
1311 last_ghash_frag = false;
1312 request_ghash = false;
1313 send_partial_ghash = false;
1314 if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM &&
1315 tlsp->sc->tlst.partial_ghash && tlsp->sc->tlst.short_records) {
1316 u_int trailer_len;
1317
1318 trailer_len = m_tls->m_epg_trllen;
1319 if (tlsp->tls13)
1320 trailer_len--;
1321 KASSERT(trailer_len == AES_GMAC_HASH_LEN,
1322 ("invalid trailer length for AES-GCM"));
1323
1324 /* Is this the start of a TLS record? */
1325 if (mtod(m_tls, vm_offset_t) <= m_tls->m_epg_hdrlen) {
1326 /*
1327 * If this is the very first TLS record or
1328 * if this is a newer TLS record, request a partial
1329 * hash, but not if we are going to send the whole
1330 * thing.
1331 */
1332 if ((tlsp->ghash_tls_seqno == 0 ||
1333 tlsp->ghash_tls_seqno < m_tls->m_epg_seqno) &&
1334 tlen < rlen) {
1335 /*
1336 * If we are only missing part or all
1337 * of the trailer, send a normal full
1338 * record but request the hash.
1339 * Otherwise, use partial GHASH mode.
1340 */
1341 if (tlen >= (rlen - trailer_len))
1342 ghash_lcb = true;
1343 else
1344 send_partial_ghash = true;
1345 request_ghash = true;
1346 tlsp->ghash_tls_seqno = m_tls->m_epg_seqno;
1347 }
1348 } else if (tlsp->ghash_tls_seqno == m_tls->m_epg_seqno &&
1349 tlsp->ghash_valid) {
1350 /*
1351 * Compute the offset of the first AES block as
1352 * is done in ktls_is_short_record.
1353 */
1354 if (rlen - tlen < trailer_len)
1355 plen = rlen - (m_tls->m_epg_hdrlen +
1356 trailer_len);
1357 else
1358 plen = tlen - m_tls->m_epg_hdrlen;
1359 offset = mtod(m_tls, vm_offset_t) - m_tls->m_epg_hdrlen;
1360 if (offset >= plen)
1361 offset = plen;
1362 else
1363 offset = rounddown2(offset, AES_BLOCK_LEN);
1364 if (tlsp->ghash_offset == offset) {
1365 if (offset == plen) {
1366 /*
1367 * Send a partial trailer as a
1368 * tunnelled packet as
1369 * immediate data.
1370 */
1371 #ifdef VERBOSE_TRACES
1372 CTR(KTR_CXGBE,
1373 "%s: %p trailer-only TLS record %u",
1374 __func__, tlsp,
1375 (u_int)m_tls->m_epg_seqno);
1376 #endif
1377
1378 txq->kern_tls_trailer++;
1379
1380 offset = mtod(m_tls, vm_offset_t) -
1381 (m_tls->m_epg_hdrlen + plen);
1382 KASSERT(offset <= AES_GMAC_HASH_LEN,
1383 ("offset outside of trailer"));
1384 return (ktls_write_tunnel_packet(txq,
1385 dst, m, tlsp->ghash + offset,
1386 m_tls->m_len, available, tcp_seqno,
1387 pidx, eh_type, last_wr));
1388 }
1389
1390 /*
1391 * If this request sends the end of
1392 * the payload, it is the last
1393 * fragment.
1394 */
1395 if (tlen >= (rlen - trailer_len)) {
1396 last_ghash_frag = true;
1397 ghash_lcb = true;
1398 }
1399
1400 /*
1401 * Only use partial GCM mode (rather
1402 * than an AES-CTR short record) if
1403 * there is input auth data to pass to
1404 * the GHASH. That is true so long as
1405 * there is at least one full block of
1406 * payload data, or if the remaining
1407 * payload data is the final partial
1408 * block.
1409 */
1410 if (plen - offset >= GMAC_BLOCK_LEN ||
1411 last_ghash_frag) {
1412 send_partial_ghash = true;
1413
1414 /*
1415 * If not sending the complete
1416 * end of the record, this is
1417 * a middle request so needs
1418 * to request an updated
1419 * partial hash.
1420 */
1421 if (tlen < rlen)
1422 request_ghash = true;
1423 }
1424 }
1425 }
1426 }
1427
1428 short_record = ktls_is_short_record(tlsp, m_tls, tlen, rlen,
1429 &header_len, &offset, &plen, &leading_waste, &trailing_waste,
1430 send_partial_ghash, request_ghash);
1431
1432 if (short_record) {
1433 #ifdef VERBOSE_TRACES
1434 CTR(KTR_CXGBE,
1435 "%s: %p short TLS record %u hdr %u offs %u plen %u",
1436 __func__, tlsp, (u_int)m_tls->m_epg_seqno, header_len,
1437 offset, plen);
1438 if (send_partial_ghash) {
1439 if (header_len != 0)
1440 CTR(KTR_CXGBE, "%s: %p sending initial GHASH",
1441 __func__, tlsp);
1442 else
1443 CTR(KTR_CXGBE, "%s: %p sending partial GHASH for offset %u%s",
1444 __func__, tlsp, tlsp->ghash_offset,
1445 last_ghash_frag ? ", last_frag" : "");
1446 }
1447 #endif
1448 KASSERT(send_partial_ghash || !request_ghash,
1449 ("requesting but not sending partial hash for short record"));
1450 } else {
1451 KASSERT(!send_partial_ghash,
1452 ("sending partial hash with full record"));
1453 }
1454
1455 if (tlen < rlen && m_tls->m_next == NULL &&
1456 (tcp->th_flags & TH_FIN) != 0) {
1457 txq->kern_tls_fin_short++;
1458 #ifdef INVARIANTS
1459 panic("%s: FIN on short TLS record", __func__);
1460 #endif
1461 }
1462
1463 /*
1464 * Use cached value for first record in chain if not using
1465 * partial GCM mode. ktls_parse_pkt() calculates nsegs based
1466 * on send_partial_ghash being false.
1467 */
1468 if (m->m_next == m_tls && !send_partial_ghash)
1469 nsegs = mbuf_nsegs(m);
1470 else
1471 nsegs = sglist_count_mbuf_epg(m_tls,
1472 m_tls->m_epg_hdrlen + offset, plen);
1473
1474 /* Determine if we need an LSO header. */
1475 need_lso = (m_tls->m_len > mss);
1476
1477 /* Calculate the size of the TLS work request. */
1478 inline_key = send_partial_ghash || tlsp->inline_key;
1479 wr_len = ktls_base_wr_size(tlsp, inline_key);
1480
1481 if (send_partial_ghash) {
1482 /* Inline key context includes partial hash in OPAD. */
1483 wr_len += AES_GMAC_HASH_LEN;
1484 }
1485
1486 /*
1487 * SplitMode is required if there is any thing we need to trim
1488 * from the crypto output, either at the front or end of the
1489 * record. Note that short records might not need trimming.
1490 */
1491 split_mode = leading_waste != 0 || trailing_waste != 0;
1492 if (split_mode) {
1493 /*
1494 * Partial records require a SplitMode
1495 * CPL_RX_PHYS_DSGL.
1496 */
1497 wr_len += sizeof(struct cpl_t7_rx_phys_dsgl);
1498 }
1499
1500 if (need_lso)
1501 wr_len += sizeof(struct cpl_tx_pkt_lso_core);
1502
1503 imm_len = m->m_len + header_len;
1504 if (short_record) {
1505 imm_len += AES_BLOCK_LEN;
1506 if (send_partial_ghash && header_len != 0)
1507 imm_len += ktls_gcm_aad_len(tlsp);
1508 } else if (tlsp->tls13)
1509 imm_len += sizeof(uint64_t);
1510 wr_len += roundup2(imm_len, 16);
1511 wr_len += ktls_sgl_size(nsegs + (last_ghash_frag ? 1 : 0));
1512 wr_len = roundup2(wr_len, 16);
1513 txpkt_lens[0] = wr_len - sizeof(*wr);
1514
1515 if (request_ghash) {
1516 /*
1517 * Requesting the hash entails a second ULP_TX_PKT
1518 * containing CPL_TX_TLS_ACK, CPL_FW6_PLD, and space
1519 * for the hash.
1520 */
1521 txpkt_lens[1] = sizeof(struct ulp_txpkt);
1522 txpkt_lens[1] += sizeof(struct ulptx_idata);
1523 txpkt_lens[1] += sizeof(struct cpl_tx_tls_ack);
1524 txpkt_lens[1] += sizeof(struct rss_header) +
1525 sizeof(struct cpl_fw6_pld);
1526 txpkt_lens[1] += AES_GMAC_HASH_LEN;
1527 wr_len += txpkt_lens[1];
1528 } else
1529 txpkt_lens[1] = 0;
1530
1531 ndesc = howmany(wr_len, EQ_ESIZE);
1532 MPASS(ndesc <= available);
1533
1534 /*
1535 * Use the per-txq scratch pad if near the end of the ring to
1536 * simplify handling of wrap-around.
1537 */
1538 using_scratch = (eq->sidx - pidx < ndesc);
1539 if (using_scratch)
1540 wr = (void *)txq->ss;
1541 else
1542 wr = dst;
1543
1544 /* FW_ULPTX_WR */
1545 wr->op_to_compl = htobe32(V_FW_WR_OP(FW_ULPTX_WR));
1546 wr->flowid_len16 = htobe32(F_FW_ULPTX_WR_DATA |
1547 V_FW_WR_LEN16(wr_len / 16));
1548 wr->cookie = 0;
1549
1550 /* ULP_TXPKT */
1551 txpkt = (void *)(wr + 1);
1552 txpkt->cmd_dest = htobe32(V_ULPTX_CMD(ULP_TX_PKT) |
1553 V_ULP_TXPKT_DATAMODIFY(0) |
1554 V_T7_ULP_TXPKT_CHANNELID(tlsp->vi->pi->port_id) |
1555 V_ULP_TXPKT_DEST(0) |
1556 V_ULP_TXPKT_CMDMORE(request_ghash ? 1 : 0) |
1557 V_ULP_TXPKT_FID(txq->eq.iqid) | V_ULP_TXPKT_RO(1));
1558 txpkt->len = htobe32(howmany(txpkt_lens[0], 16));
1559
1560 /* ULPTX_IDATA sub-command */
1561 idata = (void *)(txpkt + 1);
1562 idata->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM) |
1563 V_ULP_TX_SC_MORE(1));
1564 idata->len = sizeof(struct cpl_tx_sec_pdu);
1565
1566 /*
1567 * After the key context comes CPL_RX_PHYS_DSGL, CPL_TX_*, and
1568 * immediate data containing headers. When using an inline
1569 * key, these are counted as part of this ULPTX_IDATA. When
1570 * reading the key from memory, these are part of a separate
1571 * ULPTX_IDATA.
1572 */
1573 cpl_len = sizeof(struct cpl_tx_pkt_core);
1574 if (need_lso)
1575 cpl_len += sizeof(struct cpl_tx_pkt_lso_core);
1576 if (split_mode)
1577 cpl_len += sizeof(struct cpl_t7_rx_phys_dsgl);
1578 post_key_context_len = cpl_len + imm_len;
1579
1580 if (inline_key) {
1581 idata->len += tlsp->tx_key_info_size + post_key_context_len;
1582 if (send_partial_ghash) {
1583 /* Partial GHASH in key context. */
1584 idata->len += AES_GMAC_HASH_LEN;
1585 }
1586 }
1587 idata->len = htobe32(idata->len);
1588
1589 /* CPL_TX_SEC_PDU */
1590 sec_pdu = (void *)(idata + 1);
1591
1592 /*
1593 * Packet headers are passed through unchanged by the crypto
1594 * engine by marking them as header data in SCMD0.
1595 */
1596 crypto_hdr_len = m->m_len;
1597
1598 if (send_partial_ghash) {
1599 /*
1600 * For short records using a partial hash, the TLS
1601 * header is counted as header data in SCMD0. TLS AAD
1602 * is next (if AAD is present) followed by the AES-CTR
1603 * IV. Last is the cipher region for the payload.
1604 */
1605 if (header_len != 0) {
1606 aad_start = 1;
1607 aad_stop = ktls_gcm_aad_len(tlsp);
1608 } else {
1609 aad_start = 0;
1610 aad_stop = 0;
1611 }
1612 iv_offset = aad_stop + 1;
1613 cipher_start = iv_offset + AES_BLOCK_LEN;
1614 cipher_stop = 0;
1615 if (last_ghash_frag) {
1616 auth_start = cipher_start;
1617 auth_stop = AES_GMAC_HASH_LEN;
1618 auth_insert = auth_stop;
1619 } else if (plen < GMAC_BLOCK_LEN) {
1620 /*
1621 * A request that sends part of the first AES
1622 * block will only have AAD.
1623 */
1624 KASSERT(header_len != 0,
1625 ("%s: partial GHASH with no auth", __func__));
1626 auth_start = 0;
1627 auth_stop = 0;
1628 auth_insert = 0;
1629 } else {
1630 auth_start = cipher_start;
1631 auth_stop = plen % GMAC_BLOCK_LEN;
1632 auth_insert = 0;
1633 }
1634
1635 sec_pdu->pldlen = htobe32(aad_stop + AES_BLOCK_LEN + plen +
1636 (last_ghash_frag ? AES_GMAC_HASH_LEN : 0));
1637
1638 /*
1639 * For short records, the TLS header is treated as
1640 * header data.
1641 */
1642 crypto_hdr_len += header_len;
1643
1644 /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */
1645 sec_pdu->seqno_numivs = tlsp->scmd0_partial.seqno_numivs;
1646 sec_pdu->ivgen_hdrlen = tlsp->scmd0_partial.ivgen_hdrlen;
1647 if (last_ghash_frag)
1648 sec_pdu->ivgen_hdrlen |= V_SCMD_LAST_FRAG(1);
1649 else
1650 sec_pdu->ivgen_hdrlen |= V_SCMD_MORE_FRAGS(1);
1651 sec_pdu->ivgen_hdrlen = htobe32(sec_pdu->ivgen_hdrlen |
1652 V_SCMD_HDR_LEN(crypto_hdr_len));
1653
1654 txq->kern_tls_partial_ghash++;
1655 } else if (short_record) {
1656 /*
1657 * For short records without a partial hash, the TLS
1658 * header is counted as header data in SCMD0 and the
1659 * IV is next, followed by a cipher region for the
1660 * payload.
1661 */
1662 aad_start = 0;
1663 aad_stop = 0;
1664 iv_offset = 1;
1665 auth_start = 0;
1666 auth_stop = 0;
1667 auth_insert = 0;
1668 cipher_start = AES_BLOCK_LEN + 1;
1669 cipher_stop = 0;
1670
1671 sec_pdu->pldlen = htobe32(AES_BLOCK_LEN + plen);
1672
1673 /*
1674 * For short records, the TLS header is treated as
1675 * header data.
1676 */
1677 crypto_hdr_len += header_len;
1678
1679 /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */
1680 sec_pdu->seqno_numivs = tlsp->scmd0_short.seqno_numivs;
1681 sec_pdu->ivgen_hdrlen = htobe32(
1682 tlsp->scmd0_short.ivgen_hdrlen |
1683 V_SCMD_HDR_LEN(crypto_hdr_len));
1684
1685 txq->kern_tls_short++;
1686 } else {
1687 /*
1688 * AAD is TLS header. IV is after AAD for TLS < 1.3.
1689 * For TLS 1.3, a placeholder for the TLS sequence
1690 * number is provided as an IV before the AAD. The
1691 * cipher region starts after the AAD and IV. See
1692 * comments in ccr_authenc() and ccr_gmac() in
1693 * t4_crypto.c regarding cipher and auth start/stop
1694 * values.
1695 */
1696 if (tlsp->tls13) {
1697 iv_offset = 1;
1698 aad_start = 1 + sizeof(uint64_t);
1699 aad_stop = sizeof(uint64_t) + TLS_HEADER_LENGTH;
1700 cipher_start = aad_stop + 1;
1701 } else {
1702 aad_start = 1;
1703 aad_stop = TLS_HEADER_LENGTH;
1704 iv_offset = TLS_HEADER_LENGTH + 1;
1705 cipher_start = m_tls->m_epg_hdrlen + 1;
1706 }
1707 if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM) {
1708 cipher_stop = 0;
1709 auth_start = cipher_start;
1710 auth_stop = 0;
1711 auth_insert = 0;
1712 } else {
1713 cipher_stop = 0;
1714 auth_start = cipher_start;
1715 auth_stop = 0;
1716 auth_insert = 0;
1717 }
1718
1719 sec_pdu->pldlen = htobe32((tlsp->tls13 ? sizeof(uint64_t) : 0) +
1720 m_tls->m_epg_hdrlen + plen);
1721
1722 /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */
1723 sec_pdu->seqno_numivs = tlsp->scmd0.seqno_numivs;
1724 sec_pdu->ivgen_hdrlen = htobe32(tlsp->scmd0.ivgen_hdrlen |
1725 V_SCMD_HDR_LEN(crypto_hdr_len));
1726
1727 if (split_mode)
1728 txq->kern_tls_partial++;
1729 else
1730 txq->kern_tls_full++;
1731 }
1732 sec_pdu->op_ivinsrtofst = htobe32(
1733 V_CPL_TX_SEC_PDU_OPCODE(CPL_TX_SEC_PDU) |
1734 V_CPL_TX_SEC_PDU_CPLLEN(cpl_len / 8) |
1735 V_CPL_TX_SEC_PDU_PLACEHOLDER(send_partial_ghash ? 1 : 0) |
1736 V_CPL_TX_SEC_PDU_IVINSRTOFST(iv_offset));
1737 sec_pdu->aadstart_cipherstop_hi = htobe32(
1738 V_CPL_TX_SEC_PDU_AADSTART(aad_start) |
1739 V_CPL_TX_SEC_PDU_AADSTOP(aad_stop) |
1740 V_CPL_TX_SEC_PDU_CIPHERSTART(cipher_start) |
1741 V_CPL_TX_SEC_PDU_CIPHERSTOP_HI(cipher_stop >> 4));
1742 sec_pdu->cipherstop_lo_authinsert = htobe32(
1743 V_CPL_TX_SEC_PDU_CIPHERSTOP_LO(cipher_stop & 0xf) |
1744 V_CPL_TX_SEC_PDU_AUTHSTART(auth_start) |
1745 V_CPL_TX_SEC_PDU_AUTHSTOP(auth_stop) |
1746 V_CPL_TX_SEC_PDU_AUTHINSERT(auth_insert));
1747
1748 if (send_partial_ghash && last_ghash_frag) {
1749 uint64_t aad_len, cipher_len;
1750
1751 aad_len = ktls_gcm_aad_len(tlsp);
1752 cipher_len = rlen - (m_tls->m_epg_hdrlen + AES_GMAC_HASH_LEN);
1753 sec_pdu->scmd1 = htobe64(aad_len << 44 | cipher_len);
1754 } else
1755 sec_pdu->scmd1 = htobe64(m_tls->m_epg_seqno);
1756
1757 /* Key context */
1758 out = (void *)(sec_pdu + 1);
1759 if (inline_key) {
1760 memcpy(out, &tlsp->keyctx, tlsp->tx_key_info_size);
1761 if (send_partial_ghash) {
1762 struct tls_keyctx *keyctx = (void *)out;
1763
1764 keyctx->u.txhdr.ctxlen++;
1765 keyctx->u.txhdr.dualck_to_txvalid &= ~htobe16(
1766 V_KEY_CONTEXT_MK_SIZE(M_KEY_CONTEXT_MK_SIZE));
1767 keyctx->u.txhdr.dualck_to_txvalid |= htobe16(
1768 F_KEY_CONTEXT_OPAD_PRESENT |
1769 V_KEY_CONTEXT_MK_SIZE(0));
1770 }
1771 out += tlsp->tx_key_info_size;
1772 if (send_partial_ghash) {
1773 if (header_len != 0)
1774 memset(out, 0, AES_GMAC_HASH_LEN);
1775 else
1776 memcpy(out, tlsp->ghash, AES_GMAC_HASH_LEN);
1777 out += AES_GMAC_HASH_LEN;
1778 }
1779 } else {
1780 /* ULPTX_SC_MEMRD to read key context. */
1781 memrd = (void *)out;
1782 memrd->cmd_to_len = htobe32(V_ULPTX_CMD(ULP_TX_SC_MEMRD) |
1783 V_ULP_TX_SC_MORE(1) |
1784 V_ULPTX_LEN16(tlsp->tx_key_info_size >> 4));
1785 memrd->addr = htobe32(tlsp->tx_key_addr >> 5);
1786
1787 /* ULPTX_IDATA for CPL_TX_* and headers. */
1788 idata = (void *)(memrd + 1);
1789 idata->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM) |
1790 V_ULP_TX_SC_MORE(1));
1791 idata->len = htobe32(post_key_context_len);
1792
1793 out = (void *)(idata + 1);
1794 }
1795
1796 /* CPL_RX_PHYS_DSGL */
1797 if (split_mode) {
1798 crypto_hdr_len = sizeof(struct cpl_tx_pkt_core);
1799 if (need_lso)
1800 crypto_hdr_len += sizeof(struct cpl_tx_pkt_lso_core);
1801 crypto_hdr_len += m->m_len;
1802 out = write_split_mode_rx_phys(out, m, m_tls, crypto_hdr_len,
1803 leading_waste, trailing_waste);
1804 }
1805
1806 /* CPL_TX_PKT_LSO */
1807 if (need_lso) {
1808 out = write_lso_cpl(out, m, mss, eh_type, m->m_len +
1809 m_tls->m_len);
1810 txq->tso_wrs++;
1811 }
1812
1813 /* CPL_TX_PKT_XT */
1814 tx_pkt = (void *)out;
1815 tx_pkt->ctrl0 = txq->cpl_ctrl0;
1816 tx_pkt->ctrl1 = htobe64(pkt_ctrl1(txq, m, eh_type));
1817 tx_pkt->pack = 0;
1818 tx_pkt->len = htobe16(m->m_len + m_tls->m_len);
1819
1820 /* Copy the packet headers. */
1821 out = (void *)(tx_pkt + 1);
1822 memcpy(out, mtod(m, char *), m->m_len);
1823
1824 /* Modify the packet length in the IP header. */
1825 ip_len = m->m_len + m_tls->m_len - m->m_pkthdr.l2hlen;
1826 if (eh_type == ETHERTYPE_IP) {
1827 ip = (void *)(out + m->m_pkthdr.l2hlen);
1828 be16enc(&ip->ip_len, ip_len);
1829 } else {
1830 ip6 = (void *)(out + m->m_pkthdr.l2hlen);
1831 be16enc(&ip6->ip6_plen, ip_len - sizeof(*ip6));
1832 }
1833
1834 /* Modify sequence number and flags in TCP header. */
1835 newtcp = (void *)(out + m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen);
1836 be32enc(&newtcp->th_seq, tcp_seqno);
1837 if (!last_wr)
1838 newtcp->th_flags = tcp->th_flags & ~(TH_PUSH | TH_FIN);
1839 out += m->m_len;
1840
1841 /*
1842 * Insert placeholder for sequence number as IV for TLS 1.3
1843 * non-short records.
1844 */
1845 if (tlsp->tls13 && !short_record) {
1846 memset(out, 0, sizeof(uint64_t));
1847 out += sizeof(uint64_t);
1848 }
1849
1850 /* Populate the TLS header */
1851 memcpy(out, m_tls->m_epg_hdr, header_len);
1852 out += header_len;
1853
1854 /* TLS AAD for short records using a partial hash. */
1855 if (send_partial_ghash && header_len != 0) {
1856 if (tlsp->tls13) {
1857 struct tls_aead_data_13 ad;
1858
1859 ad.type = hdr->tls_type;
1860 ad.tls_vmajor = hdr->tls_vmajor;
1861 ad.tls_vminor = hdr->tls_vminor;
1862 ad.tls_length = hdr->tls_length;
1863 memcpy(out, &ad, sizeof(ad));
1864 out += sizeof(ad);
1865 } else {
1866 struct tls_aead_data ad;
1867 uint16_t cipher_len;
1868
1869 cipher_len = rlen -
1870 (m_tls->m_epg_hdrlen + AES_GMAC_HASH_LEN);
1871 ad.seq = htobe64(m_tls->m_epg_seqno);
1872 ad.type = hdr->tls_type;
1873 ad.tls_vmajor = hdr->tls_vmajor;
1874 ad.tls_vminor = hdr->tls_vminor;
1875 ad.tls_length = htons(cipher_len);
1876 memcpy(out, &ad, sizeof(ad));
1877 out += sizeof(ad);
1878 }
1879 }
1880
1881 /* AES IV for a short record. */
1882 if (short_record) {
1883 iv = out;
1884 if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM) {
1885 memcpy(iv, tlsp->keyctx.u.txhdr.txsalt, SALT_SIZE);
1886 if (tlsp->tls13) {
1887 uint64_t value;
1888
1889 value = be64dec(tlsp->keyctx.u.txhdr.txsalt +
1890 4);
1891 value ^= m_tls->m_epg_seqno;
1892 be64enc(iv + 4, value);
1893 } else
1894 memcpy(iv + 4, hdr + 1, 8);
1895 if (send_partial_ghash)
1896 be32enc(iv + 12, 1 + offset / AES_BLOCK_LEN);
1897 else
1898 be32enc(iv + 12, 2 + offset / AES_BLOCK_LEN);
1899 } else
1900 memcpy(iv, hdr + 1, AES_BLOCK_LEN);
1901 out += AES_BLOCK_LEN;
1902 }
1903
1904 if (imm_len % 16 != 0) {
1905 if (imm_len % 8 != 0) {
1906 /* Zero pad to an 8-byte boundary. */
1907 memset(out, 0, 8 - (imm_len % 8));
1908 out += 8 - (imm_len % 8);
1909 }
1910
1911 /*
1912 * Insert a ULP_TX_SC_NOOP if needed so the SGL is
1913 * 16-byte aligned.
1914 */
1915 if (imm_len % 16 <= 8) {
1916 idata = (void *)out;
1917 idata->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP) |
1918 V_ULP_TX_SC_MORE(1));
1919 idata->len = htobe32(0);
1920 out = (void *)(idata + 1);
1921 }
1922 }
1923
1924 /* SGL for record payload */
1925 sglist_reset(txq->gl);
1926 if (sglist_append_mbuf_epg(txq->gl, m_tls, m_tls->m_epg_hdrlen + offset,
1927 plen) != 0) {
1928 #ifdef INVARIANTS
1929 panic("%s: failed to append sglist", __func__);
1930 #endif
1931 }
1932 if (last_ghash_frag) {
1933 if (sglist_append_phys(txq->gl, zero_buffer_pa,
1934 AES_GMAC_HASH_LEN) != 0) {
1935 #ifdef INVARIANTS
1936 panic("%s: failed to append sglist (2)", __func__);
1937 #endif
1938 }
1939 }
1940 out = write_gl_to_buf(txq->gl, out);
1941
1942 if (request_ghash) {
1943 /* ULP_TXPKT */
1944 txpkt = (void *)out;
1945 txpkt->cmd_dest = htobe32(V_ULPTX_CMD(ULP_TX_PKT) |
1946 V_ULP_TXPKT_DATAMODIFY(0) |
1947 V_T7_ULP_TXPKT_CHANNELID(tlsp->vi->pi->port_id) |
1948 V_ULP_TXPKT_DEST(0) |
1949 V_ULP_TXPKT_FID(txq->eq.iqid) | V_ULP_TXPKT_RO(1));
1950 txpkt->len = htobe32(howmany(txpkt_lens[1], 16));
1951
1952 /* ULPTX_IDATA sub-command */
1953 idata = (void *)(txpkt + 1);
1954 idata->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM) |
1955 V_ULP_TX_SC_MORE(0));
1956 idata->len = sizeof(struct cpl_tx_tls_ack);
1957 idata->len += sizeof(struct rss_header) +
1958 sizeof(struct cpl_fw6_pld);
1959 idata->len += AES_GMAC_HASH_LEN;
1960 idata->len = htobe32(idata->len);
1961 out = (void *)(idata + 1);
1962
1963 /* CPL_TX_TLS_ACK */
1964 out = write_tx_tls_ack(out, tlsp->rx_chid, AES_GMAC_HASH_LEN,
1965 ghash_lcb);
1966
1967 /* CPL_FW6_PLD */
1968 out = write_fw6_pld(out, tlsp->rx_chid, tlsp->rx_qid,
1969 AES_GMAC_HASH_LEN, (uintptr_t)tlsp | CPL_FW6_COOKIE_KTLS);
1970
1971 /* Space for partial hash. */
1972 memset(out, 0, AES_GMAC_HASH_LEN);
1973 out += AES_GMAC_HASH_LEN;
1974
1975 tlsp->ghash_pending = true;
1976 tlsp->ghash_valid = false;
1977 tlsp->ghash_lcb = ghash_lcb;
1978 if (ghash_lcb)
1979 tlsp->ghash_offset = offset + plen;
1980 else
1981 tlsp->ghash_offset = rounddown2(offset + plen,
1982 GMAC_BLOCK_LEN);
1983 #ifdef VERBOSE_TRACES
1984 CTR(KTR_CXGBE, "%s: %p requesting GHASH for offset %u",
1985 __func__, tlsp, tlsp->ghash_offset);
1986 #endif
1987 m_snd_tag_ref(&tlsp->com);
1988
1989 txq->kern_tls_ghash_requested++;
1990 }
1991
1992 if (using_scratch) {
1993 out = dst;
1994 copy_to_txd(eq, txq->ss, &out, wr_len);
1995 }
1996
1997 txq->kern_tls_records++;
1998 txq->kern_tls_octets += m_tls->m_len;
1999 if (split_mode) {
2000 txq->kern_tls_splitmode++;
2001 txq->kern_tls_waste += leading_waste + trailing_waste;
2002 }
2003 if (need_lso)
2004 txq->kern_tls_lso++;
2005
2006 txsd = &txq->sdesc[pidx];
2007 if (last_wr)
2008 txsd->m = m;
2009 else
2010 txsd->m = NULL;
2011 txsd->desc_used = ndesc;
2012
2013 return (ndesc);
2014 }
2015
2016 int
t7_ktls_write_wr(struct sge_txq * txq,void * dst,struct mbuf * m,u_int available)2017 t7_ktls_write_wr(struct sge_txq *txq, void *dst, struct mbuf *m,
2018 u_int available)
2019 {
2020 struct sge_eq *eq = &txq->eq;
2021 struct tlspcb *tlsp;
2022 struct tcphdr *tcp;
2023 struct mbuf *m_tls;
2024 struct ether_header *eh;
2025 tcp_seq tcp_seqno;
2026 u_int ndesc, pidx, totdesc;
2027 uint16_t eh_type, mss;
2028
2029 TXQ_LOCK_ASSERT_OWNED(txq);
2030 M_ASSERTPKTHDR(m);
2031 MPASS(m->m_pkthdr.snd_tag != NULL);
2032 tlsp = mst_to_tls(m->m_pkthdr.snd_tag);
2033
2034 totdesc = 0;
2035 eh = mtod(m, struct ether_header *);
2036 eh_type = ntohs(eh->ether_type);
2037 if (eh_type == ETHERTYPE_VLAN) {
2038 struct ether_vlan_header *evh = (void *)eh;
2039
2040 eh_type = ntohs(evh->evl_proto);
2041 }
2042
2043 tcp = (struct tcphdr *)((char *)eh + m->m_pkthdr.l2hlen +
2044 m->m_pkthdr.l3hlen);
2045 pidx = eq->pidx;
2046
2047 /* Determine MSS. */
2048 if (m->m_pkthdr.csum_flags & CSUM_TSO) {
2049 mss = m->m_pkthdr.tso_segsz;
2050 tlsp->prev_mss = mss;
2051 } else if (tlsp->prev_mss != 0)
2052 mss = tlsp->prev_mss;
2053 else
2054 mss = if_getmtu(tlsp->vi->ifp) -
2055 (m->m_pkthdr.l3hlen + m->m_pkthdr.l4hlen);
2056
2057 /* Fetch the starting TCP sequence number for this chain. */
2058 tcp_seqno = ntohl(tcp->th_seq);
2059 #ifdef VERBOSE_TRACES
2060 CTR(KTR_CXGBE, "%s: pkt len %d TCP seq %u", __func__, m->m_pkthdr.len,
2061 tcp_seqno);
2062 #endif
2063 KASSERT(!tlsp->ghash_pending, ("%s: GHASH pending for send", __func__));
2064
2065 /*
2066 * Iterate over each TLS record constructing a work request
2067 * for that record.
2068 */
2069 for (m_tls = m->m_next; m_tls != NULL; m_tls = m_tls->m_next) {
2070 MPASS(m_tls->m_flags & M_EXTPG);
2071
2072 ndesc = ktls_write_tls_wr(tlsp, txq, dst, m, tcp, m_tls,
2073 available - totdesc, tcp_seqno, pidx, eh_type, mss);
2074 totdesc += ndesc;
2075 IDXINCR(pidx, ndesc, eq->sidx);
2076 dst = &eq->desc[pidx];
2077
2078 tcp_seqno += m_tls->m_len;
2079 }
2080
2081 /*
2082 * Queue another packet if this was a GCM request that didn't
2083 * request a GHASH response.
2084 */
2085 if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM && !tlsp->ghash_pending)
2086 ktls_queue_next_packet(tlsp, true);
2087
2088 MPASS(totdesc <= available);
2089 return (totdesc);
2090 }
2091
2092 static void
t7_tls_tag_free(struct m_snd_tag * mst)2093 t7_tls_tag_free(struct m_snd_tag *mst)
2094 {
2095 struct adapter *sc;
2096 struct tlspcb *tlsp;
2097
2098 tlsp = mst_to_tls(mst);
2099 sc = tlsp->sc;
2100
2101 CTR2(KTR_CXGBE, "%s: %p", __func__, tlsp);
2102
2103 if (tlsp->tx_key_addr >= 0)
2104 t4_free_tls_keyid(sc, tlsp->tx_key_addr);
2105
2106 KASSERT(mbufq_len(&tlsp->pending_mbufs) == 0,
2107 ("%s: pending mbufs", __func__));
2108
2109 zfree(tlsp, M_CXGBE);
2110 }
2111
2112 static int
ktls_fw6_pld(struct sge_iq * iq,const struct rss_header * rss,struct mbuf * m)2113 ktls_fw6_pld(struct sge_iq *iq, const struct rss_header *rss,
2114 struct mbuf *m)
2115 {
2116 const struct cpl_fw6_pld *cpl;
2117 struct tlspcb *tlsp;
2118 const void *ghash;
2119
2120 if (m != NULL)
2121 cpl = mtod(m, const void *);
2122 else
2123 cpl = (const void *)(rss + 1);
2124
2125 tlsp = (struct tlspcb *)(uintptr_t)CPL_FW6_PLD_COOKIE(cpl);
2126 KASSERT(cpl->data[0] == 0, ("%s: error status returned", __func__));
2127
2128 TXQ_LOCK(tlsp->txq);
2129 #ifdef VERBOSE_TRACES
2130 CTR(KTR_CXGBE, "%s: %p received GHASH for offset %u%s", __func__, tlsp,
2131 tlsp->ghash_offset, tlsp->ghash_lcb ? " in LCB" : "");
2132 #endif
2133 if (tlsp->ghash_lcb)
2134 ghash = &cpl->data[2];
2135 else
2136 ghash = cpl + 1;
2137 memcpy(tlsp->ghash, ghash, AES_GMAC_HASH_LEN);
2138 tlsp->ghash_valid = true;
2139 tlsp->ghash_pending = false;
2140 tlsp->txq->kern_tls_ghash_received++;
2141
2142 ktls_queue_next_packet(tlsp, false);
2143 TXQ_UNLOCK(tlsp->txq);
2144
2145 m_snd_tag_rele(&tlsp->com);
2146 m_freem(m);
2147 return (0);
2148 }
2149
2150 void
t7_ktls_modload(void)2151 t7_ktls_modload(void)
2152 {
2153 zero_buffer = malloc_aligned(AES_GMAC_HASH_LEN, AES_GMAC_HASH_LEN,
2154 M_CXGBE, M_ZERO | M_WAITOK);
2155 zero_buffer_pa = vtophys(zero_buffer);
2156 t4_register_shared_cpl_handler(CPL_FW6_PLD, ktls_fw6_pld,
2157 CPL_FW6_COOKIE_KTLS);
2158 }
2159
2160 void
t7_ktls_modunload(void)2161 t7_ktls_modunload(void)
2162 {
2163 free(zero_buffer, M_CXGBE);
2164 t4_register_shared_cpl_handler(CPL_FW6_PLD, NULL, CPL_FW6_COOKIE_KTLS);
2165 }
2166
2167 #else
2168
2169 int
t7_tls_tag_alloc(struct ifnet * ifp,union if_snd_tag_alloc_params * params,struct m_snd_tag ** pt)2170 t7_tls_tag_alloc(struct ifnet *ifp, union if_snd_tag_alloc_params *params,
2171 struct m_snd_tag **pt)
2172 {
2173 return (ENXIO);
2174 }
2175
2176 int
t7_ktls_parse_pkt(struct mbuf * m)2177 t7_ktls_parse_pkt(struct mbuf *m)
2178 {
2179 return (EINVAL);
2180 }
2181
2182 int
t7_ktls_write_wr(struct sge_txq * txq,void * dst,struct mbuf * m,u_int available)2183 t7_ktls_write_wr(struct sge_txq *txq, void *dst, struct mbuf *m,
2184 u_int available)
2185 {
2186 panic("can't happen");
2187 }
2188
2189 void
t7_ktls_modload(void)2190 t7_ktls_modload(void)
2191 {
2192 }
2193
2194 void
t7_ktls_modunload(void)2195 t7_ktls_modunload(void)
2196 {
2197 }
2198
2199 #endif
2200