Lines Matching +full:hdr +full:- +full:engine
1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
130 struct port_info *pi = vi->pi; in alloc_tlspcb()
131 struct adapter *sc = pi->adapter; in alloc_tlspcb()
138 m_snd_tag_init(&tlsp->com, ifp, &t7_tls_tag_sw); in alloc_tlspcb()
139 tlsp->vi = vi; in alloc_tlspcb()
140 tlsp->sc = sc; in alloc_tlspcb()
141 tlsp->tx_key_addr = -1; in alloc_tlspcb()
142 tlsp->ghash_offset = -1; in alloc_tlspcb()
143 tlsp->rx_chid = pi->rx_chan; in alloc_tlspcb()
144 tlsp->rx_qid = -1; in alloc_tlspcb()
145 tlsp->txq = NULL; in alloc_tlspcb()
146 mbufq_init(&tlsp->pending_mbufs, INT_MAX); in alloc_tlspcb()
164 tls = params->tls.tls; in t7_tls_tag_alloc()
167 if (tls->params.tls_vmajor != TLS_MAJOR_VER_ONE || in t7_tls_tag_alloc()
168 tls->params.tls_vminor < TLS_MINOR_VER_ONE || in t7_tls_tag_alloc()
169 tls->params.tls_vminor > TLS_MINOR_VER_THREE) in t7_tls_tag_alloc()
173 switch (tls->params.cipher_algorithm) { in t7_tls_tag_alloc()
176 switch (tls->params.cipher_key_len) { in t7_tls_tag_alloc()
184 switch (tls->params.auth_algorithm) { in t7_tls_tag_alloc()
196 switch (tls->params.cipher_key_len) { in t7_tls_tag_alloc()
218 sc = vi->adapter; in t7_tls_tag_alloc()
231 tlsp->tls13 = tls->params.tls_vminor == TLS_MINOR_VER_THREE; in t7_tls_tag_alloc()
233 if (sc->tlst.inline_keys) in t7_tls_tag_alloc()
234 keyid = -1; in t7_tls_tag_alloc()
240 tlsp->inline_key = true; in t7_tls_tag_alloc()
242 tlsp->tx_key_addr = keyid; in t7_tls_tag_alloc()
244 tlsp, tlsp->tx_key_addr); in t7_tls_tag_alloc()
247 inp = params->tls.inp; in t7_tls_tag_alloc()
249 if (inp->inp_flags & INP_DROPPED) { in t7_tls_tag_alloc()
255 if (inp->inp_flowtype != M_HASHTYPE_NONE) in t7_tls_tag_alloc()
256 flowid = inp->inp_flowid; in t7_tls_tag_alloc()
259 qidx = flowid % vi->nrxq + vi->first_rxq; in t7_tls_tag_alloc()
260 tlsp->rx_qid = sc->sge.rxq[qidx].iq.abs_id; in t7_tls_tag_alloc()
261 qidx = (flowid % (vi->ntxq - vi->rsrv_noflowq)) + vi->rsrv_noflowq + in t7_tls_tag_alloc()
262 vi->first_txq; in t7_tls_tag_alloc()
263 tlsp->txq = txq = &sc->sge.txq[qidx]; in t7_tls_tag_alloc()
270 tlsp->enc_mode = t4_tls_cipher_mode(tls); in t7_tls_tag_alloc()
271 tlsp->tx_key_info_size = t4_tls_key_info_size(tls); in t7_tls_tag_alloc()
274 if (tlsp->tls13) in t7_tls_tag_alloc()
275 tlsp->scmd0.seqno_numivs = V_SCMD_SEQ_NO_CTRL(0); in t7_tls_tag_alloc()
277 tlsp->scmd0.seqno_numivs = V_SCMD_SEQ_NO_CTRL(3); in t7_tls_tag_alloc()
278 tlsp->scmd0.seqno_numivs |= in t7_tls_tag_alloc()
282 V_SCMD_CIPH_MODE(tlsp->enc_mode) | in t7_tls_tag_alloc()
286 tlsp->scmd0.seqno_numivs = htobe32(tlsp->scmd0.seqno_numivs); in t7_tls_tag_alloc()
288 tlsp->scmd0.ivgen_hdrlen = V_SCMD_IV_GEN_CTRL(0) | in t7_tls_tag_alloc()
290 if (tlsp->inline_key) in t7_tls_tag_alloc()
291 tlsp->scmd0.ivgen_hdrlen |= V_SCMD_KEY_CTX_INLINE(1); in t7_tls_tag_alloc()
297 tlsp->scmd0_short.seqno_numivs = V_SCMD_SEQ_NO_CTRL(0) | in t7_tls_tag_alloc()
304 if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM) in t7_tls_tag_alloc()
305 tlsp->scmd0_short.seqno_numivs |= in t7_tls_tag_alloc()
308 tlsp->scmd0_short.seqno_numivs |= in t7_tls_tag_alloc()
309 V_SCMD_CIPH_MODE(tlsp->enc_mode); in t7_tls_tag_alloc()
310 tlsp->scmd0_short.seqno_numivs = in t7_tls_tag_alloc()
311 htobe32(tlsp->scmd0_short.seqno_numivs); in t7_tls_tag_alloc()
313 tlsp->scmd0_short.ivgen_hdrlen = V_SCMD_IV_GEN_CTRL(0) | in t7_tls_tag_alloc()
315 if (tlsp->inline_key) in t7_tls_tag_alloc()
316 tlsp->scmd0_short.ivgen_hdrlen |= V_SCMD_KEY_CTX_INLINE(1); in t7_tls_tag_alloc()
322 tlsp->scmd0_partial.seqno_numivs = V_SCMD_SEQ_NO_CTRL(0) | in t7_tls_tag_alloc()
326 V_SCMD_CIPH_MODE(tlsp->enc_mode) | in t7_tls_tag_alloc()
330 tlsp->scmd0_partial.seqno_numivs = in t7_tls_tag_alloc()
331 htobe32(tlsp->scmd0_partial.seqno_numivs); in t7_tls_tag_alloc()
333 tlsp->scmd0_partial.ivgen_hdrlen = V_SCMD_IV_GEN_CTRL(0) | in t7_tls_tag_alloc()
338 if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM) in t7_tls_tag_alloc()
339 txq->kern_tls_gcm++; in t7_tls_tag_alloc()
341 txq->kern_tls_cbc++; in t7_tls_tag_alloc()
343 *pt = &tlsp->com; in t7_tls_tag_alloc()
347 m_snd_tag_rele(&tlsp->com); in t7_tls_tag_alloc()
368 t4_tls_key_ctx(tls, KTLS_TX, &tlsp->keyctx); in ktls_setup_keys()
369 if (tlsp->inline_key) in ktls_setup_keys()
379 m->m_pkthdr.snd_tag = m_snd_tag_ref(&tlsp->com); in ktls_setup_keys()
380 m->m_pkthdr.csum_flags |= CSUM_SND_TAG; in ktls_setup_keys()
384 t4_write_tlskey_wr(tls, KTLS_TX, 0, 0, tlsp->tx_key_addr, kwr); in ktls_setup_keys()
386 memcpy(kctx, &tlsp->keyctx, sizeof(*kctx)); in ktls_setup_keys()
394 error = mp_ring_enqueue(txq->r, items, 1, 1); in ktls_setup_keys()
412 wr_len += tlsp->tx_key_info_size; in ktls_base_wr_size()
429 nsegs--; in ktls_sgl_size()
446 * *header_len - Number of bytes of TLS header to pass as immediate
449 * *offset - Start offset of TLS record payload to pass as DSGL data
451 * *plen - Length of TLS record payload to pass as DSGL data
453 * *leading_waste - amount of non-packet-header bytes to drop at the
456 * *trailing_waste - amount of crypto output to drop from the end
466 MPASS(tlen > m_tls->m_epg_hdrlen); in ktls_is_short_record()
473 trailer_len = m_tls->m_epg_trllen; in ktls_is_short_record()
474 if (tlsp->tls13) in ktls_is_short_record()
475 trailer_len--; in ktls_is_short_record()
479 * engine and relying on SplitMode to drop any waste. in ktls_is_short_record()
481 *header_len = m_tls->m_epg_hdrlen; in ktls_is_short_record()
483 *plen = rlen - (m_tls->m_epg_hdrlen + trailer_len); in ktls_is_short_record()
485 *trailing_waste = rlen - tlen; in ktls_is_short_record()
486 if (!tlsp->sc->tlst.short_records) in ktls_is_short_record()
489 if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_CBC) { in ktls_is_short_record()
491 * For AES-CBC we have to send input from the start of in ktls_is_short_record()
499 roundup2(tlen - TLS_HEADER_LENGTH, AES_BLOCK_LEN); in ktls_is_short_record()
500 if (rlen - new_tlen < trailer_len) in ktls_is_short_record()
503 *trailing_waste = new_tlen - tlen; in ktls_is_short_record()
504 *plen = new_tlen - m_tls->m_epg_hdrlen; in ktls_is_short_record()
506 if (rlen - tlen < trailer_len || in ktls_is_short_record()
507 (rlen - tlen == trailer_len && request_ghash)) { in ktls_is_short_record()
509 * For AES-GCM we have to send the full record in ktls_is_short_record()
523 * We can use AES-CTR or AES-GCM in partial GHASH in ktls_is_short_record()
530 *plen = tlen - m_tls->m_epg_hdrlen; in ktls_is_short_record()
540 if (mtod(m_tls, vm_offset_t) == m_tls->m_epg_hdrlen && in ktls_is_short_record()
549 if (mtod(m_tls, vm_offset_t) >= m_tls->m_epg_hdrlen) { in ktls_is_short_record()
551 *offset = mtod(m_tls, vm_offset_t) - in ktls_is_short_record()
552 m_tls->m_epg_hdrlen; in ktls_is_short_record()
563 *plen -= *offset; in ktls_is_short_record()
564 *leading_waste -= (m_tls->m_epg_hdrlen + *offset); in ktls_is_short_record()
570 /* Size of the AES-GCM TLS AAD for a given connection. */
574 return (tlsp->tls13 ? sizeof(struct tls_aead_data_13) : in ktls_gcm_aad_len()
582 const struct tls_record_layer *hdr; in ktls_wr_len() local
594 tlen = mtod(m_tls, vm_offset_t) + m_tls->m_len; in ktls_wr_len()
595 if (tlen <= m_tls->m_epg_hdrlen) { in ktls_wr_len()
602 roundup2(m->m_len + m_tls->m_len, 16); in ktls_wr_len()
605 "%s: %p TLS header-only packet too long (len %d)", in ktls_wr_len()
606 __func__, tlsp, m->m_len + m_tls->m_len); in ktls_wr_len()
610 MPASS(m_tls->m_next == NULL); in ktls_wr_len()
615 hdr = (void *)m_tls->m_epg_hdr; in ktls_wr_len()
616 rlen = TLS_HEADER_LENGTH + ntohs(hdr->tls_length); in ktls_wr_len()
620 * errs on the side of over-budgeting the WR size. in ktls_wr_len()
625 if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM && in ktls_wr_len()
626 tlsp->sc->tlst.partial_ghash && tlsp->sc->tlst.short_records) { in ktls_wr_len()
629 trailer_len = m_tls->m_epg_trllen; in ktls_wr_len()
630 if (tlsp->tls13) in ktls_wr_len()
631 trailer_len--; in ktls_wr_len()
633 ("invalid trailer length for AES-GCM")); in ktls_wr_len()
636 if (mtod(m_tls, vm_offset_t) <= m_tls->m_epg_hdrlen) { in ktls_wr_len()
642 if (tlen < (rlen - trailer_len)) in ktls_wr_len()
650 if (tlen >= (rlen - trailer_len)) in ktls_wr_len()
663 inline_key = send_partial_ghash || tlsp->inline_key; in ktls_wr_len()
685 * immediate data. TLS 1.3 non-short records include a in ktls_wr_len()
691 imm_len = m->m_len + header_len; in ktls_wr_len()
696 } else if (tlsp->tls13) in ktls_wr_len()
704 *nsegsp = sglist_count_mbuf_epg(m_tls, m_tls->m_epg_hdrlen + offset, in ktls_wr_len()
709 /* AES-GCM records might return a partial hash. */ in ktls_wr_len()
735 TXQ_LOCK_ASSERT_OWNED(tlsp->txq); in ktls_queue_next_packet()
736 KASSERT(tlsp->queue_mbufs, ("%s: mbufs not being queued for %p", in ktls_queue_next_packet()
739 m = mbufq_dequeue(&tlsp->pending_mbufs); in ktls_queue_next_packet()
741 tlsp->queue_mbufs = false; in ktls_queue_next_packet()
747 tcp = (struct tcphdr *)((char *)eh + m->m_pkthdr.l2hlen + in ktls_queue_next_packet()
748 m->m_pkthdr.l3hlen); in ktls_queue_next_packet()
749 tcp_seqno = ntohl(tcp->th_seq); in ktls_queue_next_packet()
752 m->m_pkthdr.len, tcp_seqno); in ktls_queue_next_packet()
758 rc = mp_ring_enqueue_only(tlsp->txq->r, items, 1); in ktls_queue_next_packet()
760 TXQ_UNLOCK(tlsp->txq); in ktls_queue_next_packet()
761 rc = mp_ring_enqueue(tlsp->txq->r, items, 1, 256); in ktls_queue_next_packet()
762 TXQ_LOCK(tlsp->txq); in ktls_queue_next_packet()
768 m->m_pkthdr.len, tcp_seqno); in ktls_queue_next_packet()
795 MPASS(m->m_pkthdr.snd_tag != NULL); in t7_ktls_parse_pkt()
796 tlsp = mst_to_tls(m->m_pkthdr.snd_tag); in t7_ktls_parse_pkt()
798 if (m->m_len <= sizeof(*eh) + sizeof(*ip)) { in t7_ktls_parse_pkt()
803 eh_type = ntohs(eh->ether_type); in t7_ktls_parse_pkt()
807 eh_type = ntohs(evh->evl_proto); in t7_ktls_parse_pkt()
808 m->m_pkthdr.l2hlen = sizeof(*evh); in t7_ktls_parse_pkt()
810 m->m_pkthdr.l2hlen = sizeof(*eh); in t7_ktls_parse_pkt()
815 if (ip->ip_p != IPPROTO_TCP) { in t7_ktls_parse_pkt()
820 m->m_pkthdr.l3hlen = ip->ip_hl * 4; in t7_ktls_parse_pkt()
824 if (ip6->ip6_nxt != IPPROTO_TCP) { in t7_ktls_parse_pkt()
826 __func__, tlsp, ip6->ip6_nxt); in t7_ktls_parse_pkt()
829 m->m_pkthdr.l3hlen = sizeof(struct ip6_hdr); in t7_ktls_parse_pkt()
836 if (m->m_len < m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen + in t7_ktls_parse_pkt()
842 tcp = (struct tcphdr *)((char *)(eh + 1) + m->m_pkthdr.l3hlen); in t7_ktls_parse_pkt()
843 m->m_pkthdr.l4hlen = tcp->th_off * 4; in t7_ktls_parse_pkt()
846 if (m->m_len != m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen + in t7_ktls_parse_pkt()
847 m->m_pkthdr.l4hlen) { in t7_ktls_parse_pkt()
850 __func__, tlsp, m->m_pkthdr.l2hlen, m->m_pkthdr.l3hlen, in t7_ktls_parse_pkt()
851 m->m_pkthdr.l4hlen, m->m_len); in t7_ktls_parse_pkt()
856 MPASS(m->m_next != NULL); in t7_ktls_parse_pkt()
857 MPASS(m->m_next->m_flags & M_EXTPG); in t7_ktls_parse_pkt()
865 for (m_tls = m->m_next; m_tls != NULL; m_tls = m_tls->m_next) { in t7_ktls_parse_pkt()
866 MPASS(m_tls->m_flags & M_EXTPG); in t7_ktls_parse_pkt()
881 if (m_tls == m->m_next) in t7_ktls_parse_pkt()
888 if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM) { in t7_ktls_parse_pkt()
890 TXQ_LOCK(tlsp->txq); in t7_ktls_parse_pkt()
891 if (tlsp->queue_mbufs) { in t7_ktls_parse_pkt()
892 error = mbufq_enqueue(&tlsp->pending_mbufs, m); in t7_ktls_parse_pkt()
898 mbuf_nsegs(m), ntohl(tcp->th_seq)); in t7_ktls_parse_pkt()
901 TXQ_UNLOCK(tlsp->txq); in t7_ktls_parse_pkt()
904 tlsp->queue_mbufs = true; in t7_ktls_parse_pkt()
905 TXQ_UNLOCK(tlsp->txq); in t7_ktls_parse_pkt()
913 error = mp_ring_enqueue(tlsp->txq->r, items, 1, 256); in t7_ktls_parse_pkt()
915 if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM) { in t7_ktls_parse_pkt()
916 TXQ_LOCK(tlsp->txq); in t7_ktls_parse_pkt()
918 TXQ_UNLOCK(tlsp->txq); in t7_ktls_parse_pkt()
930 return (m->m_flags & M_VLANTAG); in needs_vlan_insertion()
941 V_T6_TXPKT_ETHHDR_LEN(m->m_pkthdr.l2hlen - ETHER_HDR_LEN) | in pkt_ctrl1()
942 V_TXPKT_IPHDR_LEN(m->m_pkthdr.l3hlen); in pkt_ctrl1()
944 MPASS(m->m_pkthdr.l3hlen == sizeof(struct ip6_hdr)); in pkt_ctrl1()
946 V_T6_TXPKT_ETHHDR_LEN(m->m_pkthdr.l2hlen - ETHER_HDR_LEN) | in pkt_ctrl1()
947 V_TXPKT_IPHDR_LEN(m->m_pkthdr.l3hlen); in pkt_ctrl1()
949 txq->txcsum++; in pkt_ctrl1()
954 V_TXPKT_VLAN(m->m_pkthdr.ether_vtag); in pkt_ctrl1()
955 txq->vlan_insertion++; in pkt_ctrl1()
968 KASSERT(m0->m_pkthdr.l2hlen > 0 && m0->m_pkthdr.l3hlen > 0 && in write_lso_cpl()
969 m0->m_pkthdr.l4hlen > 0, in write_lso_cpl()
975 V_LSO_ETHHDR_LEN((m0->m_pkthdr.l2hlen - ETHER_HDR_LEN) >> 2) | in write_lso_cpl()
976 V_LSO_IPHDR_LEN(m0->m_pkthdr.l3hlen >> 2) | in write_lso_cpl()
977 V_LSO_TCPHDR_LEN(m0->m_pkthdr.l4hlen >> 2); in write_lso_cpl()
982 lso->lso_ctrl = htobe32(ctrl); in write_lso_cpl()
983 lso->ipid_ofst = htobe16(0); in write_lso_cpl()
984 lso->mss = htobe16(mss); in write_lso_cpl()
985 lso->seqno_offset = htobe32(0); in write_lso_cpl()
986 lso->len = htobe32(total_len); in write_lso_cpl()
999 cpl->op_to_Rsvd2 = htobe32(V_CPL_TX_TLS_ACK_OPCODE(CPL_TX_TLS_ACK) | in write_tx_tls_ack()
1004 cpl->PldLen = htobe32(V_CPL_TX_TLS_ACK_PLDLEN(32 + 16 + hash_len)); in write_tx_tls_ack()
1005 cpl->Rsvd3 = 0; in write_tx_tls_ack()
1019 rss->opcode = CPL_FW6_PLD; in write_fw6_pld()
1020 rss->qid = htobe16(rx_qid); in write_fw6_pld()
1021 rss->channel = rx_chid; in write_fw6_pld()
1025 cpl->opcode = CPL_FW6_PLD; in write_fw6_pld()
1026 cpl->len = htobe16(hash_len); in write_fw6_pld()
1027 cpl->data[1] = htobe64(cookie); in write_fw6_pld()
1044 cpl->ot.opcode = CPL_RX_PHYS_DSGL; in write_split_mode_rx_phys()
1045 cpl->PhysAddrFields_lo_to_NumSGE = in write_split_mode_rx_phys()
1049 len = (uint16_t *)(cpl->RSSCopy); in write_split_mode_rx_phys()
1053 * transmit-related CPLs. in write_split_mode_rx_phys()
1064 len[2] = htobe16(m_tls->m_len); in write_split_mode_rx_phys()
1092 nsegs = gl->sg_nseg; in write_gl_to_buf()
1095 nflits = (3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1) + 2; in write_gl_to_buf()
1097 seg = &gl->sg_segs[0]; in write_gl_to_buf()
1100 usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) | in write_gl_to_buf()
1102 usgl->len0 = htobe32(seg->ss_len); in write_gl_to_buf()
1103 usgl->addr0 = htobe64(seg->ss_paddr); in write_gl_to_buf()
1106 for (i = 0; i < nsegs - 1; i++, seg++) { in write_gl_to_buf()
1107 usgl->sge[i / 2].len[i & 1] = htobe32(seg->ss_len); in write_gl_to_buf()
1108 usgl->sge[i / 2].addr[i & 1] = htobe64(seg->ss_paddr); in write_gl_to_buf()
1111 usgl->sge[i / 2].len[1] = htobe32(0); in write_gl_to_buf()
1127 MPASS((uintptr_t)(*to) >= (uintptr_t)&eq->desc[0]); in copy_to_txd()
1128 MPASS((uintptr_t)(*to) < (uintptr_t)&eq->desc[eq->sidx]); in copy_to_txd()
1131 (uintptr_t)&eq->desc[eq->sidx])) { in copy_to_txd()
1134 if ((uintptr_t)(*to) == (uintptr_t)&eq->desc[eq->sidx]) in copy_to_txd()
1135 (*to) = (caddr_t)eq->desc; in copy_to_txd()
1137 int portion = (uintptr_t)&eq->desc[eq->sidx] - (uintptr_t)(*to); in copy_to_txd()
1141 portion = len - portion; /* remaining */ in copy_to_txd()
1142 bcopy(from, (void *)eq->desc, portion); in copy_to_txd()
1143 (*to) = (caddr_t)eq->desc + portion; in copy_to_txd()
1167 pktlen = m->m_len + len; in ktls_write_tunnel_packet()
1175 wr->op_immdlen = htobe32(V_FW_WR_OP(FW_ETH_TX_PKT_WR) | in ktls_write_tunnel_packet()
1179 wr->equiq_to_len16 = htobe32(ctrl); in ktls_write_tunnel_packet()
1180 wr->r3 = 0; in ktls_write_tunnel_packet()
1185 cpl->ctrl0 = txq->cpl_ctrl0; in ktls_write_tunnel_packet()
1186 cpl->pack = 0; in ktls_write_tunnel_packet()
1187 cpl->len = htobe16(pktlen); in ktls_write_tunnel_packet()
1193 copy_to_txd(&txq->eq, (caddr_t)eh, &out, m->m_pkthdr.l2hlen); in ktls_write_tunnel_packet()
1197 ip = (void *)((char *)eh + m->m_pkthdr.l2hlen); in ktls_write_tunnel_packet()
1199 newip.ip_len = htons(pktlen - m->m_pkthdr.l2hlen); in ktls_write_tunnel_packet()
1200 copy_to_txd(&txq->eq, (caddr_t)&newip, &out, sizeof(newip)); in ktls_write_tunnel_packet()
1201 if (m->m_pkthdr.l3hlen > sizeof(*ip)) in ktls_write_tunnel_packet()
1202 copy_to_txd(&txq->eq, (caddr_t)(ip + 1), &out, in ktls_write_tunnel_packet()
1203 m->m_pkthdr.l3hlen - sizeof(*ip)); in ktls_write_tunnel_packet()
1205 ip6 = (void *)((char *)eh + m->m_pkthdr.l2hlen); in ktls_write_tunnel_packet()
1207 newip6.ip6_plen = htons(pktlen - m->m_pkthdr.l2hlen - in ktls_write_tunnel_packet()
1209 copy_to_txd(&txq->eq, (caddr_t)&newip6, &out, sizeof(newip6)); in ktls_write_tunnel_packet()
1210 MPASS(m->m_pkthdr.l3hlen == sizeof(*ip6)); in ktls_write_tunnel_packet()
1212 cpl->ctrl1 = htobe64(pkt_ctrl1(txq, m, eh_type)); in ktls_write_tunnel_packet()
1215 tcp = (void *)((char *)eh + m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen); in ktls_write_tunnel_packet()
1218 copy_to_txd(&txq->eq, (caddr_t)&newtcp, &out, sizeof(newtcp)); in ktls_write_tunnel_packet()
1221 copy_to_txd(&txq->eq, (caddr_t)(tcp + 1), &out, m->m_len - in ktls_write_tunnel_packet()
1222 (m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen + sizeof(*tcp))); in ktls_write_tunnel_packet()
1225 copy_to_txd(&txq->eq, src, &out, len); in ktls_write_tunnel_packet()
1226 txq->imm_wrs++; in ktls_write_tunnel_packet()
1228 txq->txpkt_wrs++; in ktls_write_tunnel_packet()
1230 txsd = &txq->sdesc[pidx]; in ktls_write_tunnel_packet()
1232 txsd->m = m; in ktls_write_tunnel_packet()
1234 txsd->m = NULL; in ktls_write_tunnel_packet()
1235 txsd->desc_used = ndesc; in ktls_write_tunnel_packet()
1246 struct sge_eq *eq = &txq->eq; in ktls_write_tls_wr()
1254 const struct tls_record_layer *hdr; in ktls_write_tls_wr() local
1271 MPASS(tlsp->txq == txq); in ktls_write_tls_wr()
1275 last_wr = (m_tls->m_next == NULL); in ktls_write_tls_wr()
1281 tlen = mtod(m_tls, vm_offset_t) + m_tls->m_len; in ktls_write_tls_wr()
1282 if (tlen <= m_tls->m_epg_hdrlen) { in ktls_write_tls_wr()
1288 CTR(KTR_CXGBE, "%s: %p header-only TLS record %u", __func__, in ktls_write_tls_wr()
1289 tlsp, (u_int)m_tls->m_epg_seqno); in ktls_write_tls_wr()
1294 txq->kern_tls_header++; in ktls_write_tls_wr()
1297 (char *)m_tls->m_epg_hdr + mtod(m_tls, vm_offset_t), in ktls_write_tls_wr()
1298 m_tls->m_len, available, tcp_seqno, pidx, eh_type, in ktls_write_tls_wr()
1303 hdr = (void *)m_tls->m_epg_hdr; in ktls_write_tls_wr()
1304 rlen = TLS_HEADER_LENGTH + ntohs(hdr->tls_length); in ktls_write_tls_wr()
1308 __func__, mtod(m_tls, vm_offset_t), m_tls->m_len, tcp_seqno, in ktls_write_tls_wr()
1309 (u_int)m_tls->m_epg_seqno); in ktls_write_tls_wr()
1317 if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM && in ktls_write_tls_wr()
1318 tlsp->sc->tlst.partial_ghash && tlsp->sc->tlst.short_records) { in ktls_write_tls_wr()
1321 trailer_len = m_tls->m_epg_trllen; in ktls_write_tls_wr()
1322 if (tlsp->tls13) in ktls_write_tls_wr()
1323 trailer_len--; in ktls_write_tls_wr()
1325 ("invalid trailer length for AES-GCM")); in ktls_write_tls_wr()
1328 if (mtod(m_tls, vm_offset_t) <= m_tls->m_epg_hdrlen) { in ktls_write_tls_wr()
1335 if ((tlsp->ghash_tls_seqno == 0 || in ktls_write_tls_wr()
1336 tlsp->ghash_tls_seqno < m_tls->m_epg_seqno) && in ktls_write_tls_wr()
1344 if (tlen >= (rlen - trailer_len)) in ktls_write_tls_wr()
1349 tlsp->ghash_tls_seqno = m_tls->m_epg_seqno; in ktls_write_tls_wr()
1351 } else if (tlsp->ghash_tls_seqno == m_tls->m_epg_seqno && in ktls_write_tls_wr()
1352 tlsp->ghash_valid) { in ktls_write_tls_wr()
1357 if (rlen - tlen < trailer_len) in ktls_write_tls_wr()
1358 plen = rlen - (m_tls->m_epg_hdrlen + in ktls_write_tls_wr()
1361 plen = tlen - m_tls->m_epg_hdrlen; in ktls_write_tls_wr()
1362 offset = mtod(m_tls, vm_offset_t) - m_tls->m_epg_hdrlen; in ktls_write_tls_wr()
1367 if (tlsp->ghash_offset == offset) { in ktls_write_tls_wr()
1376 "%s: %p trailer-only TLS record %u", in ktls_write_tls_wr()
1378 (u_int)m_tls->m_epg_seqno); in ktls_write_tls_wr()
1381 txq->kern_tls_trailer++; in ktls_write_tls_wr()
1383 offset = mtod(m_tls, vm_offset_t) - in ktls_write_tls_wr()
1384 (m_tls->m_epg_hdrlen + plen); in ktls_write_tls_wr()
1388 dst, m, tlsp->ghash + offset, in ktls_write_tls_wr()
1389 m_tls->m_len, available, tcp_seqno, in ktls_write_tls_wr()
1398 if (tlen >= (rlen - trailer_len)) { in ktls_write_tls_wr()
1405 * than an AES-CTR short record) if in ktls_write_tls_wr()
1413 if (plen - offset >= GMAC_BLOCK_LEN || in ktls_write_tls_wr()
1438 "%s: %p short TLS record %u hdr %u offs %u plen %u", in ktls_write_tls_wr()
1439 __func__, tlsp, (u_int)m_tls->m_epg_seqno, header_len, in ktls_write_tls_wr()
1447 __func__, tlsp, tlsp->ghash_offset, in ktls_write_tls_wr()
1458 if (tlen < rlen && m_tls->m_next == NULL && in ktls_write_tls_wr()
1459 (tcp->th_flags & TH_FIN) != 0) { in ktls_write_tls_wr()
1460 txq->kern_tls_fin_short++; in ktls_write_tls_wr()
1471 if (m->m_next == m_tls && !send_partial_ghash) in ktls_write_tls_wr()
1475 m_tls->m_epg_hdrlen + offset, plen); in ktls_write_tls_wr()
1478 need_lso = (m_tls->m_len > mss); in ktls_write_tls_wr()
1481 inline_key = send_partial_ghash || tlsp->inline_key; in ktls_write_tls_wr()
1506 imm_len = m->m_len + header_len; in ktls_write_tls_wr()
1511 } else if (tlsp->tls13) in ktls_write_tls_wr()
1516 txpkt_lens[0] = wr_len - sizeof(*wr); in ktls_write_tls_wr()
1538 * Use the per-txq scratch pad if near the end of the ring to in ktls_write_tls_wr()
1539 * simplify handling of wrap-around. in ktls_write_tls_wr()
1541 using_scratch = (eq->sidx - pidx < ndesc); in ktls_write_tls_wr()
1543 wr = (void *)txq->ss; in ktls_write_tls_wr()
1548 wr->op_to_compl = htobe32(V_FW_WR_OP(FW_ULPTX_WR)); in ktls_write_tls_wr()
1549 wr->flowid_len16 = htobe32(F_FW_ULPTX_WR_DATA | in ktls_write_tls_wr()
1551 wr->cookie = 0; in ktls_write_tls_wr()
1555 txpkt->cmd_dest = htobe32(V_ULPTX_CMD(ULP_TX_PKT) | in ktls_write_tls_wr()
1557 V_T7_ULP_TXPKT_CHANNELID(tlsp->vi->pi->port_id) | in ktls_write_tls_wr()
1560 V_ULP_TXPKT_FID(txq->eq.cntxt_id) | V_ULP_TXPKT_RO(1)); in ktls_write_tls_wr()
1561 txpkt->len = htobe32(howmany(txpkt_lens[0], 16)); in ktls_write_tls_wr()
1563 /* ULPTX_IDATA sub-command */ in ktls_write_tls_wr()
1565 idata->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM) | in ktls_write_tls_wr()
1567 idata->len = sizeof(struct cpl_tx_sec_pdu); in ktls_write_tls_wr()
1584 idata->len += tlsp->tx_key_info_size + post_key_context_len; in ktls_write_tls_wr()
1587 idata->len += AES_GMAC_HASH_LEN; in ktls_write_tls_wr()
1590 idata->len = htobe32(idata->len); in ktls_write_tls_wr()
1597 * engine by marking them as header data in SCMD0. in ktls_write_tls_wr()
1599 crypto_hdr_len = m->m_len; in ktls_write_tls_wr()
1605 * is next (if AAD is present) followed by the AES-CTR in ktls_write_tls_wr()
1638 sec_pdu->pldlen = htobe32(aad_stop + AES_BLOCK_LEN + plen + in ktls_write_tls_wr()
1648 sec_pdu->seqno_numivs = tlsp->scmd0_partial.seqno_numivs; in ktls_write_tls_wr()
1649 sec_pdu->ivgen_hdrlen = tlsp->scmd0_partial.ivgen_hdrlen; in ktls_write_tls_wr()
1651 sec_pdu->ivgen_hdrlen |= V_SCMD_LAST_FRAG(1); in ktls_write_tls_wr()
1653 sec_pdu->ivgen_hdrlen |= V_SCMD_MORE_FRAGS(1); in ktls_write_tls_wr()
1654 sec_pdu->ivgen_hdrlen = htobe32(sec_pdu->ivgen_hdrlen | in ktls_write_tls_wr()
1657 txq->kern_tls_partial_ghash++; in ktls_write_tls_wr()
1674 sec_pdu->pldlen = htobe32(AES_BLOCK_LEN + plen); in ktls_write_tls_wr()
1683 sec_pdu->seqno_numivs = tlsp->scmd0_short.seqno_numivs; in ktls_write_tls_wr()
1684 sec_pdu->ivgen_hdrlen = htobe32( in ktls_write_tls_wr()
1685 tlsp->scmd0_short.ivgen_hdrlen | in ktls_write_tls_wr()
1688 txq->kern_tls_short++; in ktls_write_tls_wr()
1699 if (tlsp->tls13) { in ktls_write_tls_wr()
1708 cipher_start = m_tls->m_epg_hdrlen + 1; in ktls_write_tls_wr()
1710 if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM) { in ktls_write_tls_wr()
1722 sec_pdu->pldlen = htobe32((tlsp->tls13 ? sizeof(uint64_t) : 0) + in ktls_write_tls_wr()
1723 m_tls->m_epg_hdrlen + plen); in ktls_write_tls_wr()
1726 sec_pdu->seqno_numivs = tlsp->scmd0.seqno_numivs; in ktls_write_tls_wr()
1727 sec_pdu->ivgen_hdrlen = htobe32(tlsp->scmd0.ivgen_hdrlen | in ktls_write_tls_wr()
1731 txq->kern_tls_partial++; in ktls_write_tls_wr()
1733 txq->kern_tls_full++; in ktls_write_tls_wr()
1735 sec_pdu->op_ivinsrtofst = htobe32( in ktls_write_tls_wr()
1740 sec_pdu->aadstart_cipherstop_hi = htobe32( in ktls_write_tls_wr()
1745 sec_pdu->cipherstop_lo_authinsert = htobe32( in ktls_write_tls_wr()
1755 cipher_len = rlen - (m_tls->m_epg_hdrlen + AES_GMAC_HASH_LEN); in ktls_write_tls_wr()
1756 sec_pdu->scmd1 = htobe64(aad_len << 44 | cipher_len); in ktls_write_tls_wr()
1758 sec_pdu->scmd1 = htobe64(m_tls->m_epg_seqno); in ktls_write_tls_wr()
1763 memcpy(out, &tlsp->keyctx, tlsp->tx_key_info_size); in ktls_write_tls_wr()
1767 keyctx->u.txhdr.ctxlen++; in ktls_write_tls_wr()
1768 keyctx->u.txhdr.dualck_to_txvalid &= ~htobe16( in ktls_write_tls_wr()
1770 keyctx->u.txhdr.dualck_to_txvalid |= htobe16( in ktls_write_tls_wr()
1774 out += tlsp->tx_key_info_size; in ktls_write_tls_wr()
1779 memcpy(out, tlsp->ghash, AES_GMAC_HASH_LEN); in ktls_write_tls_wr()
1785 memrd->cmd_to_len = htobe32(V_ULPTX_CMD(ULP_TX_SC_MEMRD) | in ktls_write_tls_wr()
1787 V_ULPTX_LEN16(tlsp->tx_key_info_size >> 4)); in ktls_write_tls_wr()
1788 memrd->addr = htobe32(tlsp->tx_key_addr >> 5); in ktls_write_tls_wr()
1792 idata->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM) | in ktls_write_tls_wr()
1794 idata->len = htobe32(post_key_context_len); in ktls_write_tls_wr()
1804 crypto_hdr_len += m->m_len; in ktls_write_tls_wr()
1811 out = write_lso_cpl(out, m, mss, eh_type, m->m_len + in ktls_write_tls_wr()
1812 m_tls->m_len); in ktls_write_tls_wr()
1813 txq->tso_wrs++; in ktls_write_tls_wr()
1818 tx_pkt->ctrl0 = txq->cpl_ctrl0; in ktls_write_tls_wr()
1819 tx_pkt->ctrl1 = htobe64(pkt_ctrl1(txq, m, eh_type)); in ktls_write_tls_wr()
1820 tx_pkt->pack = 0; in ktls_write_tls_wr()
1821 tx_pkt->len = htobe16(m->m_len + m_tls->m_len); in ktls_write_tls_wr()
1825 memcpy(out, mtod(m, char *), m->m_len); in ktls_write_tls_wr()
1828 ip_len = m->m_len + m_tls->m_len - m->m_pkthdr.l2hlen; in ktls_write_tls_wr()
1830 ip = (void *)(out + m->m_pkthdr.l2hlen); in ktls_write_tls_wr()
1831 be16enc(&ip->ip_len, ip_len); in ktls_write_tls_wr()
1833 ip6 = (void *)(out + m->m_pkthdr.l2hlen); in ktls_write_tls_wr()
1834 be16enc(&ip6->ip6_plen, ip_len - sizeof(*ip6)); in ktls_write_tls_wr()
1838 newtcp = (void *)(out + m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen); in ktls_write_tls_wr()
1839 be32enc(&newtcp->th_seq, tcp_seqno); in ktls_write_tls_wr()
1841 newtcp->th_flags = tcp->th_flags & ~(TH_PUSH | TH_FIN); in ktls_write_tls_wr()
1842 out += m->m_len; in ktls_write_tls_wr()
1846 * non-short records. in ktls_write_tls_wr()
1848 if (tlsp->tls13 && !short_record) { in ktls_write_tls_wr()
1854 memcpy(out, m_tls->m_epg_hdr, header_len); in ktls_write_tls_wr()
1859 if (tlsp->tls13) { in ktls_write_tls_wr()
1862 ad.type = hdr->tls_type; in ktls_write_tls_wr()
1863 ad.tls_vmajor = hdr->tls_vmajor; in ktls_write_tls_wr()
1864 ad.tls_vminor = hdr->tls_vminor; in ktls_write_tls_wr()
1865 ad.tls_length = hdr->tls_length; in ktls_write_tls_wr()
1872 cipher_len = rlen - in ktls_write_tls_wr()
1873 (m_tls->m_epg_hdrlen + AES_GMAC_HASH_LEN); in ktls_write_tls_wr()
1874 ad.seq = htobe64(m_tls->m_epg_seqno); in ktls_write_tls_wr()
1875 ad.type = hdr->tls_type; in ktls_write_tls_wr()
1876 ad.tls_vmajor = hdr->tls_vmajor; in ktls_write_tls_wr()
1877 ad.tls_vminor = hdr->tls_vminor; in ktls_write_tls_wr()
1887 if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM) { in ktls_write_tls_wr()
1888 memcpy(iv, tlsp->keyctx.u.txhdr.txsalt, SALT_SIZE); in ktls_write_tls_wr()
1889 if (tlsp->tls13) { in ktls_write_tls_wr()
1892 value = be64dec(tlsp->keyctx.u.txhdr.txsalt + in ktls_write_tls_wr()
1894 value ^= m_tls->m_epg_seqno; in ktls_write_tls_wr()
1897 memcpy(iv + 4, hdr + 1, 8); in ktls_write_tls_wr()
1903 memcpy(iv, hdr + 1, AES_BLOCK_LEN); in ktls_write_tls_wr()
1909 /* Zero pad to an 8-byte boundary. */ in ktls_write_tls_wr()
1910 memset(out, 0, 8 - (imm_len % 8)); in ktls_write_tls_wr()
1911 out += 8 - (imm_len % 8); in ktls_write_tls_wr()
1916 * 16-byte aligned. in ktls_write_tls_wr()
1920 idata->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP) | in ktls_write_tls_wr()
1922 idata->len = htobe32(0); in ktls_write_tls_wr()
1928 sglist_reset(txq->gl); in ktls_write_tls_wr()
1929 if (sglist_append_mbuf_epg(txq->gl, m_tls, m_tls->m_epg_hdrlen + offset, in ktls_write_tls_wr()
1936 if (sglist_append_phys(txq->gl, zero_buffer_pa, in ktls_write_tls_wr()
1943 out = write_gl_to_buf(txq->gl, out); in ktls_write_tls_wr()
1948 txpkt->cmd_dest = htobe32(V_ULPTX_CMD(ULP_TX_PKT) | in ktls_write_tls_wr()
1950 V_T7_ULP_TXPKT_CHANNELID(tlsp->vi->pi->port_id) | in ktls_write_tls_wr()
1952 V_ULP_TXPKT_FID(txq->eq.cntxt_id) | V_ULP_TXPKT_RO(1)); in ktls_write_tls_wr()
1953 txpkt->len = htobe32(howmany(txpkt_lens[1], 16)); in ktls_write_tls_wr()
1955 /* ULPTX_IDATA sub-command */ in ktls_write_tls_wr()
1957 idata->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM) | in ktls_write_tls_wr()
1959 idata->len = sizeof(struct cpl_tx_tls_ack); in ktls_write_tls_wr()
1960 idata->len += sizeof(struct rss_header) + in ktls_write_tls_wr()
1962 idata->len += AES_GMAC_HASH_LEN; in ktls_write_tls_wr()
1963 idata->len = htobe32(idata->len); in ktls_write_tls_wr()
1967 out = write_tx_tls_ack(out, tlsp->rx_chid, AES_GMAC_HASH_LEN, in ktls_write_tls_wr()
1971 out = write_fw6_pld(out, tlsp->rx_chid, tlsp->rx_qid, in ktls_write_tls_wr()
1978 tlsp->ghash_pending = true; in ktls_write_tls_wr()
1979 tlsp->ghash_valid = false; in ktls_write_tls_wr()
1980 tlsp->ghash_lcb = ghash_lcb; in ktls_write_tls_wr()
1982 tlsp->ghash_offset = offset + plen; in ktls_write_tls_wr()
1984 tlsp->ghash_offset = rounddown2(offset + plen, in ktls_write_tls_wr()
1988 __func__, tlsp, tlsp->ghash_offset); in ktls_write_tls_wr()
1990 m_snd_tag_ref(&tlsp->com); in ktls_write_tls_wr()
1992 txq->kern_tls_ghash_requested++; in ktls_write_tls_wr()
1997 copy_to_txd(eq, txq->ss, &out, wr_len); in ktls_write_tls_wr()
2000 txq->kern_tls_records++; in ktls_write_tls_wr()
2001 txq->kern_tls_octets += m_tls->m_len; in ktls_write_tls_wr()
2003 txq->kern_tls_splitmode++; in ktls_write_tls_wr()
2004 txq->kern_tls_waste += leading_waste + trailing_waste; in ktls_write_tls_wr()
2007 txq->kern_tls_lso++; in ktls_write_tls_wr()
2009 txsd = &txq->sdesc[pidx]; in ktls_write_tls_wr()
2011 txsd->m = m; in ktls_write_tls_wr()
2013 txsd->m = NULL; in ktls_write_tls_wr()
2014 txsd->desc_used = ndesc; in ktls_write_tls_wr()
2023 struct sge_eq *eq = &txq->eq; in t7_ktls_write_wr()
2034 MPASS(m->m_pkthdr.snd_tag != NULL); in t7_ktls_write_wr()
2035 tlsp = mst_to_tls(m->m_pkthdr.snd_tag); in t7_ktls_write_wr()
2039 eh_type = ntohs(eh->ether_type); in t7_ktls_write_wr()
2043 eh_type = ntohs(evh->evl_proto); in t7_ktls_write_wr()
2046 tcp = (struct tcphdr *)((char *)eh + m->m_pkthdr.l2hlen + in t7_ktls_write_wr()
2047 m->m_pkthdr.l3hlen); in t7_ktls_write_wr()
2048 pidx = eq->pidx; in t7_ktls_write_wr()
2051 if (m->m_pkthdr.csum_flags & CSUM_TSO) { in t7_ktls_write_wr()
2052 mss = m->m_pkthdr.tso_segsz; in t7_ktls_write_wr()
2053 tlsp->prev_mss = mss; in t7_ktls_write_wr()
2054 } else if (tlsp->prev_mss != 0) in t7_ktls_write_wr()
2055 mss = tlsp->prev_mss; in t7_ktls_write_wr()
2057 mss = if_getmtu(tlsp->vi->ifp) - in t7_ktls_write_wr()
2058 (m->m_pkthdr.l3hlen + m->m_pkthdr.l4hlen); in t7_ktls_write_wr()
2061 tcp_seqno = ntohl(tcp->th_seq); in t7_ktls_write_wr()
2063 CTR(KTR_CXGBE, "%s: pkt len %d TCP seq %u", __func__, m->m_pkthdr.len, in t7_ktls_write_wr()
2066 KASSERT(!tlsp->ghash_pending, ("%s: GHASH pending for send", __func__)); in t7_ktls_write_wr()
2072 for (m_tls = m->m_next; m_tls != NULL; m_tls = m_tls->m_next) { in t7_ktls_write_wr()
2073 MPASS(m_tls->m_flags & M_EXTPG); in t7_ktls_write_wr()
2076 available - totdesc, tcp_seqno, pidx, eh_type, mss); in t7_ktls_write_wr()
2078 IDXINCR(pidx, ndesc, eq->sidx); in t7_ktls_write_wr()
2079 dst = &eq->desc[pidx]; in t7_ktls_write_wr()
2081 tcp_seqno += m_tls->m_len; in t7_ktls_write_wr()
2088 if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM && !tlsp->ghash_pending) in t7_ktls_write_wr()
2102 sc = tlsp->sc; in t7_tls_tag_free()
2106 if (tlsp->tx_key_addr >= 0) in t7_tls_tag_free()
2107 t4_free_tls_keyid(sc, tlsp->tx_key_addr); in t7_tls_tag_free()
2109 KASSERT(mbufq_len(&tlsp->pending_mbufs) == 0, in t7_tls_tag_free()
2129 KASSERT(cpl->data[0] == 0, ("%s: error status returned", __func__)); in ktls_fw6_pld()
2131 TXQ_LOCK(tlsp->txq); in ktls_fw6_pld()
2134 tlsp->ghash_offset, tlsp->ghash_lcb ? " in LCB" : ""); in ktls_fw6_pld()
2136 if (tlsp->ghash_lcb) in ktls_fw6_pld()
2137 ghash = &cpl->data[2]; in ktls_fw6_pld()
2140 memcpy(tlsp->ghash, ghash, AES_GMAC_HASH_LEN); in ktls_fw6_pld()
2141 tlsp->ghash_valid = true; in ktls_fw6_pld()
2142 tlsp->ghash_pending = false; in ktls_fw6_pld()
2143 tlsp->txq->kern_tls_ghash_received++; in ktls_fw6_pld()
2146 TXQ_UNLOCK(tlsp->txq); in ktls_fw6_pld()
2148 m_snd_tag_rele(&tlsp->com); in ktls_fw6_pld()