Lines Matching +full:hdr +full:- +full:engine
1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
4 * Copyright (c) 2018-2019 Chelsio Communications, Inc.
120 struct port_info *pi = vi->pi; in alloc_tlspcb()
121 struct adapter *sc = pi->adapter; in alloc_tlspcb()
128 m_snd_tag_init(&tlsp->com, ifp, &t6_tls_tag_sw); in alloc_tlspcb()
129 tlsp->vi = vi; in alloc_tlspcb()
130 tlsp->sc = sc; in alloc_tlspcb()
131 tlsp->ctrlq = &sc->sge.ctrlq[pi->port_id]; in alloc_tlspcb()
132 tlsp->tid = -1; in alloc_tlspcb()
133 tlsp->tx_key_addr = -1; in alloc_tlspcb()
161 qid_atid = V_TID_QID(sc->sge.fwq.abs_id) | V_TID_TID(atid) | in mk_ktls_act_open_req()
165 inp_4tuple_get(inp, &cpl->local_ip, &cpl->local_port, in mk_ktls_act_open_req()
166 &cpl->peer_ip, &cpl->peer_port); in mk_ktls_act_open_req()
169 options |= V_SMAC_SEL(vi->smt_idx) | V_TX_CHAN(vi->pi->tx_chan); in mk_ktls_act_open_req()
171 cpl->opt0 = htobe64(options); in mk_ktls_act_open_req()
173 options = V_TX_QUEUE(TX_MODQ(vi->pi->tx_chan)); in mk_ktls_act_open_req()
174 if (tp->t_flags & TF_REQ_TSTMP) in mk_ktls_act_open_req()
176 cpl->opt2 = htobe32(options); in mk_ktls_act_open_req()
192 qid_atid = V_TID_QID(sc->sge.fwq.abs_id) | V_TID_TID(atid) | in mk_ktls_act_open_req6()
196 cpl->local_port = inp->inp_lport; in mk_ktls_act_open_req6()
197 cpl->local_ip_hi = *(uint64_t *)&inp->in6p_laddr.s6_addr[0]; in mk_ktls_act_open_req6()
198 cpl->local_ip_lo = *(uint64_t *)&inp->in6p_laddr.s6_addr[8]; in mk_ktls_act_open_req6()
199 cpl->peer_port = inp->inp_fport; in mk_ktls_act_open_req6()
200 cpl->peer_ip_hi = *(uint64_t *)&inp->in6p_faddr.s6_addr[0]; in mk_ktls_act_open_req6()
201 cpl->peer_ip_lo = *(uint64_t *)&inp->in6p_faddr.s6_addr[8]; in mk_ktls_act_open_req6()
204 options |= V_SMAC_SEL(vi->smt_idx) | V_TX_CHAN(vi->pi->tx_chan); in mk_ktls_act_open_req6()
206 cpl->opt0 = htobe64(options); in mk_ktls_act_open_req6()
208 options = V_TX_QUEUE(TX_MODQ(vi->pi->tx_chan)); in mk_ktls_act_open_req6()
209 if (tp->t_flags & TF_REQ_TSTMP) in mk_ktls_act_open_req6()
211 cpl->opt2 = htobe32(options); in mk_ktls_act_open_req6()
221 isipv6 = (inp->inp_vflag & INP_IPV6) != 0; in send_ktls_act_open_req()
223 tlsp->ce = t4_get_clip_entry(sc, &inp->in6p_laddr, true); in send_ktls_act_open_req()
224 if (tlsp->ce == NULL) in send_ktls_act_open_req()
228 wr = alloc_wrqe(ktls_act_open_cpl_size(isipv6), tlsp->ctrlq); in send_ktls_act_open_req()
240 tlsp->open_pending = true; in send_ktls_act_open_req()
249 struct adapter *sc = iq->adapter; in ktls_act_open_rpl()
251 u_int atid = G_TID_TID(G_AOPEN_ATID(be32toh(cpl->atid_status))); in ktls_act_open_rpl()
252 u_int status = G_AOPEN_STATUS(be32toh(cpl->atid_status)); in ktls_act_open_rpl()
254 struct inpcb *inp = tlsp->inp; in ktls_act_open_rpl()
259 tlsp->tid = GET_TID(cpl); in ktls_act_open_rpl()
262 tlsp->open_pending = false; in ktls_act_open_rpl()
273 "CPL_SET_TCB_FIELD ULP command not 16-byte aligned");
285 txpkt->cmd_dest = htobe32(V_ULPTX_CMD(ULP_TX_PKT) | in write_set_tcb_field_ulp()
287 V_ULP_TXPKT_CHANNELID(tlsp->vi->pi->port_id) | V_ULP_TXPKT_DEST(0) | in write_set_tcb_field_ulp()
288 V_ULP_TXPKT_FID(txq->eq.cntxt_id) | V_ULP_TXPKT_RO(1)); in write_set_tcb_field_ulp()
289 txpkt->len = htobe32(howmany(LEN__SET_TCB_FIELD_ULP, 16)); in write_set_tcb_field_ulp()
291 /* ULPTX_IDATA sub-command */ in write_set_tcb_field_ulp()
293 idata->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM)); in write_set_tcb_field_ulp()
294 idata->len = htobe32(sizeof(*cpl)); in write_set_tcb_field_ulp()
298 OPCODE_TID(cpl) = htobe32(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tlsp->tid)); in write_set_tcb_field_ulp()
299 cpl->reply_ctrl = htobe16(F_NO_REPLY); in write_set_tcb_field_ulp()
300 cpl->word_cookie = htobe16(V_WORD(word)); in write_set_tcb_field_ulp()
301 cpl->mask = htobe64(mask); in write_set_tcb_field_ulp()
302 cpl->val = htobe64(val); in write_set_tcb_field_ulp()
306 idata->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP)); in write_set_tcb_field_ulp()
307 idata->len = htobe32(0); in write_set_tcb_field_ulp()
320 if (tp->t_flags & TF_REQ_TSTMP) in ktls_set_tcb_fields()
325 tlsp->tid); in ktls_set_tcb_fields()
328 m->m_pkthdr.snd_tag = m_snd_tag_ref(&tlsp->com); in ktls_set_tcb_fields()
329 m->m_pkthdr.csum_flags |= CSUM_SND_TAG; in ktls_set_tcb_fields()
333 wr->op_to_compl = htobe32(V_FW_WR_OP(FW_ULPTX_WR)); in ktls_set_tcb_fields()
334 wr->flowid_len16 = htobe32(F_FW_ULPTX_WR_DATA | in ktls_set_tcb_fields()
336 wr->cookie = 0; in ktls_set_tcb_fields()
356 if (tp->t_flags & TF_REQ_TSTMP) { in ktls_set_tcb_fields()
359 V_TCB_TIMESTAMP_OFFSET(tp->ts_offset >> 28)); in ktls_set_tcb_fields()
363 KASSERT(dst - (char *)wr == len, ("%s: length mismatch", __func__)); in ktls_set_tcb_fields()
366 error = mp_ring_enqueue(txq->r, items, 1, 1); in ktls_set_tcb_fields()
385 tls = params->tls.tls; in t6_tls_tag_alloc()
388 if (tls->params.tls_vmajor != TLS_MAJOR_VER_ONE || in t6_tls_tag_alloc()
389 tls->params.tls_vminor < TLS_MINOR_VER_ONE || in t6_tls_tag_alloc()
390 tls->params.tls_vminor > TLS_MINOR_VER_TWO) in t6_tls_tag_alloc()
394 switch (tls->params.cipher_algorithm) { in t6_tls_tag_alloc()
397 switch (tls->params.cipher_key_len) { in t6_tls_tag_alloc()
405 switch (tls->params.auth_algorithm) { in t6_tls_tag_alloc()
417 if (tls->params.iv_len != SALT_SIZE) in t6_tls_tag_alloc()
419 switch (tls->params.cipher_key_len) { in t6_tls_tag_alloc()
435 sc = vi->adapter; in t6_tls_tag_alloc()
445 if (sc->tlst.inline_keys) in t6_tls_tag_alloc()
446 keyid = -1; in t6_tls_tag_alloc()
452 tlsp->inline_key = true; in t6_tls_tag_alloc()
454 tlsp->tx_key_addr = keyid; in t6_tls_tag_alloc()
457 atid, tlsp->tx_key_addr); in t6_tls_tag_alloc()
460 inp = params->tls.inp; in t6_tls_tag_alloc()
462 if (inp->inp_flags & INP_DROPPED) { in t6_tls_tag_alloc()
467 tlsp->inp = inp; in t6_tls_tag_alloc()
470 if (tp->t_flags & TF_REQ_TSTMP) { in t6_tls_tag_alloc()
471 tlsp->using_timestamps = true; in t6_tls_tag_alloc()
472 if ((tp->ts_offset & 0xfffffff) != 0) { in t6_tls_tag_alloc()
478 tlsp->using_timestamps = false; in t6_tls_tag_alloc()
489 while (tlsp->open_pending) { in t6_tls_tag_alloc()
494 error = rw_sleep(tlsp, &inp->inp_lock, 0, "t6tlsop", 0); in t6_tls_tag_alloc()
497 atid = -1; in t6_tls_tag_alloc()
498 if (tlsp->tid < 0) { in t6_tls_tag_alloc()
504 if (inp->inp_flags & INP_DROPPED) { in t6_tls_tag_alloc()
510 txq = &sc->sge.txq[vi->first_txq]; in t6_tls_tag_alloc()
511 if (inp->inp_flowtype != M_HASHTYPE_NONE) in t6_tls_tag_alloc()
512 txq += ((inp->inp_flowid % (vi->ntxq - vi->rsrv_noflowq)) + in t6_tls_tag_alloc()
513 vi->rsrv_noflowq); in t6_tls_tag_alloc()
514 tlsp->txq = txq; in t6_tls_tag_alloc()
525 tlsp->enc_mode = t4_tls_cipher_mode(tls); in t6_tls_tag_alloc()
526 tlsp->tx_key_info_size = t4_tls_key_info_size(tls); in t6_tls_tag_alloc()
529 tlsp->scmd0.seqno_numivs = htobe32(V_SCMD_SEQ_NO_CTRL(3) | in t6_tls_tag_alloc()
533 V_SCMD_CIPH_MODE(tlsp->enc_mode) | in t6_tls_tag_alloc()
538 tlsp->scmd0.ivgen_hdrlen = V_SCMD_IV_GEN_CTRL(0) | in t6_tls_tag_alloc()
540 if (tlsp->inline_key) in t6_tls_tag_alloc()
541 tlsp->scmd0.ivgen_hdrlen |= V_SCMD_KEY_CTX_INLINE(1); in t6_tls_tag_alloc()
542 tlsp->scmd0.ivgen_hdrlen = htobe32(tlsp->scmd0.ivgen_hdrlen); in t6_tls_tag_alloc()
548 tlsp->scmd0_short.seqno_numivs = V_SCMD_SEQ_NO_CTRL(0) | in t6_tls_tag_alloc()
555 if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM) in t6_tls_tag_alloc()
556 tlsp->scmd0_short.seqno_numivs |= in t6_tls_tag_alloc()
559 tlsp->scmd0_short.seqno_numivs |= in t6_tls_tag_alloc()
560 V_SCMD_CIPH_MODE(tlsp->enc_mode); in t6_tls_tag_alloc()
561 tlsp->scmd0_short.seqno_numivs = in t6_tls_tag_alloc()
562 htobe32(tlsp->scmd0_short.seqno_numivs); in t6_tls_tag_alloc()
564 tlsp->scmd0_short.ivgen_hdrlen = V_SCMD_IV_GEN_CTRL(0) | in t6_tls_tag_alloc()
567 if (tlsp->inline_key) in t6_tls_tag_alloc()
568 tlsp->scmd0_short.ivgen_hdrlen |= V_SCMD_KEY_CTX_INLINE(1); in t6_tls_tag_alloc()
571 if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM) in t6_tls_tag_alloc()
572 txq->kern_tls_gcm++; in t6_tls_tag_alloc()
574 txq->kern_tls_cbc++; in t6_tls_tag_alloc()
576 *pt = &tlsp->com; in t6_tls_tag_alloc()
582 m_snd_tag_rele(&tlsp->com); in t6_tls_tag_alloc()
603 t4_tls_key_ctx(tls, KTLS_TX, &tlsp->keyctx); in ktls_setup_keys()
604 if (tlsp->inline_key) in ktls_setup_keys()
611 tlsp->tid); in ktls_setup_keys()
614 m->m_pkthdr.snd_tag = m_snd_tag_ref(&tlsp->com); in ktls_setup_keys()
615 m->m_pkthdr.csum_flags |= CSUM_SND_TAG; in ktls_setup_keys()
619 t4_write_tlskey_wr(tls, KTLS_TX, tlsp->tid, 0, tlsp->tx_key_addr, kwr); in ktls_setup_keys()
621 memcpy(kctx, &tlsp->keyctx, sizeof(*kctx)); in ktls_setup_keys()
629 error = mp_ring_enqueue(txq->r, items, 1, 1); in ktls_setup_keys()
633 CTR2(KTR_CXGBE, "%s: tid %d sent key WR", __func__, tlsp->tid); in ktls_setup_keys()
646 if (tlsp->inline_key) in ktls_base_wr_size()
647 wr_len += tlsp->tx_key_info_size; in ktls_base_wr_size()
660 struct tls_record_layer *hdr; in ktls_tcp_payload_length() local
664 hdr = (void *)m_tls->m_epg_hdr; in ktls_tcp_payload_length()
665 plen = ntohs(hdr->tls_length); in ktls_tcp_payload_length()
671 mlen = mtod(m_tls, vm_offset_t) + m_tls->m_len; in ktls_tcp_payload_length()
682 if (mlen > TLS_HEADER_LENGTH + plen - m_tls->m_epg_trllen) in ktls_tcp_payload_length()
683 mlen = TLS_HEADER_LENGTH + plen - m_tls->m_epg_trllen; in ktls_tcp_payload_length()
687 * For AES-CBC adjust the ciphertext length for the block in ktls_tcp_payload_length()
690 if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_CBC && in ktls_tcp_payload_length()
692 mlen = TLS_HEADER_LENGTH + rounddown(mlen - TLS_HEADER_LENGTH, in ktls_tcp_payload_length()
698 __func__, tlsp->tid, mlen, TLS_HEADER_LENGTH + plen); in ktls_tcp_payload_length()
706 * a non-zero offset implies that a header will not be sent.
711 struct tls_record_layer *hdr; in ktls_payload_offset() local
718 hdr = (void *)m_tls->m_epg_hdr; in ktls_payload_offset()
719 plen = ntohs(hdr->tls_length); in ktls_payload_offset()
721 mlen = mtod(m_tls, vm_offset_t) + m_tls->m_len; in ktls_payload_offset()
724 if (mtod(m_tls, vm_offset_t) <= m_tls->m_epg_hdrlen) in ktls_payload_offset()
726 if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM) { in ktls_payload_offset()
735 offset = min(mtod(m_tls, vm_offset_t) - m_tls->m_epg_hdrlen, in ktls_payload_offset()
736 (plen - TLS_HEADER_LENGTH - m_tls->m_epg_trllen) - 1); in ktls_payload_offset()
748 nsegs--; in ktls_sgl_size()
759 struct tls_record_layer *hdr; in ktls_wr_len() local
769 if (tlen <= m_tls->m_epg_hdrlen) { in ktls_wr_len()
776 roundup2(m->m_len + m_tls->m_len, 16); in ktls_wr_len()
779 "%s: tid %d TLS header-only packet too long (len %d)", in ktls_wr_len()
780 __func__, tlsp->tid, m->m_len + m_tls->m_len); in ktls_wr_len()
784 MPASS(m_tls->m_next == NULL); in ktls_wr_len()
789 hdr = (void *)m_tls->m_epg_hdr; in ktls_wr_len()
790 plen = TLS_HEADER_LENGTH + ntohs(hdr->tls_length) - m_tls->m_epg_trllen; in ktls_wr_len()
807 imm_len += m_tls->m_epg_hdrlen; in ktls_wr_len()
813 *nsegsp = sglist_count_mbuf_epg(m_tls, m_tls->m_epg_hdrlen + offset, in ktls_wr_len()
814 plen - (m_tls->m_epg_hdrlen + offset)); in ktls_wr_len()
822 * See if we have any TCP options requiring a dedicated options-only
832 cnt = tcp->th_off * 4 - sizeof(struct tcphdr); in ktls_has_tcp_options()
833 for (; cnt > 0; cnt -= optlen, cp += optlen) { in ktls_has_tcp_options()
867 cnt = tcp->th_off * 4 - sizeof(struct tcphdr); in ktls_find_tcp_timestamps()
868 for (; cnt > 0; cnt -= optlen, cp += optlen) { in ktls_find_tcp_timestamps()
908 MPASS(m->m_pkthdr.snd_tag != NULL); in t6_ktls_parse_pkt()
909 tlsp = mst_to_tls(m->m_pkthdr.snd_tag); in t6_ktls_parse_pkt()
911 if (m->m_len <= sizeof(*eh) + sizeof(*ip)) { in t6_ktls_parse_pkt()
913 tlsp->tid); in t6_ktls_parse_pkt()
917 if (ntohs(eh->ether_type) != ETHERTYPE_IP && in t6_ktls_parse_pkt()
918 ntohs(eh->ether_type) != ETHERTYPE_IPV6) { in t6_ktls_parse_pkt()
920 __func__, tlsp->tid); in t6_ktls_parse_pkt()
923 m->m_pkthdr.l2hlen = sizeof(*eh); in t6_ktls_parse_pkt()
926 if (ntohs(eh->ether_type) == ETHERTYPE_IP) { in t6_ktls_parse_pkt()
928 if (ip->ip_p != IPPROTO_TCP) { in t6_ktls_parse_pkt()
930 __func__, tlsp->tid); in t6_ktls_parse_pkt()
933 m->m_pkthdr.l3hlen = ip->ip_hl * 4; in t6_ktls_parse_pkt()
936 if (ip6->ip6_nxt != IPPROTO_TCP) { in t6_ktls_parse_pkt()
938 __func__, tlsp->tid, ip6->ip6_nxt); in t6_ktls_parse_pkt()
941 m->m_pkthdr.l3hlen = sizeof(struct ip6_hdr); in t6_ktls_parse_pkt()
943 if (m->m_len < m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen + in t6_ktls_parse_pkt()
946 __func__, tlsp->tid); in t6_ktls_parse_pkt()
949 tcp = (struct tcphdr *)((char *)(eh + 1) + m->m_pkthdr.l3hlen); in t6_ktls_parse_pkt()
950 m->m_pkthdr.l4hlen = tcp->th_off * 4; in t6_ktls_parse_pkt()
953 if (m->m_len != m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen + in t6_ktls_parse_pkt()
954 m->m_pkthdr.l4hlen) { in t6_ktls_parse_pkt()
957 __func__, tlsp->tid, m->m_pkthdr.l2hlen, in t6_ktls_parse_pkt()
958 m->m_pkthdr.l3hlen, m->m_pkthdr.l4hlen, m->m_len); in t6_ktls_parse_pkt()
963 MPASS(m->m_next != NULL); in t6_ktls_parse_pkt()
964 MPASS(m->m_next->m_flags & M_EXTPG); in t6_ktls_parse_pkt()
972 for (m_tls = m->m_next; m_tls != NULL; m_tls = m_tls->m_next) { in t6_ktls_parse_pkt()
973 MPASS(m_tls->m_flags & M_EXTPG); in t6_ktls_parse_pkt()
978 tlsp->tid, wr_len, nsegs); in t6_ktls_parse_pkt()
988 if (m_tls == m->m_next) in t6_ktls_parse_pkt()
1000 sizeof(struct cpl_tx_pkt_core) + roundup2(m->m_len, 16); in t6_ktls_parse_pkt()
1003 "%s: tid %d options-only packet too long (len %d)", in t6_ktls_parse_pkt()
1004 __func__, tlsp->tid, m->m_len); in t6_ktls_parse_pkt()
1025 if (tlsp->using_timestamps) in t6_ktls_parse_pkt()
1033 tlsp->tid, mbuf_len16(m), mbuf_nsegs(m)); in t6_ktls_parse_pkt()
1036 return (mp_ring_enqueue(tlsp->txq->r, items, 1, 256)); in t6_ktls_parse_pkt()
1054 nsegs = gl->sg_nseg; in write_gl_to_buf()
1057 nflits = (3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1) + 2; in write_gl_to_buf()
1059 seg = &gl->sg_segs[0]; in write_gl_to_buf()
1062 usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) | in write_gl_to_buf()
1064 usgl->len0 = htobe32(seg->ss_len); in write_gl_to_buf()
1065 usgl->addr0 = htobe64(seg->ss_paddr); in write_gl_to_buf()
1068 for (i = 0; i < nsegs - 1; i++, seg++) { in write_gl_to_buf()
1069 usgl->sge[i / 2].len[i & 1] = htobe32(seg->ss_len); in write_gl_to_buf()
1070 usgl->sge[i / 2].addr[i & 1] = htobe64(seg->ss_paddr); in write_gl_to_buf()
1073 usgl->sge[i / 2].len[1] = htobe32(0); in write_gl_to_buf()
1088 MPASS((uintptr_t)(*to) >= (uintptr_t)&eq->desc[0]); in copy_to_txd()
1089 MPASS((uintptr_t)(*to) < (uintptr_t)&eq->desc[eq->sidx]); in copy_to_txd()
1092 (uintptr_t)&eq->desc[eq->sidx])) { in copy_to_txd()
1095 if ((uintptr_t)(*to) == (uintptr_t)&eq->desc[eq->sidx]) in copy_to_txd()
1096 (*to) = (caddr_t)eq->desc; in copy_to_txd()
1098 int portion = (uintptr_t)&eq->desc[eq->sidx] - (uintptr_t)(*to); in copy_to_txd()
1102 portion = len - portion; /* remaining */ in copy_to_txd()
1103 bcopy(from, (void *)eq->desc, portion); in copy_to_txd()
1104 (*to) = (caddr_t)eq->desc + portion; in copy_to_txd()
1128 pktlen = m->m_len; in ktls_write_tcp_options()
1135 wr->op_immdlen = htobe32(V_FW_WR_OP(FW_ETH_TX_PKT_WR) | in ktls_write_tcp_options()
1139 wr->equiq_to_len16 = htobe32(ctrl); in ktls_write_tcp_options()
1140 wr->r3 = 0; in ktls_write_tcp_options()
1145 cpl->ctrl0 = txq->cpl_ctrl0; in ktls_write_tcp_options()
1146 cpl->pack = 0; in ktls_write_tcp_options()
1147 cpl->len = htobe16(pktlen); in ktls_write_tcp_options()
1153 copy_to_txd(&txq->eq, (caddr_t)eh, &out, m->m_pkthdr.l2hlen); in ktls_write_tcp_options()
1156 if (ntohs(eh->ether_type) == ETHERTYPE_IP) { in ktls_write_tcp_options()
1157 ip = (void *)((char *)eh + m->m_pkthdr.l2hlen); in ktls_write_tcp_options()
1159 newip.ip_len = htons(pktlen - m->m_pkthdr.l2hlen); in ktls_write_tcp_options()
1160 copy_to_txd(&txq->eq, (caddr_t)&newip, &out, sizeof(newip)); in ktls_write_tcp_options()
1161 if (m->m_pkthdr.l3hlen > sizeof(*ip)) in ktls_write_tcp_options()
1162 copy_to_txd(&txq->eq, (caddr_t)(ip + 1), &out, in ktls_write_tcp_options()
1163 m->m_pkthdr.l3hlen - sizeof(*ip)); in ktls_write_tcp_options()
1165 V_T6_TXPKT_ETHHDR_LEN(m->m_pkthdr.l2hlen - ETHER_HDR_LEN) | in ktls_write_tcp_options()
1166 V_TXPKT_IPHDR_LEN(m->m_pkthdr.l3hlen); in ktls_write_tcp_options()
1168 ip6 = (void *)((char *)eh + m->m_pkthdr.l2hlen); in ktls_write_tcp_options()
1170 newip6.ip6_plen = htons(pktlen - m->m_pkthdr.l2hlen); in ktls_write_tcp_options()
1171 copy_to_txd(&txq->eq, (caddr_t)&newip6, &out, sizeof(newip6)); in ktls_write_tcp_options()
1172 MPASS(m->m_pkthdr.l3hlen == sizeof(*ip6)); in ktls_write_tcp_options()
1174 V_T6_TXPKT_ETHHDR_LEN(m->m_pkthdr.l2hlen - ETHER_HDR_LEN) | in ktls_write_tcp_options()
1175 V_TXPKT_IPHDR_LEN(m->m_pkthdr.l3hlen); in ktls_write_tcp_options()
1177 cpl->ctrl1 = htobe64(ctrl1); in ktls_write_tcp_options()
1178 txq->txcsum++; in ktls_write_tcp_options()
1181 tcp = (void *)((char *)eh + m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen); in ktls_write_tcp_options()
1184 copy_to_txd(&txq->eq, (caddr_t)&newtcp, &out, sizeof(newtcp)); in ktls_write_tcp_options()
1187 copy_to_txd(&txq->eq, (caddr_t)(tcp + 1), &out, pktlen - in ktls_write_tcp_options()
1188 (m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen + sizeof(*tcp))); in ktls_write_tcp_options()
1189 txq->imm_wrs++; in ktls_write_tcp_options()
1191 txq->txpkt_wrs++; in ktls_write_tcp_options()
1193 txq->kern_tls_options++; in ktls_write_tcp_options()
1195 txsd = &txq->sdesc[pidx]; in ktls_write_tcp_options()
1196 txsd->m = NULL; in ktls_write_tcp_options()
1197 txsd->desc_used = ndesc; in ktls_write_tcp_options()
1225 MPASS(m_tls->m_next == NULL); in ktls_write_tunnel_packet()
1228 pktlen = m->m_len + m_tls->m_len; in ktls_write_tunnel_packet()
1235 wr->op_immdlen = htobe32(V_FW_WR_OP(FW_ETH_TX_PKT_WR) | in ktls_write_tunnel_packet()
1239 wr->equiq_to_len16 = htobe32(ctrl); in ktls_write_tunnel_packet()
1240 wr->r3 = 0; in ktls_write_tunnel_packet()
1245 cpl->ctrl0 = txq->cpl_ctrl0; in ktls_write_tunnel_packet()
1246 cpl->pack = 0; in ktls_write_tunnel_packet()
1247 cpl->len = htobe16(pktlen); in ktls_write_tunnel_packet()
1253 copy_to_txd(&txq->eq, (caddr_t)eh, &out, m->m_pkthdr.l2hlen); in ktls_write_tunnel_packet()
1256 if (ntohs(eh->ether_type) == ETHERTYPE_IP) { in ktls_write_tunnel_packet()
1257 ip = (void *)((char *)eh + m->m_pkthdr.l2hlen); in ktls_write_tunnel_packet()
1259 newip.ip_len = htons(pktlen - m->m_pkthdr.l2hlen); in ktls_write_tunnel_packet()
1260 copy_to_txd(&txq->eq, (caddr_t)&newip, &out, sizeof(newip)); in ktls_write_tunnel_packet()
1261 if (m->m_pkthdr.l3hlen > sizeof(*ip)) in ktls_write_tunnel_packet()
1262 copy_to_txd(&txq->eq, (caddr_t)(ip + 1), &out, in ktls_write_tunnel_packet()
1263 m->m_pkthdr.l3hlen - sizeof(*ip)); in ktls_write_tunnel_packet()
1265 V_T6_TXPKT_ETHHDR_LEN(m->m_pkthdr.l2hlen - ETHER_HDR_LEN) | in ktls_write_tunnel_packet()
1266 V_TXPKT_IPHDR_LEN(m->m_pkthdr.l3hlen); in ktls_write_tunnel_packet()
1268 ip6 = (void *)((char *)eh + m->m_pkthdr.l2hlen); in ktls_write_tunnel_packet()
1270 newip6.ip6_plen = htons(pktlen - m->m_pkthdr.l2hlen); in ktls_write_tunnel_packet()
1271 copy_to_txd(&txq->eq, (caddr_t)&newip6, &out, sizeof(newip6)); in ktls_write_tunnel_packet()
1272 MPASS(m->m_pkthdr.l3hlen == sizeof(*ip6)); in ktls_write_tunnel_packet()
1274 V_T6_TXPKT_ETHHDR_LEN(m->m_pkthdr.l2hlen - ETHER_HDR_LEN) | in ktls_write_tunnel_packet()
1275 V_TXPKT_IPHDR_LEN(m->m_pkthdr.l3hlen); in ktls_write_tunnel_packet()
1277 cpl->ctrl1 = htobe64(ctrl1); in ktls_write_tunnel_packet()
1278 txq->txcsum++; in ktls_write_tunnel_packet()
1281 tcp = (void *)((char *)eh + m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen); in ktls_write_tunnel_packet()
1284 copy_to_txd(&txq->eq, (caddr_t)&newtcp, &out, sizeof(newtcp)); in ktls_write_tunnel_packet()
1287 copy_to_txd(&txq->eq, (caddr_t)(tcp + 1), &out, m->m_len - in ktls_write_tunnel_packet()
1288 (m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen + sizeof(*tcp))); in ktls_write_tunnel_packet()
1291 copy_to_txd(&txq->eq, (char *)m_tls->m_epg_hdr + in ktls_write_tunnel_packet()
1292 mtod(m_tls, vm_offset_t), &out, m_tls->m_len); in ktls_write_tunnel_packet()
1293 txq->imm_wrs++; in ktls_write_tunnel_packet()
1295 txq->txpkt_wrs++; in ktls_write_tunnel_packet()
1297 txq->kern_tls_header++; in ktls_write_tunnel_packet()
1299 txsd = &txq->sdesc[pidx]; in ktls_write_tunnel_packet()
1300 txsd->m = m; in ktls_write_tunnel_packet()
1301 txsd->desc_used = ndesc; in ktls_write_tunnel_packet()
1316 struct sge_eq *eq = &txq->eq; in ktls_write_tls_wr()
1324 struct tls_record_layer *hdr; in ktls_write_tls_wr() local
1334 MPASS(tlsp->txq == txq); in ktls_write_tls_wr()
1336 first_wr = (tlsp->prev_seq == 0 && tlsp->prev_ack == 0 && in ktls_write_tls_wr()
1337 tlsp->prev_win == 0); in ktls_write_tls_wr()
1340 * Use the per-txq scratch pad if near the end of the ring to in ktls_write_tls_wr()
1341 * simplify handling of wrap-around. This uses a simple but in ktls_write_tls_wr()
1345 using_scratch = (eq->sidx - pidx < SGE_MAX_WR_LEN / EQ_ESIZE); in ktls_write_tls_wr()
1349 hdr = (void *)m_tls->m_epg_hdr; in ktls_write_tls_wr()
1350 plen = TLS_HEADER_LENGTH + ntohs(hdr->tls_length) - m_tls->m_epg_trllen; in ktls_write_tls_wr()
1354 if (tlen <= m_tls->m_epg_hdrlen) { in ktls_write_tls_wr()
1360 CTR3(KTR_CXGBE, "%s: tid %d header-only TLS record %u", in ktls_write_tls_wr()
1361 __func__, tlsp->tid, (u_int)m_tls->m_epg_seqno); in ktls_write_tls_wr()
1371 __func__, tlsp->tid, (u_int)m_tls->m_epg_seqno, offset); in ktls_write_tls_wr()
1373 if (m_tls->m_next == NULL && (tcp_get_flags(tcp) & TH_FIN) != 0) { in ktls_write_tls_wr()
1374 txq->kern_tls_fin_short++; in ktls_write_tls_wr()
1388 last_wr = m_tls->m_next == NULL && (tcp_get_flags(tcp) & TH_FIN) == 0; in ktls_write_tls_wr()
1395 * record.) The crypto engine must process a TLS record from in ktls_write_tls_wr()
1398 * crypto engine and via CPL_TX_DATA to TP. However, TP will in ktls_write_tls_wr()
1400 * into MSS-sized chunks if the entire sequence range of those in ktls_write_tls_wr()
1402 * TX_MAX - SND_UNA_RAW. Thus, use the offset stored in in ktls_write_tls_wr()
1425 if (tx_max_offset > TLS_HEADER_LENGTH + ntohs(hdr->tls_length) - in ktls_write_tls_wr()
1426 m_tls->m_epg_trllen) { in ktls_write_tls_wr()
1428 tx_max_offset = TLS_HEADER_LENGTH + ntohs(hdr->tls_length) - in ktls_write_tls_wr()
1429 m_tls->m_epg_trllen; in ktls_write_tls_wr()
1431 if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_CBC && in ktls_write_tls_wr()
1435 rounddown(tx_max_offset - TLS_HEADER_LENGTH, in ktls_write_tls_wr()
1446 wr = (void *)txq->ss; in ktls_write_tls_wr()
1452 KASSERT(m->m_next == m_tls, in ktls_write_tls_wr()
1456 tlsp->tid, tlsp->l2te->idx); in ktls_write_tls_wr()
1459 V_TCB_L2T_IX(M_TCB_L2T_IX), V_TCB_L2T_IX(tlsp->l2te->idx)); in ktls_write_tls_wr()
1463 if (tsopt != NULL && tlsp->prev_tsecr != ntohl(tsopt[1])) { in ktls_write_tls_wr()
1464 KASSERT(m->m_next == m_tls, in ktls_write_tls_wr()
1468 __func__, tlsp->tid); in ktls_write_tls_wr()
1476 tlsp->prev_tsecr = ntohl(tsopt[1]); in ktls_write_tls_wr()
1479 if (first_wr || tlsp->prev_seq != tx_max) { in ktls_write_tls_wr()
1480 KASSERT(m->m_next == m_tls, in ktls_write_tls_wr()
1485 __func__, tlsp->tid, tx_max, tcp_seqno); in ktls_write_tls_wr()
1498 if (tlsp->prev_seq != tx_max || mtod(m_tls, vm_offset_t) != 0) { in ktls_write_tls_wr()
1499 KASSERT(m->m_next == m_tls, in ktls_write_tls_wr()
1503 tlsp->tid); in ktls_write_tls_wr()
1516 tlsp->prev_seq = tcp_seqno + tlen; in ktls_write_tls_wr()
1518 if (first_wr || tlsp->prev_ack != ntohl(tcp->th_ack)) { in ktls_write_tls_wr()
1519 KASSERT(m->m_next == m_tls, in ktls_write_tls_wr()
1523 V_TCB_RCV_NXT(ntohl(tcp->th_ack))); in ktls_write_tls_wr()
1527 tlsp->prev_ack = ntohl(tcp->th_ack); in ktls_write_tls_wr()
1530 if (first_wr || tlsp->prev_win != ntohs(tcp->th_win)) { in ktls_write_tls_wr()
1531 KASSERT(m->m_next == m_tls, in ktls_write_tls_wr()
1535 V_TCB_RCV_WND(ntohs(tcp->th_win))); in ktls_write_tls_wr()
1539 tlsp->prev_win = ntohs(tcp->th_win); in ktls_write_tls_wr()
1543 if (m->m_next == m_tls) in ktls_write_tls_wr()
1546 nsegs = sglist_count_mbuf_epg(m_tls, m_tls->m_epg_hdrlen + in ktls_write_tls_wr()
1547 offset, plen - (m_tls->m_epg_hdrlen + offset)); in ktls_write_tls_wr()
1554 imm_len += m_tls->m_epg_hdrlen; in ktls_write_tls_wr()
1570 tlsp->sc->tlst.combo_wrs) { in ktls_write_tls_wr()
1575 wr->op_to_compl = htobe32(V_FW_WR_OP(FW_ULPTX_WR)); in ktls_write_tls_wr()
1576 wr->flowid_len16 = htobe32(F_FW_ULPTX_WR_DATA | in ktls_write_tls_wr()
1578 wr->cookie = 0; in ktls_write_tls_wr()
1586 copy_to_txd(eq, txq->ss, &out, wr_len); in ktls_write_tls_wr()
1592 txq->raw_wrs++; in ktls_write_tls_wr()
1593 txsd = &txq->sdesc[pidx]; in ktls_write_tls_wr()
1594 txsd->m = NULL; in ktls_write_tls_wr()
1595 txsd->desc_used = ndesc; in ktls_write_tls_wr()
1596 IDXINCR(pidx, ndesc, eq->sidx); in ktls_write_tls_wr()
1597 dst = &eq->desc[pidx]; in ktls_write_tls_wr()
1606 using_scratch = (eq->sidx - pidx < in ktls_write_tls_wr()
1609 wr = (void *)txq->ss; in ktls_write_tls_wr()
1623 wr->op_to_compl = htobe32(V_FW_WR_OP(FW_ULPTX_WR)); in ktls_write_tls_wr()
1624 wr->flowid_len16 = htobe32(F_FW_ULPTX_WR_DATA | in ktls_write_tls_wr()
1626 wr->cookie = 0; in ktls_write_tls_wr()
1629 txpkt->cmd_dest = htobe32(V_ULPTX_CMD(ULP_TX_PKT) | in ktls_write_tls_wr()
1631 V_ULP_TXPKT_CHANNELID(tlsp->vi->pi->port_id) | V_ULP_TXPKT_DEST(0) | in ktls_write_tls_wr()
1632 V_ULP_TXPKT_FID(txq->eq.cntxt_id) | V_ULP_TXPKT_RO(1)); in ktls_write_tls_wr()
1633 txpkt->len = htobe32(howmany(twr_len - sizeof(*wr), 16)); in ktls_write_tls_wr()
1635 /* ULPTX_IDATA sub-command */ in ktls_write_tls_wr()
1637 idata->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM) | in ktls_write_tls_wr()
1639 idata->len = sizeof(struct cpl_tx_sec_pdu); in ktls_write_tls_wr()
1647 if (tlsp->inline_key) in ktls_write_tls_wr()
1648 idata->len += tlsp->tx_key_info_size + in ktls_write_tls_wr()
1650 idata->len = htobe32(idata->len); in ktls_write_tls_wr()
1669 sec_pdu->pldlen = htobe32(16 + plen - in ktls_write_tls_wr()
1670 (m_tls->m_epg_hdrlen + offset)); in ktls_write_tls_wr()
1673 sec_pdu->seqno_numivs = tlsp->scmd0_short.seqno_numivs; in ktls_write_tls_wr()
1674 sec_pdu->ivgen_hdrlen = htobe32( in ktls_write_tls_wr()
1675 tlsp->scmd0_short.ivgen_hdrlen | in ktls_write_tls_wr()
1676 V_SCMD_HDR_LEN(offset == 0 ? m_tls->m_epg_hdrlen : 0)); in ktls_write_tls_wr()
1678 txq->kern_tls_short++; in ktls_write_tls_wr()
1689 cipher_start = m_tls->m_epg_hdrlen + 1; in ktls_write_tls_wr()
1690 if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM) { in ktls_write_tls_wr()
1702 sec_pdu->pldlen = htobe32(plen); in ktls_write_tls_wr()
1705 sec_pdu->seqno_numivs = tlsp->scmd0.seqno_numivs; in ktls_write_tls_wr()
1706 sec_pdu->ivgen_hdrlen = tlsp->scmd0.ivgen_hdrlen; in ktls_write_tls_wr()
1709 txq->kern_tls_full++; in ktls_write_tls_wr()
1711 txq->kern_tls_partial++; in ktls_write_tls_wr()
1713 sec_pdu->op_ivinsrtofst = htobe32( in ktls_write_tls_wr()
1717 sec_pdu->aadstart_cipherstop_hi = htobe32( in ktls_write_tls_wr()
1722 sec_pdu->cipherstop_lo_authinsert = htobe32( in ktls_write_tls_wr()
1728 sec_pdu->scmd1 = htobe64(m_tls->m_epg_seqno); in ktls_write_tls_wr()
1732 if (tlsp->inline_key) { in ktls_write_tls_wr()
1733 memcpy(out, &tlsp->keyctx, tlsp->tx_key_info_size); in ktls_write_tls_wr()
1734 out += tlsp->tx_key_info_size; in ktls_write_tls_wr()
1738 memrd->cmd_to_len = htobe32(V_ULPTX_CMD(ULP_TX_SC_MEMRD) | in ktls_write_tls_wr()
1740 V_ULPTX_LEN16(tlsp->tx_key_info_size >> 4)); in ktls_write_tls_wr()
1741 memrd->addr = htobe32(tlsp->tx_key_addr >> 5); in ktls_write_tls_wr()
1745 idata->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM) | in ktls_write_tls_wr()
1747 idata->len = htobe32(sizeof(struct cpl_tx_data) + imm_len); in ktls_write_tls_wr()
1754 OPCODE_TID(tx_data) = htonl(MK_OPCODE_TID(CPL_TX_DATA, tlsp->tid)); in ktls_write_tls_wr()
1755 if (m->m_pkthdr.csum_flags & CSUM_TSO) { in ktls_write_tls_wr()
1756 mss = m->m_pkthdr.tso_segsz; in ktls_write_tls_wr()
1757 tlsp->prev_mss = mss; in ktls_write_tls_wr()
1758 } else if (tlsp->prev_mss != 0) in ktls_write_tls_wr()
1759 mss = tlsp->prev_mss; in ktls_write_tls_wr()
1761 mss = if_getmtu(tlsp->vi->ifp) - in ktls_write_tls_wr()
1762 (m->m_pkthdr.l3hlen + m->m_pkthdr.l4hlen); in ktls_write_tls_wr()
1764 tx_data->len = htobe32(V_TX_DATA_MSS(mss) | V_TX_LENGTH(tlen)); in ktls_write_tls_wr()
1765 tx_data->rsvd = htobe32(tcp_seqno); in ktls_write_tls_wr()
1767 tx_data->len = htobe32(V_TX_DATA_MSS(mss) | in ktls_write_tls_wr()
1768 V_TX_LENGTH(tlen - (m_tls->m_epg_hdrlen + offset))); in ktls_write_tls_wr()
1769 tx_data->rsvd = htobe32(tcp_seqno + m_tls->m_epg_hdrlen + offset); in ktls_write_tls_wr()
1771 tx_data->flags = htobe32(F_TX_BYPASS); in ktls_write_tls_wr()
1773 tx_data->flags |= htobe32(F_TX_PUSH | F_TX_SHOVE); in ktls_write_tls_wr()
1778 memcpy(out, m_tls->m_epg_hdr, m_tls->m_epg_hdrlen); in ktls_write_tls_wr()
1779 out += m_tls->m_epg_hdrlen; in ktls_write_tls_wr()
1785 if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM) { in ktls_write_tls_wr()
1786 memcpy(iv, tlsp->keyctx.u.txhdr.txsalt, SALT_SIZE); in ktls_write_tls_wr()
1787 memcpy(iv + 4, hdr + 1, 8); in ktls_write_tls_wr()
1791 memcpy(iv, hdr + 1, AES_BLOCK_LEN); in ktls_write_tls_wr()
1796 /* Zero pad to an 8-byte boundary. */ in ktls_write_tls_wr()
1797 memset(out, 0, 8 - (imm_len % 8)); in ktls_write_tls_wr()
1798 out += 8 - (imm_len % 8); in ktls_write_tls_wr()
1802 * 16-byte aligned. in ktls_write_tls_wr()
1806 idata->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP)); in ktls_write_tls_wr()
1807 idata->len = htobe32(0); in ktls_write_tls_wr()
1813 sglist_reset(txq->gl); in ktls_write_tls_wr()
1814 if (sglist_append_mbuf_epg(txq->gl, m_tls, m_tls->m_epg_hdrlen + offset, in ktls_write_tls_wr()
1815 plen - (m_tls->m_epg_hdrlen + offset)) != 0) { in ktls_write_tls_wr()
1820 write_gl_to_buf(txq->gl, out); in ktls_write_tls_wr()
1824 copy_to_txd(eq, txq->ss, &out, wr_len); in ktls_write_tls_wr()
1830 txq->kern_tls_records++; in ktls_write_tls_wr()
1831 txq->kern_tls_octets += tlen - mtod(m_tls, vm_offset_t); in ktls_write_tls_wr()
1834 txq->kern_tls_waste += mtod(m_tls, vm_offset_t); in ktls_write_tls_wr()
1836 txq->kern_tls_waste += mtod(m_tls, vm_offset_t) - in ktls_write_tls_wr()
1837 (m_tls->m_epg_hdrlen + offset); in ktls_write_tls_wr()
1840 txsd = &txq->sdesc[pidx]; in ktls_write_tls_wr()
1842 txsd->m = m; in ktls_write_tls_wr()
1844 txsd->m = NULL; in ktls_write_tls_wr()
1845 txsd->desc_used = howmany(wr_len, EQ_ESIZE); in ktls_write_tls_wr()
1870 pktlen = m->m_len; in ktls_write_tcp_fin()
1877 wr->op_immdlen = htobe32(V_FW_WR_OP(FW_ETH_TX_PKT_WR) | in ktls_write_tcp_fin()
1881 wr->equiq_to_len16 = htobe32(ctrl); in ktls_write_tcp_fin()
1882 wr->r3 = 0; in ktls_write_tcp_fin()
1887 cpl->ctrl0 = txq->cpl_ctrl0; in ktls_write_tcp_fin()
1888 cpl->pack = 0; in ktls_write_tcp_fin()
1889 cpl->len = htobe16(pktlen); in ktls_write_tcp_fin()
1895 copy_to_txd(&txq->eq, (caddr_t)eh, &out, m->m_pkthdr.l2hlen); in ktls_write_tcp_fin()
1898 if (ntohs(eh->ether_type) == ETHERTYPE_IP) { in ktls_write_tcp_fin()
1899 ip = (void *)((char *)eh + m->m_pkthdr.l2hlen); in ktls_write_tcp_fin()
1901 newip.ip_len = htons(pktlen - m->m_pkthdr.l2hlen); in ktls_write_tcp_fin()
1902 copy_to_txd(&txq->eq, (caddr_t)&newip, &out, sizeof(newip)); in ktls_write_tcp_fin()
1903 if (m->m_pkthdr.l3hlen > sizeof(*ip)) in ktls_write_tcp_fin()
1904 copy_to_txd(&txq->eq, (caddr_t)(ip + 1), &out, in ktls_write_tcp_fin()
1905 m->m_pkthdr.l3hlen - sizeof(*ip)); in ktls_write_tcp_fin()
1907 V_T6_TXPKT_ETHHDR_LEN(m->m_pkthdr.l2hlen - ETHER_HDR_LEN) | in ktls_write_tcp_fin()
1908 V_TXPKT_IPHDR_LEN(m->m_pkthdr.l3hlen); in ktls_write_tcp_fin()
1910 ip6 = (void *)((char *)eh + m->m_pkthdr.l2hlen); in ktls_write_tcp_fin()
1912 newip6.ip6_plen = htons(pktlen - m->m_pkthdr.l2hlen); in ktls_write_tcp_fin()
1913 copy_to_txd(&txq->eq, (caddr_t)&newip6, &out, sizeof(newip6)); in ktls_write_tcp_fin()
1914 MPASS(m->m_pkthdr.l3hlen == sizeof(*ip6)); in ktls_write_tcp_fin()
1916 V_T6_TXPKT_ETHHDR_LEN(m->m_pkthdr.l2hlen - ETHER_HDR_LEN) | in ktls_write_tcp_fin()
1917 V_TXPKT_IPHDR_LEN(m->m_pkthdr.l3hlen); in ktls_write_tcp_fin()
1919 cpl->ctrl1 = htobe64(ctrl1); in ktls_write_tcp_fin()
1920 txq->txcsum++; in ktls_write_tcp_fin()
1923 tcp = (void *)((char *)eh + m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen); in ktls_write_tcp_fin()
1926 copy_to_txd(&txq->eq, (caddr_t)&newtcp, &out, sizeof(newtcp)); in ktls_write_tcp_fin()
1929 copy_to_txd(&txq->eq, (caddr_t)(tcp + 1), &out, m->m_len - in ktls_write_tcp_fin()
1930 (m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen + sizeof(*tcp))); in ktls_write_tcp_fin()
1931 txq->imm_wrs++; in ktls_write_tcp_fin()
1933 txq->txpkt_wrs++; in ktls_write_tcp_fin()
1935 txq->kern_tls_fin++; in ktls_write_tcp_fin()
1937 txsd = &txq->sdesc[pidx]; in ktls_write_tcp_fin()
1938 txsd->m = m; in ktls_write_tcp_fin()
1939 txsd->desc_used = ndesc; in ktls_write_tcp_fin()
1948 struct sge_eq *eq = &txq->eq; in t6_ktls_write_wr()
1961 MPASS(m->m_pkthdr.snd_tag != NULL); in t6_ktls_write_wr()
1962 tlsp = mst_to_tls(m->m_pkthdr.snd_tag); in t6_ktls_write_wr()
1966 tcp = (struct tcphdr *)((char *)eh + m->m_pkthdr.l2hlen + in t6_ktls_write_wr()
1967 m->m_pkthdr.l3hlen); in t6_ktls_write_wr()
1968 pidx = eq->pidx; in t6_ktls_write_wr()
1978 IDXINCR(pidx, ndesc, eq->sidx); in t6_ktls_write_wr()
1979 dst = &eq->desc[pidx]; in t6_ktls_write_wr()
1982 tlsp->tid); in t6_ktls_write_wr()
1990 if (m->m_flags & M_VLANTAG) in t6_ktls_write_wr()
1991 vlan_tag = m->m_pkthdr.ether_vtag; in t6_ktls_write_wr()
1995 if (tlsp->l2te == NULL || tlsp->l2te->vlan != vlan_tag || in t6_ktls_write_wr()
1996 memcmp(tlsp->l2te->dmac, eh->ether_dhost, ETHER_ADDR_LEN) != 0) { in t6_ktls_write_wr()
1998 if (tlsp->l2te) in t6_ktls_write_wr()
1999 t4_l2t_release(tlsp->l2te); in t6_ktls_write_wr()
2000 tlsp->l2te = t4_l2t_alloc_tls(tlsp->sc, txq, dst, &ndesc, in t6_ktls_write_wr()
2001 vlan_tag, tlsp->vi->pi->lport, eh->ether_dhost); in t6_ktls_write_wr()
2002 if (tlsp->l2te == NULL) in t6_ktls_write_wr()
2005 MPASS(ndesc <= available - totdesc); in t6_ktls_write_wr()
2007 txq->raw_wrs++; in t6_ktls_write_wr()
2008 txsd = &txq->sdesc[pidx]; in t6_ktls_write_wr()
2009 txsd->m = NULL; in t6_ktls_write_wr()
2010 txsd->desc_used = ndesc; in t6_ktls_write_wr()
2012 IDXINCR(pidx, ndesc, eq->sidx); in t6_ktls_write_wr()
2013 dst = &eq->desc[pidx]; in t6_ktls_write_wr()
2021 for (m_tls = m->m_next; m_tls != NULL; m_tls = m_tls->m_next) { in t6_ktls_write_wr()
2022 MPASS(m_tls->m_flags & M_EXTPG); in t6_ktls_write_wr()
2029 if (m_tls == m->m_next) { in t6_ktls_write_wr()
2030 tcp_seqno = ntohl(tcp->th_seq) - in t6_ktls_write_wr()
2032 if (tlsp->using_timestamps) in t6_ktls_write_wr()
2036 tcp_seqno = tlsp->prev_seq; in t6_ktls_write_wr()
2040 available - totdesc, tcp_seqno, tsopt, pidx, set_l2t_idx); in t6_ktls_write_wr()
2042 IDXINCR(pidx, ndesc, eq->sidx); in t6_ktls_write_wr()
2043 dst = &eq->desc[pidx]; in t6_ktls_write_wr()
2059 tlsp->prev_seq, pidx); in t6_ktls_write_wr()
2074 sc = tlsp->sc; in t6_tls_tag_free()
2076 CTR2(KTR_CXGBE, "%s: tid %d", __func__, tlsp->tid); in t6_tls_tag_free()
2078 if (tlsp->l2te) in t6_tls_tag_free()
2079 t4_l2t_release(tlsp->l2te); in t6_tls_tag_free()
2080 if (tlsp->tid >= 0) in t6_tls_tag_free()
2081 release_tid(sc, tlsp->tid, tlsp->ctrlq); in t6_tls_tag_free()
2082 if (tlsp->ce) in t6_tls_tag_free()
2083 t4_release_clip_entry(sc, tlsp->ce); in t6_tls_tag_free()
2084 if (tlsp->tx_key_addr >= 0) in t6_tls_tag_free()
2085 t4_free_tls_keyid(sc, tlsp->tx_key_addr); in t6_tls_tag_free()