103 static int ktls_setup_keys(struct tlspcb *tlsp,
122 	struct tlspcb *tlsp;  in alloc_tlspcb()  local
124 	tlsp = malloc(sizeof(*tlsp), M_CXGBE, M_ZERO | flags);  in alloc_tlspcb()
125 	if (tlsp == NULL)  in alloc_tlspcb()
128 	m_snd_tag_init(&tlsp->com, ifp, &t6_tls_tag_sw);  in alloc_tlspcb()
129 	tlsp->vi = vi;  in alloc_tlspcb()
130 	tlsp->sc = sc;  in alloc_tlspcb()
131 	tlsp->ctrlq = &sc->sge.ctrlq[pi->port_id];  in alloc_tlspcb()
132 	tlsp->tid = -1;  in alloc_tlspcb()
133 	tlsp->tx_key_addr = -1;  in alloc_tlspcb()
135 	return (tlsp);  in alloc_tlspcb()
150     struct tlspcb *tlsp, int atid, void *dst)  in mk_ktls_act_open_req()  argument
181     struct inpcb *inp, struct tlspcb *tlsp, int atid, void *dst)  in mk_ktls_act_open_req6()  argument
216     struct inpcb *inp, struct tlspcb *tlsp, int atid)  in send_ktls_act_open_req()  argument
223 		tlsp->ce = t4_get_clip_entry(sc, &inp->in6p_laddr, true);  in send_ktls_act_open_req()
224 		if (tlsp->ce == NULL)  in send_ktls_act_open_req()
228 	wr = alloc_wrqe(ktls_act_open_cpl_size(isipv6), tlsp->ctrlq);  in send_ktls_act_open_req()
236 		mk_ktls_act_open_req6(sc, vi, inp, tlsp, atid, wrtod(wr));  in send_ktls_act_open_req()
238 		mk_ktls_act_open_req(sc, vi, inp, tlsp, atid, wrtod(wr));  in send_ktls_act_open_req()
240 	tlsp->open_pending = true;  in send_ktls_act_open_req()
253 	struct tlspcb *tlsp = lookup_atid(sc, atid);  in ktls_act_open_rpl()  local
254 	struct inpcb *inp = tlsp->inp;  in ktls_act_open_rpl()
259 		tlsp->tid = GET_TID(cpl);  in ktls_act_open_rpl()
262 	tlsp->open_pending = false;  in ktls_act_open_rpl()
263 	wakeup(tlsp);  in ktls_act_open_rpl()
276 write_set_tcb_field_ulp(struct tlspcb *tlsp, void *dst, struct sge_txq *txq,  in write_set_tcb_field_ulp()  argument
287 	    V_ULP_TXPKT_CHANNELID(tlsp->vi->pi->port_id) | V_ULP_TXPKT_DEST(0) |  in write_set_tcb_field_ulp()
298 	OPCODE_TID(cpl) = htobe32(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tlsp->tid));  in write_set_tcb_field_ulp()
311 ktls_set_tcb_fields(struct tlspcb *tlsp, struct tcpcb *tp, struct sge_txq *txq)  in ktls_set_tcb_fields()  argument
325 		    tlsp->tid);  in ktls_set_tcb_fields()
328 	m->m_pkthdr.snd_tag = m_snd_tag_ref(&tlsp->com);  in ktls_set_tcb_fields()
340 	write_set_tcb_field_ulp(tlsp, dst, txq, W_TCB_T_FLAGS,  in ktls_set_tcb_fields()
346 	write_set_tcb_field_ulp(tlsp, dst, txq, W_TCB_SND_UNA_RAW,  in ktls_set_tcb_fields()
352 	write_set_tcb_field_ulp(tlsp, dst, txq, W_TCB_SND_MAX_RAW,  in ktls_set_tcb_fields()
357 		write_set_tcb_field_ulp(tlsp, dst, txq, W_TCB_TIMESTAMP_OFFSET,  in ktls_set_tcb_fields()
377 	struct tlspcb *tlsp;  in t6_tls_tag_alloc()  local
437 	tlsp = alloc_tlspcb(ifp, vi, M_WAITOK);  in t6_tls_tag_alloc()
439 	atid = alloc_atid(sc, tlsp);  in t6_tls_tag_alloc()
452 		tlsp->inline_key = true;  in t6_tls_tag_alloc()
454 		tlsp->tx_key_addr = keyid;  in t6_tls_tag_alloc()
457 		    atid, tlsp->tx_key_addr);  in t6_tls_tag_alloc()
467 	tlsp->inp = inp;  in t6_tls_tag_alloc()
471 		tlsp->using_timestamps = true;  in t6_tls_tag_alloc()
478 		tlsp->using_timestamps = false;  in t6_tls_tag_alloc()
480 	error = send_ktls_act_open_req(sc, vi, inp, tlsp, atid);  in t6_tls_tag_alloc()
489 	while (tlsp->open_pending) {  in t6_tls_tag_alloc()
494 		error = rw_sleep(tlsp, &inp->inp_lock, 0, "t6tlsop", 0);  in t6_tls_tag_alloc()
498 	if (tlsp->tid < 0) {  in t6_tls_tag_alloc()
514 	tlsp->txq = txq;  in t6_tls_tag_alloc()
516 	error = ktls_set_tcb_fields(tlsp, tp, txq);  in t6_tls_tag_alloc()
521 	error = ktls_setup_keys(tlsp, tls, txq);  in t6_tls_tag_alloc()
525 	tlsp->enc_mode = t4_tls_cipher_mode(tls);  in t6_tls_tag_alloc()
526 	tlsp->tx_key_info_size = t4_tls_key_info_size(tls);  in t6_tls_tag_alloc()
529 	tlsp->scmd0.seqno_numivs = htobe32(V_SCMD_SEQ_NO_CTRL(3) |  in t6_tls_tag_alloc()
533 	    V_SCMD_CIPH_MODE(tlsp->enc_mode) |  in t6_tls_tag_alloc()
538 	tlsp->scmd0.ivgen_hdrlen = V_SCMD_IV_GEN_CTRL(0) |  in t6_tls_tag_alloc()
540 	if (tlsp->inline_key)  in t6_tls_tag_alloc()
541 		tlsp->scmd0.ivgen_hdrlen |= V_SCMD_KEY_CTX_INLINE(1);  in t6_tls_tag_alloc()
542 	tlsp->scmd0.ivgen_hdrlen = htobe32(tlsp->scmd0.ivgen_hdrlen);  in t6_tls_tag_alloc()
548 	tlsp->scmd0_short.seqno_numivs = V_SCMD_SEQ_NO_CTRL(0) |  in t6_tls_tag_alloc()
555 	if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM)  in t6_tls_tag_alloc()
556 		tlsp->scmd0_short.seqno_numivs |=  in t6_tls_tag_alloc()
559 		tlsp->scmd0_short.seqno_numivs |=  in t6_tls_tag_alloc()
560 		    V_SCMD_CIPH_MODE(tlsp->enc_mode);  in t6_tls_tag_alloc()
561 	tlsp->scmd0_short.seqno_numivs =  in t6_tls_tag_alloc()
562 	    htobe32(tlsp->scmd0_short.seqno_numivs);  in t6_tls_tag_alloc()
564 	tlsp->scmd0_short.ivgen_hdrlen = V_SCMD_IV_GEN_CTRL(0) |  in t6_tls_tag_alloc()
567 	if (tlsp->inline_key)  in t6_tls_tag_alloc()
568 		tlsp->scmd0_short.ivgen_hdrlen |= V_SCMD_KEY_CTX_INLINE(1);  in t6_tls_tag_alloc()
571 	if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM)  in t6_tls_tag_alloc()
576 	*pt = &tlsp->com;  in t6_tls_tag_alloc()
582 	m_snd_tag_rele(&tlsp->com);  in t6_tls_tag_alloc()
587 ktls_setup_keys(struct tlspcb *tlsp, const struct ktls_session *tls,  in ktls_setup_keys()  argument
603 	t4_tls_key_ctx(tls, KTLS_TX, &tlsp->keyctx);  in ktls_setup_keys()
604 	if (tlsp->inline_key)  in ktls_setup_keys()
611 		    tlsp->tid);  in ktls_setup_keys()
614 	m->m_pkthdr.snd_tag = m_snd_tag_ref(&tlsp->com);  in ktls_setup_keys()
619 	t4_write_tlskey_wr(tls, KTLS_TX, tlsp->tid, 0, tlsp->tx_key_addr, kwr);  in ktls_setup_keys()
621 	memcpy(kctx, &tlsp->keyctx, sizeof(*kctx));  in ktls_setup_keys()
633 		CTR2(KTR_CXGBE, "%s: tid %d sent key WR", __func__, tlsp->tid);  in ktls_setup_keys()
638 ktls_base_wr_size(struct tlspcb *tlsp)  in ktls_base_wr_size()  argument
646 	if (tlsp->inline_key)  in ktls_base_wr_size()
647 		wr_len += tlsp->tx_key_info_size;  in ktls_base_wr_size()
658 ktls_tcp_payload_length(struct tlspcb *tlsp, struct mbuf *m_tls)  in ktls_tcp_payload_length()  argument
690 	if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_CBC &&  in ktls_tcp_payload_length()
698 	    __func__, tlsp->tid, mlen, TLS_HEADER_LENGTH + plen);  in ktls_tcp_payload_length()
709 ktls_payload_offset(struct tlspcb *tlsp, struct mbuf *m_tls)  in ktls_payload_offset()  argument
726 	if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM) {  in ktls_payload_offset()
756 ktls_wr_len(struct tlspcb *tlsp, struct mbuf *m, struct mbuf *m_tls,  in ktls_wr_len()  argument
768 	tlen = ktls_tcp_payload_length(tlsp, m_tls);  in ktls_wr_len()
780 			    __func__, tlsp->tid, m->m_len + m_tls->m_len);  in ktls_wr_len()
793 		offset = ktls_payload_offset(tlsp, m_tls);  in ktls_wr_len()
798 	wr_len = ktls_base_wr_size(tlsp);  in ktls_wr_len()
890 	struct tlspcb *tlsp;  in t6_ktls_parse_pkt()  local
909 	tlsp = mst_to_tls(m->m_pkthdr.snd_tag);  in t6_ktls_parse_pkt()
913 		    tlsp->tid);  in t6_ktls_parse_pkt()
920 		    __func__, tlsp->tid);  in t6_ktls_parse_pkt()
930 			    __func__, tlsp->tid);  in t6_ktls_parse_pkt()
938 			    __func__, tlsp->tid, ip6->ip6_nxt);  in t6_ktls_parse_pkt()
946 		    __func__, tlsp->tid);  in t6_ktls_parse_pkt()
957 		    __func__, tlsp->tid, m->m_pkthdr.l2hlen,  in t6_ktls_parse_pkt()
975 		wr_len = ktls_wr_len(tlsp, m, m_tls, &nsegs);  in t6_ktls_parse_pkt()
978 		    tlsp->tid, wr_len, nsegs);  in t6_ktls_parse_pkt()
1004 			    __func__, tlsp->tid, m->m_len);  in t6_ktls_parse_pkt()
1025 	if (tlsp->using_timestamps)  in t6_ktls_parse_pkt()
1033 	    tlsp->tid, mbuf_len16(m), mbuf_nsegs(m));  in t6_ktls_parse_pkt()
1036 	return (mp_ring_enqueue(tlsp->txq->r, items, 1, 256));  in t6_ktls_parse_pkt()
1312 ktls_write_tls_wr(struct tlspcb *tlsp, struct sge_txq *txq, void *dst,  in ktls_write_tls_wr()  argument
1334 	MPASS(tlsp->txq == txq);  in ktls_write_tls_wr()
1336 	first_wr = (tlsp->prev_seq == 0 && tlsp->prev_ack == 0 &&  in ktls_write_tls_wr()
1337 	    tlsp->prev_win == 0);  in ktls_write_tls_wr()
1353 	tlen = ktls_tcp_payload_length(tlsp, m_tls);  in ktls_write_tls_wr()
1361 		    __func__, tlsp->tid, (u_int)m_tls->m_epg_seqno);  in ktls_write_tls_wr()
1368 		offset = ktls_payload_offset(tlsp, m_tls);  in ktls_write_tls_wr()
1371 		    __func__, tlsp->tid, (u_int)m_tls->m_epg_seqno, offset);  in ktls_write_tls_wr()
1431 	if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_CBC &&  in ktls_write_tls_wr()
1456 		    tlsp->tid, tlsp->l2te->idx);  in ktls_write_tls_wr()
1458 		write_set_tcb_field_ulp(tlsp, out, txq, W_TCB_L2T_IX,  in ktls_write_tls_wr()
1459 		    V_TCB_L2T_IX(M_TCB_L2T_IX), V_TCB_L2T_IX(tlsp->l2te->idx));  in ktls_write_tls_wr()
1463 	if (tsopt != NULL && tlsp->prev_tsecr != ntohl(tsopt[1])) {  in ktls_write_tls_wr()
1468 		    __func__, tlsp->tid);  in ktls_write_tls_wr()
1470 		write_set_tcb_field_ulp(tlsp, out, txq, W_TCB_T_RTSEQ_RECENT,  in ktls_write_tls_wr()
1476 		tlsp->prev_tsecr = ntohl(tsopt[1]);  in ktls_write_tls_wr()
1479 	if (first_wr || tlsp->prev_seq != tx_max) {  in ktls_write_tls_wr()
1485 		    __func__, tlsp->tid, tx_max, tcp_seqno);  in ktls_write_tls_wr()
1487 		write_set_tcb_field_ulp(tlsp, out, txq, W_TCB_TX_MAX,  in ktls_write_tls_wr()
1498 	if (tlsp->prev_seq != tx_max || mtod(m_tls, vm_offset_t) != 0) {  in ktls_write_tls_wr()
1503 		    tlsp->tid);  in ktls_write_tls_wr()
1505 		write_set_tcb_field_ulp(tlsp, out, txq, W_TCB_SND_UNA_RAW,  in ktls_write_tls_wr()
1516 	tlsp->prev_seq = tcp_seqno + tlen;  in ktls_write_tls_wr()
1518 	if (first_wr || tlsp->prev_ack != ntohl(tcp->th_ack)) {  in ktls_write_tls_wr()
1521 		write_set_tcb_field_ulp(tlsp, out, txq, W_TCB_RCV_NXT,  in ktls_write_tls_wr()
1527 		tlsp->prev_ack = ntohl(tcp->th_ack);  in ktls_write_tls_wr()
1530 	if (first_wr || tlsp->prev_win != ntohs(tcp->th_win)) {  in ktls_write_tls_wr()
1533 		write_set_tcb_field_ulp(tlsp, out, txq, W_TCB_RCV_WND,  in ktls_write_tls_wr()
1539 		tlsp->prev_win = ntohs(tcp->th_win);  in ktls_write_tls_wr()
1550 	twr_len = ktls_base_wr_size(tlsp);  in ktls_write_tls_wr()
1570 		    tlsp->sc->tlst.combo_wrs) {  in ktls_write_tls_wr()
1631 	    V_ULP_TXPKT_CHANNELID(tlsp->vi->pi->port_id) | V_ULP_TXPKT_DEST(0) |  in ktls_write_tls_wr()
1647 	if (tlsp->inline_key)  in ktls_write_tls_wr()
1648 		idata->len += tlsp->tx_key_info_size +  in ktls_write_tls_wr()
1673 		sec_pdu->seqno_numivs = tlsp->scmd0_short.seqno_numivs;  in ktls_write_tls_wr()
1675 		    tlsp->scmd0_short.ivgen_hdrlen |  in ktls_write_tls_wr()
1690 		if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM) {  in ktls_write_tls_wr()
1705 		sec_pdu->seqno_numivs = tlsp->scmd0.seqno_numivs;  in ktls_write_tls_wr()
1706 		sec_pdu->ivgen_hdrlen = tlsp->scmd0.ivgen_hdrlen;  in ktls_write_tls_wr()
1732 	if (tlsp->inline_key) {  in ktls_write_tls_wr()
1733 		memcpy(out, &tlsp->keyctx, tlsp->tx_key_info_size);  in ktls_write_tls_wr()
1734 		out += tlsp->tx_key_info_size;  in ktls_write_tls_wr()
1740 		    V_ULPTX_LEN16(tlsp->tx_key_info_size >> 4));  in ktls_write_tls_wr()
1741 		memrd->addr = htobe32(tlsp->tx_key_addr >> 5);  in ktls_write_tls_wr()
1754 	OPCODE_TID(tx_data) = htonl(MK_OPCODE_TID(CPL_TX_DATA, tlsp->tid));  in ktls_write_tls_wr()
1757 		tlsp->prev_mss = mss;  in ktls_write_tls_wr()
1758 	} else if (tlsp->prev_mss != 0)  in ktls_write_tls_wr()
1759 		mss = tlsp->prev_mss;  in ktls_write_tls_wr()
1761 		mss = if_getmtu(tlsp->vi->ifp) -  in ktls_write_tls_wr()
1785 		if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM) {  in ktls_write_tls_wr()
1786 			memcpy(iv, tlsp->keyctx.u.txhdr.txsalt, SALT_SIZE);  in ktls_write_tls_wr()
1950 	struct tlspcb *tlsp;  in t6_ktls_write_wr()  local
1962 	tlsp = mst_to_tls(m->m_pkthdr.snd_tag);  in t6_ktls_write_wr()
1982 		    tlsp->tid);  in t6_ktls_write_wr()
1995 	if (tlsp->l2te == NULL || tlsp->l2te->vlan != vlan_tag ||  in t6_ktls_write_wr()
1996 	    memcmp(tlsp->l2te->dmac, eh->ether_dhost, ETHER_ADDR_LEN) != 0) {  in t6_ktls_write_wr()
1998 		if (tlsp->l2te)  in t6_ktls_write_wr()
1999 			t4_l2t_release(tlsp->l2te);  in t6_ktls_write_wr()
2000 		tlsp->l2te = t4_l2t_alloc_tls(tlsp->sc, txq, dst, &ndesc,  in t6_ktls_write_wr()
2001 		    vlan_tag, tlsp->vi->pi->lport, eh->ether_dhost);  in t6_ktls_write_wr()
2002 		if (tlsp->l2te == NULL)  in t6_ktls_write_wr()
2032 			if (tlsp->using_timestamps)  in t6_ktls_write_wr()
2036 			tcp_seqno = tlsp->prev_seq;  in t6_ktls_write_wr()
2039 		ndesc = ktls_write_tls_wr(tlsp, txq, dst, m, tcp, m_tls,  in t6_ktls_write_wr()
2059 		    tlsp->prev_seq, pidx);  in t6_ktls_write_wr()
2071 	struct tlspcb *tlsp;  in t6_tls_tag_free()  local
2073 	tlsp = mst_to_tls(mst);  in t6_tls_tag_free()
2074 	sc = tlsp->sc;  in t6_tls_tag_free()
2076 	CTR2(KTR_CXGBE, "%s: tid %d", __func__, tlsp->tid);  in t6_tls_tag_free()
2078 	if (tlsp->l2te)  in t6_tls_tag_free()
2079 		t4_l2t_release(tlsp->l2te);  in t6_tls_tag_free()
2080 	if (tlsp->tid >= 0)  in t6_tls_tag_free()
2081 		release_tid(sc, tlsp->tid, tlsp->ctrlq);  in t6_tls_tag_free()
2082 	if (tlsp->ce)  in t6_tls_tag_free()
2083 		t4_release_clip_entry(sc, tlsp->ce);  in t6_tls_tag_free()
2084 	if (tlsp->tx_key_addr >= 0)  in t6_tls_tag_free()
2085 		t4_free_tls_keyid(sc, tlsp->tx_key_addr);  in t6_tls_tag_free()
2087 	zfree(tlsp, M_CXGBE);  in t6_tls_tag_free()