Lines Matching defs:toep
63 t4_set_tls_tcb_field(struct toepcb *toep, uint16_t word, uint64_t mask,
66 struct adapter *sc = td_adapter(toep->td);
68 t4_set_tcb_field(sc, &toep->ofld_txq->wrq, toep, word, mask, val, 0, 0);
80 tls_tx_key(struct toepcb *toep)
82 struct tls_ofld_info *tls_ofld = &toep->tls;
89 t4_set_rx_quiesce(struct toepcb *toep)
91 struct adapter *sc = td_adapter(toep->td);
93 t4_set_tcb_field(sc, &toep->ofld_txq->wrq, toep, W_TCB_T_FLAGS,
99 t4_clear_rx_quiesce(struct toepcb *toep)
102 t4_set_tls_tcb_field(toep, W_TCB_T_FLAGS, V_TF_RX_QUIESCE(1), 0);
125 clear_tls_keyid(struct toepcb *toep)
127 struct tls_ofld_info *tls_ofld = &toep->tls;
128 struct adapter *sc = td_adapter(toep->td);
150 tls_program_key_id(struct toepcb *toep, struct ktls_session *tls,
153 struct tls_ofld_info *tls_ofld = &toep->tls;
154 struct adapter *sc = td_adapter(toep->td);
169 if (toep->txsd_avail == 0)
176 wr = alloc_wrqe(TLS_KEY_WR_SZ, &toep->ofld_txq->wrq);
184 t4_write_tlskey_wr(tls, direction, toep->tid, F_FW_WR_COMPL, keyid,
193 txsd = &toep->txsd[toep->txsd_pidx];
196 toep->tx_credits -= txsd->tx_credits;
197 if (__predict_false(++toep->txsd_pidx == toep->txsd_total))
198 toep->txsd_pidx = 0;
199 toep->txsd_avail--;
207 tls_alloc_ktls(struct toepcb *toep, struct ktls_session *tls, int direction)
209 struct adapter *sc = td_adapter(toep->td);
216 if (ulp_mode(toep) != ULP_MODE_NONE)
218 if ((toep->flags & TPF_TLS_STARTING) != 0)
221 switch (ulp_mode(toep)) {
281 if (toep->tls.tx_key_addr != -1)
284 if (toep->tls.rx_key_addr != -1)
288 error = tls_program_key_id(toep, tls, direction);
293 toep->tls.scmd0.seqno_numivs =
303 toep->tls.scmd0.ivgen_hdrlen =
308 toep->tls.iv_len = explicit_iv_size;
309 toep->tls.frag_size = tls->params.max_frame_len;
310 toep->tls.fcplenmax = get_tp_plen_max(tls);
311 toep->tls.expn_per_ulp = tls->params.tls_hlen +
313 toep->tls.pdus_per_ulp = 1;
314 toep->tls.adjusted_plen = toep->tls.expn_per_ulp +
316 toep->tls.tx_key_info_size = t4_tls_key_info_size(tls);
318 toep->flags |= TPF_TLS_STARTING | TPF_TLS_RX_QUIESCING;
319 toep->tls.rx_version = tls->params.tls_vmajor << 8 |
323 toep->tid);
324 t4_set_rx_quiesce(toep);
331 tls_init_toep(struct toepcb *toep)
333 struct tls_ofld_info *tls_ofld = &toep->tls;
340 tls_uninit_toep(struct toepcb *toep)
343 clear_tls_keyid(toep);
347 #define MIN_OFLD_TLSTX_CREDITS(toep) \
354 write_tlstx_wr(struct fw_tlstx_data_wr *txwr, struct toepcb *toep,
357 struct tls_ofld_info *tls_ofld = &toep->tls;
363 txwr->flowid_len16 = htobe32(V_FW_TLSTX_DATA_WR_FLOWID(toep->tid) |
383 write_tlstx_cpl(struct cpl_tx_tls_sfo *cpl, struct toepcb *toep,
386 struct tls_ofld_info *tls_ofld = &toep->tls;
482 t4_push_ktls(struct adapter *sc, struct toepcb *toep, int drop)
493 struct inpcb *inp = toep->inp;
502 KASSERT(toep->flags & TPF_FLOWC_WR_SENT,
503 ("%s: flowc_wr not sent for tid %u.", __func__, toep->tid));
505 KASSERT(ulp_mode(toep) == ULP_MODE_NONE ||
506 ulp_mode(toep) == ULP_MODE_TCPDDP || ulp_mode(toep) == ULP_MODE_TLS,
507 ("%s: ulp_mode %u for toep %p", __func__, ulp_mode(toep), toep));
508 KASSERT(tls_tx_key(toep),
509 ("%s: TX key not set for toep %p", __func__, toep));
512 CTR4(KTR_CXGBE, "%s: tid %d toep flags %#x tp flags %#x drop %d",
513 __func__, toep->tid, toep->flags, tp->t_flags);
515 if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN))
520 (update_tx_rate_limit(sc, toep, so->so_max_pacing_rate) == 0)) {
529 if (__predict_false(toep->flags & TPF_TX_SUSPENDED)) {
535 txsd = &toep->txsd[toep->txsd_pidx];
537 tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS);
552 if (m == NULL && toep->flags & TPF_SEND_FIN) {
558 t4_close_conn(sc, toep);
574 __func__, toep->tid);
590 MPASS(toep->tls.iv_len <= AES_BLOCK_LEN);
608 __func__, toep->tid, m, howmany(wr_len, 16),
611 toep->flags |= TPF_TX_SUSPENDED;
638 if (__predict_false(toep->flags & TPF_FIN_SENT))
641 wr = alloc_wrqe(roundup2(wr_len, 16), &toep->ofld_txq->wrq);
644 toep->flags |= TPF_TX_SUSPENDED;
651 __func__, toep->tid, m->m_epg_seqno, thdr->type,
661 write_tlstx_wr(txwr, toep, tls_size, expn_size, credits, shove);
662 write_tlstx_cpl(cpl, toep, thdr, tls_size, m->m_epg_seqno);
670 V_ULPTX_LEN16(toep->tls.tx_key_info_size >> 4));
671 memrd->addr = htobe32(toep->tls.tx_key_addr >> 5);
675 memcpy(buf, thdr + 1, toep->tls.iv_len);
680 KASSERT(toep->tx_credits >= credits,
683 toep->tx_credits -= credits;
692 toep->flags |= TPF_TX_DATA_SENT;
693 if (toep->tx_credits < MIN_OFLD_TLSTX_CREDITS(toep))
694 toep->flags |= TPF_TX_SUSPENDED;
696 KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__));
700 if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) {
701 toep->txsd_pidx = 0;
702 txsd = &toep->txsd[0];
704 toep->txsd_avail--;
706 counter_u64_add(toep->ofld_txq->tx_toe_tls_records, 1);
707 counter_u64_add(toep->ofld_txq->tx_toe_tls_octets, m->m_len);
709 t4_l2t_send(sc, wr, toep->l2te);
727 struct toepcb *toep = lookup_tid(sc, tid);
728 struct inpcb *inp = toep->inp;
733 KASSERT(!(toep->flags & TPF_SYNQE),
734 ("%s: toep %p claims to be a synq entry", __func__, toep));
736 KASSERT(toep->tid == tid, ("%s: toep tid/atid mismatch", __func__));
742 toep->ofld_rxq->rx_toe_tls_octets += len;
759 if (mbufq_enqueue(&toep->ulp_pdu_reclaimq, m)) {
789 struct toepcb *toep = lookup_tid(sc, tid);
790 struct inpcb *inp = toep->inp;
802 KASSERT(toep->tid == tid, ("%s: toep tid/atid mismatch", __func__));
803 KASSERT(!(toep->flags & TPF_SYNQE),
804 ("%s: toep %p claims to be a synq entry", __func__, toep));
812 toep->ofld_rxq->rx_toe_tls_records++;
851 tls_data = mbufq_dequeue(&toep->ulp_pdu_reclaimq);
860 __func__, toep->tid, tls_hdr_pkt->res_to_mac_error,
865 CURVNET_SET(toep->vnet);
888 CURVNET_SET(toep->vnet);
960 MPASS(toep->vnet == so->so_vnet);
961 CURVNET_SET(toep->vnet);
978 t4_rcvd_locked(&toep->td->tod, tp);
989 do_rx_data_tls(const struct cpl_rx_data *cpl, struct toepcb *toep,
992 struct inpcb *inp = toep->inp;
993 struct tls_ofld_info *tls_ofld = &toep->tls;
1008 CURVNET_SET(toep->vnet);
1017 __func__, toep->tid, len);
1026 __func__, toep->tid, be16toh(hdr->version));
1032 __func__, toep->tid, be16toh(hdr->length));
1040 __func__, toep->tid, len, be16toh(hdr->length));
1055 __func__, toep->tid, hdr->type);
1065 __func__, toep->tid, hdr->type, be16toh(hdr->length));
1083 tls_update_tcb(struct adapter *sc, struct toepcb *toep, uint64_t seqno)
1090 KASSERT(ulp_mode(toep) == ULP_MODE_NONE,
1091 ("%s: tid %d already ULP_MODE_TLS", __func__, toep->tid));
1114 wr = alloc_wrqe(len, toep->ctrlq);
1132 ulpmc = mk_set_tcb_field_ulp(sc, ulpmc, toep->tid, 26,
1140 key_offset = toep->tls.rx_key_addr - sc->vres.key.start;
1141 ulpmc = mk_set_tcb_field_ulp(sc, ulpmc, toep->tid, 30,
1146 toep->tid, seqno);
1147 ulpmc = mk_set_tcb_field_ulp(sc, ulpmc, toep->tid, W_TCB_TLS_SEQ,
1149 ulpmc = mk_set_tcb_field_ulp(sc, ulpmc, toep->tid, W_TCB_ULP_RAW,
1154 toep->flags &= ~TPF_TLS_STARTING;
1155 toep->flags |= TPF_TLS_RECEIVE;
1158 toep->params.ulp_mode = ULP_MODE_TLS;
1159 ulpmc = mk_set_tcb_field_ulp(sc, ulpmc, toep->tid, W_TCB_ULP_TYPE,
1163 ulpmc = mk_set_tcb_field_ulp(sc, ulpmc, toep->tid, W_TCB_T_FLAGS,
1174 tls_check_rx_sockbuf(struct adapter *sc, struct toepcb *toep,
1182 MPASS(toep->tls.rx_resid == 0);
1186 __func__, toep->tid, have_header, seqno, resid);
1195 toep->tid);
1196 toep->flags &= ~TPF_TLS_RX_QUIESCED;
1197 t4_clear_rx_quiesce(toep);
1201 tls_update_tcb(sc, toep, seqno);
1205 tls_received_starting_data(struct adapter *sc, struct toepcb *toep,
1208 MPASS(toep->flags & TPF_TLS_STARTING);
1211 if ((toep->flags & TPF_TLS_RX_QUIESCING) != 0)
1219 if ((toep->flags & TPF_TLS_RX_QUIESCED) == 0) {
1220 CTR(KTR_CXGBE, "%s: tid %d quiescing", __func__, toep->tid);
1221 toep->flags |= TPF_TLS_RX_QUIESCING;
1222 t4_set_rx_quiesce(toep);
1226 KASSERT(len <= toep->tls.rx_resid,
1228 toep->tls.rx_resid));
1229 toep->tls.rx_resid -= len;
1230 if (toep->tls.rx_resid != 0)
1233 tls_check_rx_sockbuf(sc, toep, sb);
1242 struct toepcb *toep;
1250 toep = lookup_tid(sc, tid);
1251 inp = toep->inp;
1255 if ((toep->flags & TPF_TLS_STARTING) == 0)
1258 MPASS((toep->flags & TPF_TLS_RX_QUIESCING) != 0);
1260 toep->flags &= ~TPF_TLS_RX_QUIESCING;
1261 toep->flags |= TPF_TLS_RX_QUIESCED;
1266 tls_check_rx_sockbuf(sc, toep, sb);