Lines Matching refs:toep
63 t4_set_tls_tcb_field(struct toepcb *toep, uint16_t word, uint64_t mask, in t4_set_tls_tcb_field() argument
66 struct adapter *sc = td_adapter(toep->td); in t4_set_tls_tcb_field()
68 t4_set_tcb_field(sc, &toep->ofld_txq->wrq, toep, word, mask, val, 0, 0); in t4_set_tls_tcb_field()
80 tls_tx_key(struct toepcb *toep) in tls_tx_key() argument
82 struct tls_ofld_info *tls_ofld = &toep->tls; in tls_tx_key()
89 t4_set_rx_quiesce(struct toepcb *toep) in t4_set_rx_quiesce() argument
91 struct adapter *sc = td_adapter(toep->td); in t4_set_rx_quiesce()
93 t4_set_tcb_field(sc, &toep->ofld_txq->wrq, toep, W_TCB_T_FLAGS, in t4_set_rx_quiesce()
99 t4_clear_rx_quiesce(struct toepcb *toep) in t4_clear_rx_quiesce() argument
102 t4_set_tls_tcb_field(toep, W_TCB_T_FLAGS, V_TF_RX_QUIESCE(1), 0); in t4_clear_rx_quiesce()
125 clear_tls_keyid(struct toepcb *toep) in clear_tls_keyid() argument
127 struct tls_ofld_info *tls_ofld = &toep->tls; in clear_tls_keyid()
128 struct adapter *sc = td_adapter(toep->td); in clear_tls_keyid()
150 tls_program_key_id(struct toepcb *toep, struct ktls_session *tls, in tls_program_key_id() argument
153 struct tls_ofld_info *tls_ofld = &toep->tls; in tls_program_key_id()
154 struct adapter *sc = td_adapter(toep->td); in tls_program_key_id()
169 if (toep->txsd_avail == 0) in tls_program_key_id()
176 wr = alloc_wrqe(TLS_KEY_WR_SZ, &toep->ofld_txq->wrq); in tls_program_key_id()
184 t4_write_tlskey_wr(tls, direction, toep->tid, F_FW_WR_COMPL, keyid, in tls_program_key_id()
193 txsd = &toep->txsd[toep->txsd_pidx]; in tls_program_key_id()
196 toep->tx_credits -= txsd->tx_credits; in tls_program_key_id()
197 if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) in tls_program_key_id()
198 toep->txsd_pidx = 0; in tls_program_key_id()
199 toep->txsd_avail--; in tls_program_key_id()
207 tls_alloc_ktls(struct toepcb *toep, struct ktls_session *tls, int direction) in tls_alloc_ktls() argument
209 struct adapter *sc = td_adapter(toep->td); in tls_alloc_ktls()
216 if (ulp_mode(toep) != ULP_MODE_NONE) in tls_alloc_ktls()
218 if ((toep->flags & TPF_TLS_STARTING) != 0) in tls_alloc_ktls()
221 switch (ulp_mode(toep)) { in tls_alloc_ktls()
281 if (toep->tls.tx_key_addr != -1) in tls_alloc_ktls()
284 if (toep->tls.rx_key_addr != -1) in tls_alloc_ktls()
288 error = tls_program_key_id(toep, tls, direction); in tls_alloc_ktls()
293 toep->tls.scmd0.seqno_numivs = in tls_alloc_ktls()
303 toep->tls.scmd0.ivgen_hdrlen = in tls_alloc_ktls()
308 toep->tls.iv_len = explicit_iv_size; in tls_alloc_ktls()
309 toep->tls.frag_size = tls->params.max_frame_len; in tls_alloc_ktls()
310 toep->tls.fcplenmax = get_tp_plen_max(tls); in tls_alloc_ktls()
311 toep->tls.expn_per_ulp = tls->params.tls_hlen + in tls_alloc_ktls()
313 toep->tls.pdus_per_ulp = 1; in tls_alloc_ktls()
314 toep->tls.adjusted_plen = toep->tls.expn_per_ulp + in tls_alloc_ktls()
316 toep->tls.tx_key_info_size = t4_tls_key_info_size(tls); in tls_alloc_ktls()
318 toep->flags |= TPF_TLS_STARTING | TPF_TLS_RX_QUIESCING; in tls_alloc_ktls()
319 toep->tls.rx_version = tls->params.tls_vmajor << 8 | in tls_alloc_ktls()
323 toep->tid); in tls_alloc_ktls()
324 t4_set_rx_quiesce(toep); in tls_alloc_ktls()
331 tls_init_toep(struct toepcb *toep) in tls_init_toep() argument
333 struct tls_ofld_info *tls_ofld = &toep->tls; in tls_init_toep()
340 tls_uninit_toep(struct toepcb *toep) in tls_uninit_toep() argument
343 clear_tls_keyid(toep); in tls_uninit_toep()
347 #define MIN_OFLD_TLSTX_CREDITS(toep) \ argument
354 write_tlstx_wr(struct fw_tlstx_data_wr *txwr, struct toepcb *toep, in write_tlstx_wr() argument
357 struct tls_ofld_info *tls_ofld = &toep->tls; in write_tlstx_wr()
363 txwr->flowid_len16 = htobe32(V_FW_TLSTX_DATA_WR_FLOWID(toep->tid) | in write_tlstx_wr()
383 write_tlstx_cpl(struct cpl_tx_tls_sfo *cpl, struct toepcb *toep, in write_tlstx_cpl() argument
386 struct tls_ofld_info *tls_ofld = &toep->tls; in write_tlstx_cpl()
482 t4_push_ktls(struct adapter *sc, struct toepcb *toep, int drop) in t4_push_ktls() argument
493 struct inpcb *inp = toep->inp; in t4_push_ktls()
502 KASSERT(toep->flags & TPF_FLOWC_WR_SENT, in t4_push_ktls()
503 ("%s: flowc_wr not sent for tid %u.", __func__, toep->tid)); in t4_push_ktls()
505 KASSERT(ulp_mode(toep) == ULP_MODE_NONE || in t4_push_ktls()
506 ulp_mode(toep) == ULP_MODE_TCPDDP || ulp_mode(toep) == ULP_MODE_TLS, in t4_push_ktls()
507 ("%s: ulp_mode %u for toep %p", __func__, ulp_mode(toep), toep)); in t4_push_ktls()
508 KASSERT(tls_tx_key(toep), in t4_push_ktls()
509 ("%s: TX key not set for toep %p", __func__, toep)); in t4_push_ktls()
513 __func__, toep->tid, toep->flags, tp->t_flags); in t4_push_ktls()
515 if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN)) in t4_push_ktls()
520 (update_tx_rate_limit(sc, toep, so->so_max_pacing_rate) == 0)) { in t4_push_ktls()
529 if (__predict_false(toep->flags & TPF_TX_SUSPENDED)) { in t4_push_ktls()
535 txsd = &toep->txsd[toep->txsd_pidx]; in t4_push_ktls()
537 tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS); in t4_push_ktls()
552 if (m == NULL && toep->flags & TPF_SEND_FIN) { in t4_push_ktls()
558 t4_close_conn(sc, toep); in t4_push_ktls()
574 __func__, toep->tid); in t4_push_ktls()
590 MPASS(toep->tls.iv_len <= AES_BLOCK_LEN); in t4_push_ktls()
608 __func__, toep->tid, m, howmany(wr_len, 16), in t4_push_ktls()
611 toep->flags |= TPF_TX_SUSPENDED; in t4_push_ktls()
638 if (__predict_false(toep->flags & TPF_FIN_SENT)) in t4_push_ktls()
641 wr = alloc_wrqe(roundup2(wr_len, 16), &toep->ofld_txq->wrq); in t4_push_ktls()
644 toep->flags |= TPF_TX_SUSPENDED; in t4_push_ktls()
651 __func__, toep->tid, m->m_epg_seqno, thdr->type, in t4_push_ktls()
661 write_tlstx_wr(txwr, toep, tls_size, expn_size, credits, shove); in t4_push_ktls()
662 write_tlstx_cpl(cpl, toep, thdr, tls_size, m->m_epg_seqno); in t4_push_ktls()
670 V_ULPTX_LEN16(toep->tls.tx_key_info_size >> 4)); in t4_push_ktls()
671 memrd->addr = htobe32(toep->tls.tx_key_addr >> 5); in t4_push_ktls()
675 memcpy(buf, thdr + 1, toep->tls.iv_len); in t4_push_ktls()
680 KASSERT(toep->tx_credits >= credits, in t4_push_ktls()
683 toep->tx_credits -= credits; in t4_push_ktls()
692 toep->flags |= TPF_TX_DATA_SENT; in t4_push_ktls()
693 if (toep->tx_credits < MIN_OFLD_TLSTX_CREDITS(toep)) in t4_push_ktls()
694 toep->flags |= TPF_TX_SUSPENDED; in t4_push_ktls()
696 KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__)); in t4_push_ktls()
700 if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) { in t4_push_ktls()
701 toep->txsd_pidx = 0; in t4_push_ktls()
702 txsd = &toep->txsd[0]; in t4_push_ktls()
704 toep->txsd_avail--; in t4_push_ktls()
706 counter_u64_add(toep->ofld_txq->tx_toe_tls_records, 1); in t4_push_ktls()
707 counter_u64_add(toep->ofld_txq->tx_toe_tls_octets, m->m_len); in t4_push_ktls()
709 t4_l2t_send(sc, wr, toep->l2te); in t4_push_ktls()
727 struct toepcb *toep = lookup_tid(sc, tid); in do_tls_data() local
728 struct inpcb *inp = toep->inp; in do_tls_data()
733 KASSERT(!(toep->flags & TPF_SYNQE), in do_tls_data()
734 ("%s: toep %p claims to be a synq entry", __func__, toep)); in do_tls_data()
736 KASSERT(toep->tid == tid, ("%s: toep tid/atid mismatch", __func__)); in do_tls_data()
742 toep->ofld_rxq->rx_toe_tls_octets += len; in do_tls_data()
759 if (mbufq_enqueue(&toep->ulp_pdu_reclaimq, m)) { in do_tls_data()
789 struct toepcb *toep = lookup_tid(sc, tid); in do_rx_tls_cmp() local
790 struct inpcb *inp = toep->inp; in do_rx_tls_cmp()
802 KASSERT(toep->tid == tid, ("%s: toep tid/atid mismatch", __func__)); in do_rx_tls_cmp()
803 KASSERT(!(toep->flags & TPF_SYNQE), in do_rx_tls_cmp()
804 ("%s: toep %p claims to be a synq entry", __func__, toep)); in do_rx_tls_cmp()
812 toep->ofld_rxq->rx_toe_tls_records++; in do_rx_tls_cmp()
851 tls_data = mbufq_dequeue(&toep->ulp_pdu_reclaimq); in do_rx_tls_cmp()
860 __func__, toep->tid, tls_hdr_pkt->res_to_mac_error, in do_rx_tls_cmp()
865 CURVNET_SET(toep->vnet); in do_rx_tls_cmp()
888 CURVNET_SET(toep->vnet); in do_rx_tls_cmp()
960 MPASS(toep->vnet == so->so_vnet); in do_rx_tls_cmp()
961 CURVNET_SET(toep->vnet); in do_rx_tls_cmp()
978 t4_rcvd_locked(&toep->td->tod, tp); in do_rx_tls_cmp()
989 do_rx_data_tls(const struct cpl_rx_data *cpl, struct toepcb *toep, in do_rx_data_tls() argument
992 struct inpcb *inp = toep->inp; in do_rx_data_tls()
993 struct tls_ofld_info *tls_ofld = &toep->tls; in do_rx_data_tls()
1008 CURVNET_SET(toep->vnet); in do_rx_data_tls()
1017 __func__, toep->tid, len); in do_rx_data_tls()
1026 __func__, toep->tid, be16toh(hdr->version)); in do_rx_data_tls()
1032 __func__, toep->tid, be16toh(hdr->length)); in do_rx_data_tls()
1040 __func__, toep->tid, len, be16toh(hdr->length)); in do_rx_data_tls()
1055 __func__, toep->tid, hdr->type); in do_rx_data_tls()
1065 __func__, toep->tid, hdr->type, be16toh(hdr->length)); in do_rx_data_tls()
1083 tls_update_tcb(struct adapter *sc, struct toepcb *toep, uint64_t seqno) in tls_update_tcb() argument
1090 KASSERT(ulp_mode(toep) == ULP_MODE_NONE, in tls_update_tcb()
1091 ("%s: tid %d already ULP_MODE_TLS", __func__, toep->tid)); in tls_update_tcb()
1114 wr = alloc_wrqe(len, toep->ctrlq); in tls_update_tcb()
1132 ulpmc = mk_set_tcb_field_ulp(sc, ulpmc, toep->tid, 26, in tls_update_tcb()
1140 key_offset = toep->tls.rx_key_addr - sc->vres.key.start; in tls_update_tcb()
1141 ulpmc = mk_set_tcb_field_ulp(sc, ulpmc, toep->tid, 30, in tls_update_tcb()
1146 toep->tid, seqno); in tls_update_tcb()
1147 ulpmc = mk_set_tcb_field_ulp(sc, ulpmc, toep->tid, W_TCB_TLS_SEQ, in tls_update_tcb()
1149 ulpmc = mk_set_tcb_field_ulp(sc, ulpmc, toep->tid, W_TCB_ULP_RAW, in tls_update_tcb()
1154 toep->flags &= ~TPF_TLS_STARTING; in tls_update_tcb()
1155 toep->flags |= TPF_TLS_RECEIVE; in tls_update_tcb()
1158 toep->params.ulp_mode = ULP_MODE_TLS; in tls_update_tcb()
1159 ulpmc = mk_set_tcb_field_ulp(sc, ulpmc, toep->tid, W_TCB_ULP_TYPE, in tls_update_tcb()
1163 ulpmc = mk_set_tcb_field_ulp(sc, ulpmc, toep->tid, W_TCB_T_FLAGS, in tls_update_tcb()
1174 tls_check_rx_sockbuf(struct adapter *sc, struct toepcb *toep, in tls_check_rx_sockbuf() argument
1182 MPASS(toep->tls.rx_resid == 0); in tls_check_rx_sockbuf()
1186 __func__, toep->tid, have_header, seqno, resid); in tls_check_rx_sockbuf()
1195 toep->tid); in tls_check_rx_sockbuf()
1196 toep->flags &= ~TPF_TLS_RX_QUIESCED; in tls_check_rx_sockbuf()
1197 t4_clear_rx_quiesce(toep); in tls_check_rx_sockbuf()
1201 tls_update_tcb(sc, toep, seqno); in tls_check_rx_sockbuf()
1205 tls_received_starting_data(struct adapter *sc, struct toepcb *toep, in tls_received_starting_data() argument
1208 MPASS(toep->flags & TPF_TLS_STARTING); in tls_received_starting_data()
1211 if ((toep->flags & TPF_TLS_RX_QUIESCING) != 0) in tls_received_starting_data()
1219 if ((toep->flags & TPF_TLS_RX_QUIESCED) == 0) { in tls_received_starting_data()
1220 CTR(KTR_CXGBE, "%s: tid %d quiescing", __func__, toep->tid); in tls_received_starting_data()
1221 toep->flags |= TPF_TLS_RX_QUIESCING; in tls_received_starting_data()
1222 t4_set_rx_quiesce(toep); in tls_received_starting_data()
1226 KASSERT(len <= toep->tls.rx_resid, in tls_received_starting_data()
1228 toep->tls.rx_resid)); in tls_received_starting_data()
1229 toep->tls.rx_resid -= len; in tls_received_starting_data()
1230 if (toep->tls.rx_resid != 0) in tls_received_starting_data()
1233 tls_check_rx_sockbuf(sc, toep, sb); in tls_received_starting_data()
1242 struct toepcb *toep; in do_tls_tcb_rpl() local
1250 toep = lookup_tid(sc, tid); in do_tls_tcb_rpl()
1251 inp = toep->inp; in do_tls_tcb_rpl()
1255 if ((toep->flags & TPF_TLS_STARTING) == 0) in do_tls_tcb_rpl()
1258 MPASS((toep->flags & TPF_TLS_RX_QUIESCING) != 0); in do_tls_tcb_rpl()
1260 toep->flags &= ~TPF_TLS_RX_QUIESCING; in do_tls_tcb_rpl()
1261 toep->flags |= TPF_TLS_RX_QUIESCED; in do_tls_tcb_rpl()
1266 tls_check_rx_sockbuf(sc, toep, sb); in do_tls_tcb_rpl()