Lines Matching refs:toep
115 struct toepcb *toep; in alloc_toepcb() local
137 toep = malloc(len, M_CXGBE, M_ZERO | flags); in alloc_toepcb()
138 if (toep == NULL) in alloc_toepcb()
141 refcount_init(&toep->refcount, 1); in alloc_toepcb()
142 toep->td = sc->tom_softc; in alloc_toepcb()
143 toep->incarnation = sc->incarnation; in alloc_toepcb()
144 toep->vi = vi; in alloc_toepcb()
145 toep->tid = -1; in alloc_toepcb()
146 toep->tx_total = tx_credits; in alloc_toepcb()
147 toep->tx_credits = tx_credits; in alloc_toepcb()
148 mbufq_init(&toep->ulp_pduq, INT_MAX); in alloc_toepcb()
149 mbufq_init(&toep->ulp_pdu_reclaimq, INT_MAX); in alloc_toepcb()
150 toep->txsd_total = txsd_total; in alloc_toepcb()
151 toep->txsd_avail = txsd_total; in alloc_toepcb()
152 toep->txsd_pidx = 0; in alloc_toepcb()
153 toep->txsd_cidx = 0; in alloc_toepcb()
154 aiotx_init_toep(toep); in alloc_toepcb()
156 return (toep); in alloc_toepcb()
163 init_toepcb(struct vi_info *vi, struct toepcb *toep) in init_toepcb() argument
165 struct conn_params *cp = &toep->params; in init_toepcb()
176 toep->tid, cp->tc_idx, tc->state); in init_toepcb()
183 toep->ofld_txq = &sc->sge.ofld_txq[cp->txq_idx]; in init_toepcb()
184 toep->ofld_rxq = &sc->sge.ofld_rxq[cp->rxq_idx]; in init_toepcb()
185 toep->ctrlq = &sc->sge.ctrlq[pi->port_id]; in init_toepcb()
187 tls_init_toep(toep); in init_toepcb()
188 MPASS(ulp_mode(toep) != ULP_MODE_TCPDDP); in init_toepcb()
190 toep->flags |= TPF_INITIALIZED; in init_toepcb()
196 hold_toepcb(struct toepcb *toep) in hold_toepcb() argument
199 refcount_acquire(&toep->refcount); in hold_toepcb()
200 return (toep); in hold_toepcb()
204 free_toepcb(struct toepcb *toep) in free_toepcb() argument
207 if (refcount_release(&toep->refcount) == 0) in free_toepcb()
210 KASSERT(!(toep->flags & TPF_ATTACHED), in free_toepcb()
212 KASSERT(!(toep->flags & TPF_CPL_PENDING), in free_toepcb()
215 if (toep->flags & TPF_INITIALIZED) { in free_toepcb()
216 if (ulp_mode(toep) == ULP_MODE_TCPDDP) in free_toepcb()
217 ddp_uninit_toep(toep); in free_toepcb()
218 tls_uninit_toep(toep); in free_toepcb()
220 free(toep, M_CXGBE); in free_toepcb()
227 offload_socket(struct socket *so, struct toepcb *toep) in offload_socket() argument
229 struct tom_data *td = toep->td; in offload_socket()
252 tp->t_toe = toep; in offload_socket()
256 toep->inp = inp; in offload_socket()
257 toep->flags |= TPF_ATTACHED; in offload_socket()
276 struct toepcb *toep = tp->t_toe; in undo_offload_socket() local
295 toep->inp = NULL; in undo_offload_socket()
296 toep->flags &= ~TPF_ATTACHED; in undo_offload_socket()
302 release_offload_resources(struct toepcb *toep) in release_offload_resources() argument
304 struct tom_data *td = toep->td; in release_offload_resources()
306 int tid = toep->tid; in release_offload_resources()
308 KASSERT(!(toep->flags & TPF_CPL_PENDING), in release_offload_resources()
309 ("%s: %p has CPL pending.", __func__, toep)); in release_offload_resources()
312 __func__, toep, tid, toep->l2te, toep->ce); in release_offload_resources()
314 if (toep->l2te) { in release_offload_resources()
315 t4_l2t_release(toep->l2te); in release_offload_resources()
316 toep->l2te = NULL; in release_offload_resources()
319 remove_tid(sc, tid, toep->ce ? 2 : 1); in release_offload_resources()
320 release_tid(sc, tid, toep->ctrlq); in release_offload_resources()
321 toep->tid = -1; in release_offload_resources()
323 if (toep->flags & TPF_IN_TOEP_LIST) { in release_offload_resources()
324 toep->flags &= ~TPF_IN_TOEP_LIST; in release_offload_resources()
325 TAILQ_REMOVE(&td->toep_list, toep, link); in release_offload_resources()
329 if (toep->ce) { in release_offload_resources()
330 t4_release_clip_entry(sc, toep->ce); in release_offload_resources()
331 toep->ce = NULL; in release_offload_resources()
333 if (toep->params.tc_idx != -1) in release_offload_resources()
334 t4_release_cl_rl(sc, toep->vi->pi->port_id, toep->params.tc_idx); in release_offload_resources()
341 done_with_toepcb(struct toepcb *toep) in done_with_toepcb() argument
343 KASSERT(!(toep->flags & TPF_CPL_PENDING), in done_with_toepcb()
344 ("%s: %p has CPL pending.", __func__, toep)); in done_with_toepcb()
345 KASSERT(!(toep->flags & TPF_ATTACHED), in done_with_toepcb()
346 ("%s: %p is still attached.", __func__, toep)); in done_with_toepcb()
348 CTR(KTR_CXGBE, "%s: toep %p (0x%x)", __func__, toep, toep->flags); in done_with_toepcb()
355 MPASS(mbufq_empty(&toep->ulp_pduq)); in done_with_toepcb()
356 MPASS(mbufq_empty(&toep->ulp_pdu_reclaimq)); in done_with_toepcb()
358 if (ulp_mode(toep) == ULP_MODE_TCPDDP) in done_with_toepcb()
359 ddp_assert_empty(toep); in done_with_toepcb()
361 MPASS(TAILQ_EMPTY(&toep->aiotx_jobq)); in done_with_toepcb()
362 MPASS(toep->tid == -1); in done_with_toepcb()
363 MPASS(toep->l2te == NULL); in done_with_toepcb()
364 MPASS(toep->ce == NULL); in done_with_toepcb()
365 MPASS((toep->flags & TPF_IN_TOEP_LIST) == 0); in done_with_toepcb()
367 free_toepcb(toep); in done_with_toepcb()
384 struct toepcb *toep = tp->t_toe; in t4_pcb_detach() local
388 KASSERT(toep != NULL, ("%s: toep is NULL", __func__)); in t4_pcb_detach()
389 KASSERT(toep->flags & TPF_ATTACHED, in t4_pcb_detach()
395 __func__, toep->tid, toep, toep->flags, inp, in t4_pcb_detach()
400 toep->tid, tcpstates[tp->t_state], toep, toep->flags, inp, in t4_pcb_detach()
408 toep->flags &= ~TPF_ATTACHED; in t4_pcb_detach()
410 if (!(toep->flags & TPF_CPL_PENDING)) in t4_pcb_detach()
411 done_with_toepcb(toep); in t4_pcb_detach()
421 struct toepcb *toep = tp->t_toe; in t4_ctloutput() local
432 toep->params.nagle = tp->t_flags & TF_NODELAY ? 0 : 1; in t4_ctloutput()
433 t4_set_tcb_field(sc, toep->ctrlq, toep, W_TCB_T_FLAGS, in t4_ctloutput()
434 V_TF_NAGLE(1), V_TF_NAGLE(toep->params.nagle), 0, 0); in t4_ctloutput()
835 struct toepcb *toep = tp->t_toe; in t4_tcp_info() local
840 fill_tcp_info(sc, toep->tid, ti); in t4_tcp_info()
848 struct toepcb *toep = tp->t_toe; in t4_alloc_tls_session() local
853 return (tls_alloc_ktls(toep, tls, direction)); in t4_alloc_tls_session()
858 send_mss_flowc_wr(struct adapter *sc, struct toepcb *toep) in send_mss_flowc_wr() argument
866 if (toep->tx_credits < flowclen16 || toep->txsd_avail == 0) { in send_mss_flowc_wr()
868 toep->tid, toep->tx_credits, toep->txsd_avail); in send_mss_flowc_wr()
872 flowc = start_wrq_wr(&toep->ofld_txq->wrq, flowclen16, &cookie); in send_mss_flowc_wr()
874 CH_ERR(sc, "ENOMEM in %s for tid %u.\n", __func__, toep->tid); in send_mss_flowc_wr()
880 V_FW_WR_FLOWID(toep->tid)); in send_mss_flowc_wr()
882 flowc->mnemval[0].val = htobe32(toep->params.emss); in send_mss_flowc_wr()
884 txsd = &toep->txsd[toep->txsd_pidx]; in send_mss_flowc_wr()
887 toep->tx_credits -= txsd->tx_credits; in send_mss_flowc_wr()
888 if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) in send_mss_flowc_wr()
889 toep->txsd_pidx = 0; in send_mss_flowc_wr()
890 toep->txsd_avail--; in send_mss_flowc_wr()
891 commit_wrq_wr(&toep->ofld_txq->wrq, flowc, &cookie); in send_mss_flowc_wr()
902 struct toepcb *toep = tp->t_toe; in t4_pmtu_update() local
903 struct adapter *sc = td_adapter(toep->td); in t4_pmtu_update()
913 __func__, toep->tid, seq, mtu, toep->params.mtu_idx, in t4_pmtu_update()
914 mtus[toep->params.mtu_idx]); in t4_pmtu_update()
916 if (ulp_mode(toep) == ULP_MODE_NONE && /* XXX: Read TCB otherwise? */ in t4_pmtu_update()
920 __func__, toep->tid, seq, tp->snd_una, tp->snd_max); in t4_pmtu_update()
927 if (idx >= toep->params.mtu_idx) in t4_pmtu_update()
935 wrh = start_wrq_wr(toep->ctrlq, howmany(len, 16), &cookie); in t4_pmtu_update()
938 toep->tid, toep->params.mtu_idx, idx); in t4_pmtu_update()
943 ulpmc = mk_set_tcb_field_ulp(sc, ulpmc, toep->tid, W_TCB_T_MAXSEG, in t4_pmtu_update()
945 ulpmc = mk_set_tcb_field_ulp(sc, ulpmc, toep->tid, W_TCB_TIMESTAMP, in t4_pmtu_update()
947 commit_wrq_wr(toep->ctrlq, wrh, &cookie); in t4_pmtu_update()
950 toep->params.mtu_idx = idx; in t4_pmtu_update()
951 tp->t_maxseg = mtus[toep->params.mtu_idx]; in t4_pmtu_update()
956 toep->params.emss = tp->t_maxseg; in t4_pmtu_update()
958 toep->params.emss -= TCPOLEN_TSTAMP_APPA; in t4_pmtu_update()
961 send_mss_flowc_wr(sc, toep); in t4_pmtu_update()
978 __func__, toep->tid, toep->params.mtu_idx, in t4_pmtu_update()
979 mtus[toep->params.mtu_idx], tp->t_maxseg, toep->params.emss); in t4_pmtu_update()
987 final_cpl_received(struct toepcb *toep) in final_cpl_received() argument
989 struct inpcb *inp = toep->inp; in final_cpl_received()
994 KASSERT(toep->flags & TPF_CPL_PENDING, in final_cpl_received()
998 __func__, toep->tid, toep, toep->flags, inp, inp->inp_flags); in final_cpl_received()
1000 if (ulp_mode(toep) == ULP_MODE_TCPDDP) in final_cpl_received()
1001 release_ddp_resources(toep); in final_cpl_received()
1002 toep->inp = NULL; in final_cpl_received()
1003 need_wakeup = (toep->flags & TPF_WAITING_FOR_FINAL) != 0; in final_cpl_received()
1004 toep->flags &= ~(TPF_CPL_PENDING | TPF_WAITING_FOR_FINAL); in final_cpl_received()
1005 mbufq_drain(&toep->ulp_pduq); in final_cpl_received()
1006 mbufq_drain(&toep->ulp_pdu_reclaimq); in final_cpl_received()
1007 release_offload_resources(toep); in final_cpl_received()
1008 if (!(toep->flags & TPF_ATTACHED)) in final_cpl_received()
1009 done_with_toepcb(toep); in final_cpl_received()
1015 struct mtx *lock = mtx_pool_find(mtxpool_sleep, toep); in final_cpl_received()
1018 wakeup(toep); in final_cpl_received()
1784 live_tid_failure_cleanup(struct adapter *sc, struct toepcb *toep, u_int status) in live_tid_failure_cleanup() argument
1790 MPASS(!(toep->flags & TPF_SYNQE)); in live_tid_failure_cleanup()
1792 inp = toep->inp; in live_tid_failure_cleanup()
1793 CURVNET_SET(toep->vnet); in live_tid_failure_cleanup()
1797 toep->flags |= TPF_ABORT_SHUTDOWN; in live_tid_failure_cleanup()
1807 final_cpl_received(toep); in live_tid_failure_cleanup()
1819 struct toepcb *toep; in cleanup_stranded_tids() local
1837 while ((toep = TAILQ_FIRST(&tlist)) != NULL) { in cleanup_stranded_tids()
1838 TAILQ_REMOVE(&tlist, toep, link); in cleanup_stranded_tids()
1839 MPASS(toep->tid >= 0); /* stale, was kept around for debug */ in cleanup_stranded_tids()
1840 toep->tid = -1; in cleanup_stranded_tids()
1841 act_open_failure_cleanup(sc, toep, EHOSTUNREACH); in cleanup_stranded_tids()
1848 while ((toep = TAILQ_FIRST(&tlist)) != NULL) { in cleanup_stranded_tids()
1849 TAILQ_REMOVE(&tlist, toep, link); in cleanup_stranded_tids()
1850 MPASS(toep->tid >= 0); /* stale, was kept around for debug */ in cleanup_stranded_tids()
1851 toep->tid = -1; in cleanup_stranded_tids()
1852 live_tid_failure_cleanup(sc, toep, ECONNABORTED); in cleanup_stranded_tids()
2005 struct toepcb *toep; in stop_atids() local
2032 toep = lookup_atid(sc, atid); in stop_atids()
2033 if ((uintptr_t)toep >= (uintptr_t)&t->atid_tab[0] && in stop_atids()
2034 (uintptr_t)toep < (uintptr_t)&t->atid_tab[t->natids]) in stop_atids()
2036 if (__predict_false(toep == NULL)) in stop_atids()
2038 MPASS(toep->tid == atid); in stop_atids()
2039 MPASS(toep->incarnation == sc->incarnation); in stop_atids()
2045 __func__, atid, toep->incarnation); in stop_atids()
2046 free_atid(sc, toep->tid); in stop_atids()
2048 toep->tid = -1; in stop_atids()
2051 toep->flags &= ~TPF_IN_TOEP_LIST; in stop_atids()
2052 TAILQ_REMOVE(&td->toep_list, toep, link); in stop_atids()
2053 TAILQ_INSERT_TAIL(&td->stranded_atids, toep, link); in stop_atids()
2063 struct toepcb *toep; in stop_tids() local
2076 TAILQ_FOREACH(toep, &td->toep_list, link) { in stop_tids()
2077 MPASS(sc->incarnation == toep->incarnation); in stop_tids()
2078 MPASS(toep->tid >= 0); in stop_tids()
2079 MPASS(toep == lookup_tid(sc, toep->tid)); in stop_tids()
2082 __func__, toep->tid, toep->incarnation); in stop_tids()
2083 remove_tid(sc, toep->tid, toep->ce ? 2 : 1); in stop_tids()
2086 toep->tid = -1; in stop_tids()
2089 toep->flags &= ~TPF_IN_TOEP_LIST; in stop_tids()
2180 struct toepcb *toep = tp->t_toe; in t4_ctloutput_tom() local
2198 return (t4_enable_ddp_rcv(so, toep)); in t4_ctloutput_tom()
2209 struct toepcb *toep = tp->t_toe; in t4_aio_queue_tom() local
2219 if (ulp_mode(toep) == ULP_MODE_TCPDDP || in t4_aio_queue_tom()
2220 ulp_mode(toep) == ULP_MODE_NONE) { in t4_aio_queue_tom()