15190f052SMike Marciniszyn /* SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) */ 25190f052SMike Marciniszyn /* 35190f052SMike Marciniszyn * Copyright(c) 2018 Intel Corporation. 45190f052SMike Marciniszyn * 55190f052SMike Marciniszyn */ 65190f052SMike Marciniszyn #ifndef HFI1_TID_RDMA_H 75190f052SMike Marciniszyn #define HFI1_TID_RDMA_H 85190f052SMike Marciniszyn 9838b6fd2SKaike Wan #include <linux/circ_buf.h> 10838b6fd2SKaike Wan #include "common.h" 11838b6fd2SKaike Wan 12838b6fd2SKaike Wan /* Add a convenience helper */ 13838b6fd2SKaike Wan #define CIRC_ADD(val, add, size) (((val) + (add)) & ((size) - 1)) 14838b6fd2SKaike Wan #define CIRC_NEXT(val, size) CIRC_ADD(val, 1, size) 15838b6fd2SKaike Wan #define CIRC_PREV(val, size) CIRC_ADD(val, -1, size) 16838b6fd2SKaike Wan 17f1ab4efaSKaike Wan #define TID_RDMA_MIN_SEGMENT_SIZE BIT(18) /* 256 KiB (for now) */ 18d22a207dSKaike Wan #define TID_RDMA_MAX_SEGMENT_SIZE BIT(18) /* 256 KiB (for now) */ 19838b6fd2SKaike Wan #define TID_RDMA_MAX_PAGES (BIT(18) >> PAGE_SHIFT) 20d22a207dSKaike Wan 21a0b34f75SKaike Wan /* 22a0b34f75SKaike Wan * Bit definitions for priv->s_flags. 23a0b34f75SKaike Wan * These bit flags overload the bit flags defined for the QP's s_flags. 24a0b34f75SKaike Wan * Due to the fact that these bit fields are used only for the QP priv 25a0b34f75SKaike Wan * s_flags, there are no collisions. 26a0b34f75SKaike Wan * 27a0b34f75SKaike Wan * HFI1_S_TID_WAIT_INTERLCK - QP is waiting for requester interlock 28a0b34f75SKaike Wan */ 2907b92370SKaike Wan /* BIT(4) reserved for RVT_S_ACK_PENDING. */ 30a0b34f75SKaike Wan #define HFI1_S_TID_WAIT_INTERLCK BIT(5) 3107b92370SKaike Wan #define HFI1_R_TID_SW_PSN BIT(19) 32a0b34f75SKaike Wan 33f5a4a95fSKaike Wan /* 34f5a4a95fSKaike Wan * Unlike regular IB RDMA VERBS, which do not require an entry 35f5a4a95fSKaike Wan * in the s_ack_queue, TID RDMA WRITE requests do because they 36f5a4a95fSKaike Wan * generate responses. 37f5a4a95fSKaike Wan * Therefore, the s_ack_queue needs to be extended by a certain 38f5a4a95fSKaike Wan * amount. The key point is that the queue needs to be extended 39f5a4a95fSKaike Wan * without letting the "user" know so they user doesn't end up 40f5a4a95fSKaike Wan * using these extra entries. 41f5a4a95fSKaike Wan */ 42f5a4a95fSKaike Wan #define HFI1_TID_RDMA_WRITE_CNT 8 43f5a4a95fSKaike Wan 44d22a207dSKaike Wan struct tid_rdma_params { 45d22a207dSKaike Wan struct rcu_head rcu_head; 46d22a207dSKaike Wan u32 qp; 47d22a207dSKaike Wan u32 max_len; 48d22a207dSKaike Wan u16 jkey; 49d22a207dSKaike Wan u8 max_read; 50d22a207dSKaike Wan u8 max_write; 51d22a207dSKaike Wan u8 timeout; 52d22a207dSKaike Wan u8 urg; 53d22a207dSKaike Wan u8 version; 54d22a207dSKaike Wan }; 55d22a207dSKaike Wan 56d22a207dSKaike Wan struct tid_rdma_qp_params { 5737356e78SKaike Wan struct work_struct trigger_work; 58d22a207dSKaike Wan struct tid_rdma_params local; 59d22a207dSKaike Wan struct tid_rdma_params __rcu *remote; 60d22a207dSKaike Wan }; 61d22a207dSKaike Wan 6237356e78SKaike Wan /* Track state for each hardware flow */ 6337356e78SKaike Wan struct tid_flow_state { 6437356e78SKaike Wan u32 generation; 6537356e78SKaike Wan u32 psn; 6637356e78SKaike Wan u32 r_next_psn; /* next PSN to be received (in TID space) */ 6737356e78SKaike Wan u8 index; 6837356e78SKaike Wan u8 last_index; 6937356e78SKaike Wan u8 flags; 7037356e78SKaike Wan }; 7137356e78SKaike Wan 72742a3826SKaike Wan enum tid_rdma_req_state { 73742a3826SKaike Wan TID_REQUEST_INACTIVE = 0, 74742a3826SKaike Wan TID_REQUEST_INIT, 75742a3826SKaike Wan TID_REQUEST_INIT_RESEND, 76742a3826SKaike Wan TID_REQUEST_ACTIVE, 77742a3826SKaike Wan TID_REQUEST_RESEND, 78742a3826SKaike Wan TID_REQUEST_RESEND_ACTIVE, 79742a3826SKaike Wan TID_REQUEST_QUEUED, 80742a3826SKaike Wan TID_REQUEST_SYNC, 81742a3826SKaike Wan TID_REQUEST_RNR_NAK, 82742a3826SKaike Wan TID_REQUEST_COMPLETE, 83742a3826SKaike Wan }; 84742a3826SKaike Wan 85838b6fd2SKaike Wan struct tid_rdma_request { 86838b6fd2SKaike Wan struct rvt_qp *qp; 87838b6fd2SKaike Wan struct hfi1_ctxtdata *rcd; 88838b6fd2SKaike Wan union { 89838b6fd2SKaike Wan struct rvt_swqe *swqe; 90838b6fd2SKaike Wan struct rvt_ack_entry *ack; 91838b6fd2SKaike Wan } e; 92838b6fd2SKaike Wan 93838b6fd2SKaike Wan struct tid_rdma_flow *flows; /* array of tid flows */ 9407b92370SKaike Wan struct rvt_sge_state ss; /* SGE state for TID RDMA requests */ 95838b6fd2SKaike Wan u16 n_flows; /* size of the flow buffer window */ 96838b6fd2SKaike Wan u16 setup_head; /* flow index we are setting up */ 97838b6fd2SKaike Wan u16 clear_tail; /* flow index we are clearing */ 98838b6fd2SKaike Wan u16 flow_idx; /* flow index most recently set up */ 9907b92370SKaike Wan u16 acked_tail; 100838b6fd2SKaike Wan 101838b6fd2SKaike Wan u32 seg_len; 102d0d564a1SKaike Wan u32 total_len; 103d0d564a1SKaike Wan u32 r_flow_psn; /* IB PSN of next segment start */ 104742a3826SKaike Wan u32 s_next_psn; /* IB PSN of next segment start for read */ 105838b6fd2SKaike Wan 106d0d564a1SKaike Wan u32 total_segs; /* segments required to complete a request */ 107742a3826SKaike Wan u32 cur_seg; /* index of current segment */ 108d0d564a1SKaike Wan u32 comp_seg; /* index of last completed segment */ 109d0d564a1SKaike Wan u32 ack_seg; /* index of last ack'ed segment */ 11007b92370SKaike Wan u32 alloc_seg; /* index of next segment to be allocated */ 111838b6fd2SKaike Wan u32 isge; /* index of "current" sge */ 112742a3826SKaike Wan u32 ack_pending; /* num acks pending for this request */ 113742a3826SKaike Wan 114742a3826SKaike Wan enum tid_rdma_req_state state; 115838b6fd2SKaike Wan }; 116838b6fd2SKaike Wan 117838b6fd2SKaike Wan /* 118838b6fd2SKaike Wan * When header suppression is used, PSNs associated with a "flow" are 119838b6fd2SKaike Wan * relevant (and not the PSNs maintained by verbs). Track per-flow 120838b6fd2SKaike Wan * PSNs here for a TID RDMA segment. 121838b6fd2SKaike Wan * 122838b6fd2SKaike Wan */ 123838b6fd2SKaike Wan struct flow_state { 124838b6fd2SKaike Wan u32 flags; 125838b6fd2SKaike Wan u32 resp_ib_psn; /* The IB PSN of the response for this flow */ 126838b6fd2SKaike Wan u32 generation; /* generation of flow */ 127838b6fd2SKaike Wan u32 spsn; /* starting PSN in TID space */ 128838b6fd2SKaike Wan u32 lpsn; /* last PSN in TID space */ 129838b6fd2SKaike Wan u32 r_next_psn; /* next PSN to be received (in TID space) */ 130742a3826SKaike Wan 131742a3826SKaike Wan /* For tid rdma read */ 132742a3826SKaike Wan u32 ib_spsn; /* starting PSN in Verbs space */ 133742a3826SKaike Wan u32 ib_lpsn; /* last PSn in Verbs space */ 134838b6fd2SKaike Wan }; 135838b6fd2SKaike Wan 136838b6fd2SKaike Wan struct tid_rdma_pageset { 137838b6fd2SKaike Wan dma_addr_t addr : 48; /* Only needed for the first page */ 138838b6fd2SKaike Wan u8 idx: 8; 139838b6fd2SKaike Wan u8 count : 7; 140838b6fd2SKaike Wan u8 mapped: 1; 141838b6fd2SKaike Wan }; 142838b6fd2SKaike Wan 143838b6fd2SKaike Wan /** 144838b6fd2SKaike Wan * kern_tid_node - used for managing TID's in TID groups 145838b6fd2SKaike Wan * 146838b6fd2SKaike Wan * @grp_idx: rcd relative index to tid_group 147838b6fd2SKaike Wan * @map: grp->map captured prior to programming this TID group in HW 148838b6fd2SKaike Wan * @cnt: Only @cnt of available group entries are actually programmed 149838b6fd2SKaike Wan */ 150838b6fd2SKaike Wan struct kern_tid_node { 151838b6fd2SKaike Wan struct tid_group *grp; 152838b6fd2SKaike Wan u8 map; 153838b6fd2SKaike Wan u8 cnt; 154838b6fd2SKaike Wan }; 155838b6fd2SKaike Wan 156838b6fd2SKaike Wan /* Overall info for a TID RDMA segment */ 157838b6fd2SKaike Wan struct tid_rdma_flow { 158838b6fd2SKaike Wan /* 159838b6fd2SKaike Wan * While a TID RDMA segment is being transferred, it uses a QP number 160838b6fd2SKaike Wan * from the "KDETH section of QP numbers" (which is different from the 161838b6fd2SKaike Wan * QP number that originated the request). Bits 11-15 of these QP 162838b6fd2SKaike Wan * numbers identify the "TID flow" for the segment. 163838b6fd2SKaike Wan */ 164838b6fd2SKaike Wan struct flow_state flow_state; 165838b6fd2SKaike Wan struct tid_rdma_request *req; 166d0d564a1SKaike Wan u32 tid_qpn; 167d0d564a1SKaike Wan u32 tid_offset; 168838b6fd2SKaike Wan u32 length; 169742a3826SKaike Wan u32 sent; 170838b6fd2SKaike Wan u8 tnode_cnt; 171838b6fd2SKaike Wan u8 tidcnt; 172742a3826SKaike Wan u8 tid_idx; 173838b6fd2SKaike Wan u8 idx; 174838b6fd2SKaike Wan u8 npagesets; 175838b6fd2SKaike Wan u8 npkts; 176742a3826SKaike Wan u8 pkt; 177838b6fd2SKaike Wan struct kern_tid_node tnode[TID_RDMA_MAX_PAGES]; 178838b6fd2SKaike Wan struct tid_rdma_pageset pagesets[TID_RDMA_MAX_PAGES]; 179838b6fd2SKaike Wan u32 tid_entry[TID_RDMA_MAX_PAGES]; 180838b6fd2SKaike Wan }; 181838b6fd2SKaike Wan 18207b92370SKaike Wan enum tid_rnr_nak_state { 18307b92370SKaike Wan TID_RNR_NAK_INIT = 0, 18407b92370SKaike Wan TID_RNR_NAK_SEND, 18507b92370SKaike Wan TID_RNR_NAK_SENT, 18607b92370SKaike Wan }; 18707b92370SKaike Wan 188d22a207dSKaike Wan bool tid_rdma_conn_req(struct rvt_qp *qp, u64 *data); 189d22a207dSKaike Wan bool tid_rdma_conn_reply(struct rvt_qp *qp, u64 data); 190d22a207dSKaike Wan bool tid_rdma_conn_resp(struct rvt_qp *qp, u64 *data); 191d22a207dSKaike Wan void tid_rdma_conn_error(struct rvt_qp *qp); 192d22a207dSKaike Wan void tid_rdma_opfn_init(struct rvt_qp *qp, struct tid_rdma_params *p); 193d22a207dSKaike Wan 194d22a207dSKaike Wan int hfi1_kern_exp_rcv_init(struct hfi1_ctxtdata *rcd, int reinit); 195838b6fd2SKaike Wan int hfi1_kern_exp_rcv_setup(struct tid_rdma_request *req, 196838b6fd2SKaike Wan struct rvt_sge_state *ss, bool *last); 197838b6fd2SKaike Wan int hfi1_kern_exp_rcv_clear(struct tid_rdma_request *req); 198838b6fd2SKaike Wan void hfi1_kern_exp_rcv_clear_all(struct tid_rdma_request *req); 199838b6fd2SKaike Wan void __trdma_clean_swqe(struct rvt_qp *qp, struct rvt_swqe *wqe); 200838b6fd2SKaike Wan 201838b6fd2SKaike Wan /** 202838b6fd2SKaike Wan * trdma_clean_swqe - clean flows for swqe if large send queue 203838b6fd2SKaike Wan * @qp: the qp 204838b6fd2SKaike Wan * @wqe: the send wqe 205838b6fd2SKaike Wan */ 206838b6fd2SKaike Wan static inline void trdma_clean_swqe(struct rvt_qp *qp, struct rvt_swqe *wqe) 207838b6fd2SKaike Wan { 208838b6fd2SKaike Wan if (!wqe->priv) 209838b6fd2SKaike Wan return; 210838b6fd2SKaike Wan __trdma_clean_swqe(qp, wqe); 211838b6fd2SKaike Wan } 212d22a207dSKaike Wan 2139905bf06SKaike Wan void hfi1_kern_read_tid_flow_free(struct rvt_qp *qp); 2149905bf06SKaike Wan 2155190f052SMike Marciniszyn int hfi1_qp_priv_init(struct rvt_dev_info *rdi, struct rvt_qp *qp, 2165190f052SMike Marciniszyn struct ib_qp_init_attr *init_attr); 21748a615dcSKaike Wan void hfi1_qp_priv_tid_free(struct rvt_dev_info *rdi, struct rvt_qp *qp); 2185190f052SMike Marciniszyn 21937356e78SKaike Wan void hfi1_tid_rdma_flush_wait(struct rvt_qp *qp); 22037356e78SKaike Wan 22137356e78SKaike Wan int hfi1_kern_setup_hw_flow(struct hfi1_ctxtdata *rcd, struct rvt_qp *qp); 22237356e78SKaike Wan void hfi1_kern_clear_hw_flow(struct hfi1_ctxtdata *rcd, struct rvt_qp *qp); 22337356e78SKaike Wan void hfi1_kern_init_ctxt_generations(struct hfi1_ctxtdata *rcd); 22437356e78SKaike Wan 2252f16a696SKaike Wan struct cntr_entry; 2262f16a696SKaike Wan u64 hfi1_access_sw_tid_wait(const struct cntr_entry *entry, 2272f16a696SKaike Wan void *context, int vl, int mode, u64 data); 2282f16a696SKaike Wan 229742a3826SKaike Wan u32 hfi1_build_tid_rdma_read_packet(struct rvt_swqe *wqe, 230742a3826SKaike Wan struct ib_other_headers *ohdr, 231742a3826SKaike Wan u32 *bth1, u32 *bth2, u32 *len); 232742a3826SKaike Wan u32 hfi1_build_tid_rdma_read_req(struct rvt_qp *qp, struct rvt_swqe *wqe, 233742a3826SKaike Wan struct ib_other_headers *ohdr, u32 *bth1, 234742a3826SKaike Wan u32 *bth2, u32 *len); 235d0d564a1SKaike Wan void hfi1_rc_rcv_tid_rdma_read_req(struct hfi1_packet *packet); 2361db21b50SKaike Wan u32 hfi1_build_tid_rdma_read_resp(struct rvt_qp *qp, struct rvt_ack_entry *e, 2371db21b50SKaike Wan struct ib_other_headers *ohdr, u32 *bth0, 2381db21b50SKaike Wan u32 *bth1, u32 *bth2, u32 *len, bool *last); 2399905bf06SKaike Wan void hfi1_rc_rcv_tid_rdma_read_resp(struct hfi1_packet *packet); 2409905bf06SKaike Wan bool hfi1_handle_kdeth_eflags(struct hfi1_ctxtdata *rcd, 2419905bf06SKaike Wan struct hfi1_pportdata *ppd, 2429905bf06SKaike Wan struct hfi1_packet *packet); 243b126078eSKaike Wan void hfi1_tid_rdma_restart_req(struct rvt_qp *qp, struct rvt_swqe *wqe, 244b126078eSKaike Wan u32 *bth2); 24524b11923SKaike Wan void hfi1_qp_kern_exp_rcv_clear_all(struct rvt_qp *qp); 246a0b34f75SKaike Wan bool hfi1_tid_rdma_wqe_interlock(struct rvt_qp *qp, struct rvt_swqe *wqe); 247742a3826SKaike Wan 248f1ab4efaSKaike Wan void setup_tid_rdma_wqe(struct rvt_qp *qp, struct rvt_swqe *wqe); 249f1ab4efaSKaike Wan static inline void hfi1_setup_tid_rdma_wqe(struct rvt_qp *qp, 250f1ab4efaSKaike Wan struct rvt_swqe *wqe) 251f1ab4efaSKaike Wan { 252f1ab4efaSKaike Wan if (wqe->priv && 253f1ab4efaSKaike Wan wqe->wr.opcode == IB_WR_RDMA_READ && 254f1ab4efaSKaike Wan wqe->length >= TID_RDMA_MIN_SEGMENT_SIZE) 255f1ab4efaSKaike Wan setup_tid_rdma_wqe(qp, wqe); 256f1ab4efaSKaike Wan } 257f1ab4efaSKaike Wan 258c098bbb0SKaike Wan u32 hfi1_build_tid_rdma_write_req(struct rvt_qp *qp, struct rvt_swqe *wqe, 259c098bbb0SKaike Wan struct ib_other_headers *ohdr, 260c098bbb0SKaike Wan u32 *bth1, u32 *bth2, u32 *len); 26107b92370SKaike Wan 26207b92370SKaike Wan void hfi1_compute_tid_rdma_flow_wt(void); 26307b92370SKaike Wan 26407b92370SKaike Wan void hfi1_rc_rcv_tid_rdma_write_req(struct hfi1_packet *packet); 26507b92370SKaike Wan 266*38d46d36SKaike Wan u32 hfi1_build_tid_rdma_write_resp(struct rvt_qp *qp, struct rvt_ack_entry *e, 267*38d46d36SKaike Wan struct ib_other_headers *ohdr, u32 *bth1, 268*38d46d36SKaike Wan u32 bth2, u32 *len, 269*38d46d36SKaike Wan struct rvt_sge_state **ss); 270*38d46d36SKaike Wan 2715190f052SMike Marciniszyn #endif /* HFI1_TID_RDMA_H */ 272